Merge branch 'fix/p0-reader-numeric' into fix/phase0-correctness
This commit is contained in:
osobh
@@ -0,0 +1,374 @@
|
||||
//! Numeric conversions on read, checked against h5py/libhdf5.
|
||||
//!
|
||||
//! h5py writes each file and prints what libhdf5 converts the data to
|
||||
//! (`Dataset.astype`); the typed readers must return the same values.
|
||||
//! Skipped when python3 with h5py is unavailable, unless
|
||||
//! `CLAWHDF5_REQUIRE_INTEROP=1`.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::process::Command;
|
||||
|
||||
use clawhdf5::File;
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn interop_required() -> bool {
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
||||
}
|
||||
|
||||
fn python_available() -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", "import h5py, numpy"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
macro_rules! skip_if_no_python {
|
||||
() => {
|
||||
if !python_available() {
|
||||
assert!(
|
||||
!interop_required(),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
||||
);
|
||||
eprintln!("SKIP: python3 with h5py not available");
|
||||
return;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// Run `script` (which writes the file at `path`) and return its stdout as
|
||||
/// `key -> values`, one `key v1 v2 ...` line per key.
|
||||
fn run_python(script: &str) -> HashMap<String, Vec<String>> {
|
||||
let output = Command::new(python())
|
||||
.args(["-c", script])
|
||||
.output()
|
||||
.expect("failed to run python");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"python failed:\n{}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
String::from_utf8_lossy(&output.stdout)
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
let mut words = line.split_whitespace().map(str::to_string);
|
||||
Some((words.next()?, words.collect()))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn parse<T: std::str::FromStr>(values: &[String]) -> Vec<T>
|
||||
where
|
||||
T::Err: std::fmt::Debug,
|
||||
{
|
||||
values.iter().map(|v| v.parse().unwrap()).collect()
|
||||
}
|
||||
|
||||
/// Python prelude: `emit(key, array)` prints one line of integers.
|
||||
const PRELUDE: &str = r#"
|
||||
import h5py, numpy as np
|
||||
def emit(key, arr):
|
||||
print(key, *[int(v) for v in np.asarray(arr).ravel()])
|
||||
"#;
|
||||
|
||||
#[test]
|
||||
fn float_dataset_read_as_integers_converts_like_libhdf5() {
|
||||
// read_i32/read_i64/read_u64 on a float dataset used to return the raw
|
||||
// IEEE bit patterns (1.5 read as i64 was 4609434218613702656).
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("float_to_int.h5");
|
||||
let script = format!(
|
||||
r#"{PRELUDE}
|
||||
vals = [1.5, -2.75, 3e9, 1e300, -1e300, -0.5, 0.0, 7.99, np.inf, -np.inf, 1e19, -1e19]
|
||||
with h5py.File("{path}", "w") as f:
|
||||
for name, dt in (("f8", "<f8"), ("f8be", ">f8"), ("f4", "<f4")):
|
||||
f.create_dataset(name, data=np.array(vals).astype(dt))
|
||||
# libhdf5's half conversions are not saturating (an infinite half becomes
|
||||
# INT_MIN whatever its sign, a negative one wraps as u64), so the half
|
||||
# case stays finite and its u64 read is checked separately below.
|
||||
f.create_dataset("f2", data=np.array([1.5, -2.75, -0.5, 0.0, 7.99, 65504, -65504], "<f2"))
|
||||
with h5py.File("{path}", "r") as f:
|
||||
for name in ("f8", "f8be", "f4", "f2"):
|
||||
d = f[name]
|
||||
emit(name + ":i32", d.astype("<i4")[()])
|
||||
emit(name + ":i64", d.astype("<i8")[()])
|
||||
emit(name + ":u64", d.astype("<u8")[()])
|
||||
"#,
|
||||
path = path.display()
|
||||
);
|
||||
let expected = run_python(&script);
|
||||
|
||||
let file = File::open(&path).unwrap();
|
||||
for name in ["f8", "f8be", "f4", "f2"] {
|
||||
let ds = file.dataset(name).unwrap();
|
||||
assert_eq!(
|
||||
ds.read_i32().unwrap(),
|
||||
parse::<i32>(&expected[&format!("{name}:i32")]),
|
||||
"{name} as i32"
|
||||
);
|
||||
assert_eq!(
|
||||
ds.read_i64().unwrap(),
|
||||
parse::<i64>(&expected[&format!("{name}:i64")]),
|
||||
"{name} as i64"
|
||||
);
|
||||
if name != "f2" {
|
||||
assert_eq!(
|
||||
ds.read_u64().unwrap(),
|
||||
parse::<u64>(&expected[&format!("{name}:u64")]),
|
||||
"{name} as u64"
|
||||
);
|
||||
}
|
||||
}
|
||||
// Negative values saturate at 0 rather than wrapping.
|
||||
let f2 = file.dataset("f2").unwrap();
|
||||
assert_eq!(f2.read_u64().unwrap(), vec![1, 0, 0, 0, 7, 65504, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn integer_reads_saturate_out_of_range_values_like_libhdf5() {
|
||||
// Narrowing reads used to keep the low bits (i64 2^40+5 read as i32 was
|
||||
// 5, u64::MAX read as i64 was -1) and signed-to-unsigned reads wrapped
|
||||
// (-1 read as u64 was 4294967295).
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("int_narrowing.h5");
|
||||
let script = format!(
|
||||
r#"{PRELUDE}
|
||||
data = {{
|
||||
"i8": np.array([2**40 + 5, -(2**35), 7, -1, 2**63 - 1, -(2**63)], "<i8"),
|
||||
"i8be": np.array([2**40 + 5, -(2**35), 7, -1], ">i8"),
|
||||
"u8": np.array([2**64 - 1, 2**63, 5, 0], "<u8"),
|
||||
"u4": np.array([2**32 - 1, 2**31, 2**31 - 1, 3], "<u4"),
|
||||
"i4": np.array([-1, 5, -(2**31), 2**31 - 1], "<i4"),
|
||||
"i2be": np.array([-300, 300, -1], ">i2"),
|
||||
"u1": np.array([255, 0, 128], "u1"),
|
||||
}}
|
||||
with h5py.File("{path}", "w") as f:
|
||||
for name, arr in data.items():
|
||||
f.create_dataset(name, data=arr)
|
||||
with h5py.File("{path}", "r") as f:
|
||||
for name in data:
|
||||
d = f[name]
|
||||
emit(name + ":i32", d.astype("<i4")[()])
|
||||
emit(name + ":i64", d.astype("<i8")[()])
|
||||
emit(name + ":u64", d.astype("<u8")[()])
|
||||
"#,
|
||||
path = path.display()
|
||||
);
|
||||
let expected = run_python(&script);
|
||||
|
||||
let file = File::open(&path).unwrap();
|
||||
for name in ["i8", "i8be", "u8", "u4", "i4", "i2be", "u1"] {
|
||||
let ds = file.dataset(name).unwrap();
|
||||
assert_eq!(
|
||||
ds.read_i32().unwrap(),
|
||||
parse::<i32>(&expected[&format!("{name}:i32")]),
|
||||
"{name} as i32"
|
||||
);
|
||||
assert_eq!(
|
||||
ds.read_i64().unwrap(),
|
||||
parse::<i64>(&expected[&format!("{name}:i64")]),
|
||||
"{name} as i64"
|
||||
);
|
||||
// libhdf5 wraps a negative big-endian i64 read as little-endian u64
|
||||
// (it only byte-swaps when the sizes match and the order differs);
|
||||
// every other signed-to-unsigned read saturates at 0, so do that.
|
||||
if name != "i8be" {
|
||||
assert_eq!(
|
||||
ds.read_u64().unwrap(),
|
||||
parse::<u64>(&expected[&format!("{name}:u64")]),
|
||||
"{name} as u64"
|
||||
);
|
||||
}
|
||||
}
|
||||
let i8be = file.dataset("i8be").unwrap();
|
||||
assert_eq!(i8be.read_u64().unwrap(), vec![(1 << 40) + 5, 0, 7, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn floats_decode_by_their_datatype_fields() {
|
||||
// Every 2-byte float used to decode as IEEE half, so bfloat16 1.5 read as
|
||||
// 1.9375 and +inf as NaN; 1-byte FP8 floats were refused.
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("float_layouts.h5");
|
||||
let script = format!(
|
||||
r#"{PRELUDE}
|
||||
def custom(base, fields, bias, size):
|
||||
# fields: (sign pos, exponent pos, exponent size, mantissa pos, mantissa size)
|
||||
t = base.copy()
|
||||
t.set_fields(*fields)
|
||||
t.set_ebias(bias)
|
||||
t.set_precision(size * 8)
|
||||
t.set_size(size)
|
||||
return t
|
||||
|
||||
def write(f, name, ftype, raw):
|
||||
raw = np.ascontiguousarray(raw)
|
||||
space = h5py.h5s.create_simple(raw.shape)
|
||||
ds = h5py.h5d.create(f.id, name.encode(), ftype, space)
|
||||
ds.write(h5py.h5s.ALL, h5py.h5s.ALL, raw, mtype=ftype)
|
||||
|
||||
# bfloat16: 1.5, -2.25, +inf, 0, 3.140625, 1, -0, smallest subnormal,
|
||||
# largest finite, NaN
|
||||
bf16 = np.array([0x3FC0, 0xC010, 0x7F80, 0x0000, 0x4049, 0x3F80, 0x8000, 0x0001,
|
||||
0x7F7F, 0x7FC1], "<u2")
|
||||
# 8-bit patterns, all 256 of them
|
||||
fp8 = np.arange(256, dtype="u1")
|
||||
types = {{
|
||||
"bf16_le": (custom(h5py.h5t.IEEE_F32LE, (15, 7, 8, 0, 7), 127, 2), bf16),
|
||||
"bf16_be": (custom(h5py.h5t.IEEE_F32BE, (15, 7, 8, 0, 7), 127, 2), bf16.byteswap()),
|
||||
"e4m3": (custom(h5py.h5t.IEEE_F32LE, (7, 3, 4, 0, 3), 7, 1), fp8),
|
||||
"e5m2": (custom(h5py.h5t.IEEE_F32LE, (7, 2, 5, 0, 2), 15, 1), fp8),
|
||||
}}
|
||||
with h5py.File("{path}", "w") as f:
|
||||
for name, (t, raw) in types.items():
|
||||
write(f, name, t, raw)
|
||||
vals = np.array([1.5, -2.25, np.inf, -np.inf, 0.0, -0.0, 6e-8, 65504, 1e-40, 1e300])
|
||||
f.create_dataset("f2_be", data=vals.astype(">f2"))
|
||||
f.create_dataset("f4_be", data=vals.astype(">f4"))
|
||||
f.create_dataset("f8_le", data=vals.astype("<f8"))
|
||||
with h5py.File("{path}", "r") as f:
|
||||
for name in list(types) + ["f2_be", "f4_be", "f8_le"]:
|
||||
v = f[name].astype("<f8")[()]
|
||||
# NaN payloads differ between converters; compare NaN as one value.
|
||||
v[np.isnan(v)] = np.nan
|
||||
emit(name, v.view("<u8"))
|
||||
"#,
|
||||
path = path.display()
|
||||
);
|
||||
let expected = run_python(&script);
|
||||
|
||||
let canonical = |v: f64| if v.is_nan() { f64::NAN } else { v };
|
||||
let file = File::open(&path).unwrap();
|
||||
for name in [
|
||||
"bf16_le", "bf16_be", "e4m3", "e5m2", "f2_be", "f4_be", "f8_le",
|
||||
] {
|
||||
let ds = file.dataset(name).unwrap();
|
||||
let want: Vec<u64> = parse(&expected[name]);
|
||||
let got: Vec<u64> = ds
|
||||
.read_f64()
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|v| canonical(v).to_bits())
|
||||
.collect();
|
||||
assert_eq!(got, want, "{name} as f64");
|
||||
// f32 reads agree too (every value here is exact in f32 except the
|
||||
// f64 dataset, which rounds like `as f32`).
|
||||
let got32: Vec<u32> = ds
|
||||
.read_f32()
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|v| if v.is_nan() { f32::NAN } else { v }.to_bits())
|
||||
.collect();
|
||||
let want32: Vec<u32> = want
|
||||
.iter()
|
||||
.map(|&b| canonical(f64::from_bits(b)) as f32)
|
||||
.map(|v| if v.is_nan() { f32::NAN } else { v }.to_bits())
|
||||
.collect();
|
||||
assert_eq!(got32, want32, "{name} as f32");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enum_and_bool_datasets_read_as_their_integer_values() {
|
||||
// Enumerations (h5py stores bool as an enum of int8) were refused by the
|
||||
// numeric readers with a type mismatch.
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("enums.h5");
|
||||
let script = format!(
|
||||
r#"{PRELUDE}
|
||||
with h5py.File("{path}", "w") as f:
|
||||
f.create_dataset("bool", data=np.array([True, False, True]))
|
||||
e = h5py.enum_dtype({{"RED": 0, "GREEN": 7, "BLUE": -3}}, basetype=">i2")
|
||||
f.create_dataset("enum_i2be", data=np.array([0, 7, -3, 7], ">i2"), dtype=e)
|
||||
e = h5py.enum_dtype({{"LOW": 0, "HIGH": 200}}, basetype="u1")
|
||||
f.create_dataset("enum_u1", data=np.array([200, 0, 200], "u1"), dtype=e)
|
||||
e = h5py.enum_dtype({{"A": -(2**40), "B": 2**40}}, basetype="<i8")
|
||||
f.create_dataset("enum_i8", data=np.array([2**40, -(2**40)], "<i8"), dtype=e)
|
||||
with h5py.File("{path}", "r") as f:
|
||||
for name in ("bool", "enum_i2be", "enum_u1", "enum_i8"):
|
||||
emit(name, np.asarray(f[name][()]).astype(np.int64))
|
||||
"#,
|
||||
path = path.display()
|
||||
);
|
||||
let expected = run_python(&script);
|
||||
|
||||
let file = File::open(&path).unwrap();
|
||||
for name in ["bool", "enum_i2be", "enum_u1", "enum_i8"] {
|
||||
let ds = file.dataset(name).unwrap();
|
||||
let want: Vec<i64> = parse(&expected[name]);
|
||||
assert_eq!(ds.read_i64().unwrap(), want, "{name} as i64");
|
||||
let want_f64: Vec<f64> = want.iter().map(|&v| v as f64).collect();
|
||||
assert_eq!(ds.read_f64().unwrap(), want_f64, "{name} as f64");
|
||||
}
|
||||
let bools = file.dataset("bool").unwrap();
|
||||
assert_eq!(bools.read_u64().unwrap(), vec![1, 0, 1]);
|
||||
assert_eq!(bools.read_i32().unwrap(), vec![1, 0, 1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vl_sequences_of_wide_base_types_read_whole() {
|
||||
// read_vl_bytes took the sequence's element count as its byte length, so
|
||||
// [1, 2, 3] as VL int32 came back as 3 bytes instead of 12.
|
||||
use clawhdf5::Selection;
|
||||
use clawhdf5_format::datatype::{Datatype, DatatypeByteOrder};
|
||||
use clawhdf5_format::vl_data::read_vl_bytes;
|
||||
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("vlen.h5");
|
||||
let script = format!(
|
||||
r#"{PRELUDE}
|
||||
data = {{
|
||||
"i4": (h5py.vlen_dtype("<i4"), [[1, 2, 3], [], [-5], list(range(40))]),
|
||||
"f8": (h5py.vlen_dtype("<f8"), [[1.5, -2.0], [0.25]]),
|
||||
"u1": (h5py.vlen_dtype("u1"), [[1, 2, 255], []]),
|
||||
}}
|
||||
with h5py.File("{path}", "w") as f:
|
||||
for name, (dt, seqs) in data.items():
|
||||
ds = f.create_dataset(name, (len(seqs),), dtype=dt)
|
||||
for i, s in enumerate(seqs):
|
||||
ds[i] = s
|
||||
with h5py.File("{path}", "r") as f:
|
||||
for name in data:
|
||||
for i, s in enumerate(f[name][()]):
|
||||
emit(f"{{name}}:{{i}}", np.frombuffer(np.asarray(s).tobytes(), "u1"))
|
||||
"#,
|
||||
path = path.display()
|
||||
);
|
||||
let expected = run_python(&script);
|
||||
|
||||
let file = File::open(&path).unwrap();
|
||||
let sb = file.superblock();
|
||||
for (name, count) in [("i4", 4), ("f8", 2), ("u1", 2)] {
|
||||
let raw = file
|
||||
.dataset(name)
|
||||
.unwrap()
|
||||
.read_selection(&Selection::All)
|
||||
.unwrap();
|
||||
let got =
|
||||
read_vl_bytes(file.as_bytes(), &raw, count, sb.offset_size, sb.length_size).unwrap();
|
||||
let want: Vec<Vec<u8>> = (0..count)
|
||||
.map(|i| parse(expected.get(&format!("{name}:{i}")).unwrap()))
|
||||
.collect();
|
||||
assert_eq!(got, want, "{name}");
|
||||
if name == "i4" {
|
||||
let i32_le = Datatype::FixedPoint {
|
||||
size: 4,
|
||||
byte_order: DatatypeByteOrder::LittleEndian,
|
||||
signed: true,
|
||||
bit_offset: 0,
|
||||
bit_precision: 32,
|
||||
};
|
||||
let values = clawhdf5_format::data_read::read_as_i64(&got[0], &i32_le).unwrap();
|
||||
assert_eq!(values, vec![1, 2, 3]);
|
||||
assert_eq!(got[3].len(), 40 * 4);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user