feat: write complex numbers, incl. HDF5 2.0 native complex (class 11)
- Datatype::Complex serializes class 11 version 5 byte-identically to
libhdf5 2.2.0; containers holding it are written as version 5.
- DatasetBuilder::with_complex_f32/f64_data (h5py's {r, i} compound,
default) and with_native_complex_f32/f64_data (class 11, opt-in);
make_(native_)complex_f32/f64_type for attributes.
- Dataset::read_complex_f64/f32 read either form.
- Python create_dataset accepts complex64/complex128 (compound form).
- Parsing unchanged: class 11 still surfaces as {r, i}.
- Tests vs h5py 3.16 / libhdf5 2.0.0 and h5dump 2.2.0; docs.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -383,6 +383,169 @@ else:
|
||||
assert_eq!((fields[1].name.as_str(), im), ("i", vec![2.0, 4.0]));
|
||||
}
|
||||
|
||||
/// Complex data both ways we write it: h5py's compound `{r, i}` (the
|
||||
/// default, readable everywhere) and HDF5 2.0's native complex type (class
|
||||
/// 11, opt-in). h5py on libhdf5 2.0+ must read the native datasets and
|
||||
/// attributes as numpy `complex64`/`complex128`, and a compound holding a
|
||||
/// native complex member (written as datatype version 5, as libhdf5 does).
|
||||
/// Skips the native checks when h5py's libhdf5 predates 2.0; also runs
|
||||
/// h5dump 2.x when `CLAWHDF5_H5DUMP2` names one.
|
||||
#[test]
|
||||
#[ignore = "requires Python h5py module"]
|
||||
fn h5py_reads_our_complex_datasets_and_attributes() {
|
||||
use clawhdf5_format::type_builders::{
|
||||
make_complex_f64_type, make_native_complex_f32_type, make_native_complex_f64_type,
|
||||
};
|
||||
let path = std::env::temp_dir().join("clawhdf5_test_complex.h5");
|
||||
let native_ok = h5py_read(
|
||||
&path,
|
||||
"import h5py; print(int(getattr(h5py.get_config(), 'has_native_complex', False)))",
|
||||
) == "1";
|
||||
|
||||
let z64 = [[1.5f32, -2.0], [0.0, 3.25], [-7.0, 1.0e-3]];
|
||||
let z128 = [[1.0f64, 2.0], [-3.5, 4.0e300], [0.25, -0.0]];
|
||||
let big: Vec<[f64; 2]> = (0..600).map(|k| [k as f64, -(k as f64) / 4.0]).collect();
|
||||
let c128 = |re: f64, im: f64| [re.to_le_bytes(), im.to_le_bytes()].concat();
|
||||
|
||||
let mut fw = FileWriter::new();
|
||||
fw.create_dataset("compound128")
|
||||
.with_complex_f64_data(&z128)
|
||||
.set_attr(
|
||||
"c",
|
||||
AttrValue::Raw {
|
||||
datatype: make_complex_f64_type(),
|
||||
shape: vec![],
|
||||
data: c128(1.0, -1.0),
|
||||
},
|
||||
);
|
||||
if native_ok {
|
||||
fw.create_dataset("native64")
|
||||
.with_native_complex_f32_data(&z64)
|
||||
.set_attr(
|
||||
"c",
|
||||
AttrValue::Raw {
|
||||
datatype: make_native_complex_f64_type(),
|
||||
shape: vec![2],
|
||||
data: [c128(0.5, -1.5), c128(2.0, 3.0)].concat(),
|
||||
},
|
||||
);
|
||||
fw.create_dataset("native128")
|
||||
.with_native_complex_f64_data(&z128);
|
||||
fw.create_dataset("native_chunked")
|
||||
.with_native_complex_f64_data(&big)
|
||||
.with_shape(&[20, 30])
|
||||
.with_chunks(&[7, 16])
|
||||
.with_deflate(4);
|
||||
// A compound with a native complex member, and an array of them.
|
||||
let rec = CompoundTypeBuilder::new()
|
||||
.field("z", make_native_complex_f64_type())
|
||||
.i64_field("k")
|
||||
.build();
|
||||
let raw = [
|
||||
c128(1.0, 2.0),
|
||||
7i64.to_le_bytes().to_vec(),
|
||||
c128(-1.0, 0.5),
|
||||
(-8i64).to_le_bytes().to_vec(),
|
||||
]
|
||||
.concat();
|
||||
fw.create_dataset("records").with_compound_data(rec, raw, 2);
|
||||
let arr: Vec<u8> = [1.0f32, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]
|
||||
.iter()
|
||||
.flat_map(|v| v.to_le_bytes())
|
||||
.collect();
|
||||
fw.create_dataset("arrays")
|
||||
.with_array_data(make_native_complex_f32_type(), &[2], arr, 2);
|
||||
fw.set_root_attr(
|
||||
"zroot",
|
||||
AttrValue::Raw {
|
||||
datatype: make_native_complex_f32_type(),
|
||||
shape: vec![],
|
||||
data: [4.0f32.to_le_bytes(), (-4.0f32).to_le_bytes()].concat(),
|
||||
},
|
||||
);
|
||||
}
|
||||
std::fs::write(&path, fw.finish().unwrap()).unwrap();
|
||||
|
||||
let script = format!(
|
||||
r#"
|
||||
import h5py, json, numpy as np
|
||||
f = h5py.File('{}', 'r')
|
||||
def z(a): return [[float(np.real(v)), float(np.imag(v))] for v in np.asarray(a).ravel()]
|
||||
out = {{}}
|
||||
d = f['compound128']
|
||||
out['compound128'] = [str(d.dtype), z(d[()]), str(d.attrs['c'].dtype), z(d.attrs['c'])]
|
||||
if {native}:
|
||||
for n in ['native64', 'native128', 'native_chunked']:
|
||||
d = f[n]
|
||||
out[n] = [str(d.dtype), list(d.shape), z(d[()]), d.id.get_type().get_class()]
|
||||
a = f['native64'].attrs['c']
|
||||
out['attr'] = [str(a.dtype), z(a)]
|
||||
r = f['records'][()]
|
||||
out['records'] = [str(r.dtype['z']), z(r['z']), r['k'].tolist()]
|
||||
a = f['arrays'][()]
|
||||
out['arrays'] = [str(a.dtype), list(a.shape), z(a)]
|
||||
a = f.attrs['zroot']
|
||||
out['zroot'] = [str(a.dtype), z(a)]
|
||||
out['CLASS'] = h5py.h5t.COMPLEX
|
||||
print(json.dumps(out))
|
||||
"#,
|
||||
path.display(),
|
||||
native = if native_ok { "True" } else { "False" },
|
||||
);
|
||||
let v: serde_json::Value = serde_json::from_str(&h5py_read(&path, &script)).unwrap();
|
||||
let pairs = |p: &[[f64; 2]]| serde_json::json!(p);
|
||||
assert_eq!(v["compound128"][0], "complex128");
|
||||
assert_eq!(v["compound128"][1], pairs(&z128));
|
||||
assert_eq!(v["compound128"][2], "complex128");
|
||||
assert_eq!(v["compound128"][3], pairs(&[[1.0, -1.0]]));
|
||||
if !native_ok {
|
||||
eprintln!("HDF5 < 2.0: native complex not checked");
|
||||
return;
|
||||
}
|
||||
let z64_wide: Vec<[f64; 2]> = z64.iter().map(|p| [p[0].into(), p[1].into()]).collect();
|
||||
let class_complex = v["CLASS"].clone();
|
||||
for (name, dtype, shape, values) in [
|
||||
("native64", "complex64", vec![3], z64_wide.clone()),
|
||||
("native128", "complex128", vec![3], z128.to_vec()),
|
||||
("native_chunked", "complex128", vec![20, 30], big.clone()),
|
||||
] {
|
||||
assert_eq!(v[name][0], dtype, "{name}");
|
||||
assert_eq!(v[name][1], serde_json::json!(shape), "{name}");
|
||||
assert_eq!(v[name][2], pairs(&values), "{name}");
|
||||
// Stored as class 11, not converted from a compound.
|
||||
assert_eq!(v[name][3], class_complex, "{name}");
|
||||
}
|
||||
assert_eq!(v["attr"][0], "complex128");
|
||||
assert_eq!(v["attr"][1], pairs(&[[0.5, -1.5], [2.0, 3.0]]));
|
||||
assert_eq!(v["records"][0], "complex128");
|
||||
assert_eq!(v["records"][1], pairs(&[[1.0, 2.0], [-1.0, 0.5]]));
|
||||
assert_eq!(v["records"][2], serde_json::json!([7, -8]));
|
||||
assert_eq!(v["arrays"][0], "complex64");
|
||||
assert_eq!(v["arrays"][1], serde_json::json!([2, 2]));
|
||||
assert_eq!(
|
||||
v["arrays"][2],
|
||||
pairs(&[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]])
|
||||
);
|
||||
assert_eq!(v["zroot"][0], "complex64");
|
||||
assert_eq!(v["zroot"][1], pairs(&[[4.0, -4.0]]));
|
||||
|
||||
// h5dump from libhdf5 2.x, when available (Debian's 1.14 cannot read
|
||||
// class 11 at all).
|
||||
if let Ok(h5dump) = std::env::var("CLAWHDF5_H5DUMP2") {
|
||||
let o = std::process::Command::new(&h5dump)
|
||||
.arg(&path)
|
||||
.output()
|
||||
.expect("run h5dump");
|
||||
let text = String::from_utf8_lossy(&o.stdout);
|
||||
assert!(
|
||||
o.status.success(),
|
||||
"h5dump: {}",
|
||||
String::from_utf8_lossy(&o.stderr)
|
||||
);
|
||||
assert!(text.contains("H5T_COMPLEX"), "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires Python h5py module"]
|
||||
fn read_h5py_generated_enum() {
|
||||
|
||||
Reference in New Issue
Block a user