osobhandClaude Fable 5.1
d668e45ab5
With libver='latest', a chunked dataset with two or more unlimited dimensions indexes its chunks with a v2 B-tree (layout v4, index type 5). Reading one failed with "unsupported chunked layout version=4, index_type=Some(5)". read_btree_v2_chunks decodes record types 10 (address + scaled offsets) and 11 (address, stored size, filter mask, scaled offsets). The width of the stored-size field is taken from the record size the tree header declares rather than re-deriving the library's formula. Scaled offsets are multiplied back by the chunk dimensions with overflow checks. The chunk-index dispatch existed four times (uncached, cached, sweep and indexed readers). The three copies outside list_chunks now call it, so every read path — and fill-value handling and partial reads — supports every index type from one place. h5py interop test: plain, gzip+shuffle, a 2500-chunk tree with internal nodes, a sparse dataset with a fill value, and a strided hyperslab. Co-Authored-By: Claude Fable 5.1 <[email protected]>
982 lines
34 KiB
Rust
982 lines
34 KiB
Rust
//! Bidirectional interop tests between clawhdf5 and h5py (Python).
|
|
//!
|
|
//! Tests are skipped if python3 or h5py are not available.
|
|
|
|
use std::process::Command;
|
|
|
|
use clawhdf5::{AttrValue, CompoundTypeBuilder, DType, File, FileBuilder};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helpers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// When `CLAWHDF5_REQUIRE_INTEROP=1` (set in CI), a missing Python dependency
|
|
/// is a test failure instead of a silent skip.
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
fn python_available() -> bool {
|
|
Command::new("python3")
|
|
.args(["-c", "import h5py; print(h5py.__version__)"])
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
macro_rules! skip_if_no_python {
|
|
() => {
|
|
if !python_available() {
|
|
assert!(
|
|
!interop_required(),
|
|
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
|
);
|
|
eprintln!("SKIP: python3 with h5py not available");
|
|
return;
|
|
}
|
|
};
|
|
}
|
|
|
|
/// Run a Python script and panic if it fails.
|
|
fn run_python(script: &str) {
|
|
let output = Command::new("python3")
|
|
.args(["-c", script])
|
|
.output()
|
|
.expect("failed to run python3");
|
|
if !output.status.success() {
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
|
panic!("Python script failed:\nSTDOUT: {stdout}\nSTDERR: {stderr}");
|
|
}
|
|
}
|
|
|
|
/// Run a Python script and return stdout as a trimmed string.
|
|
fn run_python_output(script: &str) -> String {
|
|
let output = Command::new("python3")
|
|
.args(["-c", script])
|
|
.output()
|
|
.expect("failed to run python3");
|
|
if !output.status.success() {
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
panic!("Python script failed:\nSTDERR: {stderr}");
|
|
}
|
|
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
|
}
|
|
|
|
// ===========================================================================
|
|
// Part A: clawhdf5 writes -> h5py reads
|
|
// ===========================================================================
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A1. Write f64 dataset -> h5py reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_f64_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("f64_test.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let data = vec![1.5, 2.5, 3.5, -4.5, 0.0];
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("values").with_f64_data(&data);
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np, json
|
|
with h5py.File("{path_str}", "r") as f:
|
|
ds = f["values"]
|
|
assert ds.dtype == np.float64, f"expected float64, got {{ds.dtype}}"
|
|
assert ds.shape == (5,), f"expected (5,), got {{ds.shape}}"
|
|
vals = ds[()].tolist()
|
|
print(json.dumps(vals))
|
|
"#
|
|
);
|
|
let output = run_python_output(&script);
|
|
let vals: Vec<f64> = serde_json_minimal_parse(&output);
|
|
assert_eq!(vals, data);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A2. Write string dataset -> h5py reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_strings_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("strings_test.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
// clawhdf5 doesn't have a direct with_string_data, so we write f64 and
|
|
// test string attributes instead
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("data").with_f64_data(&[1.0]);
|
|
b.set_attr("title", AttrValue::String("Hello World".into()));
|
|
b.set_attr(
|
|
"tags",
|
|
AttrValue::StringArray(vec!["alpha".into(), "beta".into()]),
|
|
);
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "r") as f:
|
|
title = f.attrs["title"]
|
|
if isinstance(title, bytes):
|
|
title = title.decode()
|
|
print(title)
|
|
tags = f.attrs["tags"]
|
|
tag_list = [t.decode() if isinstance(t, bytes) else t for t in tags]
|
|
print(",".join(tag_list))
|
|
"#
|
|
);
|
|
let output = run_python_output(&script);
|
|
let lines: Vec<&str> = output.lines().collect();
|
|
assert_eq!(lines[0], "Hello World");
|
|
assert_eq!(lines[1], "alpha,beta");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A3. Write chunked+compressed dataset -> h5py reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_chunked_compressed_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("chunked_test.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let data: Vec<f64> = (0..1000).map(|i| i as f64 * 0.01).collect();
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("compressed")
|
|
.with_f64_data(&data)
|
|
.with_chunks(&[100])
|
|
.with_deflate(6);
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "r") as f:
|
|
ds = f["compressed"]
|
|
assert ds.chunks is not None, "expected chunked dataset"
|
|
assert ds.compression == "gzip", f"expected gzip, got {{ds.compression}}"
|
|
vals = ds[()].tolist()
|
|
assert len(vals) == 1000, f"expected 1000 values, got {{len(vals)}}"
|
|
# Check first and last
|
|
assert abs(vals[0] - 0.0) < 1e-10
|
|
assert abs(vals[999] - 9.99) < 1e-10
|
|
print("OK")
|
|
"#
|
|
);
|
|
let output = run_python_output(&script);
|
|
assert_eq!(output, "OK");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A4. Write groups with attributes -> h5py reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_groups_attrs_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("groups_test.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let mut b = FileBuilder::new();
|
|
b.set_attr("file_version", AttrValue::I64(3));
|
|
|
|
let mut g = b.create_group("experiment");
|
|
g.set_attr("run_id", AttrValue::I64(42));
|
|
g.set_attr("description", AttrValue::String("test run".into()));
|
|
g.create_dataset("measurements")
|
|
.with_f64_data(&[10.0, 20.0, 30.0]);
|
|
b.add_group(g.finish());
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "r") as f:
|
|
assert f.attrs["file_version"] == 3
|
|
g = f["experiment"]
|
|
assert g.attrs["run_id"] == 42
|
|
desc = g.attrs["description"]
|
|
if isinstance(desc, bytes):
|
|
desc = desc.decode()
|
|
assert desc == "test run", f"got {{desc}}"
|
|
vals = g["measurements"][()].tolist()
|
|
assert vals == [10.0, 20.0, 30.0]
|
|
print("OK")
|
|
"#
|
|
);
|
|
let output = run_python_output(&script);
|
|
assert_eq!(output, "OK");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A5. Write compound type -> h5py reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_compound_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("compound_test.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let dt = CompoundTypeBuilder::new()
|
|
.f64_field("x")
|
|
.f64_field("y")
|
|
.i32_field("id")
|
|
.build();
|
|
|
|
let mut raw = Vec::new();
|
|
for &(x, y, id) in &[(1.0f64, 2.0f64, 10i32), (3.0, 4.0, 20)] {
|
|
raw.extend_from_slice(&x.to_le_bytes());
|
|
raw.extend_from_slice(&y.to_le_bytes());
|
|
raw.extend_from_slice(&id.to_le_bytes());
|
|
}
|
|
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("points").with_compound_data(dt, raw, 2);
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "r") as f:
|
|
ds = f["points"]
|
|
assert ds.dtype.names == ("x", "y", "id"), f"got {{ds.dtype.names}}"
|
|
assert len(ds) == 2
|
|
row0 = ds[0]
|
|
assert abs(float(row0["x"]) - 1.0) < 1e-10
|
|
assert abs(float(row0["y"]) - 2.0) < 1e-10
|
|
assert int(row0["id"]) == 10
|
|
row1 = ds[1]
|
|
assert abs(float(row1["x"]) - 3.0) < 1e-10
|
|
assert int(row1["id"]) == 20
|
|
print("OK")
|
|
"#
|
|
);
|
|
let output = run_python_output(&script);
|
|
assert_eq!(output, "OK");
|
|
}
|
|
|
|
// ===========================================================================
|
|
// Part B: h5py writes -> clawhdf5 reads
|
|
// ===========================================================================
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// B6. h5py creates f64 dataset -> clawhdf5 reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn h5py_writes_f64_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("h5py_f64.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w") as f:
|
|
f.create_dataset("values", data=np.array([1.5, 2.5, 3.5, -4.5, 0.0], dtype=np.float64))
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let ds = file.dataset("values").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::F64);
|
|
assert_eq!(ds.shape().unwrap(), vec![5]);
|
|
assert_eq!(ds.read_f64().unwrap(), vec![1.5, 2.5, 3.5, -4.5, 0.0]);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// B7. h5py creates string dataset -> clawhdf5 reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn h5py_writes_strings_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("h5py_strings.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
// Use fixed-length strings (S10) since clawhdf5 doesn't support vlen strings yet
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w") as f:
|
|
dt = np.dtype("S10")
|
|
f.create_dataset("names", data=np.array([b"alice", b"bob", b"charlie"], dtype=dt))
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let ds = file.dataset("names").unwrap();
|
|
let strings = ds.read_string().unwrap();
|
|
assert_eq!(strings, vec!["alice", "bob", "charlie"]);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// B8. h5py creates chunked+compressed+shuffled -> clawhdf5 reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn h5py_writes_chunked_compressed_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("h5py_chunked.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
data = np.arange(2000, dtype=np.float64) * 0.1
|
|
with h5py.File("{path_str}", "w") as f:
|
|
f.create_dataset("compressed", data=data, chunks=(200,),
|
|
compression="gzip", compression_opts=4, shuffle=True)
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let ds = file.dataset("compressed").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::F64);
|
|
assert_eq!(ds.shape().unwrap(), vec![2000]);
|
|
let values = ds.read_f64().unwrap();
|
|
assert_eq!(values.len(), 2000);
|
|
assert!((values[0] - 0.0).abs() < 1e-10);
|
|
assert!((values[1999] - 199.9).abs() < 1e-10);
|
|
assert!((values[1000] - 100.0).abs() < 1e-10);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// B9. h5py creates nested groups with attrs -> clawhdf5 reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn h5py_writes_nested_groups_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("h5py_groups.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w") as f:
|
|
f.attrs["file_attr"] = "root_value"
|
|
g1 = f.create_group("level1")
|
|
g1.attrs["g1_attr"] = 42
|
|
g2 = g1.create_group("level2")
|
|
g2.attrs["g2_attr"] = 3.25
|
|
g2.create_dataset("deep_data", data=np.array([100.0, 200.0], dtype=np.float64))
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
|
|
// Root attrs
|
|
let root_attrs = file.root().attrs().unwrap();
|
|
assert!(matches!(root_attrs.get("file_attr"), Some(AttrValue::String(s)) if s == "root_value"));
|
|
|
|
// Level1 group
|
|
let g1 = file.group("level1").unwrap();
|
|
let g1_attrs = g1.attrs().unwrap();
|
|
assert!(matches!(g1_attrs.get("g1_attr"), Some(AttrValue::I64(42))));
|
|
|
|
// Level2 group (nested)
|
|
let g2 = file.group("level1/level2").unwrap();
|
|
let g2_attrs = g2.attrs().unwrap();
|
|
assert!(
|
|
matches!(g2_attrs.get("g2_attr"), Some(AttrValue::F64(v)) if (*v - 3.25).abs() < 1e-10)
|
|
);
|
|
|
|
// Deep dataset
|
|
let ds = file.dataset("level1/level2/deep_data").unwrap();
|
|
assert_eq!(ds.read_f64().unwrap(), vec![100.0, 200.0]);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// B10. h5py creates multiple dtypes -> clawhdf5 reads all -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn h5py_writes_multiple_dtypes_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("h5py_dtypes.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w") as f:
|
|
f.create_dataset("int8", data=np.array([1, -1, 127], dtype=np.int8))
|
|
f.create_dataset("int16", data=np.array([256, -256], dtype=np.int16))
|
|
f.create_dataset("int32", data=np.array([100000, -100000], dtype=np.int32))
|
|
f.create_dataset("int64", data=np.array([2**40, -2**40], dtype=np.int64))
|
|
f.create_dataset("float32", data=np.array([1.5, 2.5], dtype=np.float32))
|
|
f.create_dataset("float64", data=np.array([3.25, 2.75], dtype=np.float64))
|
|
f.create_dataset("uint8", data=np.array([0, 128, 255], dtype=np.uint8))
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
|
|
// int8 -> read as i32 (upcast)
|
|
let ds = file.dataset("int8").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::I8);
|
|
let vals = ds.read_i32().unwrap();
|
|
assert_eq!(vals, vec![1, -1, 127]);
|
|
|
|
// int16
|
|
let ds = file.dataset("int16").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::I16);
|
|
let vals = ds.read_i32().unwrap();
|
|
assert_eq!(vals, vec![256, -256]);
|
|
|
|
// int32
|
|
let ds = file.dataset("int32").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::I32);
|
|
assert_eq!(ds.read_i32().unwrap(), vec![100000, -100000]);
|
|
|
|
// int64
|
|
let ds = file.dataset("int64").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::I64);
|
|
assert_eq!(ds.read_i64().unwrap(), vec![1i64 << 40, -(1i64 << 40)]);
|
|
|
|
// float32
|
|
let ds = file.dataset("float32").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::F32);
|
|
assert_eq!(ds.read_f32().unwrap(), vec![1.5f32, 2.5]);
|
|
|
|
// float64
|
|
let ds = file.dataset("float64").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::F64);
|
|
let vals = ds.read_f64().unwrap();
|
|
assert!((vals[0] - 3.25).abs() < 1e-10);
|
|
assert!((vals[1] - 2.75).abs() < 1e-10);
|
|
|
|
// uint8
|
|
let ds = file.dataset("uint8").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::U8);
|
|
let vals = ds.read_u64().unwrap();
|
|
assert_eq!(vals, vec![0, 128, 255]);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Minimal JSON parsing (avoid serde dependency)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn serde_json_minimal_parse(s: &str) -> Vec<f64> {
|
|
// Parse a JSON array of numbers like "[1.5, 2.5, 3.5]"
|
|
let s = s.trim();
|
|
let s = s.strip_prefix('[').unwrap_or(s);
|
|
let s = s.strip_suffix(']').unwrap_or(s);
|
|
s.split(',')
|
|
.map(|v| v.trim().parse::<f64>().unwrap())
|
|
.collect()
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A_dense. Write a group with many links (dense storage) -> h5py reads all
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_dense_group_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("dense_group.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
// 20 links exceeds the compact threshold (8) -> dense fractal-heap storage.
|
|
let mut b = FileBuilder::new();
|
|
let mut g = b.create_group("big");
|
|
for i in 0..20 {
|
|
g.create_dataset(&format!("dataset_{i:03}"))
|
|
.with_i32_data(&[i, i * 10]);
|
|
}
|
|
b.add_group(g.finish());
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "r") as f:
|
|
big = f["big"]
|
|
names = sorted(big.keys())
|
|
assert len(names) == 20, f"expected 20 links, got {{len(names)}}"
|
|
for i in range(20):
|
|
v = big[f"dataset_{{i:03}}"][()].tolist()
|
|
assert v == [i, i*10], f"link {{i}} = {{v}}"
|
|
print("OK")
|
|
"#
|
|
);
|
|
let out = run_python_output(&script);
|
|
assert_eq!(out, "OK");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A_multiblock. Write a multi-direct-block fractal heap -> h5py reads
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_multiblock_heap_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("multiblock.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
// ~1600 dense attributes overflow a single 64KiB fractal-heap direct block.
|
|
let mut b = FileBuilder::new();
|
|
let mut g = b.create_group("g");
|
|
for i in 0..1600i64 {
|
|
g.set_attr(&format!("attribute_number_{i:05}"), AttrValue::I64(i * 2));
|
|
}
|
|
g.create_dataset("d").with_i32_data(&[1]);
|
|
b.add_group(g.finish());
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "r") as f:
|
|
a = f["g"].attrs
|
|
assert len(a) == 1600, f"expected 1600 attrs, got {{len(a)}}"
|
|
for i in (0, 1, 999, 1599):
|
|
v = int(a[f"attribute_number_{{i:05}}"])
|
|
assert v == i*2, f"attr {{i}} = {{v}}"
|
|
print("OK")
|
|
"#
|
|
);
|
|
assert_eq!(run_python_output(&script), "OK");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// h5py uses committed (named) datatypes -> clawhdf5 reads
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// A dataset or attribute created from a committed datatype stores only a
|
|
/// *shared message* reference to it. These used to be parsed as the datatype
|
|
/// itself (yielding `Time { size: 0 }` and unreadable data) and the attribute
|
|
/// was silently dropped.
|
|
#[test]
|
|
fn h5py_committed_datatypes_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
for (tag, kwargs) in [("default", ""), ("latest", ", libver='latest'")] {
|
|
let path = dir.path().join(format!("committed_{tag}.h5"));
|
|
let path_str = path.display().to_string();
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w"{kwargs}) as f:
|
|
f["f8type"] = np.dtype("<f8")
|
|
f["cmpd"] = np.dtype([("a", "<i4"), ("b", "<f8")])
|
|
f.create_dataset("d", data=np.arange(6, dtype="<f8"), dtype=f["f8type"])
|
|
f.create_dataset("c", data=np.array([(1, 2.5), (3, 4.5)], dtype=f["cmpd"].dtype), dtype=f["cmpd"])
|
|
f["d"].attrs.create("att", 7.0, dtype=f["f8type"])
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let d = file.dataset("d").unwrap();
|
|
assert_eq!(d.dtype().unwrap(), DType::F64, "{tag}");
|
|
assert_eq!(
|
|
d.read_f64().unwrap(),
|
|
vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0],
|
|
"{tag}"
|
|
);
|
|
assert!(
|
|
matches!(d.attrs().unwrap().get("att"), Some(AttrValue::F64(v)) if *v == 7.0),
|
|
"{tag}: attribute with a committed datatype"
|
|
);
|
|
let c = file.dataset("c").unwrap();
|
|
assert_eq!(
|
|
c.dtype().unwrap(),
|
|
DType::Compound(vec![("a".into(), DType::I32), ("b".into(), DType::F64)]),
|
|
"{tag}"
|
|
);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// h5py writes sparse / never-written datasets -> clawhdf5 applies fill values
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Parse h5py's `print(arr.ravel().tolist())` output for integer data.
|
|
fn parse_int_list(s: &str) -> Vec<i32> {
|
|
s.trim()
|
|
.trim_matches(|c| c == '[' || c == ']')
|
|
.split(',')
|
|
.filter(|t| !t.trim().is_empty())
|
|
.map(|t| t.trim().parse().unwrap())
|
|
.collect()
|
|
}
|
|
|
|
/// Storage HDF5 never allocated must read as the dataset's fill value. These
|
|
/// used to read as zeros (silently wrong for a non-zero fill value) or fail
|
|
/// outright (`NoDataAllocated`) for a dataset that was never written.
|
|
#[test]
|
|
fn h5py_fill_values_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
for (tag, kwargs) in [("default", ""), ("latest", ", libver='latest'")] {
|
|
let path = dir.path().join(format!("fill_{tag}.h5"));
|
|
let path_str = path.display().to_string();
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w"{kwargs}) as f:
|
|
d = f.create_dataset("partial", shape=(20,), dtype="<i4", chunks=(5,), fillvalue=-1)
|
|
d[0:5] = np.arange(5)
|
|
f.create_dataset("never", shape=(4,), dtype="<i4", fillvalue=25)
|
|
f.create_dataset("never_chunked", shape=(6,), dtype="<i4", chunks=(3,), fillvalue=9)
|
|
f.create_dataset("default_fill", shape=(3,), dtype="<i4")
|
|
g = f.create_dataset("gz", shape=(20,), dtype="<i4", chunks=(5,), fillvalue=7, compression="gzip")
|
|
g[10:15] = 1
|
|
s = f.create_dataset("sparse2d", shape=(5, 7), dtype="<i4", chunks=(2, 3), fillvalue=-3)
|
|
s[2:4, 3:6] = 8
|
|
s[4, 6] = 5
|
|
with h5py.File("{path_str}", "r") as f:
|
|
for name in ["partial", "never", "never_chunked", "default_fill", "gz", "sparse2d"]:
|
|
print(name, f[name][...].ravel().tolist())
|
|
print("slab", f["sparse2d"][1:5, 2:7].ravel().tolist())
|
|
"#
|
|
);
|
|
let expected: std::collections::HashMap<String, Vec<i32>> = run_python_output(&script)
|
|
.lines()
|
|
.map(|line| {
|
|
let (name, list) = line.split_once(' ').unwrap();
|
|
(name.to_string(), parse_int_list(list))
|
|
})
|
|
.collect();
|
|
|
|
let file = File::open(&path).unwrap();
|
|
for name in [
|
|
"partial",
|
|
"never",
|
|
"never_chunked",
|
|
"default_fill",
|
|
"gz",
|
|
"sparse2d",
|
|
] {
|
|
assert_eq!(
|
|
file.dataset(name).unwrap().read_i32().unwrap(),
|
|
expected[name],
|
|
"{tag}/{name}"
|
|
);
|
|
}
|
|
// A hyperslab straddling allocated and unallocated chunks.
|
|
let slab = clawhdf5_format::selection::Selection::Hyperslab {
|
|
start: vec![1, 2],
|
|
stride: vec![1, 1],
|
|
count: vec![4, 5],
|
|
block: vec![1, 1],
|
|
};
|
|
assert_eq!(
|
|
file.dataset("sparse2d")
|
|
.unwrap()
|
|
.read_i32_selection(&slab)
|
|
.unwrap(),
|
|
expected["slab"],
|
|
"{tag}/sparse2d hyperslab"
|
|
);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// h5py writes soft / external links and external raw data -> clawhdf5
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Soft links are followed (absolute, relative, through groups, with a cycle
|
|
/// guard). Things this reader does not follow — external links, and datasets
|
|
/// whose raw data lives in another file — are explicit errors. They used to
|
|
/// surface as a misleading `PathNotFound`, and external raw data could read
|
|
/// back as fill values.
|
|
#[test]
|
|
fn h5py_links_clawhdf5_resolves_or_refuses() {
|
|
use clawhdf5::Error;
|
|
use clawhdf5_format::error::FormatError;
|
|
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let dir_str = dir.path().display().to_string();
|
|
for (tag, kwargs) in [("default", ""), ("latest", ", libver='latest'")] {
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np, os
|
|
os.chdir("{dir_str}")
|
|
with h5py.File("other_{tag}.h5", "w"{kwargs}) as o:
|
|
o.create_dataset("remote", data=np.arange(3, dtype="<i4"))
|
|
with h5py.File("links_{tag}.h5", "w"{kwargs}) as f:
|
|
f.create_dataset("real", data=np.arange(4, dtype="<i4"))
|
|
g = f.create_group("grp")
|
|
g.create_dataset("inner", data=np.arange(2, dtype="<i4"))
|
|
g["rel"] = h5py.SoftLink("inner")
|
|
f["soft"] = h5py.SoftLink("/real")
|
|
f["soft_grp"] = h5py.SoftLink("/grp")
|
|
f["dangling"] = h5py.SoftLink("/nope")
|
|
f["loop_a"] = h5py.SoftLink("/loop_b")
|
|
f["loop_b"] = h5py.SoftLink("/loop_a")
|
|
f["ext"] = h5py.ExternalLink("other_{tag}.h5", "/remote")
|
|
f.create_dataset("extdata", shape=(4,), dtype="<i4", external=[("raw_{tag}.bin", 0, 16)])
|
|
f["extdata"][...] = np.array([11, 22, 33, 44], dtype="<i4")
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(dir.path().join(format!("links_{tag}.h5"))).unwrap();
|
|
let read = |path: &str| file.dataset(path).and_then(|d| d.read_i32());
|
|
assert_eq!(read("soft").unwrap(), vec![0, 1, 2, 3], "{tag}");
|
|
assert_eq!(read("soft_grp/inner").unwrap(), vec![0, 1], "{tag}");
|
|
assert_eq!(
|
|
read("grp/rel").unwrap(),
|
|
vec![0, 1],
|
|
"{tag}: relative target"
|
|
);
|
|
assert_eq!(
|
|
read("soft_grp/rel").unwrap(),
|
|
vec![0, 1],
|
|
"{tag}: link via link"
|
|
);
|
|
|
|
assert!(
|
|
matches!(read("dangling"), Err(Error::Format(FormatError::PathNotFound(p))) if p == "nope"),
|
|
"{tag}: dangling link names its missing target"
|
|
);
|
|
assert!(
|
|
matches!(
|
|
read("loop_a"),
|
|
Err(Error::Format(FormatError::NestingDepthExceeded))
|
|
),
|
|
"{tag}: link cycle"
|
|
);
|
|
assert!(
|
|
matches!(
|
|
read("ext"),
|
|
Err(Error::Format(FormatError::ExternalLinkUnsupported { ref object_path, .. }))
|
|
if object_path == "/remote"
|
|
),
|
|
"{tag}: external link"
|
|
);
|
|
assert!(
|
|
matches!(
|
|
read("extdata"),
|
|
Err(Error::Format(FormatError::ExternalDataFilesUnsupported))
|
|
),
|
|
"{tag}: external raw data must not read as fill values"
|
|
);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Attribute fidelity: nothing is dropped, unsigned stays unsigned
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// `attrs()` used to omit, without any error, every attribute whose datatype
|
|
/// had no `AttrValue` variant — including every Python `bool` (stored as an
|
|
/// enum) — and to wrap unsigned 64-bit arrays into negative `i64`s.
|
|
#[test]
|
|
fn h5py_attribute_kinds_are_all_reported() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("attrs_kinds.h5");
|
|
let path_str = path.display().to_string();
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w") as f:
|
|
d = f.create_dataset("d", data=np.arange(3))
|
|
a = d.attrs
|
|
a["float"] = 1.5; a["int"] = 7; a["str"] = "hello"; a["int_array"] = np.arange(4)
|
|
a["str_list"] = ["a", "bc"]
|
|
a["uint64_array"] = np.array([2**63, 1], dtype="u8")
|
|
a["bool_true"] = True
|
|
a["bool_false"] = False
|
|
a["bool_array"] = np.array([True, False, True])
|
|
a["complex"] = 1 + 2j
|
|
a["compound"] = np.array([(1, 2.5)], dtype=[("a", "<i4"), ("b", "<f8")])
|
|
a["object_ref"] = d.ref
|
|
a["color"] = np.array(2, dtype=h5py.enum_dtype({{"RED": 0, "GREEN": 1, "BLUE": 2}}, basetype="i1"))
|
|
print(len(a))
|
|
"#
|
|
);
|
|
let written: usize = run_python_output(&script).trim().parse().unwrap();
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let attrs = file.dataset("d").unwrap().attrs().unwrap();
|
|
assert_eq!(
|
|
attrs.len(),
|
|
written,
|
|
"every attribute is reported: {attrs:?}"
|
|
);
|
|
|
|
// Booleans decode as 0/1.
|
|
assert!(matches!(attrs["bool_true"], AttrValue::I64(1)));
|
|
assert!(matches!(attrs["bool_false"], AttrValue::I64(0)));
|
|
assert!(matches!(&attrs["bool_array"], AttrValue::I64Array(v) if v == &[1, 0, 1]));
|
|
// Unsigned stays unsigned.
|
|
assert!(matches!(&attrs["uint64_array"], AttrValue::U64Array(v) if v == &[1u64 << 63, 1]));
|
|
|
|
// A general enum is not a boolean: kept verbatim, member names intact.
|
|
match &attrs["color"] {
|
|
AttrValue::Raw { datatype, data, .. } => {
|
|
assert_eq!(data, &[2]);
|
|
assert!(format!("{datatype:?}").contains("BLUE"));
|
|
}
|
|
other => panic!("color: {other:?}"),
|
|
}
|
|
// A compound attribute is kept verbatim and decodes against its datatype.
|
|
match &attrs["compound"] {
|
|
AttrValue::Raw {
|
|
datatype,
|
|
shape,
|
|
data,
|
|
} => {
|
|
assert_eq!(shape, &[1]);
|
|
let fields = clawhdf5_format::data_read::read_compound_fields(data, datatype).unwrap();
|
|
assert_eq!(fields[0].name, "a");
|
|
let b =
|
|
clawhdf5_format::data_read::read_as_f64(&fields[1].raw_data, &fields[1].datatype)
|
|
.unwrap();
|
|
assert_eq!(b, vec![2.5]);
|
|
}
|
|
other => panic!("compound: {other:?}"),
|
|
}
|
|
for name in ["complex", "object_ref"] {
|
|
assert!(
|
|
matches!(attrs[name], AttrValue::Raw { .. }),
|
|
"{name}: {:?}",
|
|
attrs[name]
|
|
);
|
|
}
|
|
}
|
|
|
|
/// `Raw` and `U64Array` are writable, so an attribute read from one file can
|
|
/// be stored in another unchanged.
|
|
#[test]
|
|
fn clawhdf5_writes_raw_and_unsigned_attrs_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let src = dir.path().join("src.h5");
|
|
let dst = dir.path().join("dst.h5");
|
|
let (src_str, dst_str) = (src.display().to_string(), dst.display().to_string());
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{src_str}", "w") as f:
|
|
d = f.create_dataset("d", data=np.arange(3))
|
|
d.attrs["compound"] = np.array([(1, 2.5), (3, 4.5)], dtype=[("a", "<i4"), ("b", "<f8")])
|
|
"#
|
|
));
|
|
let compound = File::open(&src)
|
|
.unwrap()
|
|
.dataset("d")
|
|
.unwrap()
|
|
.attrs()
|
|
.unwrap()["compound"]
|
|
.clone();
|
|
|
|
let mut builder = FileBuilder::new();
|
|
let ds = builder.create_dataset("d");
|
|
ds.with_f64_data(&[1.0]);
|
|
ds.set_attr("compound", compound);
|
|
ds.set_attr("big", AttrValue::U64Array(vec![u64::MAX, 0, 1 << 63]));
|
|
builder.write(&dst).unwrap();
|
|
|
|
let out = run_python_output(&format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{dst_str}", "r") as f:
|
|
a = f["d"].attrs
|
|
print(a["compound"].tolist(), a["compound"].dtype.names, a["big"].tolist(), a["big"].dtype)
|
|
"#
|
|
));
|
|
assert_eq!(
|
|
out.trim(),
|
|
"[(1, 2.5), (3, 4.5)] ('a', 'b') [18446744073709551615, 0, 9223372036854775808] uint64"
|
|
);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// h5py writes datasets indexed by a version-2 B-tree -> clawhdf5 reads
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// With `libver='latest'`, a chunked dataset with two or more unlimited
|
|
/// dimensions indexes its chunks with a version-2 B-tree (layout v4, index
|
|
/// type 5). These used to fail with "unsupported chunked layout".
|
|
#[test]
|
|
fn h5py_btree_v2_chunk_index_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("bt2.h5");
|
|
let path_str = path.display().to_string();
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w", libver="latest") as f:
|
|
a = np.arange(60 * 45, dtype="<i4").reshape(60, 45)
|
|
f.create_dataset("plain", data=a, chunks=(7, 8), maxshape=(None, None))
|
|
f.create_dataset("gz", data=a, chunks=(7, 8), maxshape=(None, None), compression="gzip", shuffle=True)
|
|
# Enough chunks (2500) that the tree has internal nodes.
|
|
big = np.arange(200 * 200, dtype="<i4").reshape(200, 200)
|
|
f.create_dataset("deep", data=big, chunks=(4, 4), maxshape=(None, None))
|
|
s = f.create_dataset("sparse", shape=(30, 30), dtype="<i4", chunks=(5, 5), maxshape=(None, None), fillvalue=-9)
|
|
s[10:15, 20:25] = 4
|
|
s[29, 29] = 1
|
|
with h5py.File("{path_str}", "r") as f:
|
|
print("sparse", f["sparse"][...].ravel().tolist())
|
|
print("slab", f["deep"][37:141:13, 5:190:31].ravel().tolist())
|
|
"#
|
|
);
|
|
let out = run_python_output(&script);
|
|
let expected: std::collections::HashMap<&str, Vec<i32>> = out
|
|
.lines()
|
|
.map(|l| {
|
|
let (name, list) = l.split_once(' ').unwrap();
|
|
(name, parse_int_list(list))
|
|
})
|
|
.collect();
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let small: Vec<i32> = (0..60 * 45).collect();
|
|
assert_eq!(file.dataset("plain").unwrap().read_i32().unwrap(), small);
|
|
assert_eq!(file.dataset("gz").unwrap().read_i32().unwrap(), small);
|
|
let deep: Vec<i32> = (0..200 * 200).collect();
|
|
assert_eq!(file.dataset("deep").unwrap().read_i32().unwrap(), deep);
|
|
assert_eq!(
|
|
file.dataset("sparse").unwrap().read_i32().unwrap(),
|
|
expected["sparse"]
|
|
);
|
|
// Partial read through the same index: rows 37,50,..,128 x cols 5,36,..,160.
|
|
let slab = clawhdf5_format::selection::Selection::Hyperslab {
|
|
start: vec![37, 5],
|
|
stride: vec![13, 31],
|
|
count: vec![8, 6],
|
|
block: vec![1, 1],
|
|
};
|
|
assert_eq!(
|
|
file.dataset("deep")
|
|
.unwrap()
|
|
.read_i32_selection(&slab)
|
|
.unwrap(),
|
|
expected["slab"]
|
|
);
|
|
}
|