osobhandClaude Opus 5.5
5e4aa1c6bf
Two write-side bugs, both present in every release (the first at least since v2.1.0), made libhdf5 refuse files written by clawhdf5. Our own reader ignores both fields, and the interop suites only ever wrote f64 from our side, so nothing here caught them. - Every f32 dataset: "sign bit position out of bounds". The float datatype encoder hard-coded the sign bit's position (bits 8-15 of the class bit field) to 63, which is right only for f64. It is now derived from the type: bit_offset + bit_precision - 1. This covered every agent store's embeddings, norms and activation weights. - Every empty dataset: "invalid dataset size, likely file corruption". It was written with a real address and size 0, which trips libhdf5's `addr + size <= addr` overflow check. An empty contiguous dataset now gets the undefined address, as libhdf5 writes it. This covered every agent store without sessions or a knowledge graph. Agent stores are rewritten in full at each checkpoint, so they become readable at their next checkpoint on a fixed build; other files with f32 or empty datasets need rewriting. Both are recorded in docs/known-issues.md. Tests: the sign position byte for f32/f64, and h5py reading our f32 datasets (plain and chunked + deflate) bit for bit. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1429 lines
50 KiB
Rust
1429 lines
50 KiB
Rust
//! Bidirectional interop tests between clawhdf5 and h5py (Python).
|
|
//!
|
|
//! Tests are skipped if python3 or h5py are not available.
|
|
|
|
use std::process::Command;
|
|
|
|
use clawhdf5::{AttrValue, CompoundTypeBuilder, DType, File, FileBuilder};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helpers
|
|
// ---------------------------------------------------------------------------
|
|
/// The Python interpreter to drive interop checks with.
|
|
///
|
|
/// `CLAWHDF5_PYTHON` lets these run against a virtualenv holding h5py, which
|
|
/// on a PEP 668 "externally managed" system is the only place it can be
|
|
/// installed. Without it the suite silently skips, and a silent skip here is
|
|
/// how a datatype bug once reached a release.
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
/// When `CLAWHDF5_REQUIRE_INTEROP=1` (set in CI), a missing Python dependency
|
|
/// is a test failure instead of a silent skip.
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
fn python_available() -> bool {
|
|
Command::new(python())
|
|
.args(["-c", "import h5py; print(h5py.__version__)"])
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
macro_rules! skip_if_no_python {
|
|
() => {
|
|
if !python_available() {
|
|
assert!(
|
|
!interop_required(),
|
|
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
|
);
|
|
eprintln!("SKIP: python3 with h5py not available");
|
|
return;
|
|
}
|
|
};
|
|
}
|
|
|
|
/// Run a Python script and panic if it fails.
|
|
fn run_python(script: &str) {
|
|
let output = Command::new(python())
|
|
.args(["-c", script])
|
|
.output()
|
|
.expect("failed to run python3");
|
|
if !output.status.success() {
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
|
panic!("Python script failed:\nSTDOUT: {stdout}\nSTDERR: {stderr}");
|
|
}
|
|
}
|
|
|
|
/// Run a Python script and return stdout as a trimmed string.
|
|
fn run_python_output(script: &str) -> String {
|
|
let output = Command::new(python())
|
|
.args(["-c", script])
|
|
.output()
|
|
.expect("failed to run python3");
|
|
if !output.status.success() {
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
panic!("Python script failed:\nSTDERR: {stderr}");
|
|
}
|
|
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
|
}
|
|
|
|
// ===========================================================================
|
|
// Part A: clawhdf5 writes -> h5py reads
|
|
// ===========================================================================
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A1. Write f64 dataset -> h5py reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_f64_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("f64_test.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let data = vec![1.5, 2.5, 3.5, -4.5, 0.0];
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("values").with_f64_data(&data);
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np, json
|
|
with h5py.File("{path_str}", "r") as f:
|
|
ds = f["values"]
|
|
assert ds.dtype == np.float64, f"expected float64, got {{ds.dtype}}"
|
|
assert ds.shape == (5,), f"expected (5,), got {{ds.shape}}"
|
|
vals = ds[()].tolist()
|
|
print(json.dumps(vals))
|
|
"#
|
|
);
|
|
let output = run_python_output(&script);
|
|
let vals: Vec<f64> = serde_json_minimal_parse(&output);
|
|
assert_eq!(vals, data);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A2. Write string dataset -> h5py reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_strings_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("strings_test.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
// clawhdf5 doesn't have a direct with_string_data, so we write f64 and
|
|
// test string attributes instead
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("data").with_f64_data(&[1.0]);
|
|
b.set_attr("title", AttrValue::String("Hello World".into()));
|
|
b.set_attr(
|
|
"tags",
|
|
AttrValue::StringArray(vec!["alpha".into(), "beta".into()]),
|
|
);
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "r") as f:
|
|
title = f.attrs["title"]
|
|
if isinstance(title, bytes):
|
|
title = title.decode()
|
|
print(title)
|
|
tags = f.attrs["tags"]
|
|
tag_list = [t.decode() if isinstance(t, bytes) else t for t in tags]
|
|
print(",".join(tag_list))
|
|
"#
|
|
);
|
|
let output = run_python_output(&script);
|
|
let lines: Vec<&str> = output.lines().collect();
|
|
assert_eq!(lines[0], "Hello World");
|
|
assert_eq!(lines[1], "alpha,beta");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A3. Write chunked+compressed dataset -> h5py reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_chunked_compressed_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("chunked_test.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let data: Vec<f64> = (0..1000).map(|i| i as f64 * 0.01).collect();
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("compressed")
|
|
.with_f64_data(&data)
|
|
.with_chunks(&[100])
|
|
.with_deflate(6);
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "r") as f:
|
|
ds = f["compressed"]
|
|
assert ds.chunks is not None, "expected chunked dataset"
|
|
assert ds.compression == "gzip", f"expected gzip, got {{ds.compression}}"
|
|
vals = ds[()].tolist()
|
|
assert len(vals) == 1000, f"expected 1000 values, got {{len(vals)}}"
|
|
# Check first and last
|
|
assert abs(vals[0] - 0.0) < 1e-10
|
|
assert abs(vals[999] - 9.99) < 1e-10
|
|
print("OK")
|
|
"#
|
|
);
|
|
let output = run_python_output(&script);
|
|
assert_eq!(output, "OK");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A4. Write groups with attributes -> h5py reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_groups_attrs_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("groups_test.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let mut b = FileBuilder::new();
|
|
b.set_attr("file_version", AttrValue::I64(3));
|
|
|
|
let mut g = b.create_group("experiment");
|
|
g.set_attr("run_id", AttrValue::I64(42));
|
|
g.set_attr("description", AttrValue::String("test run".into()));
|
|
g.create_dataset("measurements")
|
|
.with_f64_data(&[10.0, 20.0, 30.0]);
|
|
b.add_group(g.finish());
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "r") as f:
|
|
assert f.attrs["file_version"] == 3
|
|
g = f["experiment"]
|
|
assert g.attrs["run_id"] == 42
|
|
desc = g.attrs["description"]
|
|
if isinstance(desc, bytes):
|
|
desc = desc.decode()
|
|
assert desc == "test run", f"got {{desc}}"
|
|
vals = g["measurements"][()].tolist()
|
|
assert vals == [10.0, 20.0, 30.0]
|
|
print("OK")
|
|
"#
|
|
);
|
|
let output = run_python_output(&script);
|
|
assert_eq!(output, "OK");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A5. Write compound type -> h5py reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_compound_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("compound_test.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let dt = CompoundTypeBuilder::new()
|
|
.f64_field("x")
|
|
.f64_field("y")
|
|
.i32_field("id")
|
|
.build();
|
|
|
|
let mut raw = Vec::new();
|
|
for &(x, y, id) in &[(1.0f64, 2.0f64, 10i32), (3.0, 4.0, 20)] {
|
|
raw.extend_from_slice(&x.to_le_bytes());
|
|
raw.extend_from_slice(&y.to_le_bytes());
|
|
raw.extend_from_slice(&id.to_le_bytes());
|
|
}
|
|
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("points").with_compound_data(dt, raw, 2);
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "r") as f:
|
|
ds = f["points"]
|
|
assert ds.dtype.names == ("x", "y", "id"), f"got {{ds.dtype.names}}"
|
|
assert len(ds) == 2
|
|
row0 = ds[0]
|
|
assert abs(float(row0["x"]) - 1.0) < 1e-10
|
|
assert abs(float(row0["y"]) - 2.0) < 1e-10
|
|
assert int(row0["id"]) == 10
|
|
row1 = ds[1]
|
|
assert abs(float(row1["x"]) - 3.0) < 1e-10
|
|
assert int(row1["id"]) == 20
|
|
print("OK")
|
|
"#
|
|
);
|
|
let output = run_python_output(&script);
|
|
assert_eq!(output, "OK");
|
|
}
|
|
|
|
// ===========================================================================
|
|
// Part B: h5py writes -> clawhdf5 reads
|
|
// ===========================================================================
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// B6. h5py creates f64 dataset -> clawhdf5 reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn h5py_writes_f64_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("h5py_f64.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w") as f:
|
|
f.create_dataset("values", data=np.array([1.5, 2.5, 3.5, -4.5, 0.0], dtype=np.float64))
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let ds = file.dataset("values").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::F64);
|
|
assert_eq!(ds.shape().unwrap(), vec![5]);
|
|
assert_eq!(ds.read_f64().unwrap(), vec![1.5, 2.5, 3.5, -4.5, 0.0]);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// B7. h5py creates string dataset -> clawhdf5 reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn h5py_writes_strings_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("h5py_strings.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
// Use fixed-length strings (S10) since clawhdf5 doesn't support vlen strings yet
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w") as f:
|
|
dt = np.dtype("S10")
|
|
f.create_dataset("names", data=np.array([b"alice", b"bob", b"charlie"], dtype=dt))
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let ds = file.dataset("names").unwrap();
|
|
let strings = ds.read_string().unwrap();
|
|
assert_eq!(strings, vec!["alice", "bob", "charlie"]);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// B8. h5py creates chunked+compressed+shuffled -> clawhdf5 reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn h5py_writes_chunked_compressed_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("h5py_chunked.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
data = np.arange(2000, dtype=np.float64) * 0.1
|
|
with h5py.File("{path_str}", "w") as f:
|
|
f.create_dataset("compressed", data=data, chunks=(200,),
|
|
compression="gzip", compression_opts=4, shuffle=True)
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let ds = file.dataset("compressed").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::F64);
|
|
assert_eq!(ds.shape().unwrap(), vec![2000]);
|
|
let values = ds.read_f64().unwrap();
|
|
assert_eq!(values.len(), 2000);
|
|
assert!((values[0] - 0.0).abs() < 1e-10);
|
|
assert!((values[1999] - 199.9).abs() < 1e-10);
|
|
assert!((values[1000] - 100.0).abs() < 1e-10);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// B9. h5py creates nested groups with attrs -> clawhdf5 reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn h5py_writes_nested_groups_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("h5py_groups.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w") as f:
|
|
f.attrs["file_attr"] = "root_value"
|
|
g1 = f.create_group("level1")
|
|
g1.attrs["g1_attr"] = 42
|
|
g2 = g1.create_group("level2")
|
|
g2.attrs["g2_attr"] = 3.25
|
|
g2.create_dataset("deep_data", data=np.array([100.0, 200.0], dtype=np.float64))
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
|
|
// Root attrs
|
|
let root_attrs = file.root().attrs().unwrap();
|
|
assert!(matches!(root_attrs.get("file_attr"), Some(AttrValue::String(s)) if s == "root_value"));
|
|
|
|
// Level1 group
|
|
let g1 = file.group("level1").unwrap();
|
|
let g1_attrs = g1.attrs().unwrap();
|
|
assert!(matches!(g1_attrs.get("g1_attr"), Some(AttrValue::I64(42))));
|
|
|
|
// Level2 group (nested)
|
|
let g2 = file.group("level1/level2").unwrap();
|
|
let g2_attrs = g2.attrs().unwrap();
|
|
assert!(
|
|
matches!(g2_attrs.get("g2_attr"), Some(AttrValue::F64(v)) if (*v - 3.25).abs() < 1e-10)
|
|
);
|
|
|
|
// Deep dataset
|
|
let ds = file.dataset("level1/level2/deep_data").unwrap();
|
|
assert_eq!(ds.read_f64().unwrap(), vec![100.0, 200.0]);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// B10. h5py creates multiple dtypes -> clawhdf5 reads all -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn h5py_writes_multiple_dtypes_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("h5py_dtypes.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w") as f:
|
|
f.create_dataset("int8", data=np.array([1, -1, 127], dtype=np.int8))
|
|
f.create_dataset("int16", data=np.array([256, -256], dtype=np.int16))
|
|
f.create_dataset("int32", data=np.array([100000, -100000], dtype=np.int32))
|
|
f.create_dataset("int64", data=np.array([2**40, -2**40], dtype=np.int64))
|
|
f.create_dataset("float32", data=np.array([1.5, 2.5], dtype=np.float32))
|
|
f.create_dataset("float64", data=np.array([3.25, 2.75], dtype=np.float64))
|
|
f.create_dataset("uint8", data=np.array([0, 128, 255], dtype=np.uint8))
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
|
|
// int8 -> read as i32 (upcast)
|
|
let ds = file.dataset("int8").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::I8);
|
|
let vals = ds.read_i32().unwrap();
|
|
assert_eq!(vals, vec![1, -1, 127]);
|
|
|
|
// int16
|
|
let ds = file.dataset("int16").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::I16);
|
|
let vals = ds.read_i32().unwrap();
|
|
assert_eq!(vals, vec![256, -256]);
|
|
|
|
// int32
|
|
let ds = file.dataset("int32").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::I32);
|
|
assert_eq!(ds.read_i32().unwrap(), vec![100000, -100000]);
|
|
|
|
// int64
|
|
let ds = file.dataset("int64").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::I64);
|
|
assert_eq!(ds.read_i64().unwrap(), vec![1i64 << 40, -(1i64 << 40)]);
|
|
|
|
// float32
|
|
let ds = file.dataset("float32").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::F32);
|
|
assert_eq!(ds.read_f32().unwrap(), vec![1.5f32, 2.5]);
|
|
|
|
// float64
|
|
let ds = file.dataset("float64").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::F64);
|
|
let vals = ds.read_f64().unwrap();
|
|
assert!((vals[0] - 3.25).abs() < 1e-10);
|
|
assert!((vals[1] - 2.75).abs() < 1e-10);
|
|
|
|
// uint8
|
|
let ds = file.dataset("uint8").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::U8);
|
|
let vals = ds.read_u64().unwrap();
|
|
assert_eq!(vals, vec![0, 128, 255]);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Minimal JSON parsing (avoid serde dependency)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn serde_json_minimal_parse(s: &str) -> Vec<f64> {
|
|
// Parse a JSON array of numbers like "[1.5, 2.5, 3.5]"
|
|
let s = s.trim();
|
|
let s = s.strip_prefix('[').unwrap_or(s);
|
|
let s = s.strip_suffix(']').unwrap_or(s);
|
|
s.split(',')
|
|
.map(|v| v.trim().parse::<f64>().unwrap())
|
|
.collect()
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A_dense. Write a group with many links (dense storage) -> h5py reads all
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_dense_group_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("dense_group.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
// 20 links exceeds the compact threshold (8) -> dense fractal-heap storage.
|
|
let mut b = FileBuilder::new();
|
|
let mut g = b.create_group("big");
|
|
for i in 0..20 {
|
|
g.create_dataset(&format!("dataset_{i:03}"))
|
|
.with_i32_data(&[i, i * 10]);
|
|
}
|
|
b.add_group(g.finish());
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "r") as f:
|
|
big = f["big"]
|
|
names = sorted(big.keys())
|
|
assert len(names) == 20, f"expected 20 links, got {{len(names)}}"
|
|
for i in range(20):
|
|
v = big[f"dataset_{{i:03}}"][()].tolist()
|
|
assert v == [i, i*10], f"link {{i}} = {{v}}"
|
|
print("OK")
|
|
"#
|
|
);
|
|
let out = run_python_output(&script);
|
|
assert_eq!(out, "OK");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A_multiblock. Write a multi-direct-block fractal heap -> h5py reads
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_multiblock_heap_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("multiblock.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
// ~1600 dense attributes overflow a single 64KiB fractal-heap direct block.
|
|
let mut b = FileBuilder::new();
|
|
let mut g = b.create_group("g");
|
|
for i in 0..1600i64 {
|
|
g.set_attr(&format!("attribute_number_{i:05}"), AttrValue::I64(i * 2));
|
|
}
|
|
g.create_dataset("d").with_i32_data(&[1]);
|
|
b.add_group(g.finish());
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "r") as f:
|
|
a = f["g"].attrs
|
|
assert len(a) == 1600, f"expected 1600 attrs, got {{len(a)}}"
|
|
for i in (0, 1, 999, 1599):
|
|
v = int(a[f"attribute_number_{{i:05}}"])
|
|
assert v == i*2, f"attr {{i}} = {{v}}"
|
|
print("OK")
|
|
"#
|
|
);
|
|
assert_eq!(run_python_output(&script), "OK");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// h5py uses committed (named) datatypes -> clawhdf5 reads
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// A dataset or attribute created from a committed datatype stores only a
|
|
/// *shared message* reference to it. These used to be parsed as the datatype
|
|
/// itself (yielding `Time { size: 0 }` and unreadable data) and the attribute
|
|
/// was silently dropped.
|
|
#[test]
|
|
fn h5py_committed_datatypes_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
for (tag, kwargs) in [("default", ""), ("latest", ", libver='latest'")] {
|
|
let path = dir.path().join(format!("committed_{tag}.h5"));
|
|
let path_str = path.display().to_string();
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w"{kwargs}) as f:
|
|
f["f8type"] = np.dtype("<f8")
|
|
f["cmpd"] = np.dtype([("a", "<i4"), ("b", "<f8")])
|
|
f.create_dataset("d", data=np.arange(6, dtype="<f8"), dtype=f["f8type"])
|
|
f.create_dataset("c", data=np.array([(1, 2.5), (3, 4.5)], dtype=f["cmpd"].dtype), dtype=f["cmpd"])
|
|
f["d"].attrs.create("att", 7.0, dtype=f["f8type"])
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let d = file.dataset("d").unwrap();
|
|
assert_eq!(d.dtype().unwrap(), DType::F64, "{tag}");
|
|
assert_eq!(
|
|
d.read_f64().unwrap(),
|
|
vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0],
|
|
"{tag}"
|
|
);
|
|
assert!(
|
|
matches!(d.attrs().unwrap().get("att"), Some(AttrValue::F64(v)) if *v == 7.0),
|
|
"{tag}: attribute with a committed datatype"
|
|
);
|
|
let c = file.dataset("c").unwrap();
|
|
assert_eq!(
|
|
c.dtype().unwrap(),
|
|
DType::Compound(vec![("a".into(), DType::I32), ("b".into(), DType::F64)]),
|
|
"{tag}"
|
|
);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// h5py writes sparse / never-written datasets -> clawhdf5 applies fill values
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Parse h5py's `print(arr.ravel().tolist())` output for integer data.
|
|
fn parse_int_list(s: &str) -> Vec<i32> {
|
|
s.trim()
|
|
.trim_matches(|c| c == '[' || c == ']')
|
|
.split(',')
|
|
.filter(|t| !t.trim().is_empty())
|
|
.map(|t| t.trim().parse().unwrap())
|
|
.collect()
|
|
}
|
|
|
|
/// Storage HDF5 never allocated must read as the dataset's fill value. These
|
|
/// used to read as zeros (silently wrong for a non-zero fill value) or fail
|
|
/// outright (`NoDataAllocated`) for a dataset that was never written.
|
|
#[test]
|
|
fn h5py_fill_values_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
for (tag, kwargs) in [("default", ""), ("latest", ", libver='latest'")] {
|
|
let path = dir.path().join(format!("fill_{tag}.h5"));
|
|
let path_str = path.display().to_string();
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w"{kwargs}) as f:
|
|
d = f.create_dataset("partial", shape=(20,), dtype="<i4", chunks=(5,), fillvalue=-1)
|
|
d[0:5] = np.arange(5)
|
|
f.create_dataset("never", shape=(4,), dtype="<i4", fillvalue=25)
|
|
f.create_dataset("never_chunked", shape=(6,), dtype="<i4", chunks=(3,), fillvalue=9)
|
|
f.create_dataset("default_fill", shape=(3,), dtype="<i4")
|
|
g = f.create_dataset("gz", shape=(20,), dtype="<i4", chunks=(5,), fillvalue=7, compression="gzip")
|
|
g[10:15] = 1
|
|
s = f.create_dataset("sparse2d", shape=(5, 7), dtype="<i4", chunks=(2, 3), fillvalue=-3)
|
|
s[2:4, 3:6] = 8
|
|
s[4, 6] = 5
|
|
with h5py.File("{path_str}", "r") as f:
|
|
for name in ["partial", "never", "never_chunked", "default_fill", "gz", "sparse2d"]:
|
|
print(name, f[name][...].ravel().tolist())
|
|
print("slab", f["sparse2d"][1:5, 2:7].ravel().tolist())
|
|
"#
|
|
);
|
|
let expected: std::collections::HashMap<String, Vec<i32>> = run_python_output(&script)
|
|
.lines()
|
|
.map(|line| {
|
|
let (name, list) = line.split_once(' ').unwrap();
|
|
(name.to_string(), parse_int_list(list))
|
|
})
|
|
.collect();
|
|
|
|
let file = File::open(&path).unwrap();
|
|
for name in [
|
|
"partial",
|
|
"never",
|
|
"never_chunked",
|
|
"default_fill",
|
|
"gz",
|
|
"sparse2d",
|
|
] {
|
|
assert_eq!(
|
|
file.dataset(name).unwrap().read_i32().unwrap(),
|
|
expected[name],
|
|
"{tag}/{name}"
|
|
);
|
|
}
|
|
// A hyperslab straddling allocated and unallocated chunks.
|
|
let slab = clawhdf5_format::selection::Selection::Hyperslab {
|
|
start: vec![1, 2],
|
|
stride: vec![1, 1],
|
|
count: vec![4, 5],
|
|
block: vec![1, 1],
|
|
};
|
|
assert_eq!(
|
|
file.dataset("sparse2d")
|
|
.unwrap()
|
|
.read_i32_selection(&slab)
|
|
.unwrap(),
|
|
expected["slab"],
|
|
"{tag}/sparse2d hyperslab"
|
|
);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// h5py writes soft / external links and external raw data -> clawhdf5
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Soft links are followed (absolute, relative, through groups, with a cycle
|
|
/// guard). Things this reader does not follow — external links, and datasets
|
|
/// whose raw data lives in another file — are explicit errors. They used to
|
|
/// surface as a misleading `PathNotFound`, and external raw data could read
|
|
/// back as fill values.
|
|
#[test]
|
|
fn h5py_links_clawhdf5_resolves_or_refuses() {
|
|
use clawhdf5::Error;
|
|
use clawhdf5_format::error::FormatError;
|
|
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let dir_str = dir.path().display().to_string();
|
|
for (tag, kwargs) in [("default", ""), ("latest", ", libver='latest'")] {
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np, os
|
|
os.chdir("{dir_str}")
|
|
with h5py.File("other_{tag}.h5", "w"{kwargs}) as o:
|
|
o.create_dataset("remote", data=np.arange(3, dtype="<i4"))
|
|
with h5py.File("links_{tag}.h5", "w"{kwargs}) as f:
|
|
f.create_dataset("real", data=np.arange(4, dtype="<i4"))
|
|
g = f.create_group("grp")
|
|
g.create_dataset("inner", data=np.arange(2, dtype="<i4"))
|
|
g["rel"] = h5py.SoftLink("inner")
|
|
f["soft"] = h5py.SoftLink("/real")
|
|
f["soft_grp"] = h5py.SoftLink("/grp")
|
|
f["dangling"] = h5py.SoftLink("/nope")
|
|
f["loop_a"] = h5py.SoftLink("/loop_b")
|
|
f["loop_b"] = h5py.SoftLink("/loop_a")
|
|
f["ext"] = h5py.ExternalLink("other_{tag}.h5", "/remote")
|
|
f.create_dataset("extdata", shape=(4,), dtype="<i4", external=[("raw_{tag}.bin", 0, 16)])
|
|
f["extdata"][...] = np.array([11, 22, 33, 44], dtype="<i4")
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(dir.path().join(format!("links_{tag}.h5"))).unwrap();
|
|
let read = |path: &str| file.dataset(path).and_then(|d| d.read_i32());
|
|
assert_eq!(read("soft").unwrap(), vec![0, 1, 2, 3], "{tag}");
|
|
assert_eq!(read("soft_grp/inner").unwrap(), vec![0, 1], "{tag}");
|
|
assert_eq!(
|
|
read("grp/rel").unwrap(),
|
|
vec![0, 1],
|
|
"{tag}: relative target"
|
|
);
|
|
assert_eq!(
|
|
read("soft_grp/rel").unwrap(),
|
|
vec![0, 1],
|
|
"{tag}: link via link"
|
|
);
|
|
|
|
assert!(
|
|
matches!(read("dangling"), Err(Error::Format(FormatError::PathNotFound(p))) if p == "nope"),
|
|
"{tag}: dangling link names its missing target"
|
|
);
|
|
assert!(
|
|
matches!(
|
|
read("loop_a"),
|
|
Err(Error::Format(FormatError::NestingDepthExceeded))
|
|
),
|
|
"{tag}: link cycle"
|
|
);
|
|
assert!(
|
|
matches!(
|
|
read("ext"),
|
|
Err(Error::Format(FormatError::ExternalLinkUnsupported { ref object_path, .. }))
|
|
if object_path == "/remote"
|
|
),
|
|
"{tag}: external link"
|
|
);
|
|
assert!(
|
|
matches!(
|
|
read("extdata"),
|
|
Err(Error::Format(FormatError::ExternalDataFilesUnsupported))
|
|
),
|
|
"{tag}: external raw data must not read as fill values"
|
|
);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Attribute fidelity: nothing is dropped, unsigned stays unsigned
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// `attrs()` used to omit, without any error, every attribute whose datatype
|
|
/// had no `AttrValue` variant — including every Python `bool` (stored as an
|
|
/// enum) — and to wrap unsigned 64-bit arrays into negative `i64`s.
|
|
#[test]
|
|
fn h5py_attribute_kinds_are_all_reported() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("attrs_kinds.h5");
|
|
let path_str = path.display().to_string();
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w") as f:
|
|
d = f.create_dataset("d", data=np.arange(3))
|
|
a = d.attrs
|
|
a["float"] = 1.5; a["int"] = 7; a["str"] = "hello"; a["int_array"] = np.arange(4)
|
|
a["str_list"] = ["a", "bc"]
|
|
a["uint64_array"] = np.array([2**63, 1], dtype="u8")
|
|
a["bool_true"] = True
|
|
a["bool_false"] = False
|
|
a["bool_array"] = np.array([True, False, True])
|
|
a["complex"] = 1 + 2j
|
|
a["compound"] = np.array([(1, 2.5)], dtype=[("a", "<i4"), ("b", "<f8")])
|
|
a["object_ref"] = d.ref
|
|
a["color"] = np.array(2, dtype=h5py.enum_dtype({{"RED": 0, "GREEN": 1, "BLUE": 2}}, basetype="i1"))
|
|
print(len(a))
|
|
"#
|
|
);
|
|
let written: usize = run_python_output(&script).trim().parse().unwrap();
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let attrs = file.dataset("d").unwrap().attrs().unwrap();
|
|
assert_eq!(
|
|
attrs.len(),
|
|
written,
|
|
"every attribute is reported: {attrs:?}"
|
|
);
|
|
|
|
// Booleans decode as 0/1.
|
|
assert!(matches!(attrs["bool_true"], AttrValue::I64(1)));
|
|
assert!(matches!(attrs["bool_false"], AttrValue::I64(0)));
|
|
assert!(matches!(&attrs["bool_array"], AttrValue::I64Array(v) if v == &[1, 0, 1]));
|
|
// Unsigned stays unsigned.
|
|
assert!(matches!(&attrs["uint64_array"], AttrValue::U64Array(v) if v == &[1u64 << 63, 1]));
|
|
|
|
// A general enum is not a boolean: kept verbatim, member names intact.
|
|
match &attrs["color"] {
|
|
AttrValue::Raw { datatype, data, .. } => {
|
|
assert_eq!(data, &[2]);
|
|
assert!(format!("{datatype:?}").contains("BLUE"));
|
|
}
|
|
other => panic!("color: {other:?}"),
|
|
}
|
|
// A compound attribute is kept verbatim and decodes against its datatype.
|
|
match &attrs["compound"] {
|
|
AttrValue::Raw {
|
|
datatype,
|
|
shape,
|
|
data,
|
|
} => {
|
|
assert_eq!(shape, &[1]);
|
|
let fields = clawhdf5_format::data_read::read_compound_fields(data, datatype).unwrap();
|
|
assert_eq!(fields[0].name, "a");
|
|
let b =
|
|
clawhdf5_format::data_read::read_as_f64(&fields[1].raw_data, &fields[1].datatype)
|
|
.unwrap();
|
|
assert_eq!(b, vec![2.5]);
|
|
}
|
|
other => panic!("compound: {other:?}"),
|
|
}
|
|
for name in ["complex", "object_ref"] {
|
|
assert!(
|
|
matches!(attrs[name], AttrValue::Raw { .. }),
|
|
"{name}: {:?}",
|
|
attrs[name]
|
|
);
|
|
}
|
|
}
|
|
|
|
/// `Raw` and `U64Array` are writable, so an attribute read from one file can
|
|
/// be stored in another unchanged.
|
|
#[test]
|
|
fn clawhdf5_writes_raw_and_unsigned_attrs_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let src = dir.path().join("src.h5");
|
|
let dst = dir.path().join("dst.h5");
|
|
let (src_str, dst_str) = (src.display().to_string(), dst.display().to_string());
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{src_str}", "w") as f:
|
|
d = f.create_dataset("d", data=np.arange(3))
|
|
d.attrs["compound"] = np.array([(1, 2.5), (3, 4.5)], dtype=[("a", "<i4"), ("b", "<f8")])
|
|
"#
|
|
));
|
|
let compound = File::open(&src)
|
|
.unwrap()
|
|
.dataset("d")
|
|
.unwrap()
|
|
.attrs()
|
|
.unwrap()["compound"]
|
|
.clone();
|
|
|
|
let mut builder = FileBuilder::new();
|
|
let ds = builder.create_dataset("d");
|
|
ds.with_f64_data(&[1.0]);
|
|
ds.set_attr("compound", compound);
|
|
ds.set_attr("big", AttrValue::U64Array(vec![u64::MAX, 0, 1 << 63]));
|
|
builder.write(&dst).unwrap();
|
|
|
|
let out = run_python_output(&format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{dst_str}", "r") as f:
|
|
a = f["d"].attrs
|
|
print(a["compound"].tolist(), a["compound"].dtype.names, a["big"].tolist(), a["big"].dtype)
|
|
"#
|
|
));
|
|
assert_eq!(
|
|
out.trim(),
|
|
"[(1, 2.5), (3, 4.5)] ('a', 'b') [18446744073709551615, 0, 9223372036854775808] uint64"
|
|
);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// h5py writes datasets indexed by a version-2 B-tree -> clawhdf5 reads
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// With `libver='latest'`, a chunked dataset with two or more unlimited
|
|
/// dimensions indexes its chunks with a version-2 B-tree (layout v4, index
|
|
/// type 5). These used to fail with "unsupported chunked layout".
|
|
#[test]
|
|
fn h5py_btree_v2_chunk_index_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("bt2.h5");
|
|
let path_str = path.display().to_string();
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w", libver="latest") as f:
|
|
a = np.arange(60 * 45, dtype="<i4").reshape(60, 45)
|
|
f.create_dataset("plain", data=a, chunks=(7, 8), maxshape=(None, None))
|
|
f.create_dataset("gz", data=a, chunks=(7, 8), maxshape=(None, None), compression="gzip", shuffle=True)
|
|
# Enough chunks (2500) that the tree has internal nodes.
|
|
big = np.arange(200 * 200, dtype="<i4").reshape(200, 200)
|
|
f.create_dataset("deep", data=big, chunks=(4, 4), maxshape=(None, None))
|
|
s = f.create_dataset("sparse", shape=(30, 30), dtype="<i4", chunks=(5, 5), maxshape=(None, None), fillvalue=-9)
|
|
s[10:15, 20:25] = 4
|
|
s[29, 29] = 1
|
|
with h5py.File("{path_str}", "r") as f:
|
|
print("sparse", f["sparse"][...].ravel().tolist())
|
|
print("slab", f["deep"][37:141:13, 5:190:31].ravel().tolist())
|
|
"#
|
|
);
|
|
let out = run_python_output(&script);
|
|
let expected: std::collections::HashMap<&str, Vec<i32>> = out
|
|
.lines()
|
|
.map(|l| {
|
|
let (name, list) = l.split_once(' ').unwrap();
|
|
(name, parse_int_list(list))
|
|
})
|
|
.collect();
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let small: Vec<i32> = (0..60 * 45).collect();
|
|
assert_eq!(file.dataset("plain").unwrap().read_i32().unwrap(), small);
|
|
assert_eq!(file.dataset("gz").unwrap().read_i32().unwrap(), small);
|
|
let deep: Vec<i32> = (0..200 * 200).collect();
|
|
assert_eq!(file.dataset("deep").unwrap().read_i32().unwrap(), deep);
|
|
assert_eq!(
|
|
file.dataset("sparse").unwrap().read_i32().unwrap(),
|
|
expected["sparse"]
|
|
);
|
|
// Partial read through the same index: rows 37,50,..,128 x cols 5,36,..,160.
|
|
let slab = clawhdf5_format::selection::Selection::Hyperslab {
|
|
start: vec![37, 5],
|
|
stride: vec![13, 31],
|
|
count: vec![8, 6],
|
|
block: vec![1, 1],
|
|
};
|
|
assert_eq!(
|
|
file.dataset("deep")
|
|
.unwrap()
|
|
.read_i32_selection(&slab)
|
|
.unwrap(),
|
|
expected["slab"]
|
|
);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// clawhdf5 auto-chunks a large compressed dataset -> h5py reads
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Compression without explicit chunk dimensions used to store the whole
|
|
/// dataset as a single chunk. Large datasets are now split automatically;
|
|
/// h5py must read the result and see sensibly sized chunks.
|
|
#[test]
|
|
fn clawhdf5_auto_chunked_dataset_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("auto_chunk.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let (rows, cols) = (1500u64, 1100u64); // 13.2 MB of f64
|
|
let data: Vec<f64> = (0..rows * cols).map(|i| (i % 9973) as f64 * 0.25).collect();
|
|
let mut builder = FileBuilder::new();
|
|
builder
|
|
.create_dataset("big")
|
|
.with_f64_data(&data)
|
|
.with_shape(&[rows, cols])
|
|
.with_deflate(4);
|
|
builder
|
|
.create_dataset("small")
|
|
.with_f64_data(&data[..600])
|
|
.with_shape(&[20, 30])
|
|
.with_deflate(4);
|
|
builder.write(&path).unwrap();
|
|
|
|
let out = run_python_output(&format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "r") as f:
|
|
big, small = f["big"], f["small"]
|
|
expect = (np.arange(1500 * 1100) % 9973) * 0.25
|
|
ok = bool(np.array_equal(big[...].ravel(), expect)) and bool(np.array_equal(small[...].ravel(), expect[:600]))
|
|
chunk_bytes = int(np.prod(big.chunks)) * 8
|
|
print(ok, chunk_bytes <= 1 << 20, chunk_bytes >= 1 << 17, small.chunks == (20, 30), big.compression)
|
|
"#
|
|
));
|
|
assert_eq!(out.trim(), "True True True True gzip");
|
|
|
|
// And it reads back here, in full and partially.
|
|
let file = File::open(&path).unwrap();
|
|
let ds = file.dataset("big").unwrap();
|
|
assert_eq!(ds.read_f64().unwrap(), data);
|
|
let row = clawhdf5_format::selection::Selection::Hyperslab {
|
|
start: vec![777, 0],
|
|
stride: vec![1, 1],
|
|
count: vec![1, cols],
|
|
block: vec![1, 1],
|
|
};
|
|
let start = (777 * cols) as usize;
|
|
assert_eq!(
|
|
ds.read_f64_selection(&row).unwrap(),
|
|
data[start..start + cols as usize]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn h5py_deep_btree_v2_chunk_index_clawhdf5_reads() {
|
|
// Two unlimited dimensions give a B-tree v2 chunk index, and 2x2 chunks
|
|
// over 400x400 give 40 000 index records — enough for HDF5 to build a
|
|
// tree of depth 2. Small h5py files only ever produce depth-0 trees, so
|
|
// this is the one fixture that walks internal nodes: the path where the
|
|
// traversal's record budget (the guard against crafted shared-subtree
|
|
// trees) is spent, which must never refuse a real file.
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("deep_btree.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w", libver="latest") as f:
|
|
d = f.create_dataset("x", shape=(400, 400), maxshape=(None, None),
|
|
chunks=(2, 2), dtype="i4")
|
|
d[...] = np.arange(160000, dtype="i4").reshape(400, 400)
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
// The fixture is only meaningful if HDF5 really built internal nodes.
|
|
let bytes = std::fs::read(&path).unwrap();
|
|
let at = bytes
|
|
.windows(4)
|
|
.position(|w| w == b"BTHD")
|
|
.expect("expected a B-tree v2 chunk index");
|
|
let depth = u16::from_le_bytes([bytes[at + 12], bytes[at + 13]]);
|
|
assert!(
|
|
depth >= 1,
|
|
"fixture tree has depth {depth}; it tests nothing"
|
|
);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let values = file.dataset("x").unwrap().read_i32().unwrap();
|
|
assert_eq!(values.len(), 160_000);
|
|
for (i, &v) in values.iter().enumerate() {
|
|
assert_eq!(v, i as i32, "element {i}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn h5py_extensible_array_chunk_index_clawhdf5_reads() {
|
|
// One unlimited dimension means an Extensible Array chunk index. Only its
|
|
// first few elements live inline in the index block (4 by default), and
|
|
// every other fixture here is small enough to stop there — which is how
|
|
// the data block and super block layouts came to be wrong without a test
|
|
// noticing. The counts below step over each boundary in turn:
|
|
// 4 inline elements only
|
|
// 37 past the first direct data block
|
|
// 400 into the first super block
|
|
// 5000 several super block levels
|
|
// 200000 data blocks large enough to be paged
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
|
|
for n in [4usize, 37, 400, 5_000, 200_000] {
|
|
let path = dir.path().join(format!("ea_{n}.h5"));
|
|
let path_str = path.display().to_string();
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w", libver="latest") as f:
|
|
d = f.create_dataset("x", shape=({n},), maxshape=(None,), chunks=(1,), dtype="i4")
|
|
d[...] = np.arange({n}, dtype="i4")
|
|
"#
|
|
));
|
|
|
|
let bytes = std::fs::read(&path).unwrap();
|
|
assert!(
|
|
bytes.windows(4).any(|w| w == b"EAHD"),
|
|
"n={n}: fixture is not indexed by an Extensible Array"
|
|
);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let values = file.dataset("x").unwrap().read_i32().unwrap();
|
|
assert_eq!(values.len(), n, "n={n}");
|
|
let wrong = values
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|&(i, &v)| v != i as i32)
|
|
.count();
|
|
assert_eq!(wrong, 0, "n={n}: {wrong} of {n} elements read back wrong");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn h5py_sparse_extensible_array_leaves_pages_uninitialised() {
|
|
// Writing a scattered subset leaves whole pages of a paged data block
|
|
// never initialised. Those pages still occupy their slot on disk, so the
|
|
// reader has to skip them by stride and take the fill value instead —
|
|
// driven by the page-init bitmap, which is packed one bit per page across
|
|
// the whole super block, MSB first.
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("ea_sparse.h5");
|
|
let path_str = path.display().to_string();
|
|
let n = 200_000usize;
|
|
let step = 997usize;
|
|
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "w", libver="latest") as f:
|
|
d = f.create_dataset("x", shape=({n},), maxshape=(None,), chunks=(1,),
|
|
dtype="i4", fillvalue=-1)
|
|
for i in list(range(0, {n}, {step})) + list(range(0, 40)):
|
|
d[i] = i
|
|
"#
|
|
));
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let values = file.dataset("x").unwrap().read_i32().unwrap();
|
|
assert_eq!(values.len(), n);
|
|
let wrong = values
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|&(i, &v)| {
|
|
let expected = if i % step == 0 || i < 40 {
|
|
i as i32
|
|
} else {
|
|
-1
|
|
};
|
|
v != expected
|
|
})
|
|
.count();
|
|
assert_eq!(wrong, 0, "{wrong} of {n} elements read back wrong");
|
|
}
|
|
|
|
#[test]
|
|
fn h5py_filtered_and_2d_extensible_array_clawhdf5_reads() {
|
|
// Filtered elements carry a size and filter mask beside the address, and
|
|
// a second (fixed) dimension changes how a linear index maps back to
|
|
// chunk offsets. Both run through the same traversal.
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
|
|
let gz = dir.path().join("ea_gzip.h5");
|
|
let gz_str = gz.display().to_string();
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{gz_str}", "w", libver="latest") as f:
|
|
d = f.create_dataset("x", shape=(5000,), maxshape=(None,), chunks=(1,),
|
|
dtype="i4", compression="gzip", compression_opts=4)
|
|
d[...] = np.arange(5000, dtype="i4")
|
|
"#
|
|
));
|
|
let values = File::open(&gz)
|
|
.unwrap()
|
|
.dataset("x")
|
|
.unwrap()
|
|
.read_i32()
|
|
.unwrap();
|
|
assert_eq!(values.len(), 5000);
|
|
assert_eq!(
|
|
values
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|&(i, &v)| v != i as i32)
|
|
.count(),
|
|
0
|
|
);
|
|
|
|
let two_d = dir.path().join("ea_2d.h5");
|
|
let two_d_str = two_d.display().to_string();
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{two_d_str}", "w", libver="latest") as f:
|
|
d = f.create_dataset("x", shape=(3000, 4), maxshape=(None, 4), chunks=(1, 4), dtype="i4")
|
|
d[...] = np.arange(12000, dtype="i4").reshape(3000, 4)
|
|
"#
|
|
));
|
|
let values = File::open(&two_d)
|
|
.unwrap()
|
|
.dataset("x")
|
|
.unwrap()
|
|
.read_i32()
|
|
.unwrap();
|
|
assert_eq!(values.len(), 12_000);
|
|
assert_eq!(
|
|
values
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|&(i, &v)| v != i as i32)
|
|
.count(),
|
|
0
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn h5py_fixed_array_chunk_index_clawhdf5_reads() {
|
|
// Fixed dimensions plus libver='latest' give a Fixed Array chunk index.
|
|
// Its data blocks are paged above 2^page_bits elements (1024 by default),
|
|
// and unlike the Extensible Array it keeps the page-init bitmap in the
|
|
// data block itself — a difference worth pinning down, since assuming
|
|
// otherwise is exactly what made the Extensible Array reader wrong. The
|
|
// sparse case leaves whole pages uninitialised so the bitmap is actually
|
|
// consulted rather than being all ones.
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
|
|
for n in [100usize, 5_000, 200_000] {
|
|
let path = dir.path().join(format!("fa_{n}.h5"));
|
|
let path_str = path.display().to_string();
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w", libver="latest") as f:
|
|
d = f.create_dataset("x", shape=({n},), chunks=(1,), dtype="i4")
|
|
d[...] = np.arange({n}, dtype="i4")
|
|
"#
|
|
));
|
|
let bytes = std::fs::read(&path).unwrap();
|
|
assert!(
|
|
bytes.windows(4).any(|w| w == b"FAHD"),
|
|
"n={n}: fixture is not indexed by a Fixed Array"
|
|
);
|
|
let values = File::open(&path)
|
|
.unwrap()
|
|
.dataset("x")
|
|
.unwrap()
|
|
.read_i32()
|
|
.unwrap();
|
|
assert_eq!(values.len(), n, "n={n}");
|
|
let wrong = values
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|&(i, &v)| v != i as i32)
|
|
.count();
|
|
assert_eq!(wrong, 0, "n={n}: {wrong} elements read back wrong");
|
|
}
|
|
|
|
let sparse = dir.path().join("fa_sparse.h5");
|
|
let sparse_str = sparse.display().to_string();
|
|
let (n, step) = (200_000usize, 997usize);
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{sparse_str}", "w", libver="latest") as f:
|
|
d = f.create_dataset("x", shape=({n},), chunks=(1,), dtype="i4", fillvalue=-1)
|
|
for i in list(range(0, {n}, {step})) + list(range(0, 40)):
|
|
d[i] = i
|
|
"#
|
|
));
|
|
let values = File::open(&sparse)
|
|
.unwrap()
|
|
.dataset("x")
|
|
.unwrap()
|
|
.read_i32()
|
|
.unwrap();
|
|
assert_eq!(values.len(), n);
|
|
let wrong = values
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|&(i, &v)| {
|
|
let expected = if i % step == 0 || i < 40 {
|
|
i as i32
|
|
} else {
|
|
-1
|
|
};
|
|
v != expected
|
|
})
|
|
.count();
|
|
assert_eq!(wrong, 0, "sparse: {wrong} of {n} elements read back wrong");
|
|
}
|
|
|
|
#[test]
|
|
fn corrupting_a_chunk_index_is_an_error_not_wrong_data() {
|
|
// Every Fixed/Extensible Array structure carries a Jenkins checksum, and
|
|
// the reader now verifies it. The point is not the checksum itself but
|
|
// what it prevents: a damaged index otherwise yields addresses pointing
|
|
// at the wrong bytes, and the caller receives another chunk's data as if
|
|
// it were the one asked for.
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
|
|
for (name, maxshape) in [("fixed", "None"), ("extensible", "(None,)")] {
|
|
let path = dir.path().join(format!("{name}.h5"));
|
|
let path_str = path.display().to_string();
|
|
let shape_arg = if maxshape == "None" {
|
|
String::new()
|
|
} else {
|
|
format!(", maxshape={maxshape}")
|
|
};
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w", libver="latest") as f:
|
|
d = f.create_dataset("x", shape=(400,), chunks=(1,), dtype="i4"{shape_arg})
|
|
d[...] = np.arange(400, dtype="i4")
|
|
"#
|
|
));
|
|
|
|
let clean = std::fs::read(&path).unwrap();
|
|
assert_eq!(
|
|
File::open(&path)
|
|
.unwrap()
|
|
.dataset("x")
|
|
.unwrap()
|
|
.read_i32()
|
|
.unwrap()
|
|
.len(),
|
|
400,
|
|
"{name}: the intact file must read"
|
|
);
|
|
|
|
// Flip a low bit of a chunk address inside a data block. Structurally
|
|
// everything still parses — the index still has the right shape and
|
|
// the address still lands inside the file — so nothing but the
|
|
// checksum can notice. Without it the read succeeds and hands back
|
|
// whatever bytes now sit at that address.
|
|
let sig: &[u8] = if name == "fixed" { b"FADB" } else { b"EADB" };
|
|
let block = clean
|
|
.windows(4)
|
|
.position(|w| w == sig)
|
|
.unwrap_or_else(|| panic!("{name}: no data block in the fixture"));
|
|
// Past the prefix (signature, version, client id, header address, and
|
|
// for the Extensible Array a block offset), into the first address.
|
|
let at = block + 4 + 1 + 1 + 8 + if name == "fixed" { 0 } else { 4 } + 1;
|
|
let mut damaged = clean.clone();
|
|
damaged[at] ^= 0x10;
|
|
let damaged_path = dir.path().join(format!("{name}_damaged.h5"));
|
|
std::fs::write(&damaged_path, &damaged).unwrap();
|
|
|
|
let result = File::open(&damaged_path)
|
|
.unwrap()
|
|
.dataset("x")
|
|
.and_then(|d| d.read_i32());
|
|
assert!(
|
|
result.is_err(),
|
|
"{name}: corruption produced data instead of an error"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_f32_h5py_reads() {
|
|
// Every f32 dataset used to be unreadable by h5py ("sign bit position out
|
|
// of bounds"): the float datatype's sign position was hard-coded for f64.
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("ours_f32.h5");
|
|
let path_str = path.display().to_string();
|
|
let values: Vec<f32> = vec![1.5, -2.25, 3.0e-7, 65536.5, f32::MAX, -0.0];
|
|
|
|
let mut fb = FileBuilder::new();
|
|
fb.create_dataset("plain").with_f32_data(&values);
|
|
fb.create_dataset("chunked")
|
|
.with_f32_data(&values)
|
|
.with_shape(&[values.len() as u64])
|
|
.with_chunks(&[4])
|
|
.with_deflate(6);
|
|
fb.write(&path).unwrap();
|
|
|
|
let bits = values
|
|
.iter()
|
|
.map(|v| v.to_bits().to_string())
|
|
.collect::<Vec<_>>()
|
|
.join(",");
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
expected = np.array([{bits}], dtype=np.uint32)
|
|
with h5py.File("{path_str}", "r") as f:
|
|
for name in ("plain", "chunked"):
|
|
d = f[name]
|
|
assert d.dtype == np.float32, (name, d.dtype)
|
|
assert (d[:].view(np.uint32) == expected).all(), (name, d[:])
|
|
print("ok")
|
|
"#
|
|
);
|
|
assert_eq!(run_python_output(&script), "ok");
|
|
}
|