osobhandClaude Opus 5.5
d0db83812b
The setting was persisted in /meta and otherwise ignored: embeddings
were always written as f32. It now does what it says.
clawhdf5-format:
- `DatasetBuilder::with_f16_data` writes IEEE binary16 (numpy float16),
rounding to nearest-even, and `make_f16_type`.
- `clawhdf5_format::float16` holds the f32 <-> f16 conversions, the one
implementation the writer, the reader and the agent all use. Checked
against the `half` crate on 16.7M f32 values and round-trips all 65536
half values; the h5py interop tests confirm the rounding matches
numpy's bit for bit (4020 values incl. ties, subnormals, overflow).
- Reading little-endian float16 as f32 has a fast path.
clawhdf5-agent:
- A float16 store writes /memory/embeddings as half precision, and
`MemoryCache::half_precision` rounds each embedding as it enters the
cache (save, update, WAL replay, and on load of a store still f32 on
disk), so memory and file agree bit for bit and a store searches the
same before and after a reopen (tested).
- Values beyond +-65504 are refused with the new
`MemoryError::InvalidEntry` rather than stored as infinity, on every
save path; batches are all or nothing, and a rejected ephemeral entry
stays in the ephemeral tier. Breaking for exhaustive matches.
- CLI: `create --float16`. Off by default.
Measured on tank, 384-dim, six runs alternating order, medians
(search_harness --float16-study --full): at 100K the file goes from
154.0 to 80.8 MiB (-48%), checkpoint 752 -> 512 ms, open 300 -> 252 ms;
vector recall@10 against an exact scan and hybrid_search latency do not
change. At 10K open is 3 ms slower. Also a test that h5py opens a whole
agent store, f32 and float16, and decodes every dataset.
Docs: README, BENCHMARKS.md ("float16 embedding storage"), CHANGELOG
(including the h5py interop fixes in the previous commit), CLAUDE.md.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1553 lines
54 KiB
Rust
1553 lines
54 KiB
Rust
//! Bidirectional interop tests between clawhdf5 and h5py (Python).
|
|
//!
|
|
//! Tests are skipped if python3 or h5py are not available.
|
|
|
|
use std::process::Command;
|
|
|
|
use clawhdf5::{AttrValue, CompoundTypeBuilder, DType, File, FileBuilder};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helpers
|
|
// ---------------------------------------------------------------------------
|
|
/// The Python interpreter to drive interop checks with.
|
|
///
|
|
/// `CLAWHDF5_PYTHON` lets these run against a virtualenv holding h5py, which
|
|
/// on a PEP 668 "externally managed" system is the only place it can be
|
|
/// installed. Without it the suite silently skips, and a silent skip here is
|
|
/// how a datatype bug once reached a release.
|
|
fn python() -> String {
|
|
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
|
}
|
|
|
|
/// When `CLAWHDF5_REQUIRE_INTEROP=1` (set in CI), a missing Python dependency
|
|
/// is a test failure instead of a silent skip.
|
|
fn interop_required() -> bool {
|
|
std::env::var("CLAWHDF5_REQUIRE_INTEROP").is_ok_and(|v| v == "1")
|
|
}
|
|
|
|
fn python_available() -> bool {
|
|
Command::new(python())
|
|
.args(["-c", "import h5py; print(h5py.__version__)"])
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
macro_rules! skip_if_no_python {
|
|
() => {
|
|
if !python_available() {
|
|
assert!(
|
|
!interop_required(),
|
|
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
|
);
|
|
eprintln!("SKIP: python3 with h5py not available");
|
|
return;
|
|
}
|
|
};
|
|
}
|
|
|
|
/// Run a Python script and panic if it fails.
|
|
fn run_python(script: &str) {
|
|
let output = Command::new(python())
|
|
.args(["-c", script])
|
|
.output()
|
|
.expect("failed to run python3");
|
|
if !output.status.success() {
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
|
panic!("Python script failed:\nSTDOUT: {stdout}\nSTDERR: {stderr}");
|
|
}
|
|
}
|
|
|
|
/// Run a Python script and return stdout as a trimmed string.
|
|
fn run_python_output(script: &str) -> String {
|
|
let output = Command::new(python())
|
|
.args(["-c", script])
|
|
.output()
|
|
.expect("failed to run python3");
|
|
if !output.status.success() {
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
panic!("Python script failed:\nSTDERR: {stderr}");
|
|
}
|
|
String::from_utf8_lossy(&output.stdout).trim().to_string()
|
|
}
|
|
|
|
// ===========================================================================
|
|
// Part A: clawhdf5 writes -> h5py reads
|
|
// ===========================================================================
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A1. Write f64 dataset -> h5py reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_f64_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("f64_test.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let data = vec![1.5, 2.5, 3.5, -4.5, 0.0];
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("values").with_f64_data(&data);
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np, json
|
|
with h5py.File("{path_str}", "r") as f:
|
|
ds = f["values"]
|
|
assert ds.dtype == np.float64, f"expected float64, got {{ds.dtype}}"
|
|
assert ds.shape == (5,), f"expected (5,), got {{ds.shape}}"
|
|
vals = ds[()].tolist()
|
|
print(json.dumps(vals))
|
|
"#
|
|
);
|
|
let output = run_python_output(&script);
|
|
let vals: Vec<f64> = serde_json_minimal_parse(&output);
|
|
assert_eq!(vals, data);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A2. Write string dataset -> h5py reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_strings_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("strings_test.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
// clawhdf5 doesn't have a direct with_string_data, so we write f64 and
|
|
// test string attributes instead
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("data").with_f64_data(&[1.0]);
|
|
b.set_attr("title", AttrValue::String("Hello World".into()));
|
|
b.set_attr(
|
|
"tags",
|
|
AttrValue::StringArray(vec!["alpha".into(), "beta".into()]),
|
|
);
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "r") as f:
|
|
title = f.attrs["title"]
|
|
if isinstance(title, bytes):
|
|
title = title.decode()
|
|
print(title)
|
|
tags = f.attrs["tags"]
|
|
tag_list = [t.decode() if isinstance(t, bytes) else t for t in tags]
|
|
print(",".join(tag_list))
|
|
"#
|
|
);
|
|
let output = run_python_output(&script);
|
|
let lines: Vec<&str> = output.lines().collect();
|
|
assert_eq!(lines[0], "Hello World");
|
|
assert_eq!(lines[1], "alpha,beta");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A3. Write chunked+compressed dataset -> h5py reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_chunked_compressed_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("chunked_test.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let data: Vec<f64> = (0..1000).map(|i| i as f64 * 0.01).collect();
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("compressed")
|
|
.with_f64_data(&data)
|
|
.with_chunks(&[100])
|
|
.with_deflate(6);
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "r") as f:
|
|
ds = f["compressed"]
|
|
assert ds.chunks is not None, "expected chunked dataset"
|
|
assert ds.compression == "gzip", f"expected gzip, got {{ds.compression}}"
|
|
vals = ds[()].tolist()
|
|
assert len(vals) == 1000, f"expected 1000 values, got {{len(vals)}}"
|
|
# Check first and last
|
|
assert abs(vals[0] - 0.0) < 1e-10
|
|
assert abs(vals[999] - 9.99) < 1e-10
|
|
print("OK")
|
|
"#
|
|
);
|
|
let output = run_python_output(&script);
|
|
assert_eq!(output, "OK");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A4. Write groups with attributes -> h5py reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_groups_attrs_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("groups_test.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let mut b = FileBuilder::new();
|
|
b.set_attr("file_version", AttrValue::I64(3));
|
|
|
|
let mut g = b.create_group("experiment");
|
|
g.set_attr("run_id", AttrValue::I64(42));
|
|
g.set_attr("description", AttrValue::String("test run".into()));
|
|
g.create_dataset("measurements")
|
|
.with_f64_data(&[10.0, 20.0, 30.0]);
|
|
b.add_group(g.finish());
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "r") as f:
|
|
assert f.attrs["file_version"] == 3
|
|
g = f["experiment"]
|
|
assert g.attrs["run_id"] == 42
|
|
desc = g.attrs["description"]
|
|
if isinstance(desc, bytes):
|
|
desc = desc.decode()
|
|
assert desc == "test run", f"got {{desc}}"
|
|
vals = g["measurements"][()].tolist()
|
|
assert vals == [10.0, 20.0, 30.0]
|
|
print("OK")
|
|
"#
|
|
);
|
|
let output = run_python_output(&script);
|
|
assert_eq!(output, "OK");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A5. Write compound type -> h5py reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_compound_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("compound_test.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let dt = CompoundTypeBuilder::new()
|
|
.f64_field("x")
|
|
.f64_field("y")
|
|
.i32_field("id")
|
|
.build();
|
|
|
|
let mut raw = Vec::new();
|
|
for &(x, y, id) in &[(1.0f64, 2.0f64, 10i32), (3.0, 4.0, 20)] {
|
|
raw.extend_from_slice(&x.to_le_bytes());
|
|
raw.extend_from_slice(&y.to_le_bytes());
|
|
raw.extend_from_slice(&id.to_le_bytes());
|
|
}
|
|
|
|
let mut b = FileBuilder::new();
|
|
b.create_dataset("points").with_compound_data(dt, raw, 2);
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "r") as f:
|
|
ds = f["points"]
|
|
assert ds.dtype.names == ("x", "y", "id"), f"got {{ds.dtype.names}}"
|
|
assert len(ds) == 2
|
|
row0 = ds[0]
|
|
assert abs(float(row0["x"]) - 1.0) < 1e-10
|
|
assert abs(float(row0["y"]) - 2.0) < 1e-10
|
|
assert int(row0["id"]) == 10
|
|
row1 = ds[1]
|
|
assert abs(float(row1["x"]) - 3.0) < 1e-10
|
|
assert int(row1["id"]) == 20
|
|
print("OK")
|
|
"#
|
|
);
|
|
let output = run_python_output(&script);
|
|
assert_eq!(output, "OK");
|
|
}
|
|
|
|
// ===========================================================================
|
|
// Part B: h5py writes -> clawhdf5 reads
|
|
// ===========================================================================
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// B6. h5py creates f64 dataset -> clawhdf5 reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn h5py_writes_f64_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("h5py_f64.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w") as f:
|
|
f.create_dataset("values", data=np.array([1.5, 2.5, 3.5, -4.5, 0.0], dtype=np.float64))
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let ds = file.dataset("values").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::F64);
|
|
assert_eq!(ds.shape().unwrap(), vec![5]);
|
|
assert_eq!(ds.read_f64().unwrap(), vec![1.5, 2.5, 3.5, -4.5, 0.0]);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// B7. h5py creates string dataset -> clawhdf5 reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn h5py_writes_strings_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("h5py_strings.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
// Use fixed-length strings (S10) since clawhdf5 doesn't support vlen strings yet
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w") as f:
|
|
dt = np.dtype("S10")
|
|
f.create_dataset("names", data=np.array([b"alice", b"bob", b"charlie"], dtype=dt))
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let ds = file.dataset("names").unwrap();
|
|
let strings = ds.read_string().unwrap();
|
|
assert_eq!(strings, vec!["alice", "bob", "charlie"]);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// B8. h5py creates chunked+compressed+shuffled -> clawhdf5 reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn h5py_writes_chunked_compressed_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("h5py_chunked.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
data = np.arange(2000, dtype=np.float64) * 0.1
|
|
with h5py.File("{path_str}", "w") as f:
|
|
f.create_dataset("compressed", data=data, chunks=(200,),
|
|
compression="gzip", compression_opts=4, shuffle=True)
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let ds = file.dataset("compressed").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::F64);
|
|
assert_eq!(ds.shape().unwrap(), vec![2000]);
|
|
let values = ds.read_f64().unwrap();
|
|
assert_eq!(values.len(), 2000);
|
|
assert!((values[0] - 0.0).abs() < 1e-10);
|
|
assert!((values[1999] - 199.9).abs() < 1e-10);
|
|
assert!((values[1000] - 100.0).abs() < 1e-10);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// B9. h5py creates nested groups with attrs -> clawhdf5 reads -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn h5py_writes_nested_groups_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("h5py_groups.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w") as f:
|
|
f.attrs["file_attr"] = "root_value"
|
|
g1 = f.create_group("level1")
|
|
g1.attrs["g1_attr"] = 42
|
|
g2 = g1.create_group("level2")
|
|
g2.attrs["g2_attr"] = 3.25
|
|
g2.create_dataset("deep_data", data=np.array([100.0, 200.0], dtype=np.float64))
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
|
|
// Root attrs
|
|
let root_attrs = file.root().attrs().unwrap();
|
|
assert!(matches!(root_attrs.get("file_attr"), Some(AttrValue::String(s)) if s == "root_value"));
|
|
|
|
// Level1 group
|
|
let g1 = file.group("level1").unwrap();
|
|
let g1_attrs = g1.attrs().unwrap();
|
|
assert!(matches!(g1_attrs.get("g1_attr"), Some(AttrValue::I64(42))));
|
|
|
|
// Level2 group (nested)
|
|
let g2 = file.group("level1/level2").unwrap();
|
|
let g2_attrs = g2.attrs().unwrap();
|
|
assert!(
|
|
matches!(g2_attrs.get("g2_attr"), Some(AttrValue::F64(v)) if (*v - 3.25).abs() < 1e-10)
|
|
);
|
|
|
|
// Deep dataset
|
|
let ds = file.dataset("level1/level2/deep_data").unwrap();
|
|
assert_eq!(ds.read_f64().unwrap(), vec![100.0, 200.0]);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// B10. h5py creates multiple dtypes -> clawhdf5 reads all -> verify
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn h5py_writes_multiple_dtypes_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("h5py_dtypes.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w") as f:
|
|
f.create_dataset("int8", data=np.array([1, -1, 127], dtype=np.int8))
|
|
f.create_dataset("int16", data=np.array([256, -256], dtype=np.int16))
|
|
f.create_dataset("int32", data=np.array([100000, -100000], dtype=np.int32))
|
|
f.create_dataset("int64", data=np.array([2**40, -2**40], dtype=np.int64))
|
|
f.create_dataset("float32", data=np.array([1.5, 2.5], dtype=np.float32))
|
|
f.create_dataset("float64", data=np.array([3.25, 2.75], dtype=np.float64))
|
|
f.create_dataset("uint8", data=np.array([0, 128, 255], dtype=np.uint8))
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
|
|
// int8 -> read as i32 (upcast)
|
|
let ds = file.dataset("int8").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::I8);
|
|
let vals = ds.read_i32().unwrap();
|
|
assert_eq!(vals, vec![1, -1, 127]);
|
|
|
|
// int16
|
|
let ds = file.dataset("int16").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::I16);
|
|
let vals = ds.read_i32().unwrap();
|
|
assert_eq!(vals, vec![256, -256]);
|
|
|
|
// int32
|
|
let ds = file.dataset("int32").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::I32);
|
|
assert_eq!(ds.read_i32().unwrap(), vec![100000, -100000]);
|
|
|
|
// int64
|
|
let ds = file.dataset("int64").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::I64);
|
|
assert_eq!(ds.read_i64().unwrap(), vec![1i64 << 40, -(1i64 << 40)]);
|
|
|
|
// float32
|
|
let ds = file.dataset("float32").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::F32);
|
|
assert_eq!(ds.read_f32().unwrap(), vec![1.5f32, 2.5]);
|
|
|
|
// float64
|
|
let ds = file.dataset("float64").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::F64);
|
|
let vals = ds.read_f64().unwrap();
|
|
assert!((vals[0] - 3.25).abs() < 1e-10);
|
|
assert!((vals[1] - 2.75).abs() < 1e-10);
|
|
|
|
// uint8
|
|
let ds = file.dataset("uint8").unwrap();
|
|
assert_eq!(ds.dtype().unwrap(), DType::U8);
|
|
let vals = ds.read_u64().unwrap();
|
|
assert_eq!(vals, vec![0, 128, 255]);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Minimal JSON parsing (avoid serde dependency)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn serde_json_minimal_parse(s: &str) -> Vec<f64> {
|
|
// Parse a JSON array of numbers like "[1.5, 2.5, 3.5]"
|
|
let s = s.trim();
|
|
let s = s.strip_prefix('[').unwrap_or(s);
|
|
let s = s.strip_suffix(']').unwrap_or(s);
|
|
s.split(',')
|
|
.map(|v| v.trim().parse::<f64>().unwrap())
|
|
.collect()
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A_dense. Write a group with many links (dense storage) -> h5py reads all
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_dense_group_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("dense_group.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
// 20 links exceeds the compact threshold (8) -> dense fractal-heap storage.
|
|
let mut b = FileBuilder::new();
|
|
let mut g = b.create_group("big");
|
|
for i in 0..20 {
|
|
g.create_dataset(&format!("dataset_{i:03}"))
|
|
.with_i32_data(&[i, i * 10]);
|
|
}
|
|
b.add_group(g.finish());
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "r") as f:
|
|
big = f["big"]
|
|
names = sorted(big.keys())
|
|
assert len(names) == 20, f"expected 20 links, got {{len(names)}}"
|
|
for i in range(20):
|
|
v = big[f"dataset_{{i:03}}"][()].tolist()
|
|
assert v == [i, i*10], f"link {{i}} = {{v}}"
|
|
print("OK")
|
|
"#
|
|
);
|
|
let out = run_python_output(&script);
|
|
assert_eq!(out, "OK");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// A_multiblock. Write a multi-direct-block fractal heap -> h5py reads
|
|
// ---------------------------------------------------------------------------
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_multiblock_heap_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("multiblock.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
// ~1600 dense attributes overflow a single 64KiB fractal-heap direct block.
|
|
let mut b = FileBuilder::new();
|
|
let mut g = b.create_group("g");
|
|
for i in 0..1600i64 {
|
|
g.set_attr(&format!("attribute_number_{i:05}"), AttrValue::I64(i * 2));
|
|
}
|
|
g.create_dataset("d").with_i32_data(&[1]);
|
|
b.add_group(g.finish());
|
|
b.write(&path).unwrap();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "r") as f:
|
|
a = f["g"].attrs
|
|
assert len(a) == 1600, f"expected 1600 attrs, got {{len(a)}}"
|
|
for i in (0, 1, 999, 1599):
|
|
v = int(a[f"attribute_number_{{i:05}}"])
|
|
assert v == i*2, f"attr {{i}} = {{v}}"
|
|
print("OK")
|
|
"#
|
|
);
|
|
assert_eq!(run_python_output(&script), "OK");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// h5py uses committed (named) datatypes -> clawhdf5 reads
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// A dataset or attribute created from a committed datatype stores only a
|
|
/// *shared message* reference to it. These used to be parsed as the datatype
|
|
/// itself (yielding `Time { size: 0 }` and unreadable data) and the attribute
|
|
/// was silently dropped.
|
|
#[test]
|
|
fn h5py_committed_datatypes_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
for (tag, kwargs) in [("default", ""), ("latest", ", libver='latest'")] {
|
|
let path = dir.path().join(format!("committed_{tag}.h5"));
|
|
let path_str = path.display().to_string();
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w"{kwargs}) as f:
|
|
f["f8type"] = np.dtype("<f8")
|
|
f["cmpd"] = np.dtype([("a", "<i4"), ("b", "<f8")])
|
|
f.create_dataset("d", data=np.arange(6, dtype="<f8"), dtype=f["f8type"])
|
|
f.create_dataset("c", data=np.array([(1, 2.5), (3, 4.5)], dtype=f["cmpd"].dtype), dtype=f["cmpd"])
|
|
f["d"].attrs.create("att", 7.0, dtype=f["f8type"])
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let d = file.dataset("d").unwrap();
|
|
assert_eq!(d.dtype().unwrap(), DType::F64, "{tag}");
|
|
assert_eq!(
|
|
d.read_f64().unwrap(),
|
|
vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0],
|
|
"{tag}"
|
|
);
|
|
assert!(
|
|
matches!(d.attrs().unwrap().get("att"), Some(AttrValue::F64(v)) if *v == 7.0),
|
|
"{tag}: attribute with a committed datatype"
|
|
);
|
|
let c = file.dataset("c").unwrap();
|
|
assert_eq!(
|
|
c.dtype().unwrap(),
|
|
DType::Compound(vec![("a".into(), DType::I32), ("b".into(), DType::F64)]),
|
|
"{tag}"
|
|
);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// h5py writes sparse / never-written datasets -> clawhdf5 applies fill values
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Parse h5py's `print(arr.ravel().tolist())` output for integer data.
|
|
fn parse_int_list(s: &str) -> Vec<i32> {
|
|
s.trim()
|
|
.trim_matches(|c| c == '[' || c == ']')
|
|
.split(',')
|
|
.filter(|t| !t.trim().is_empty())
|
|
.map(|t| t.trim().parse().unwrap())
|
|
.collect()
|
|
}
|
|
|
|
/// Storage HDF5 never allocated must read as the dataset's fill value. These
|
|
/// used to read as zeros (silently wrong for a non-zero fill value) or fail
|
|
/// outright (`NoDataAllocated`) for a dataset that was never written.
|
|
#[test]
|
|
fn h5py_fill_values_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
for (tag, kwargs) in [("default", ""), ("latest", ", libver='latest'")] {
|
|
let path = dir.path().join(format!("fill_{tag}.h5"));
|
|
let path_str = path.display().to_string();
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w"{kwargs}) as f:
|
|
d = f.create_dataset("partial", shape=(20,), dtype="<i4", chunks=(5,), fillvalue=-1)
|
|
d[0:5] = np.arange(5)
|
|
f.create_dataset("never", shape=(4,), dtype="<i4", fillvalue=25)
|
|
f.create_dataset("never_chunked", shape=(6,), dtype="<i4", chunks=(3,), fillvalue=9)
|
|
f.create_dataset("default_fill", shape=(3,), dtype="<i4")
|
|
g = f.create_dataset("gz", shape=(20,), dtype="<i4", chunks=(5,), fillvalue=7, compression="gzip")
|
|
g[10:15] = 1
|
|
s = f.create_dataset("sparse2d", shape=(5, 7), dtype="<i4", chunks=(2, 3), fillvalue=-3)
|
|
s[2:4, 3:6] = 8
|
|
s[4, 6] = 5
|
|
with h5py.File("{path_str}", "r") as f:
|
|
for name in ["partial", "never", "never_chunked", "default_fill", "gz", "sparse2d"]:
|
|
print(name, f[name][...].ravel().tolist())
|
|
print("slab", f["sparse2d"][1:5, 2:7].ravel().tolist())
|
|
"#
|
|
);
|
|
let expected: std::collections::HashMap<String, Vec<i32>> = run_python_output(&script)
|
|
.lines()
|
|
.map(|line| {
|
|
let (name, list) = line.split_once(' ').unwrap();
|
|
(name.to_string(), parse_int_list(list))
|
|
})
|
|
.collect();
|
|
|
|
let file = File::open(&path).unwrap();
|
|
for name in [
|
|
"partial",
|
|
"never",
|
|
"never_chunked",
|
|
"default_fill",
|
|
"gz",
|
|
"sparse2d",
|
|
] {
|
|
assert_eq!(
|
|
file.dataset(name).unwrap().read_i32().unwrap(),
|
|
expected[name],
|
|
"{tag}/{name}"
|
|
);
|
|
}
|
|
// A hyperslab straddling allocated and unallocated chunks.
|
|
let slab = clawhdf5_format::selection::Selection::Hyperslab {
|
|
start: vec![1, 2],
|
|
stride: vec![1, 1],
|
|
count: vec![4, 5],
|
|
block: vec![1, 1],
|
|
};
|
|
assert_eq!(
|
|
file.dataset("sparse2d")
|
|
.unwrap()
|
|
.read_i32_selection(&slab)
|
|
.unwrap(),
|
|
expected["slab"],
|
|
"{tag}/sparse2d hyperslab"
|
|
);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// h5py writes soft / external links and external raw data -> clawhdf5
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Soft links are followed (absolute, relative, through groups, with a cycle
|
|
/// guard). Things this reader does not follow — external links, and datasets
|
|
/// whose raw data lives in another file — are explicit errors. They used to
|
|
/// surface as a misleading `PathNotFound`, and external raw data could read
|
|
/// back as fill values.
|
|
#[test]
|
|
fn h5py_links_clawhdf5_resolves_or_refuses() {
|
|
use clawhdf5::Error;
|
|
use clawhdf5_format::error::FormatError;
|
|
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let dir_str = dir.path().display().to_string();
|
|
for (tag, kwargs) in [("default", ""), ("latest", ", libver='latest'")] {
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np, os
|
|
os.chdir("{dir_str}")
|
|
with h5py.File("other_{tag}.h5", "w"{kwargs}) as o:
|
|
o.create_dataset("remote", data=np.arange(3, dtype="<i4"))
|
|
with h5py.File("links_{tag}.h5", "w"{kwargs}) as f:
|
|
f.create_dataset("real", data=np.arange(4, dtype="<i4"))
|
|
g = f.create_group("grp")
|
|
g.create_dataset("inner", data=np.arange(2, dtype="<i4"))
|
|
g["rel"] = h5py.SoftLink("inner")
|
|
f["soft"] = h5py.SoftLink("/real")
|
|
f["soft_grp"] = h5py.SoftLink("/grp")
|
|
f["dangling"] = h5py.SoftLink("/nope")
|
|
f["loop_a"] = h5py.SoftLink("/loop_b")
|
|
f["loop_b"] = h5py.SoftLink("/loop_a")
|
|
f["ext"] = h5py.ExternalLink("other_{tag}.h5", "/remote")
|
|
f.create_dataset("extdata", shape=(4,), dtype="<i4", external=[("raw_{tag}.bin", 0, 16)])
|
|
f["extdata"][...] = np.array([11, 22, 33, 44], dtype="<i4")
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let file = File::open(dir.path().join(format!("links_{tag}.h5"))).unwrap();
|
|
let read = |path: &str| file.dataset(path).and_then(|d| d.read_i32());
|
|
assert_eq!(read("soft").unwrap(), vec![0, 1, 2, 3], "{tag}");
|
|
assert_eq!(read("soft_grp/inner").unwrap(), vec![0, 1], "{tag}");
|
|
assert_eq!(
|
|
read("grp/rel").unwrap(),
|
|
vec![0, 1],
|
|
"{tag}: relative target"
|
|
);
|
|
assert_eq!(
|
|
read("soft_grp/rel").unwrap(),
|
|
vec![0, 1],
|
|
"{tag}: link via link"
|
|
);
|
|
|
|
assert!(
|
|
matches!(read("dangling"), Err(Error::Format(FormatError::PathNotFound(p))) if p == "nope"),
|
|
"{tag}: dangling link names its missing target"
|
|
);
|
|
assert!(
|
|
matches!(
|
|
read("loop_a"),
|
|
Err(Error::Format(FormatError::NestingDepthExceeded))
|
|
),
|
|
"{tag}: link cycle"
|
|
);
|
|
assert!(
|
|
matches!(
|
|
read("ext"),
|
|
Err(Error::Format(FormatError::ExternalLinkUnsupported { ref object_path, .. }))
|
|
if object_path == "/remote"
|
|
),
|
|
"{tag}: external link"
|
|
);
|
|
assert!(
|
|
matches!(
|
|
read("extdata"),
|
|
Err(Error::Format(FormatError::ExternalDataFilesUnsupported))
|
|
),
|
|
"{tag}: external raw data must not read as fill values"
|
|
);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Attribute fidelity: nothing is dropped, unsigned stays unsigned
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// `attrs()` used to omit, without any error, every attribute whose datatype
|
|
/// had no `AttrValue` variant — including every Python `bool` (stored as an
|
|
/// enum) — and to wrap unsigned 64-bit arrays into negative `i64`s.
|
|
#[test]
|
|
fn h5py_attribute_kinds_are_all_reported() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("attrs_kinds.h5");
|
|
let path_str = path.display().to_string();
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w") as f:
|
|
d = f.create_dataset("d", data=np.arange(3))
|
|
a = d.attrs
|
|
a["float"] = 1.5; a["int"] = 7; a["str"] = "hello"; a["int_array"] = np.arange(4)
|
|
a["str_list"] = ["a", "bc"]
|
|
a["uint64_array"] = np.array([2**63, 1], dtype="u8")
|
|
a["bool_true"] = True
|
|
a["bool_false"] = False
|
|
a["bool_array"] = np.array([True, False, True])
|
|
a["complex"] = 1 + 2j
|
|
a["compound"] = np.array([(1, 2.5)], dtype=[("a", "<i4"), ("b", "<f8")])
|
|
a["object_ref"] = d.ref
|
|
a["color"] = np.array(2, dtype=h5py.enum_dtype({{"RED": 0, "GREEN": 1, "BLUE": 2}}, basetype="i1"))
|
|
print(len(a))
|
|
"#
|
|
);
|
|
let written: usize = run_python_output(&script).trim().parse().unwrap();
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let attrs = file.dataset("d").unwrap().attrs().unwrap();
|
|
assert_eq!(
|
|
attrs.len(),
|
|
written,
|
|
"every attribute is reported: {attrs:?}"
|
|
);
|
|
|
|
// Booleans decode as 0/1.
|
|
assert!(matches!(attrs["bool_true"], AttrValue::I64(1)));
|
|
assert!(matches!(attrs["bool_false"], AttrValue::I64(0)));
|
|
assert!(matches!(&attrs["bool_array"], AttrValue::I64Array(v) if v == &[1, 0, 1]));
|
|
// Unsigned stays unsigned.
|
|
assert!(matches!(&attrs["uint64_array"], AttrValue::U64Array(v) if v == &[1u64 << 63, 1]));
|
|
|
|
// A general enum is not a boolean: kept verbatim, member names intact.
|
|
match &attrs["color"] {
|
|
AttrValue::Raw { datatype, data, .. } => {
|
|
assert_eq!(data, &[2]);
|
|
assert!(format!("{datatype:?}").contains("BLUE"));
|
|
}
|
|
other => panic!("color: {other:?}"),
|
|
}
|
|
// A compound attribute is kept verbatim and decodes against its datatype.
|
|
match &attrs["compound"] {
|
|
AttrValue::Raw {
|
|
datatype,
|
|
shape,
|
|
data,
|
|
} => {
|
|
assert_eq!(shape, &[1]);
|
|
let fields = clawhdf5_format::data_read::read_compound_fields(data, datatype).unwrap();
|
|
assert_eq!(fields[0].name, "a");
|
|
let b =
|
|
clawhdf5_format::data_read::read_as_f64(&fields[1].raw_data, &fields[1].datatype)
|
|
.unwrap();
|
|
assert_eq!(b, vec![2.5]);
|
|
}
|
|
other => panic!("compound: {other:?}"),
|
|
}
|
|
for name in ["complex", "object_ref"] {
|
|
assert!(
|
|
matches!(attrs[name], AttrValue::Raw { .. }),
|
|
"{name}: {:?}",
|
|
attrs[name]
|
|
);
|
|
}
|
|
}
|
|
|
|
/// `Raw` and `U64Array` are writable, so an attribute read from one file can
|
|
/// be stored in another unchanged.
|
|
#[test]
|
|
fn clawhdf5_writes_raw_and_unsigned_attrs_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let src = dir.path().join("src.h5");
|
|
let dst = dir.path().join("dst.h5");
|
|
let (src_str, dst_str) = (src.display().to_string(), dst.display().to_string());
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{src_str}", "w") as f:
|
|
d = f.create_dataset("d", data=np.arange(3))
|
|
d.attrs["compound"] = np.array([(1, 2.5), (3, 4.5)], dtype=[("a", "<i4"), ("b", "<f8")])
|
|
"#
|
|
));
|
|
let compound = File::open(&src)
|
|
.unwrap()
|
|
.dataset("d")
|
|
.unwrap()
|
|
.attrs()
|
|
.unwrap()["compound"]
|
|
.clone();
|
|
|
|
let mut builder = FileBuilder::new();
|
|
let ds = builder.create_dataset("d");
|
|
ds.with_f64_data(&[1.0]);
|
|
ds.set_attr("compound", compound);
|
|
ds.set_attr("big", AttrValue::U64Array(vec![u64::MAX, 0, 1 << 63]));
|
|
builder.write(&dst).unwrap();
|
|
|
|
let out = run_python_output(&format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{dst_str}", "r") as f:
|
|
a = f["d"].attrs
|
|
print(a["compound"].tolist(), a["compound"].dtype.names, a["big"].tolist(), a["big"].dtype)
|
|
"#
|
|
));
|
|
assert_eq!(
|
|
out.trim(),
|
|
"[(1, 2.5), (3, 4.5)] ('a', 'b') [18446744073709551615, 0, 9223372036854775808] uint64"
|
|
);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// h5py writes datasets indexed by a version-2 B-tree -> clawhdf5 reads
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// With `libver='latest'`, a chunked dataset with two or more unlimited
|
|
/// dimensions indexes its chunks with a version-2 B-tree (layout v4, index
|
|
/// type 5). These used to fail with "unsupported chunked layout".
|
|
#[test]
|
|
fn h5py_btree_v2_chunk_index_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("bt2.h5");
|
|
let path_str = path.display().to_string();
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w", libver="latest") as f:
|
|
a = np.arange(60 * 45, dtype="<i4").reshape(60, 45)
|
|
f.create_dataset("plain", data=a, chunks=(7, 8), maxshape=(None, None))
|
|
f.create_dataset("gz", data=a, chunks=(7, 8), maxshape=(None, None), compression="gzip", shuffle=True)
|
|
# Enough chunks (2500) that the tree has internal nodes.
|
|
big = np.arange(200 * 200, dtype="<i4").reshape(200, 200)
|
|
f.create_dataset("deep", data=big, chunks=(4, 4), maxshape=(None, None))
|
|
s = f.create_dataset("sparse", shape=(30, 30), dtype="<i4", chunks=(5, 5), maxshape=(None, None), fillvalue=-9)
|
|
s[10:15, 20:25] = 4
|
|
s[29, 29] = 1
|
|
with h5py.File("{path_str}", "r") as f:
|
|
print("sparse", f["sparse"][...].ravel().tolist())
|
|
print("slab", f["deep"][37:141:13, 5:190:31].ravel().tolist())
|
|
"#
|
|
);
|
|
let out = run_python_output(&script);
|
|
let expected: std::collections::HashMap<&str, Vec<i32>> = out
|
|
.lines()
|
|
.map(|l| {
|
|
let (name, list) = l.split_once(' ').unwrap();
|
|
(name, parse_int_list(list))
|
|
})
|
|
.collect();
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let small: Vec<i32> = (0..60 * 45).collect();
|
|
assert_eq!(file.dataset("plain").unwrap().read_i32().unwrap(), small);
|
|
assert_eq!(file.dataset("gz").unwrap().read_i32().unwrap(), small);
|
|
let deep: Vec<i32> = (0..200 * 200).collect();
|
|
assert_eq!(file.dataset("deep").unwrap().read_i32().unwrap(), deep);
|
|
assert_eq!(
|
|
file.dataset("sparse").unwrap().read_i32().unwrap(),
|
|
expected["sparse"]
|
|
);
|
|
// Partial read through the same index: rows 37,50,..,128 x cols 5,36,..,160.
|
|
let slab = clawhdf5_format::selection::Selection::Hyperslab {
|
|
start: vec![37, 5],
|
|
stride: vec![13, 31],
|
|
count: vec![8, 6],
|
|
block: vec![1, 1],
|
|
};
|
|
assert_eq!(
|
|
file.dataset("deep")
|
|
.unwrap()
|
|
.read_i32_selection(&slab)
|
|
.unwrap(),
|
|
expected["slab"]
|
|
);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// clawhdf5 auto-chunks a large compressed dataset -> h5py reads
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Compression without explicit chunk dimensions used to store the whole
|
|
/// dataset as a single chunk. Large datasets are now split automatically;
|
|
/// h5py must read the result and see sensibly sized chunks.
|
|
#[test]
|
|
fn clawhdf5_auto_chunked_dataset_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("auto_chunk.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let (rows, cols) = (1500u64, 1100u64); // 13.2 MB of f64
|
|
let data: Vec<f64> = (0..rows * cols).map(|i| (i % 9973) as f64 * 0.25).collect();
|
|
let mut builder = FileBuilder::new();
|
|
builder
|
|
.create_dataset("big")
|
|
.with_f64_data(&data)
|
|
.with_shape(&[rows, cols])
|
|
.with_deflate(4);
|
|
builder
|
|
.create_dataset("small")
|
|
.with_f64_data(&data[..600])
|
|
.with_shape(&[20, 30])
|
|
.with_deflate(4);
|
|
builder.write(&path).unwrap();
|
|
|
|
let out = run_python_output(&format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "r") as f:
|
|
big, small = f["big"], f["small"]
|
|
expect = (np.arange(1500 * 1100) % 9973) * 0.25
|
|
ok = bool(np.array_equal(big[...].ravel(), expect)) and bool(np.array_equal(small[...].ravel(), expect[:600]))
|
|
chunk_bytes = int(np.prod(big.chunks)) * 8
|
|
print(ok, chunk_bytes <= 1 << 20, chunk_bytes >= 1 << 17, small.chunks == (20, 30), big.compression)
|
|
"#
|
|
));
|
|
assert_eq!(out.trim(), "True True True True gzip");
|
|
|
|
// And it reads back here, in full and partially.
|
|
let file = File::open(&path).unwrap();
|
|
let ds = file.dataset("big").unwrap();
|
|
assert_eq!(ds.read_f64().unwrap(), data);
|
|
let row = clawhdf5_format::selection::Selection::Hyperslab {
|
|
start: vec![777, 0],
|
|
stride: vec![1, 1],
|
|
count: vec![1, cols],
|
|
block: vec![1, 1],
|
|
};
|
|
let start = (777 * cols) as usize;
|
|
assert_eq!(
|
|
ds.read_f64_selection(&row).unwrap(),
|
|
data[start..start + cols as usize]
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn h5py_deep_btree_v2_chunk_index_clawhdf5_reads() {
|
|
// Two unlimited dimensions give a B-tree v2 chunk index, and 2x2 chunks
|
|
// over 400x400 give 40 000 index records — enough for HDF5 to build a
|
|
// tree of depth 2. Small h5py files only ever produce depth-0 trees, so
|
|
// this is the one fixture that walks internal nodes: the path where the
|
|
// traversal's record budget (the guard against crafted shared-subtree
|
|
// trees) is spent, which must never refuse a real file.
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("deep_btree.h5");
|
|
let path_str = path.display().to_string();
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w", libver="latest") as f:
|
|
d = f.create_dataset("x", shape=(400, 400), maxshape=(None, None),
|
|
chunks=(2, 2), dtype="i4")
|
|
d[...] = np.arange(160000, dtype="i4").reshape(400, 400)
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
// The fixture is only meaningful if HDF5 really built internal nodes.
|
|
let bytes = std::fs::read(&path).unwrap();
|
|
let at = bytes
|
|
.windows(4)
|
|
.position(|w| w == b"BTHD")
|
|
.expect("expected a B-tree v2 chunk index");
|
|
let depth = u16::from_le_bytes([bytes[at + 12], bytes[at + 13]]);
|
|
assert!(
|
|
depth >= 1,
|
|
"fixture tree has depth {depth}; it tests nothing"
|
|
);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let values = file.dataset("x").unwrap().read_i32().unwrap();
|
|
assert_eq!(values.len(), 160_000);
|
|
for (i, &v) in values.iter().enumerate() {
|
|
assert_eq!(v, i as i32, "element {i}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn h5py_extensible_array_chunk_index_clawhdf5_reads() {
|
|
// One unlimited dimension means an Extensible Array chunk index. Only its
|
|
// first few elements live inline in the index block (4 by default), and
|
|
// every other fixture here is small enough to stop there — which is how
|
|
// the data block and super block layouts came to be wrong without a test
|
|
// noticing. The counts below step over each boundary in turn:
|
|
// 4 inline elements only
|
|
// 37 past the first direct data block
|
|
// 400 into the first super block
|
|
// 5000 several super block levels
|
|
// 200000 data blocks large enough to be paged
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
|
|
for n in [4usize, 37, 400, 5_000, 200_000] {
|
|
let path = dir.path().join(format!("ea_{n}.h5"));
|
|
let path_str = path.display().to_string();
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w", libver="latest") as f:
|
|
d = f.create_dataset("x", shape=({n},), maxshape=(None,), chunks=(1,), dtype="i4")
|
|
d[...] = np.arange({n}, dtype="i4")
|
|
"#
|
|
));
|
|
|
|
let bytes = std::fs::read(&path).unwrap();
|
|
assert!(
|
|
bytes.windows(4).any(|w| w == b"EAHD"),
|
|
"n={n}: fixture is not indexed by an Extensible Array"
|
|
);
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let values = file.dataset("x").unwrap().read_i32().unwrap();
|
|
assert_eq!(values.len(), n, "n={n}");
|
|
let wrong = values
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|&(i, &v)| v != i as i32)
|
|
.count();
|
|
assert_eq!(wrong, 0, "n={n}: {wrong} of {n} elements read back wrong");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn h5py_sparse_extensible_array_leaves_pages_uninitialised() {
|
|
// Writing a scattered subset leaves whole pages of a paged data block
|
|
// never initialised. Those pages still occupy their slot on disk, so the
|
|
// reader has to skip them by stride and take the fill value instead —
|
|
// driven by the page-init bitmap, which is packed one bit per page across
|
|
// the whole super block, MSB first.
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("ea_sparse.h5");
|
|
let path_str = path.display().to_string();
|
|
let n = 200_000usize;
|
|
let step = 997usize;
|
|
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{path_str}", "w", libver="latest") as f:
|
|
d = f.create_dataset("x", shape=({n},), maxshape=(None,), chunks=(1,),
|
|
dtype="i4", fillvalue=-1)
|
|
for i in list(range(0, {n}, {step})) + list(range(0, 40)):
|
|
d[i] = i
|
|
"#
|
|
));
|
|
|
|
let file = File::open(&path).unwrap();
|
|
let values = file.dataset("x").unwrap().read_i32().unwrap();
|
|
assert_eq!(values.len(), n);
|
|
let wrong = values
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|&(i, &v)| {
|
|
let expected = if i % step == 0 || i < 40 {
|
|
i as i32
|
|
} else {
|
|
-1
|
|
};
|
|
v != expected
|
|
})
|
|
.count();
|
|
assert_eq!(wrong, 0, "{wrong} of {n} elements read back wrong");
|
|
}
|
|
|
|
#[test]
|
|
fn h5py_filtered_and_2d_extensible_array_clawhdf5_reads() {
|
|
// Filtered elements carry a size and filter mask beside the address, and
|
|
// a second (fixed) dimension changes how a linear index maps back to
|
|
// chunk offsets. Both run through the same traversal.
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
|
|
let gz = dir.path().join("ea_gzip.h5");
|
|
let gz_str = gz.display().to_string();
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{gz_str}", "w", libver="latest") as f:
|
|
d = f.create_dataset("x", shape=(5000,), maxshape=(None,), chunks=(1,),
|
|
dtype="i4", compression="gzip", compression_opts=4)
|
|
d[...] = np.arange(5000, dtype="i4")
|
|
"#
|
|
));
|
|
let values = File::open(&gz)
|
|
.unwrap()
|
|
.dataset("x")
|
|
.unwrap()
|
|
.read_i32()
|
|
.unwrap();
|
|
assert_eq!(values.len(), 5000);
|
|
assert_eq!(
|
|
values
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|&(i, &v)| v != i as i32)
|
|
.count(),
|
|
0
|
|
);
|
|
|
|
let two_d = dir.path().join("ea_2d.h5");
|
|
let two_d_str = two_d.display().to_string();
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{two_d_str}", "w", libver="latest") as f:
|
|
d = f.create_dataset("x", shape=(3000, 4), maxshape=(None, 4), chunks=(1, 4), dtype="i4")
|
|
d[...] = np.arange(12000, dtype="i4").reshape(3000, 4)
|
|
"#
|
|
));
|
|
let values = File::open(&two_d)
|
|
.unwrap()
|
|
.dataset("x")
|
|
.unwrap()
|
|
.read_i32()
|
|
.unwrap();
|
|
assert_eq!(values.len(), 12_000);
|
|
assert_eq!(
|
|
values
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|&(i, &v)| v != i as i32)
|
|
.count(),
|
|
0
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn h5py_fixed_array_chunk_index_clawhdf5_reads() {
|
|
// Fixed dimensions plus libver='latest' give a Fixed Array chunk index.
|
|
// Its data blocks are paged above 2^page_bits elements (1024 by default),
|
|
// and unlike the Extensible Array it keeps the page-init bitmap in the
|
|
// data block itself — a difference worth pinning down, since assuming
|
|
// otherwise is exactly what made the Extensible Array reader wrong. The
|
|
// sparse case leaves whole pages uninitialised so the bitmap is actually
|
|
// consulted rather than being all ones.
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
|
|
for n in [100usize, 5_000, 200_000] {
|
|
let path = dir.path().join(format!("fa_{n}.h5"));
|
|
let path_str = path.display().to_string();
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w", libver="latest") as f:
|
|
d = f.create_dataset("x", shape=({n},), chunks=(1,), dtype="i4")
|
|
d[...] = np.arange({n}, dtype="i4")
|
|
"#
|
|
));
|
|
let bytes = std::fs::read(&path).unwrap();
|
|
assert!(
|
|
bytes.windows(4).any(|w| w == b"FAHD"),
|
|
"n={n}: fixture is not indexed by a Fixed Array"
|
|
);
|
|
let values = File::open(&path)
|
|
.unwrap()
|
|
.dataset("x")
|
|
.unwrap()
|
|
.read_i32()
|
|
.unwrap();
|
|
assert_eq!(values.len(), n, "n={n}");
|
|
let wrong = values
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|&(i, &v)| v != i as i32)
|
|
.count();
|
|
assert_eq!(wrong, 0, "n={n}: {wrong} elements read back wrong");
|
|
}
|
|
|
|
let sparse = dir.path().join("fa_sparse.h5");
|
|
let sparse_str = sparse.display().to_string();
|
|
let (n, step) = (200_000usize, 997usize);
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py
|
|
with h5py.File("{sparse_str}", "w", libver="latest") as f:
|
|
d = f.create_dataset("x", shape=({n},), chunks=(1,), dtype="i4", fillvalue=-1)
|
|
for i in list(range(0, {n}, {step})) + list(range(0, 40)):
|
|
d[i] = i
|
|
"#
|
|
));
|
|
let values = File::open(&sparse)
|
|
.unwrap()
|
|
.dataset("x")
|
|
.unwrap()
|
|
.read_i32()
|
|
.unwrap();
|
|
assert_eq!(values.len(), n);
|
|
let wrong = values
|
|
.iter()
|
|
.enumerate()
|
|
.filter(|&(i, &v)| {
|
|
let expected = if i % step == 0 || i < 40 {
|
|
i as i32
|
|
} else {
|
|
-1
|
|
};
|
|
v != expected
|
|
})
|
|
.count();
|
|
assert_eq!(wrong, 0, "sparse: {wrong} of {n} elements read back wrong");
|
|
}
|
|
|
|
#[test]
|
|
fn corrupting_a_chunk_index_is_an_error_not_wrong_data() {
|
|
// Every Fixed/Extensible Array structure carries a Jenkins checksum, and
|
|
// the reader now verifies it. The point is not the checksum itself but
|
|
// what it prevents: a damaged index otherwise yields addresses pointing
|
|
// at the wrong bytes, and the caller receives another chunk's data as if
|
|
// it were the one asked for.
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
|
|
for (name, maxshape) in [("fixed", "None"), ("extensible", "(None,)")] {
|
|
let path = dir.path().join(format!("{name}.h5"));
|
|
let path_str = path.display().to_string();
|
|
let shape_arg = if maxshape == "None" {
|
|
String::new()
|
|
} else {
|
|
format!(", maxshape={maxshape}")
|
|
};
|
|
run_python(&format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
with h5py.File("{path_str}", "w", libver="latest") as f:
|
|
d = f.create_dataset("x", shape=(400,), chunks=(1,), dtype="i4"{shape_arg})
|
|
d[...] = np.arange(400, dtype="i4")
|
|
"#
|
|
));
|
|
|
|
let clean = std::fs::read(&path).unwrap();
|
|
assert_eq!(
|
|
File::open(&path)
|
|
.unwrap()
|
|
.dataset("x")
|
|
.unwrap()
|
|
.read_i32()
|
|
.unwrap()
|
|
.len(),
|
|
400,
|
|
"{name}: the intact file must read"
|
|
);
|
|
|
|
// Flip a low bit of a chunk address inside a data block. Structurally
|
|
// everything still parses — the index still has the right shape and
|
|
// the address still lands inside the file — so nothing but the
|
|
// checksum can notice. Without it the read succeeds and hands back
|
|
// whatever bytes now sit at that address.
|
|
let sig: &[u8] = if name == "fixed" { b"FADB" } else { b"EADB" };
|
|
let block = clean
|
|
.windows(4)
|
|
.position(|w| w == sig)
|
|
.unwrap_or_else(|| panic!("{name}: no data block in the fixture"));
|
|
// Past the prefix (signature, version, client id, header address, and
|
|
// for the Extensible Array a block offset), into the first address.
|
|
let at = block + 4 + 1 + 1 + 8 + if name == "fixed" { 0 } else { 4 } + 1;
|
|
let mut damaged = clean.clone();
|
|
damaged[at] ^= 0x10;
|
|
let damaged_path = dir.path().join(format!("{name}_damaged.h5"));
|
|
std::fs::write(&damaged_path, &damaged).unwrap();
|
|
|
|
let result = File::open(&damaged_path)
|
|
.unwrap()
|
|
.dataset("x")
|
|
.and_then(|d| d.read_i32());
|
|
assert!(
|
|
result.is_err(),
|
|
"{name}: corruption produced data instead of an error"
|
|
);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Half precision (float16) in both directions
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/// Values that exercise rounding: ties, subnormals, the overflow boundary and
|
|
/// ordinary embedding-sized components.
|
|
fn f16_probe_values() -> Vec<f32> {
|
|
let mut v = vec![
|
|
0.0,
|
|
-0.0,
|
|
1.0,
|
|
-1.0,
|
|
0.5,
|
|
1.0 + 2f32.powi(-11),
|
|
1.0 + 3.0 * 2f32.powi(-11),
|
|
65504.0,
|
|
65519.0,
|
|
65520.0,
|
|
-70000.0,
|
|
6.0e-8,
|
|
3.0e-8,
|
|
1.0e-9,
|
|
1.0e-5,
|
|
0.1,
|
|
0.333_333,
|
|
1234.567,
|
|
f32::INFINITY,
|
|
f32::NEG_INFINITY,
|
|
];
|
|
// A deterministic spread of embedding-like values.
|
|
let mut x = 0x2545_F491u32;
|
|
for _ in 0..4000 {
|
|
x ^= x << 13;
|
|
x ^= x >> 17;
|
|
x ^= x << 5;
|
|
v.push((x as f32 / u32::MAX as f32 - 0.5) * 0.4);
|
|
}
|
|
v
|
|
}
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_f16_h5py_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("ours_f16.h5");
|
|
let path_str = path.display().to_string();
|
|
let values = f16_probe_values();
|
|
|
|
let mut fb = FileBuilder::new();
|
|
fb.create_dataset("plain").with_f16_data(&values);
|
|
fb.create_dataset("chunked")
|
|
.with_f16_data(&values)
|
|
.with_shape(&[values.len() as u64])
|
|
.with_chunks(&[512])
|
|
.with_deflate(6);
|
|
fb.write(&path).unwrap();
|
|
|
|
// h5py must see a genuine float16 dataset, and our rounding must agree
|
|
// with numpy's own float32 -> float16 conversion bit for bit.
|
|
let input = values
|
|
.iter()
|
|
.map(|v| format!("{:?}", v.to_bits()))
|
|
.collect::<Vec<_>>()
|
|
.join(",");
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
src = np.array([{input}], dtype=np.uint32).view(np.float32)
|
|
expected = src.astype(np.float16).view(np.uint16)
|
|
with h5py.File("{path_str}", "r") as f:
|
|
for name in ("plain", "chunked"):
|
|
d = f[name]
|
|
assert d.dtype == np.float16, (name, d.dtype)
|
|
got = d[:].view(np.uint16)
|
|
bad = np.nonzero(got != expected)[0]
|
|
assert bad.size == 0, (name, bad[:5], got[bad[:5]], expected[bad[:5]])
|
|
print("ok")
|
|
"#
|
|
);
|
|
assert_eq!(run_python_output(&script), "ok");
|
|
}
|
|
|
|
#[test]
|
|
fn h5py_writes_f16_clawhdf5_reads() {
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("h5py_f16.h5");
|
|
let path_str = path.display().to_string();
|
|
let values = f16_probe_values();
|
|
let input = values
|
|
.iter()
|
|
.map(|v| format!("{:?}", v.to_bits()))
|
|
.collect::<Vec<_>>()
|
|
.join(",");
|
|
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
src = np.array([{input}], dtype=np.uint32).view(np.float32).astype(np.float16)
|
|
with h5py.File("{path_str}", "w") as f:
|
|
f.create_dataset("plain", data=src)
|
|
f.create_dataset("chunked", data=src, chunks=(512,), compression="gzip", shuffle=True)
|
|
f.create_dataset("big_endian", data=src.astype(">f2"))
|
|
"#
|
|
);
|
|
run_python(&script);
|
|
|
|
let expected: Vec<u32> = values
|
|
.iter()
|
|
.map(|&v| clawhdf5_format::float16::round_to_f16(v).to_bits())
|
|
.collect();
|
|
let file = File::open(&path).unwrap();
|
|
for name in ["plain", "chunked", "big_endian"] {
|
|
let ds = file.dataset(name).unwrap();
|
|
assert_eq!(
|
|
ds.dtype().unwrap(),
|
|
DType::Other("float16".into()),
|
|
"{name}"
|
|
);
|
|
let got: Vec<u32> = ds.read_f32().unwrap().iter().map(|v| v.to_bits()).collect();
|
|
assert_eq!(got, expected, "{name}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn clawhdf5_writes_f32_h5py_reads() {
|
|
// Every f32 dataset used to be unreadable by h5py ("sign bit position out
|
|
// of bounds"): the float datatype's sign position was hard-coded for f64.
|
|
skip_if_no_python!();
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let path = dir.path().join("ours_f32.h5");
|
|
let path_str = path.display().to_string();
|
|
let values: Vec<f32> = vec![1.5, -2.25, 3.0e-7, 65536.5, f32::MAX, -0.0];
|
|
|
|
let mut fb = FileBuilder::new();
|
|
fb.create_dataset("plain").with_f32_data(&values);
|
|
fb.create_dataset("chunked")
|
|
.with_f32_data(&values)
|
|
.with_shape(&[values.len() as u64])
|
|
.with_chunks(&[4])
|
|
.with_deflate(6);
|
|
fb.write(&path).unwrap();
|
|
|
|
let bits = values
|
|
.iter()
|
|
.map(|v| v.to_bits().to_string())
|
|
.collect::<Vec<_>>()
|
|
.join(",");
|
|
let script = format!(
|
|
r#"
|
|
import h5py, numpy as np
|
|
expected = np.array([{bits}], dtype=np.uint32)
|
|
with h5py.File("{path_str}", "r") as f:
|
|
for name in ("plain", "chunked"):
|
|
d = f[name]
|
|
assert d.dtype == np.float32, (name, d.dtype)
|
|
assert (d[:].view(np.uint32) == expected).all(), (name, d[:])
|
|
print("ok")
|
|
"#
|
|
);
|
|
assert_eq!(run_python_output(&script), "ok");
|
|
}
|