fix(format): files we write now open in h5py and libhdf5
Two write-side bugs, both present in every release (the first at least since v2.1.0), made libhdf5 refuse files written by clawhdf5. Our own reader ignores both fields, and the interop suites only ever wrote f64 from our side, so nothing here caught them. - Every f32 dataset: "sign bit position out of bounds". The float datatype encoder hard-coded the sign bit's position (bits 8-15 of the class bit field) to 63, which is right only for f64. It is now derived from the type: bit_offset + bit_precision - 1. This covered every agent store's embeddings, norms and activation weights. - Every empty dataset: "invalid dataset size, likely file corruption". It was written with a real address and size 0, which trips libhdf5's `addr + size <= addr` overflow check. An empty contiguous dataset now gets the undefined address, as libhdf5 writes it. This covered every agent store without sessions or a knowledge graph. Agent stores are rewritten in full at each checkpoint, so they become readable at their next checkpoint on a fixed build; other files with f32 or empty datasets need rewriting. Both are recorded in docs/known-issues.md. Tests: the sign position byte for f32/f64, and h5py reading our f32 datasets (plain and chunked + deflate) bit for bit. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -1387,3 +1387,42 @@ with h5py.File("{path_str}", "w", libver="latest") as f:
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clawhdf5_writes_f32_h5py_reads() {
|
||||
// Every f32 dataset used to be unreadable by h5py ("sign bit position out
|
||||
// of bounds"): the float datatype's sign position was hard-coded for f64.
|
||||
skip_if_no_python!();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("ours_f32.h5");
|
||||
let path_str = path.display().to_string();
|
||||
let values: Vec<f32> = vec![1.5, -2.25, 3.0e-7, 65536.5, f32::MAX, -0.0];
|
||||
|
||||
let mut fb = FileBuilder::new();
|
||||
fb.create_dataset("plain").with_f32_data(&values);
|
||||
fb.create_dataset("chunked")
|
||||
.with_f32_data(&values)
|
||||
.with_shape(&[values.len() as u64])
|
||||
.with_chunks(&[4])
|
||||
.with_deflate(6);
|
||||
fb.write(&path).unwrap();
|
||||
|
||||
let bits = values
|
||||
.iter()
|
||||
.map(|v| v.to_bits().to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
let script = format!(
|
||||
r#"
|
||||
import h5py, numpy as np
|
||||
expected = np.array([{bits}], dtype=np.uint32)
|
||||
with h5py.File("{path_str}", "r") as f:
|
||||
for name in ("plain", "chunked"):
|
||||
d = f[name]
|
||||
assert d.dtype == np.float32, (name, d.dtype)
|
||||
assert (d[:].view(np.uint32) == expected).all(), (name, d[:])
|
||||
print("ok")
|
||||
"#
|
||||
);
|
||||
assert_eq!(run_python_output(&script), "ok");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user