feat(agent): MemoryConfig::float16 stores half-precision embeddings
The setting was persisted in /meta and otherwise ignored: embeddings
were always written as f32. It now does what it says.
clawhdf5-format:
- `DatasetBuilder::with_f16_data` writes IEEE binary16 (numpy float16),
rounding to nearest-even, and `make_f16_type`.
- `clawhdf5_format::float16` holds the f32 <-> f16 conversions, the one
implementation the writer, the reader and the agent all use. Checked
against the `half` crate on 16.7M f32 values and round-trips all 65536
half values; the h5py interop tests confirm the rounding matches
numpy's bit for bit (4020 values incl. ties, subnormals, overflow).
- Reading little-endian float16 as f32 has a fast path.
clawhdf5-agent:
- A float16 store writes /memory/embeddings as half precision, and
`MemoryCache::half_precision` rounds each embedding as it enters the
cache (save, update, WAL replay, and on load of a store still f32 on
disk), so memory and file agree bit for bit and a store searches the
same before and after a reopen (tested).
- Values beyond +-65504 are refused with the new
`MemoryError::InvalidEntry` rather than stored as infinity, on every
save path; batches are all or nothing, and a rejected ephemeral entry
stays in the ephemeral tier. Breaking for exhaustive matches.
- CLI: `create --float16`. Off by default.
Measured on tank, 384-dim, six runs alternating order, medians
(search_harness --float16-study --full): at 100K the file goes from
154.0 to 80.8 MiB (-48%), checkpoint 752 -> 512 ms, open 300 -> 252 ms;
vector recall@10 against an exact scan and hybrid_search latency do not
change. At 10K open is 3 ms slower. Also a test that h5py opens a whole
agent store, f32 and float16, and decodes every dataset.
Docs: README, BENCHMARKS.md ("float16 embedding storage"), CHANGELOG
(including the h5py interop fixes in the previous commit), CLAUDE.md.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
@@ -1,6 +1,7 @@
|
||||
//! In-memory cache for memory entries, sessions, and knowledge graph.
|
||||
|
||||
use crate::vector_search;
|
||||
use clawhdf5_format::float16::round_to_f16;
|
||||
|
||||
/// Every entry's embedding, in one contiguous `[N x dim]` buffer.
|
||||
///
|
||||
@@ -149,6 +150,11 @@ pub struct MemoryCache {
|
||||
pub norms: Vec<f32>,
|
||||
/// Hebbian activation weights (default 1.0 per entry).
|
||||
pub activation_weights: Vec<f32>,
|
||||
/// Round every embedding to IEEE half precision as it enters the cache,
|
||||
/// so the cache holds exactly what a `float16` store writes to disk. Set
|
||||
/// it with [`MemoryCache::set_half_precision`], which also rounds the
|
||||
/// rows already held.
|
||||
pub half_precision: bool,
|
||||
}
|
||||
|
||||
impl MemoryCache {
|
||||
@@ -164,9 +170,44 @@ impl MemoryCache {
|
||||
embedding_dim,
|
||||
norms: Vec::new(),
|
||||
activation_weights: Vec::new(),
|
||||
half_precision: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Switch half-precision rounding on or off. Turning it on rounds every
|
||||
/// embedding already held (and recomputes norms where one changed) —
|
||||
/// e.g. a `float16` store whose last checkpoint predates half-precision
|
||||
/// storage and so is still `f32` on disk.
|
||||
pub fn set_half_precision(&mut self, on: bool) {
|
||||
self.half_precision = on;
|
||||
if !on {
|
||||
return;
|
||||
}
|
||||
for i in 0..self.embeddings.len() {
|
||||
let row = &self.embeddings[i];
|
||||
if row
|
||||
.iter()
|
||||
.all(|&v| round_to_f16(v).to_bits() == v.to_bits())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let rounded: Vec<f32> = row.iter().map(|&v| round_to_f16(v)).collect();
|
||||
self.norms[i] = vector_search::compute_norm(&rounded);
|
||||
self.embeddings.set(i, &rounded);
|
||||
}
|
||||
}
|
||||
|
||||
/// The embedding as the cache will hold it: rounded to half precision
|
||||
/// when [`Self::half_precision`] is on, otherwise unchanged.
|
||||
fn stored_form(&self, mut embedding: Vec<f32>) -> Vec<f32> {
|
||||
if self.half_precision {
|
||||
for v in &mut embedding {
|
||||
*v = round_to_f16(*v);
|
||||
}
|
||||
}
|
||||
embedding
|
||||
}
|
||||
|
||||
/// Kept for callers that used to have to re-flatten after a bulk load.
|
||||
/// The buffer is always flat now, so there is nothing to rebuild.
|
||||
#[deprecated(note = "embeddings are stored flat; this is a no-op")]
|
||||
@@ -202,6 +243,7 @@ impl MemoryCache {
|
||||
tags: String,
|
||||
) -> usize {
|
||||
let idx = self.chunks.len();
|
||||
let embedding = self.stored_form(embedding);
|
||||
let norm = vector_search::compute_norm(&embedding);
|
||||
self.chunks.push(chunk);
|
||||
self.embeddings.push(&embedding);
|
||||
@@ -240,6 +282,7 @@ impl MemoryCache {
|
||||
session_id: String,
|
||||
) {
|
||||
if idx < self.chunks.len() {
|
||||
let embedding = self.stored_form(embedding);
|
||||
let norm = vector_search::compute_norm(&embedding);
|
||||
self.chunks[idx] = chunk;
|
||||
self.embeddings.set(idx, &embedding);
|
||||
@@ -439,4 +482,60 @@ mod tests {
|
||||
.reset_from(2, vec![vec![1.0, 2.0], vec![3.0, 4.0]]);
|
||||
assert_eq!(cache.embeddings.as_flat(), vec![1.0, 2.0, 3.0, 4.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_half_precision_rounds_existing_rows_and_their_norms() {
|
||||
// A store with float16 set whose checkpoint is still f32 on disk
|
||||
// loads full-precision rows; switching rounding on must bring them to
|
||||
// exactly what the next checkpoint will write.
|
||||
let mut cache = MemoryCache::new(3);
|
||||
cache.push(
|
||||
"a".into(),
|
||||
vec![0.1, 0.2, 0.3],
|
||||
"c".into(),
|
||||
0.0,
|
||||
"s".into(),
|
||||
"".into(),
|
||||
);
|
||||
cache.push(
|
||||
"b".into(),
|
||||
vec![0.5, 0.25, 1.0],
|
||||
"c".into(),
|
||||
0.0,
|
||||
"s".into(),
|
||||
"".into(),
|
||||
);
|
||||
let exact_norm = cache.norms[0];
|
||||
|
||||
cache.set_half_precision(true);
|
||||
let row0: Vec<f32> = [0.1f32, 0.2, 0.3]
|
||||
.iter()
|
||||
.map(|&v| round_to_f16(v))
|
||||
.collect();
|
||||
assert_eq!(&cache.embeddings[0], row0.as_slice());
|
||||
assert_eq!(cache.norms[0], vector_search::compute_norm(&row0));
|
||||
assert_ne!(cache.norms[0], exact_norm);
|
||||
// Already representable: untouched.
|
||||
assert_eq!(&cache.embeddings[1], &[0.5, 0.25, 1.0]);
|
||||
|
||||
// New rows are rounded as they arrive, and updates too.
|
||||
cache.push(
|
||||
"c".into(),
|
||||
vec![0.1, 0.0, 0.0],
|
||||
"c".into(),
|
||||
0.0,
|
||||
"s".into(),
|
||||
"".into(),
|
||||
);
|
||||
assert_eq!(cache.embeddings[2][0], round_to_f16(0.1));
|
||||
cache.update(
|
||||
2,
|
||||
"c".into(),
|
||||
vec![0.3, 0.0, 0.0],
|
||||
"c".into(),
|
||||
0.0,
|
||||
"s".into(),
|
||||
);
|
||||
assert_eq!(cache.embeddings[2][0], round_to_f16(0.3));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -63,6 +63,7 @@ use std::path::{Path, PathBuf};
|
||||
use cache::MemoryCache;
|
||||
#[cfg(feature = "hnsw")]
|
||||
use clawhdf5_ann::{DistanceMetric, HnswIndex, Storage};
|
||||
use clawhdf5_format::float16::round_to_f16;
|
||||
use ephemeral::{EphemeralConfig, EphemeralStore};
|
||||
|
||||
// EphemeralEntry and EphemeralStats are part of the crate public API via
|
||||
@@ -83,6 +84,9 @@ pub enum MemoryError {
|
||||
NotFound(String),
|
||||
/// Another `HDF5Memory` (in this or another process) has the store open.
|
||||
Locked(String),
|
||||
/// A record the store cannot hold as given, e.g. an embedding value
|
||||
/// outside the half-precision range of a `float16` store.
|
||||
InvalidEntry(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for MemoryError {
|
||||
@@ -93,6 +97,7 @@ impl std::fmt::Display for MemoryError {
|
||||
MemoryError::Schema(e) => write!(f, "schema error: {e}"),
|
||||
MemoryError::NotFound(e) => write!(f, "not found: {e}"),
|
||||
MemoryError::Locked(e) => write!(f, "store is locked: {e}"),
|
||||
MemoryError::InvalidEntry(e) => write!(f, "invalid entry: {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -124,6 +129,12 @@ pub struct MemoryConfig {
|
||||
pub embedding_dim: usize,
|
||||
pub chunk_size: usize,
|
||||
pub overlap: usize,
|
||||
/// Store embeddings as IEEE half precision (numpy `float16`): half the
|
||||
/// bytes of the embeddings dataset on disk. Every embedding is rounded to
|
||||
/// the nearest half as it enters the store, in memory as well as on disk,
|
||||
/// so search results are the same before and after a reopen. Values must
|
||||
/// lie within ±65504; a save outside that is `MemoryError::InvalidEntry`.
|
||||
/// Fixed when the store is created (persisted in `/meta`).
|
||||
pub float16: bool,
|
||||
pub compression: bool,
|
||||
pub compression_level: u32,
|
||||
@@ -317,7 +328,8 @@ impl HDF5Memory {
|
||||
/// Create a new HDF5 memory file with the given configuration.
|
||||
pub fn create(config: MemoryConfig) -> Result<Self> {
|
||||
let lock = store_lock::StoreLock::acquire(&config.path)?;
|
||||
let cache = MemoryCache::new(config.embedding_dim);
|
||||
let mut cache = MemoryCache::new(config.embedding_dim);
|
||||
cache.set_half_precision(config.float16);
|
||||
let sessions = SessionCache::new();
|
||||
let knowledge = KnowledgeCache::new();
|
||||
|
||||
@@ -1070,7 +1082,29 @@ impl HDF5Memory {
|
||||
/// Upsert: if an active entry with the same tags (key) exists, update it in-place.
|
||||
/// Otherwise append a new entry. Use this for key-based memory stores where
|
||||
/// the same key should not create duplicates.
|
||||
/// A `float16` store holds embeddings as IEEE half precision, which has no
|
||||
/// finite value beyond ±65504. Refuse such an embedding rather than
|
||||
/// silently store infinity. (Values that are already infinite or NaN are
|
||||
/// stored as they are, as in an `f32` store.)
|
||||
fn check_embedding(&self, embedding: &[f32]) -> Result<()> {
|
||||
if !self.config.float16 {
|
||||
return Ok(());
|
||||
}
|
||||
let overflow = embedding
|
||||
.iter()
|
||||
.enumerate()
|
||||
.find(|&(_, &v)| v.is_finite() && round_to_f16(v).is_infinite());
|
||||
match overflow {
|
||||
None => Ok(()),
|
||||
Some((i, v)) => Err(MemoryError::InvalidEntry(format!(
|
||||
"embedding[{i}] = {v} is outside the half-precision range (±65504) \
|
||||
of this float16 store"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn save_or_update(&mut self, entry: MemoryEntry) -> Result<usize> {
|
||||
self.check_embedding(&entry.embedding)?;
|
||||
if let Some(existing_idx) = self.cache.find_by_tags(&entry.tags) {
|
||||
if let Some(ref mut w) = self.wal {
|
||||
let wal_entry = wal::WalEntry {
|
||||
@@ -1126,6 +1160,7 @@ impl HDF5Memory {
|
||||
|
||||
impl AgentMemory for HDF5Memory {
|
||||
fn save(&mut self, entry: MemoryEntry) -> Result<usize> {
|
||||
self.check_embedding(&entry.embedding)?;
|
||||
if let Some(ref mut w) = self.wal {
|
||||
let wal_entry = wal::WalEntry {
|
||||
entry_type: wal::WalEntryType::Save,
|
||||
@@ -1167,6 +1202,10 @@ impl AgentMemory for HDF5Memory {
|
||||
}
|
||||
|
||||
fn save_batch(&mut self, entries: Vec<MemoryEntry>) -> Result<Vec<usize>> {
|
||||
// All or nothing: check every entry before storing any.
|
||||
for entry in &entries {
|
||||
self.check_embedding(&entry.embedding)?;
|
||||
}
|
||||
let mut indices = Vec::with_capacity(entries.len());
|
||||
for entry in entries {
|
||||
let idx = self.cache.push(
|
||||
@@ -1327,6 +1366,9 @@ impl HDF5Memory {
|
||||
})?;
|
||||
let view = memory_strategy::CacheStoreView::new(&self.cache, &self.knowledge);
|
||||
let output = strat.evaluate(&exchange, &view);
|
||||
for e in &output.entries {
|
||||
self.check_embedding(&e.embedding)?;
|
||||
}
|
||||
for e in &output.entries {
|
||||
self.cache.push(
|
||||
e.chunk.clone(),
|
||||
@@ -1411,6 +1453,16 @@ impl HDF5Memory {
|
||||
let mut promoted = 0;
|
||||
|
||||
for key in candidates {
|
||||
// Check before taking, so a rejected entry stays in the ephemeral
|
||||
// tier rather than being lost.
|
||||
if let Some(emb) = self
|
||||
.ephemeral
|
||||
.as_ref()
|
||||
.and_then(|s| s.get_entry(&key))
|
||||
.and_then(|e| e.embedding.as_deref())
|
||||
{
|
||||
self.check_embedding(emb)?;
|
||||
}
|
||||
let entry = match self
|
||||
.ephemeral
|
||||
.as_mut()
|
||||
|
||||
@@ -159,20 +159,27 @@ fn build_memory_group(
|
||||
// chunks: fixed-length string array
|
||||
write_string_dataset(&mut group, "chunks", &cache.chunks);
|
||||
|
||||
// embeddings: f32 [N x D]
|
||||
// embeddings: [N x D], f32 — or IEEE half precision for a `float16`
|
||||
// store. The cache already holds half-rounded values then, so this
|
||||
// conversion is exact and a reopened store sees the same numbers.
|
||||
let n = cache.embeddings.len() as u64;
|
||||
let d = cache.embedding_dim as u64;
|
||||
let flat = cache.flat_embeddings();
|
||||
{
|
||||
let ds = group
|
||||
.create_dataset("embeddings")
|
||||
.with_f32_data(flat)
|
||||
.with_shape(&[n, d]);
|
||||
let ds = group.create_dataset("embeddings");
|
||||
let elem_bytes: u64 = if config.float16 {
|
||||
ds.with_f16_data(flat);
|
||||
2
|
||||
} else {
|
||||
ds.with_f32_data(flat);
|
||||
4
|
||||
};
|
||||
ds.with_shape(&[n, d]);
|
||||
|
||||
// Chunk size tuning: target ~256KB per chunk for optimal I/O
|
||||
if n > 0 && d > 0 {
|
||||
let target_chunk_bytes: u64 = 256 * 1024;
|
||||
let rows_per_chunk = (target_chunk_bytes / (d * 4)).max(1).min(n);
|
||||
let rows_per_chunk = (target_chunk_bytes / (d * elem_bytes)).max(1).min(n);
|
||||
ds.with_chunks(&[rows_per_chunk, d]);
|
||||
|
||||
// Compression. Shuffle is applied automatically (auto-shuffle
|
||||
@@ -513,7 +520,16 @@ pub fn validate_and_load(
|
||||
};
|
||||
|
||||
// Load /memory group
|
||||
let memory_cache = load_memory_group(file, embedding_dim)?;
|
||||
let mut memory_cache = load_memory_group(file, embedding_dim)?;
|
||||
// A float16 store's cache holds half-rounded embeddings. Embeddings read
|
||||
// from an f16 dataset already are; a float16 store whose last checkpoint
|
||||
// predates half-precision storage is still f32 on disk and is rounded
|
||||
// here.
|
||||
if config.float16 && embeddings_are_f16(file) {
|
||||
memory_cache.half_precision = true;
|
||||
} else {
|
||||
memory_cache.set_half_precision(config.float16);
|
||||
}
|
||||
|
||||
// Load /sessions group
|
||||
let session_cache = load_sessions_group(file)?;
|
||||
@@ -756,6 +772,13 @@ fn read_string_dataset_from_group(
|
||||
.map_err(|e| MemoryError::Hdf5(format!("cannot read strings from {name}: {e}")))
|
||||
}
|
||||
|
||||
/// Whether `/memory/embeddings` is stored as IEEE half precision.
|
||||
fn embeddings_are_f16(file: &clawhdf5::File) -> bool {
|
||||
file.dataset("memory/embeddings")
|
||||
.and_then(|ds| ds.dtype())
|
||||
.is_ok_and(|dt| matches!(dt, clawhdf5::DType::Other(ref s) if s == "float16"))
|
||||
}
|
||||
|
||||
fn read_f32_dataset(group: &clawhdf5::Group<'_>, name: &str) -> Result<Vec<f32>, MemoryError> {
|
||||
let ds = group
|
||||
.dataset(name)
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
//! `MemoryConfig::float16`: embeddings stored as IEEE half precision.
|
||||
//!
|
||||
//! The setting used to be recorded in `/meta` and otherwise ignored — the
|
||||
//! embeddings dataset was always `f32`. These tests pin what it now does: the
|
||||
//! dataset is `float16`, the in-memory cache holds exactly the values the file
|
||||
//! holds (so search results survive a reopen bit for bit), and a value half
|
||||
//! precision cannot represent is refused rather than stored as infinity.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry, MemoryError};
|
||||
use clawhdf5_format::float16::round_to_f16;
|
||||
use tempfile::TempDir;
|
||||
|
||||
const DIM: usize = 64;
|
||||
|
||||
/// Deterministic, embedding-like unit vectors.
|
||||
fn embedding(seed: u64) -> Vec<f32> {
|
||||
let mut x = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1;
|
||||
let v: Vec<f32> = (0..DIM)
|
||||
.map(|_| {
|
||||
x ^= x << 13;
|
||||
x ^= x >> 7;
|
||||
x ^= x << 17;
|
||||
(x >> 40) as f32 / (1u64 << 24) as f32 - 0.5
|
||||
})
|
||||
.collect();
|
||||
let norm = v.iter().map(|a| a * a).sum::<f32>().sqrt();
|
||||
v.iter().map(|a| a / norm).collect()
|
||||
}
|
||||
|
||||
fn entry(i: u64) -> MemoryEntry {
|
||||
MemoryEntry {
|
||||
chunk: format!("memory number {i} about topic {}", i % 7),
|
||||
embedding: embedding(i),
|
||||
source_channel: "test".into(),
|
||||
timestamp: i as f64,
|
||||
session_id: "s".into(),
|
||||
tags: format!("t{i}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn config(dir: &TempDir, name: &str, float16: bool) -> MemoryConfig {
|
||||
let mut c = MemoryConfig::new(dir.path().join(name), "agent", DIM);
|
||||
c.float16 = float16;
|
||||
c
|
||||
}
|
||||
|
||||
fn embeddings_dtype_and_values(path: &Path) -> (String, Vec<f32>) {
|
||||
let file = clawhdf5::File::open(path).unwrap();
|
||||
let ds = file.dataset("memory/embeddings").unwrap();
|
||||
(format!("{:?}", ds.dtype().unwrap()), ds.read_f32().unwrap())
|
||||
}
|
||||
|
||||
fn search_bits(m: &mut HDF5Memory, q: u64) -> Vec<(usize, u32)> {
|
||||
m.hybrid_search(&embedding(q), "memory topic 3", 0.4, 0.6, 10)
|
||||
.iter()
|
||||
.map(|r| (r.index, r.score.to_bits()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn float16_store_writes_half_precision_and_reopens_identically() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
// Two identical stores. Search is not read-only (it boosts the Hebbian
|
||||
// activation of what it returns, and checkpoints persist that), so each
|
||||
// is queried exactly once: one live, one after a checkpoint and reopen.
|
||||
let live_cfg = config(&dir, "live.h5", true);
|
||||
let cfg = config(&dir, "f16.h5", true);
|
||||
let path: PathBuf = cfg.path.clone();
|
||||
|
||||
let mut live = HDF5Memory::create(live_cfg).unwrap();
|
||||
live.save_batch((0..200).map(entry).collect()).unwrap();
|
||||
let mut m = HDF5Memory::create(cfg).unwrap();
|
||||
m.save_batch((0..200).map(entry).collect()).unwrap();
|
||||
drop(m);
|
||||
|
||||
// On disk: a genuine float16 dataset holding the rounded inputs.
|
||||
let (dtype, values) = embeddings_dtype_and_values(&path);
|
||||
assert_eq!(dtype, "Other(\"float16\")");
|
||||
let expected: Vec<u32> = (0..200)
|
||||
.flat_map(|i| embedding(i).into_iter().map(|v| round_to_f16(v).to_bits()))
|
||||
.collect();
|
||||
let got: Vec<u32> = values.iter().map(|v| v.to_bits()).collect();
|
||||
assert_eq!(got, expected);
|
||||
|
||||
// Reopened, the store answers exactly as the live one does: the cache
|
||||
// held the half-rounded values before the checkpoint.
|
||||
let mut reopened = HDF5Memory::open(&path).unwrap();
|
||||
for q in 0..5 {
|
||||
assert_eq!(
|
||||
search_bits(&mut live, 1000 + q),
|
||||
search_bits(&mut reopened, 1000 + q),
|
||||
"query {q}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn float16_halves_the_embeddings_on_disk() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut sizes = Vec::new();
|
||||
for float16 in [false, true] {
|
||||
let cfg = config(&dir, &format!("s{float16}.h5"), float16);
|
||||
let path = cfg.path.clone();
|
||||
let mut m = HDF5Memory::create(cfg).unwrap();
|
||||
m.save_batch((0..2000).map(entry).collect()).unwrap();
|
||||
drop(m);
|
||||
sizes.push(std::fs::metadata(&path).unwrap().len());
|
||||
}
|
||||
let embedding_bytes_f32 = (2000 * DIM * 4) as u64;
|
||||
let saved = sizes[0] - sizes[1];
|
||||
// Half of the f32 embeddings, give or take metadata and alignment.
|
||||
assert!(
|
||||
saved.abs_diff(embedding_bytes_f32 / 2) < 16 * 1024,
|
||||
"f32 {} B, f16 {} B, saved {saved} B, expected ~{} B",
|
||||
sizes[0],
|
||||
sizes[1],
|
||||
embedding_bytes_f32 / 2
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn f32_store_is_unchanged() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let cfg = config(&dir, "f32.h5", false);
|
||||
let path = cfg.path.clone();
|
||||
let mut m = HDF5Memory::create(cfg).unwrap();
|
||||
m.save_batch((0..50).map(entry).collect()).unwrap();
|
||||
drop(m);
|
||||
let (dtype, values) = embeddings_dtype_and_values(&path);
|
||||
assert_eq!(dtype, "F32");
|
||||
let expected: Vec<f32> = (0..50).flat_map(embedding).collect();
|
||||
assert_eq!(values, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn out_of_range_values_are_refused_not_stored_as_infinity() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut cfg = config(&dir, "range.h5", true);
|
||||
cfg.wal_enabled = true;
|
||||
let path = cfg.path.clone();
|
||||
let mut m = HDF5Memory::create(cfg).unwrap();
|
||||
m.save(entry(1)).unwrap();
|
||||
|
||||
let mut bad = entry(2);
|
||||
bad.embedding[5] = 70_000.0;
|
||||
match m.save(bad.clone()) {
|
||||
Err(MemoryError::InvalidEntry(msg)) => assert!(msg.contains("embedding[5]"), "{msg}"),
|
||||
other => panic!("expected InvalidEntry, got {other:?}"),
|
||||
}
|
||||
assert!(matches!(
|
||||
m.save_or_update(bad.clone()),
|
||||
Err(MemoryError::InvalidEntry(_))
|
||||
));
|
||||
// A batch is all or nothing.
|
||||
assert!(matches!(
|
||||
m.save_batch(vec![entry(3), bad.clone(), entry(4)]),
|
||||
Err(MemoryError::InvalidEntry(_))
|
||||
));
|
||||
assert_eq!(m.count(), 1);
|
||||
|
||||
// The largest finite half, and values that round down to it, are fine.
|
||||
let mut edge = entry(5);
|
||||
edge.embedding[0] = 65504.0;
|
||||
edge.embedding[1] = -65519.0;
|
||||
m.save(edge).unwrap();
|
||||
assert_eq!(m.count(), 2);
|
||||
drop(m);
|
||||
|
||||
// Nothing rejected reached the WAL or the file.
|
||||
let m = HDF5Memory::open(&path).unwrap();
|
||||
assert_eq!(m.count(), 2);
|
||||
|
||||
// An f32 store takes the same value as it always did.
|
||||
let mut m32 = HDF5Memory::create(config(&dir, "range32.h5", false)).unwrap();
|
||||
m32.save(bad).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wal_replay_rounds_like_a_live_save() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let mut cfg = config(&dir, "wal.h5", true);
|
||||
cfg.wal_enabled = true;
|
||||
cfg.wal_max_entries = 10_000; // keep everything in the WAL
|
||||
let path = cfg.path.clone();
|
||||
let mut m = HDF5Memory::create(cfg).unwrap();
|
||||
for i in 0..30 {
|
||||
m.save(entry(i)).unwrap();
|
||||
}
|
||||
let live = search_bits(&mut m, 77);
|
||||
|
||||
// Crash image: the .h5 is still the empty checkpoint; everything is in
|
||||
// the WAL, which holds the caller's f32 values.
|
||||
let crash = TempDir::new().unwrap();
|
||||
let image = crash.path().join("image.h5");
|
||||
std::fs::copy(&path, &image).unwrap();
|
||||
std::fs::copy(
|
||||
path.with_extension("h5.wal"),
|
||||
image.with_extension("h5.wal"),
|
||||
)
|
||||
.unwrap();
|
||||
drop(m);
|
||||
|
||||
let mut recovered = HDF5Memory::open(&image).unwrap();
|
||||
assert_eq!(recovered.count(), 30);
|
||||
assert_eq!(search_bits(&mut recovered, 77), live);
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
//! An agent store is a standard HDF5 file: h5py can open it and read every
|
||||
//! dataset.
|
||||
//!
|
||||
//! It could not: the float datatype's sign-bit position was hard-coded for
|
||||
//! f64, so every f32 dataset (embeddings, norms, activation weights) made
|
||||
//! libhdf5 refuse the file with "sign bit position out of bounds".
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry};
|
||||
|
||||
fn python() -> String {
|
||||
std::env::var("CLAWHDF5_PYTHON").unwrap_or_else(|_| "python3".to_string())
|
||||
}
|
||||
|
||||
fn h5py_available() -> bool {
|
||||
Command::new(python())
|
||||
.args(["-c", "import h5py"])
|
||||
.output()
|
||||
.map(|o| o.status.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn h5py_reads_every_dataset_of_an_agent_store() {
|
||||
if !h5py_available() {
|
||||
assert!(
|
||||
std::env::var("CLAWHDF5_REQUIRE_INTEROP").as_deref() != Ok("1"),
|
||||
"CLAWHDF5_REQUIRE_INTEROP=1 but python3 with h5py is not available"
|
||||
);
|
||||
eprintln!("SKIP: python3 with h5py not available");
|
||||
return;
|
||||
}
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
for float16 in [false, true] {
|
||||
let path = dir.path().join(format!("store_{float16}.h5"));
|
||||
let mut cfg = MemoryConfig::new(path.clone(), "agent", 8);
|
||||
cfg.float16 = float16;
|
||||
let mut m = HDF5Memory::create(cfg).unwrap();
|
||||
// save_batch checkpoints, so the records are in the .h5, not the WAL.
|
||||
m.save_batch(
|
||||
(0..20)
|
||||
.map(|i| MemoryEntry {
|
||||
chunk: format!("memory {i}"),
|
||||
embedding: (0..8).map(|j| ((i * 8 + j) as f32).sin()).collect(),
|
||||
source_channel: "test".into(),
|
||||
timestamp: i as f64,
|
||||
session_id: "s".into(),
|
||||
tags: String::new(),
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
.unwrap();
|
||||
drop(m);
|
||||
|
||||
// Exact expected values, as bits: numpy's sin need not match Rust's
|
||||
// to the last place.
|
||||
let bits = (0..160)
|
||||
.map(|k| (k as f32).sin().to_bits().to_string())
|
||||
.collect::<Vec<_>>()
|
||||
.join(",");
|
||||
let script = format!(
|
||||
r#"
|
||||
import h5py, numpy as np
|
||||
want = np.float16 if {py_bool} else np.float32
|
||||
with h5py.File("{path}", "r") as f:
|
||||
names = []
|
||||
f.visititems(lambda n, o: names.append(n) if isinstance(o, h5py.Dataset) else None)
|
||||
for n in names:
|
||||
f[n][()] # every dataset must decode
|
||||
e = f["memory/embeddings"]
|
||||
assert e.dtype == want, e.dtype
|
||||
assert e.shape == (20, 8), e.shape
|
||||
ref = np.array([{bits}], dtype=np.uint32).view(np.float32).astype(want).reshape(20, 8)
|
||||
assert (e[()] == ref).all()
|
||||
assert f["memory/norms"].dtype == np.float32
|
||||
print(len(names))
|
||||
"#,
|
||||
py_bool = if float16 { "True" } else { "False" },
|
||||
path = path.display()
|
||||
);
|
||||
let out = Command::new(python())
|
||||
.args(["-c", &script])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"float16={float16}: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
let n: usize = String::from_utf8_lossy(&out.stdout).trim().parse().unwrap();
|
||||
assert!(n >= 10, "only {n} datasets");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user