From 5c8323cb1e0a0fcaf137771b4fb79c9fbe0d9c2e Mon Sep 17 00:00:00 2001 From: osobh Date: Thu, 24 Sep 2026 19:39:33 -0500 Subject: [PATCH] feat(agent): new stores default to float16 embeddings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MemoryConfig::float16 now defaults to true for new stores, on measurement: on the full LongMemEval haystack with real MiniLM embeddings every retrieval metric matched f32 (previous commit), and at 100K the file is 48% smaller with faster checkpoints and opens. Existing stores are unaffected: every agent store has recorded `float16 = false` in /meta and keeps it. A test opens the v2.5.0 fixture, saves and checkpoints, and checks the embeddings are still f32 with the old rows bit-identical; another checks a new store is float16. CLI: `create --f32` opts out; like `--f32-index` it only ever switches the default off. `--float16` is still accepted and now a no-op. Values beyond +-65504 are refused, so f32 remains the choice for unnormalised vectors — the upgrade note says so. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 23 ++++++--- CLAUDE.md | 10 ++-- README.md | 20 +++++--- crates/clawhdf5-agent/src/lib.rs | 9 +++- crates/clawhdf5-agent/tests/float16_store.rs | 52 ++++++++++++++++++++ crates/clawhdf5-cli/src/main.rs | 20 +++++--- 6 files changed, 109 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8525407..2996da6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,18 @@ each checkpoint, so it becomes readable at its next checkpoint on this version; other files with `f32` or empty datasets need rewriting. Details in `docs/known-issues.md`. -- **`MemoryConfig::float16` now does what it says.** It was persisted and - otherwise ignored; embeddings were always stored as `f32`. A store created - with it on now writes half-precision embeddings (48% smaller files) and - rounds embeddings to half precision as they are saved. A store that already - had `float16 = true` rounds its embeddings when next opened and writes them - as `float16` at its next checkpoint. Off by default. +- **New stores store embeddings as half precision by default.** + `MemoryConfig::float16` was persisted and otherwise ignored; it now writes + `float16` embeddings (48% smaller files at 100K) and rounds each embedding + to half precision as it is saved — and it defaults to `true` for new + stores. On the full LongMemEval haystack with real MiniLM embeddings every + retrieval metric matched `f32`. **Existing stores are unaffected**: every + agent store has recorded `float16 = false`, and keeps it (a v2.5.0 fixture + guards this). A store that already had `float16 = true` rounds its + embeddings when next opened and writes them as `float16` at its next + checkpoint. Opt out with `float16 = false` or `create --f32`; the CLI's + `--float16` is still accepted and now a no-op. Values beyond ±65504 are + refused, so keep `f32` for unnormalised vectors. - **Breaking:** `MemoryError` gained `InvalidEntry`, returned when a `float16` store is given an embedding value beyond ±65504. Exhaustive matches need the new arm. @@ -123,6 +129,11 @@ now an error. ### Defaults +- `clawhdf5-agent`: `MemoryConfig::float16` defaults to `true` for new stores, + measured rather than assumed: identical LongMemEval retrieval on real + embeddings, 48% smaller files and faster checkpoints and opens at 100K. + `clawhdf5-cli create --f32` opts out; like `--f32-index`, it only ever + switches the default off. - `clawhdf5-agent`: `MemoryConfig::quantized_index` defaults to `true` for new stores. The reason it had been off — that int8 search was slower on ARM — did not survive measurement (see Corrections). Stores that predate the diff --git a/CLAUDE.md b/CLAUDE.md index 6859442..21b47d8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -87,10 +87,12 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F `export` do). An unreadable WAL (torn header, bad magic) is quarantined to `.h5.wal.corrupt-` rather than blocking `open()`; a WAL with an unknown *newer* version still fails and is left untouched. -- `MemoryConfig::float16` (off by default, persisted; CLI `create --float16`) - writes `/memory/embeddings` as IEEE half precision (48% smaller file at - 100K, same recall). `MemoryCache::half_precision` rounds each embedding as - it enters the cache (push, update, WAL replay, and on load of a store still +- `MemoryConfig::float16` (**on by default** for new stores, persisted; + existing stores keep their recorded `false` — guarded by the v2.5.0 + fixture in `tests/float16_store.rs`; CLI opt-out is `create --f32`) writes + `/memory/embeddings` as IEEE half precision (48% smaller file at 100K; + LongMemEval with real MiniLM embeddings identical to f32). + `MemoryCache::half_precision` rounds each embedding as it enters the cache (push, update, WAL replay, and on load of a store still `f32` on disk), so memory and file agree bit for bit; the conversions live in `clawhdf5_format::float16` and must stay the single implementation. Values beyond ±65504 are `MemoryError::InvalidEntry`. Interop: every file diff --git a/README.md b/README.md index eff3587..a96553b 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,8 @@ breaking change, are in [CHANGELOG.md](CHANGELOG.md). fix themselves at their next checkpoint. See [docs/known-issues.md](docs/known-issues.md). - `MemoryConfig::float16` now stores half-precision embeddings (it was - ignored): 48% smaller files at the same recall. + ignored), and is on by default for new stores: 48% smaller files, and + identical LongMemEval retrieval on real embeddings. - `HDF5Memory::search` with `SearchOptions`: filter by source channel (exact filtered top-k, never slower than unfiltered), and opt-in re-ranking and confidence rejection, which used to be OpenClaw-only. @@ -256,8 +257,9 @@ retrieval recall reported as QA accuracy typically overstates by 20–30 points. | 10K | 17.0 MB | 1.7 KB | 2.7 MB (6.2x) | | 100K | 169.8 MB | 1.7 KB | 26.9 MB (6.2x) | -With `MemoryConfig::float16` the embeddings take half the space: an agent -store of 100K × 384 records is 80.8 MiB instead of 154.0. +These figures are `f32` embeddings. New agent stores default to +`MemoryConfig::float16`, which halves them: 100K × 384 records take 80.8 MiB +instead of 154.0. **In memory** — a store reopened from disk, 384-dim `f32`, measured with a counting allocator ([BENCHMARKS.md § Memory footprint](BENCHMARKS.md#memory-footprint)): @@ -610,12 +612,14 @@ setting existed keep their `f32` index; opt out for new stores with `quantized_index = false` or `clawhdf5-cli create --f32-index`. See [BENCHMARKS.md § Quantising the index copy](BENCHMARKS.md#quantising-the-index-copy-quantized_index). -`MemoryConfig::float16` (off by default; CLI `create --float16`) stores the +`MemoryConfig::float16` (**on by default** for new stores) stores the embeddings on disk as IEEE half precision (numpy `float16`): at 100K × 384 the -file drops from 154 to 81 MiB, checkpoints and opens get faster, and vector -recall and search latency do not change. Embeddings are rounded as they are -saved, so the store searches the same before and after a reopen; values must -lie within ±65504. See +file drops from 154 to 81 MiB, checkpoints and opens get faster, and on the +full LongMemEval haystack with real MiniLM embeddings every retrieval metric +matches `f32`. Embeddings are rounded as they are saved, so the store searches +the same before and after a reopen; values must lie within ±65504. Existing +stores keep their setting. Opt out with `float16 = false` or +`clawhdf5-cli create --f32` — e.g. for unnormalised vectors. See [BENCHMARKS.md § float16 embedding storage](BENCHMARKS.md#float16-embedding-storage-memoryconfigfloat16). ### `clawhdf5-format` diff --git a/crates/clawhdf5-agent/src/lib.rs b/crates/clawhdf5-agent/src/lib.rs index 601a8a7..b74507f 100644 --- a/crates/clawhdf5-agent/src/lib.rs +++ b/crates/clawhdf5-agent/src/lib.rs @@ -136,6 +136,13 @@ pub struct MemoryConfig { /// so search results are the same before and after a reopen. Values must /// lie within ±65504; a save outside that is `MemoryError::InvalidEntry`. /// Fixed when the store is created (persisted in `/meta`). + /// + /// **On by default for new stores**: on the full LongMemEval haystack with + /// real MiniLM embeddings every retrieval metric matched `f32`, and at + /// 100K records the file is 48% smaller (`BENCHMARKS.md`). Existing + /// stores keep the setting they were created with. Set it to `false` for + /// full-precision embeddings, e.g. for unnormalised vectors that may + /// exceed the half-precision range. pub float16: bool, pub compression: bool, pub compression_level: u32, @@ -191,7 +198,7 @@ impl MemoryConfig { embedding_dim, chunk_size: 512, overlap: 50, - float16: false, + float16: true, compression: false, compression_level: 0, compact_threshold: 0.3, diff --git a/crates/clawhdf5-agent/tests/float16_store.rs b/crates/clawhdf5-agent/tests/float16_store.rs index 5c83107..0c20d62 100644 --- a/crates/clawhdf5-agent/tests/float16_store.rs +++ b/crates/clawhdf5-agent/tests/float16_store.rs @@ -206,3 +206,55 @@ fn wal_replay_rounds_like_a_live_save() { assert_eq!(recovered.count(), 30); assert_eq!(search_bits(&mut recovered, 77), live); } + +#[test] +fn new_stores_default_to_float16() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("default.h5"); + let mut m = HDF5Memory::create(MemoryConfig::new(path.clone(), "agent", DIM)).unwrap(); + assert!(m.config().float16); + m.save_batch((0..10).map(entry).collect()).unwrap(); + drop(m); + assert_eq!(embeddings_dtype_and_values(&path).0, "Other(\"float16\")"); + assert!(HDF5Memory::open(&path).unwrap().config().float16); +} + +#[test] +fn an_existing_f32_store_stays_f32() { + // Written by the v2.5.0 CLI, with `float16 = 0` in /meta (every agent + // store has recorded it). Flipping the default for new stores must not + // reach back and round an existing store's embeddings. + let dir = TempDir::new().unwrap(); + let path = dir.path().join("legacy.h5"); + std::fs::copy( + concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/store_v2_5_0.h5" + ), + &path, + ) + .unwrap(); + let before = embeddings_dtype_and_values(&path); + assert_eq!(before.0, "F32"); + + let mut m = HDF5Memory::open(&path).unwrap(); + assert!(!m.config().float16, "an old store must reopen as f32"); + let dim = m.config().embedding_dim; + let odd: Vec = (0..dim).map(|i| 0.1 + i as f32 * 1e-4).collect(); + m.save_batch(vec![MemoryEntry { + chunk: "added after the upgrade".into(), + embedding: odd.clone(), + source_channel: "test".into(), + timestamp: 1.0, + session_id: "s".into(), + tags: String::new(), + }]) + .unwrap(); + drop(m); + + // Checkpointed: still f32, the old rows untouched and the new one exact. + let (dtype, values) = embeddings_dtype_and_values(&path); + assert_eq!(dtype, "F32"); + assert_eq!(&values[..before.1.len()], before.1.as_slice()); + assert_eq!(&values[before.1.len()..], odd.as_slice()); +} diff --git a/crates/clawhdf5-cli/src/main.rs b/crates/clawhdf5-cli/src/main.rs index c3f15e3..2fc07c0 100644 --- a/crates/clawhdf5-cli/src/main.rs +++ b/crates/clawhdf5-cli/src/main.rs @@ -36,10 +36,13 @@ enum Commands { /// Accepted for compatibility; int8 is now the default #[arg(long, hide = true, conflicts_with = "f32_index")] quantized_index: bool, - /// Store embeddings on disk as IEEE half precision (float16): half - /// the bytes, about three significant digits; values must lie within - /// ±65504 + /// Store embeddings as full-precision f32 instead of the default + /// half precision (float16: half the bytes, about three significant + /// digits, values within ±65504) #[arg(long)] + f32: bool, + /// Accepted for compatibility; float16 is now the default + #[arg(long, hide = true, conflicts_with = "f32")] float16: bool, }, /// Save a memory entry (reads JSON from stdin or --json) @@ -107,11 +110,16 @@ fn run(cli: Cli) -> Result<(), Box> { wal, f32_index, quantized_index: _, - float16, + f32, + float16: _, } => { let mut config = MemoryConfig::new(cli.path.clone(), &agent_id, dim); config.wal_enabled = wal; - config.float16 = float16; + // As with --f32-index: only ever switch the library default off. + if f32 { + config.float16 = false; + } + let config_float16 = config.float16; // Only ever switch *off* the library default: assigning the flag // outright would force every CLI-created store back to f32 unless // the caller knew to ask for int8. @@ -127,7 +135,7 @@ fn run(cli: Cli) -> Result<(), Box> { "embedding_dim": dim, "wal_enabled": wal, "quantized_index": config_quantized, - "float16": float16, + "float16": config_float16, "count": mem.count(), }); println!("{}", serde_json::to_string_pretty(&j)?);