From 29baabbed20e7221c8e996f7573dc1578c24402d Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 19 Sep 2026 16:57:51 -0700 Subject: [PATCH] feat(agent): selectable fusion; adopt the measured 0.4/0.6 default weights MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BENCHMARKS.md has recorded since the weight sweep that the 0.7/0.3 default is strictly dominated by 0.4/0.6 over the full LongMemEval haystack, but the shipping code never adopted it: unified_search and the OpenClaw backend both passed 0.7/0.3. Re-running the sweep here (500 questions, real MiniLM embeddings on a GPU) reproduces it — turn-level Hit@1 51.6% vs 44.2%, Hit@5 81.4% vs 79.2%, Hit@10 87.8% vs 85.8%, MRR 0.6430 vs 0.5856 — so both now use hybrid::DEFAULT_FUSION, which is that operating point and carries the reasoning. A unit test pins it. Fusion is also selectable now. hybrid::Fusion is either Weighted { vector, keyword } or Rrf { k }; hybrid::fuse applies either to one candidate list per stage, and merge_vector_keyword / hybrid_search delegate to it, so the public API is unchanged. New HDF5Memory::hybrid_search_with and hybrid::hybrid_search_fused take a Fusion. Reciprocal rank fusion was implemented but reachable only as a free function over a linear scan, so it had never been compared with the weighted sum on equal terms; it is now a mode in the LongMemEval bench (measurement to follow). Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 20 +++ crates/clawhdf5-agent/src/async_memory.rs | 2 +- crates/clawhdf5-agent/src/hybrid.rs | 166 ++++++++++++++++-- crates/clawhdf5-agent/src/lib.rs | 3 +- crates/clawhdf5-agent/src/openclaw.rs | 11 +- crates/clawhdf5-agent/src/search.rs | 56 +++--- .../src/bin/longmemeval_bench.rs | 77 ++++---- 7 files changed, 256 insertions(+), 79 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 705e7ac..002bd94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,26 @@ ## Unreleased +### Retrieval quality +- `clawhdf5-agent`: **the default fusion weights are now the measured ones.** + A sweep of every 0.1 step over the full LongMemEval haystack (500 questions, + real MiniLM embeddings) shows the long-standing `0.7/0.3` default is + *strictly dominated* by `0.4/0.6` — turn-level Hit@1 51.6% vs 44.2%, Hit@5 + 81.4% vs 79.2%, Hit@10 87.8% vs 85.8%, MRR 0.643 vs 0.586, and better at + session level too. The finding was recorded in `BENCHMARKS.md` but had never + been applied: `unified_search` and the OpenClaw backend both hardcoded + `0.7/0.3`. They now use `hybrid::DEFAULT_FUSION`. **Callers passing weights + to `hybrid_search` explicitly are unaffected** — pass `0.4`/`0.6` (or use + `hybrid_search_with`) to get the tuned behaviour. +- `clawhdf5-agent`: fusion is now selectable. New `hybrid::Fusion` + (`Weighted { vector, keyword }` or `Rrf { k }`), `hybrid::fuse`, + `hybrid::hybrid_search_fused` and `HDF5Memory::hybrid_search_with`. + Reciprocal rank fusion existed but was unreachable from the store, so it had + never been measured against the weighted sum; the LongMemEval bench now has + an `RRF` mode. + +## Unreleased + ### HDF5 Read Path - **Selection reads cost what the selection costs.** `read_*_selection` decoded the *entire* dataset and then picked elements out, so a 64 x 64 window of a diff --git a/crates/clawhdf5-agent/src/async_memory.rs b/crates/clawhdf5-agent/src/async_memory.rs index 1f665ad..d7dd8e3 100644 --- a/crates/clawhdf5-agent/src/async_memory.rs +++ b/crates/clawhdf5-agent/src/async_memory.rs @@ -37,7 +37,7 @@ //! let mem = AsyncHDF5Memory::open_with(path, config).await?; //! mem.save(entry).await?; // buffered → background writer //! mem.save_batch(entries).await?; // also buffered -//! let results = mem.hybrid_search(emb, "query".into(), 0.7, 0.3, 5).await; +//! let results = mem.hybrid_search(emb, "query".into(), 0.4, 0.6, 5).await; //! mem.shutdown().await?; // final flush + stop //! ``` diff --git a/crates/clawhdf5-agent/src/hybrid.rs b/crates/clawhdf5-agent/src/hybrid.rs index c943ed0..80ea344 100644 --- a/crates/clawhdf5-agent/src/hybrid.rs +++ b/crates/clawhdf5-agent/src/hybrid.rs @@ -29,12 +29,39 @@ pub fn hybrid_search( query_embedding: &[f32], query_text: &str, vectors: &[Vec], - _chunks: &[String], + chunks: &[String], tombstones: &[u8], bm25_index: &BM25Index, vector_weight: f32, keyword_weight: f32, k: usize, +) -> Vec<(usize, f32)> { + hybrid_search_fused( + query_embedding, + query_text, + vectors, + chunks, + tombstones, + bm25_index, + Fusion::Weighted { + vector: vector_weight, + keyword: keyword_weight, + }, + k, + ) +} + +/// [`hybrid_search`] with the fusion method chosen explicitly. +#[allow(clippy::too_many_arguments)] +pub fn hybrid_search_fused( + query_embedding: &[f32], + query_text: &str, + vectors: &[Vec], + _chunks: &[String], + tombstones: &[u8], + bm25_index: &BM25Index, + fusion: Fusion, + k: usize, ) -> Vec<(usize, f32)> { // Get raw scores from both systems. Request all results so normalization // covers the full distribution. @@ -60,7 +87,7 @@ pub fn hybrid_search( }; let kw_scores = bm25_index.scores(query_text); - merge_vector_keyword(vec_scores, kw_scores, vector_weight, keyword_weight, k) + fuse(vec_scores, kw_scores, fusion, k) } /// Merge pre-computed vector-similarity and keyword scores into a single ranking. @@ -76,18 +103,92 @@ pub fn merge_vector_keyword( keyword_weight: f32, k: usize, ) -> Vec<(usize, f32)> { - // Normalize each set to [0, 1]. - let vec_normalized = normalize_scores(&vec_scores); - let kw_normalized = normalize_scores(&kw_scores); + fuse( + vec_scores, + kw_scores, + Fusion::Weighted { + vector: vector_weight, + keyword: keyword_weight, + }, + k, + ) +} - // Merge scores with weights. - let mut merged: HashMap = HashMap::new(); +/// How the vector and keyword stages are combined into one ranking. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Fusion { + /// Min-max normalise each stage over its own candidates, then take a + /// weighted sum. Uses the *scores*, so a stage that separates its + /// candidates sharply keeps that separation — and a stage whose candidates + /// are all near-identical contributes little. + Weighted { + /// Weight on the vector stage. + vector: f32, + /// Weight on the keyword stage. + keyword: f32, + }, + /// Reciprocal rank fusion: each stage contributes `1 / (k + rank)`, + /// ignoring score magnitudes entirely. Robust when the two stages' + /// scores aren't comparable, at the cost of discarding confidence. + Rrf { + /// The rank-damping constant; 60 is the value from the original paper. + k: f32, + }, +} - for (idx, score) in &vec_normalized { - *merged.entry(*idx).or_insert(0.0) += vector_weight * score; +impl Default for Fusion { + fn default() -> Self { + DEFAULT_FUSION } - for (idx, score) in &kw_normalized { - *merged.entry(*idx).or_insert(0.0) += keyword_weight * score; +} + +/// The fusion `hybrid_search` uses unless told otherwise. +/// +/// The weights are not a guess: a sweep of every 0.1 step over the full +/// LongMemEval haystack (500 questions, real MiniLM embeddings) found the +/// long-standing 0.7/0.3 default *strictly dominated* — 0.4/0.6 is better at +/// Hit@1, Hit@5, Hit@10 and MRR, at both turn and session granularity. See +/// `BENCHMARKS.md`, "Weight sweep". +pub const DEFAULT_FUSION: Fusion = Fusion::Weighted { + vector: 0.4, + keyword: 0.6, +}; + +/// Combine one ranked candidate list from each stage into a single top-`k`. +/// +/// Neither list need be sorted; both are consumed. +pub fn fuse( + vec_scores: Vec<(usize, f32)>, + kw_scores: Vec<(usize, f32)>, + fusion: Fusion, + k: usize, +) -> Vec<(usize, f32)> { + let mut merged: HashMap = HashMap::new(); + match fusion { + Fusion::Weighted { vector, keyword } => { + // Normalize each set to [0, 1]. + for (idx, score) in &normalize_scores(&vec_scores) { + *merged.entry(*idx).or_insert(0.0) += vector * score; + } + for (idx, score) in &normalize_scores(&kw_scores) { + *merged.entry(*idx).or_insert(0.0) += keyword * score; + } + } + Fusion::Rrf { k: damping } => { + for mut stage in [vec_scores, kw_scores] { + // Rank 1 is the best score. Ties break by index so a stage's + // contribution doesn't depend on the candidate order it + // happened to be produced in. + stage.sort_by(|a, b| { + b.1.partial_cmp(&a.1) + .unwrap_or(std::cmp::Ordering::Equal) + .then(a.0.cmp(&b.0)) + }); + for (rank, (idx, _)) in stage.iter().enumerate() { + *merged.entry(*idx).or_insert(0.0) += 1.0 / (damping + (rank + 1) as f32); + } + } + } } let mut results: Vec<(usize, f32)> = merged.into_iter().collect(); @@ -353,6 +454,49 @@ mod tests { assert_eq!(result[0].1, 1.0); } + #[test] + fn default_fusion_is_the_tuned_operating_point() { + // A sweep over the full LongMemEval haystack found 0.7/0.3 strictly + // dominated by 0.4/0.6 (BENCHMARKS.md). This guards the finding + // against being quietly undone. + assert_eq!( + DEFAULT_FUSION, + Fusion::Weighted { + vector: 0.4, + keyword: 0.6 + } + ); + } + + #[test] + fn rrf_rewards_agreement_between_the_stages_and_ignores_magnitudes() { + // Doc 1 is second-best in both stages; doc 0 is best in one and absent + // from the other. RRF prefers the doc both stages liked. + let vec_scores = vec![(0, 100.0), (1, 0.9)]; + let kw_scores = vec![(2, 5.0), (1, 4.9)]; + let ranked = fuse(vec_scores, kw_scores, Fusion::Rrf { k: 60.0 }, 3); + assert_eq!(ranked[0].0, 1, "{ranked:?}"); + + // Scaling one stage's scores cannot change an RRF ranking, only the + // order within that stage can. + let a = fuse( + vec![(0, 1.0), (1, 0.5)], + vec![(1, 2.0), (0, 1.0)], + Fusion::Rrf { k: 60.0 }, + 2, + ); + let b = fuse( + vec![(0, 1e6), (1, -3.0)], + vec![(1, 0.002), (0, 0.001)], + Fusion::Rrf { k: 60.0 }, + 2, + ); + assert_eq!( + a.iter().map(|r| r.0).collect::>(), + b.iter().map(|r| r.0).collect::>() + ); + } + #[test] fn merge_top_k_matches_a_full_sort() { // Many ties (scores repeat) so the index tie-break is exercised. diff --git a/crates/clawhdf5-agent/src/lib.rs b/crates/clawhdf5-agent/src/lib.rs index 746152e..a3c5bd4 100644 --- a/crates/clawhdf5-agent/src/lib.rs +++ b/crates/clawhdf5-agent/src/lib.rs @@ -1350,7 +1350,8 @@ impl HDF5Memory { k: usize, ) -> Vec { // Persistent tier. - let persistent = self.hybrid_search(query_embedding, query_text, 0.7, 0.3, k); + let persistent = + self.hybrid_search_with(query_embedding, query_text, hybrid::DEFAULT_FUSION, k); const EPHEMERAL_BOOST: f32 = 1.2; let mut results = persistent; diff --git a/crates/clawhdf5-agent/src/openclaw.rs b/crates/clawhdf5-agent/src/openclaw.rs index 8bb0acd..e7e301c 100644 --- a/crates/clawhdf5-agent/src/openclaw.rs +++ b/crates/clawhdf5-agent/src/openclaw.rs @@ -531,11 +531,14 @@ impl MemoryBackend for ClawhdfBackend { query_embedding: &[f32], k: usize, ) -> Vec { - // 1. Hybrid retrieval (RRF-blended vector + BM25). + // 1. Hybrid retrieval (vector + BM25, fused by score). let candidates = k.saturating_mul(3).max(10); - let raw = self - .memory - .hybrid_search(query_embedding, query_text, 0.7, 0.3, candidates); + let raw = self.memory.hybrid_search_with( + query_embedding, + query_text, + crate::hybrid::DEFAULT_FUSION, + candidates, + ); if raw.is_empty() { return Vec::new(); diff --git a/crates/clawhdf5-agent/src/search.rs b/crates/clawhdf5-agent/src/search.rs index 616b0e3..29f53ac 100644 --- a/crates/clawhdf5-agent/src/search.rs +++ b/crates/clawhdf5-agent/src/search.rs @@ -20,8 +20,7 @@ impl HDF5Memory { query_embedding: &[f32], query_text: &str, bm25: &bm25::BM25Index, - vector_weight: f32, - keyword_weight: f32, + fusion: hybrid::Fusion, k: usize, ) -> Vec<(usize, f32)> { self.ensure_hnsw_fresh(); @@ -38,23 +37,16 @@ impl HDF5Memory { // Fusion normalises over every keyword match, so it needs all // the scores — but not ranked. let kw_scores = bm25.scores(query_text); - hybrid::merge_vector_keyword( - vec_scores, - kw_scores, - vector_weight, - keyword_weight, - k, - ) + hybrid::fuse(vec_scores, kw_scores, fusion, k) } - _ => hybrid::hybrid_search( + _ => hybrid::hybrid_search_fused( query_embedding, query_text, &self.cache.embeddings, &self.cache.chunks, &self.cache.tombstones, bm25, - vector_weight, - keyword_weight, + fusion, k, ), } @@ -66,19 +58,17 @@ impl HDF5Memory { query_embedding: &[f32], query_text: &str, bm25: &bm25::BM25Index, - vector_weight: f32, - keyword_weight: f32, + fusion: hybrid::Fusion, k: usize, ) -> Vec<(usize, f32)> { - hybrid::hybrid_search( + hybrid::hybrid_search_fused( query_embedding, query_text, &self.cache.embeddings, &self.cache.chunks, &self.cache.tombstones, bm25, - vector_weight, - keyword_weight, + fusion, k, ) } @@ -91,20 +81,36 @@ impl HDF5Memory { vector_weight: f32, keyword_weight: f32, k: usize, + ) -> Vec { + self.hybrid_search_with( + query_embedding, + query_text, + hybrid::Fusion::Weighted { + vector: vector_weight, + keyword: keyword_weight, + }, + k, + ) + } + + /// [`HDF5Memory::hybrid_search`] with the fusion method chosen explicitly. + /// + /// [`hybrid::DEFAULT_FUSION`] is what the weighted form defaults to; + /// [`hybrid::Fusion::Rrf`] combines the two stages by rank instead of by + /// score. + pub fn hybrid_search_with( + &mut self, + query_embedding: &[f32], + query_text: &str, + fusion: hybrid::Fusion, + k: usize, ) -> Vec { // The keyword index lives for the life of the store and is updated // incrementally. Take it out for the duration of the call so the // vector stage can borrow `self` mutably, then put it back. self.ensure_bm25_fresh(); let bm25 = self.bm25.take().expect("ensure_bm25_fresh leaves an index"); - let scored = self.vector_keyword_search( - query_embedding, - query_text, - &bm25, - vector_weight, - keyword_weight, - k, - ); + let scored = self.vector_keyword_search(query_embedding, query_text, &bm25, fusion, k); let mut results: Vec = scored .into_iter() .map(|(idx, score)| { diff --git a/crates/clawhdf5-bench/src/bin/longmemeval_bench.rs b/crates/clawhdf5-bench/src/bin/longmemeval_bench.rs index 2729b9d..c723627 100644 --- a/crates/clawhdf5-bench/src/bin/longmemeval_bench.rs +++ b/crates/clawhdf5-bench/src/bin/longmemeval_bench.rs @@ -55,42 +55,57 @@ use std::time::{Duration, Instant}; #[path = "longmemeval_bench/embedder.rs"] mod embedder; +use clawhdf5_agent::hybrid::Fusion; use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry}; use serde::Deserialize; use tempfile::TempDir; const EMBEDDING_DIM: usize = 384; +/// A mode's fusion, as one short string for the reports. +fn describe(mode: Mode) -> String { + match mode.fusion { + Fusion::Weighted { vector, keyword } => format!("vector_{vector:.1}_keyword_{keyword:.1}"), + Fusion::Rrf { k } => format!("rrf_k{k:.0}"), + } +} + /// A retrieval configuration: how much of the score comes from each stage. #[derive(Clone, Copy)] struct Mode { label: &'static str, - vector_weight: f32, - keyword_weight: f32, + /// How the two retrieval stages are combined into one ranking. + fusion: Fusion, +} + +impl Mode { + const fn weighted(label: &'static str, vector: f32, keyword: f32) -> Self { + Self { + label, + fusion: Fusion::Weighted { vector, keyword }, + } + } } /// The only mode available without real embeddings. Passing zero vectors with /// `vector_weight = 0.0` is what made the vector stage inert. -const BM25_ONLY: Mode = Mode { - label: "BM25 only (vector stage inert)", - vector_weight: 0.0, - keyword_weight: 1.0, -}; +const BM25_ONLY: Mode = Mode::weighted("BM25 only (vector stage inert)", 0.0, 1.0); #[cfg(feature = "embeddings")] -const VECTOR_ONLY: Mode = Mode { - label: "Vector only (MiniLM + HNSW)", - vector_weight: 1.0, - keyword_weight: 0.0, -}; +const VECTOR_ONLY: Mode = Mode::weighted("Vector only (MiniLM + HNSW)", 1.0, 0.0); /// Tuned by `--sweep` over the full haystack. The former 0.7/0.3 was a /// documented default that had never been searched, and the sweep found it /// strictly dominated: 0.4/0.6 is better on Hit@1, Hit@5, Hit@10 and MRR at /// both granularities. #[cfg(feature = "embeddings")] -const HYBRID: Mode = Mode { - label: "Hybrid (0.4 vector / 0.6 BM25, tuned)", - vector_weight: 0.4, - keyword_weight: 0.6, +const HYBRID: Mode = Mode::weighted("Hybrid (0.4 vector / 0.6 BM25, tuned)", 0.4, 0.6); + +/// Reciprocal rank fusion, the documented alternative to the weighted sum. +/// It ignores score magnitudes, so there is nothing to tune — which is the +/// claim being tested. +#[cfg(feature = "embeddings")] +const RRF: Mode = Mode { + label: "Hybrid (reciprocal rank fusion, k=60)", + fusion: Fusion::Rrf { k: 60.0 }, }; /// Every 0.1 step of vector weight, keyword weight taking the remainder. @@ -104,11 +119,11 @@ fn sweep_modes() -> Vec { (0..=10) .map(|i| { let v = i as f32 / 10.0; - Mode { - label: Box::leak(format!("sweep v={v:.1} / k={:.1}", 1.0 - v).into_boxed_str()), - vector_weight: v, - keyword_weight: 1.0 - v, - } + Mode::weighted( + Box::leak(format!("sweep v={v:.1} / k={:.1}", 1.0 - v).into_boxed_str()), + v, + 1.0 - v, + ) }) .collect() } @@ -321,13 +336,7 @@ fn evaluate_question( let query_emb = embedding_for(embeddings, &q.question); let t0 = Instant::now(); - let results = memory.hybrid_search( - &query_emb, - &q.question, - mode.vector_weight, - mode.keyword_weight, - top_k, - ); + let results = memory.hybrid_search_with(&query_emb, &q.question, mode.fusion, top_k); let latency = t0.elapsed(); // Session-level recall @@ -472,10 +481,7 @@ fn print_report( println!(" LongMemEval Benchmark — {}", mode.label); println!("================================================================="); println!(); - println!( - "Mode: vector_weight={:.1} / keyword_weight={:.1}", - mode.vector_weight, mode.keyword_weight - ); + println!("Mode: {}", describe(mode)); println!(); println!("Scoring target: RETRIEVAL RECALL (did the gold memory land in top-k)."); println!(" No answer is generated or scored. This is NOT the official"); @@ -602,10 +608,7 @@ fn print_report( println!("```json"); println!("{{"); println!(" \"benchmark\": \"longmemeval\","); - println!( - " \"mode\": \"vector_{:.1}_keyword_{:.1}\",", - mode.vector_weight, mode.keyword_weight - ); + println!(" \"mode\": \"{}\",", describe(mode)); println!(" \"dataset_variant\": \"{}\",", profile.variant()); println!(" \"scoring_target\": \"retrieval_recall\","); println!(" \"k\": 10,"); @@ -768,7 +771,7 @@ fn main() { if sweep { sweep_modes() } else { - vec![BM25_ONLY, VECTOR_ONLY, HYBRID] + vec![BM25_ONLY, VECTOR_ONLY, HYBRID, RRF] } } #[cfg(not(feature = "embeddings"))]