quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat(agent): selectable fusion; adopt the measured 0.4/0.6 default weights

Browse Source 
BENCHMARKS.md has recorded since the weight sweep that the 0.7/0.3 default is
strictly dominated by 0.4/0.6 over the full LongMemEval haystack, but the
shipping code never adopted it: unified_search and the OpenClaw backend both
passed 0.7/0.3. Re-running the sweep here (500 questions, real MiniLM
embeddings on a GPU) reproduces it — turn-level Hit@1 51.6% vs 44.2%, Hit@5
81.4% vs 79.2%, Hit@10 87.8% vs 85.8%, MRR 0.6430 vs 0.5856 — so both now use
hybrid::DEFAULT_FUSION, which is that operating point and carries the
reasoning. A unit test pins it.

Fusion is also selectable now. hybrid::Fusion is either Weighted { vector,
keyword } or Rrf { k }; hybrid::fuse applies either to one candidate list per
stage, and merge_vector_keyword / hybrid_search delegate to it, so the public
API is unchanged. New HDF5Memory::hybrid_search_with and
hybrid::hybrid_search_fused take a Fusion. Reciprocal rank fusion was
implemented but reachable only as a free function over a linear scan, so it
had never been compared with the weighted sum on equal terms; it is now a mode
in the LongMemEval bench (measurement to follow).

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-19 16:57:51 -07:00
co-authored by Claude Opus 5
parent b23946e62d
commit 29baabbed2
7 changed files with 256 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-bench/src/bin/longmemeval_bench.rs
+40 -37
View File
@@ -55,42 +55,57 @@ use std::time::{Duration, Instant};
#[path = "longmemeval_bench/embedder.rs"]
mod embedder;
use clawhdf5_agent::hybrid::Fusion;
use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry};
use serde::Deserialize;
use tempfile::TempDir;
const EMBEDDING_DIM: usize = 384;
/// A mode's fusion, as one short string for the reports.
fn describe(mode: Mode) -> String {
match mode.fusion {
Fusion::Weighted { vector, keyword } => format!("vector_{vector:.1}_keyword_{keyword:.1}"),
Fusion::Rrf { k } => format!("rrf_k{k:.0}"),
}
}
/// A retrieval configuration: how much of the score comes from each stage.
#[derive(Clone, Copy)]
struct Mode {
label: &'static str,
vector_weight: f32,
keyword_weight: f32,
/// How the two retrieval stages are combined into one ranking.
fusion: Fusion,
}
impl Mode {
const fn weighted(label: &'static str, vector: f32, keyword: f32) -> Self {
Self {
label,
fusion: Fusion::Weighted { vector, keyword },
}
}
}
/// The only mode available without real embeddings. Passing zero vectors with
/// `vector_weight = 0.0` is what made the vector stage inert.
const BM25_ONLY: Mode = Mode {
label: "BM25 only (vector stage inert)",
vector_weight: 0.0,
keyword_weight: 1.0,
};
const BM25_ONLY: Mode = Mode::weighted("BM25 only (vector stage inert)", 0.0, 1.0);
#[cfg(feature = "embeddings")]
const VECTOR_ONLY: Mode = Mode {
label: "Vector only (MiniLM + HNSW)",
vector_weight: 1.0,
keyword_weight: 0.0,
};
const VECTOR_ONLY: Mode = Mode::weighted("Vector only (MiniLM + HNSW)", 1.0, 0.0);
/// Tuned by `--sweep` over the full haystack. The former 0.7/0.3 was a
/// documented default that had never been searched, and the sweep found it
/// strictly dominated: 0.4/0.6 is better on Hit@1, Hit@5, Hit@10 and MRR at
/// both granularities.
#[cfg(feature = "embeddings")]
const HYBRID: Mode = Mode {
label: "Hybrid (0.4 vector / 0.6 BM25, tuned)",
vector_weight: 0.4,
keyword_weight: 0.6,
const HYBRID: Mode = Mode::weighted("Hybrid (0.4 vector / 0.6 BM25, tuned)", 0.4, 0.6);
/// Reciprocal rank fusion, the documented alternative to the weighted sum.
/// It ignores score magnitudes, so there is nothing to tune — which is the
/// claim being tested.
#[cfg(feature = "embeddings")]
const RRF: Mode = Mode {
label: "Hybrid (reciprocal rank fusion, k=60)",
fusion: Fusion::Rrf { k: 60.0 },
};
/// Every 0.1 step of vector weight, keyword weight taking the remainder.
@@ -104,11 +119,11 @@ fn sweep_modes() -> Vec<Mode> {
(0..=10)
.map(|i| {
let v = i as f32 / 10.0;
Mode {
label: Box::leak(format!("sweep v={v:.1} / k={:.1}", 1.0 - v).into_boxed_str()),
vector_weight: v,
keyword_weight: 1.0 - v,
}
Mode::weighted(
Box::leak(format!("sweep v={v:.1} / k={:.1}", 1.0 - v).into_boxed_str()),
v,
1.0 - v,
)
})
.collect()
}
@@ -321,13 +336,7 @@ fn evaluate_question(
let query_emb = embedding_for(embeddings, &q.question);
let t0 = Instant::now();
let results = memory.hybrid_search(
&query_emb,
&q.question,
mode.vector_weight,
mode.keyword_weight,
top_k,
);
let results = memory.hybrid_search_with(&query_emb, &q.question, mode.fusion, top_k);
let latency = t0.elapsed();
// Session-level recall
@@ -472,10 +481,7 @@ fn print_report(
println!(" LongMemEval Benchmark — {}", mode.label);
println!("=================================================================");
println!();
println!(
"Mode: vector_weight={:.1} / keyword_weight={:.1}",
mode.vector_weight, mode.keyword_weight
);
println!("Mode: {}", describe(mode));
println!();
println!("Scoring target: RETRIEVAL RECALL (did the gold memory land in top-k).");
println!(" No answer is generated or scored. This is NOT the official");
@@ -602,10 +608,7 @@ fn print_report(
println!("```json");
println!("{{");
println!(" \"benchmark\": \"longmemeval\",");
println!(
" \"mode\": \"vector_{:.1}_keyword_{:.1}\",",
mode.vector_weight, mode.keyword_weight
);
println!(" \"mode\": \"{}\",", describe(mode));
println!(" \"dataset_variant\": \"{}\",", profile.variant());
println!(" \"scoring_target\": \"retrieval_recall\",");
println!(" \"k\": 10,");
@@ -768,7 +771,7 @@ fn main() {
if sweep {
sweep_modes()
} else {
vec![BM25_ONLY, VECTOR_ONLY, HYBRID]
vec![BM25_ONLY, VECTOR_ONLY, HYBRID, RRF]
}
}
#[cfg(not(feature = "embeddings"))]