From 1e18ff5a86c08837611ec46363e9d33628877e1c Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 19 Sep 2026 19:42:05 -0700 Subject: [PATCH] style(bench): gate the rerank-sweep flag and helper on the embeddings feature Co-Authored-By: Claude Opus 5 (1M context) --- crates/clawhdf5-agent/src/openclaw.rs | 1 + crates/clawhdf5-agent/src/reranker.rs | 53 ++++++++++++++++++- .../src/bin/longmemeval_bench.rs | 13 ++++- 3 files changed, 64 insertions(+), 3 deletions(-) diff --git a/crates/clawhdf5-agent/src/openclaw.rs b/crates/clawhdf5-agent/src/openclaw.rs index e7e301c..59f4382 100644 --- a/crates/clawhdf5-agent/src/openclaw.rs +++ b/crates/clawhdf5-agent/src/openclaw.rs @@ -554,6 +554,7 @@ impl MemoryBackend for ClawhdfBackend { timestamp: r.timestamp, source_channel: r.source_channel.clone(), raw_activation: r.activation, + relevance: r.score, }) .collect(); diff --git a/crates/clawhdf5-agent/src/reranker.rs b/crates/clawhdf5-agent/src/reranker.rs index b528714..51c3852 100644 --- a/crates/clawhdf5-agent/src/reranker.rs +++ b/crates/clawhdf5-agent/src/reranker.rs @@ -4,8 +4,10 @@ //! into a single composite score for each retrieved result. /// Configuration for the multi-factor re-ranker. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub struct ReRankConfig { + /// Weight applied to the retrieval score the candidate arrived with. + pub relevance_weight: f32, /// Weight applied to the temporal decay score (0.0–1.0). pub temporal_weight: f32, /// Weight applied to the source authority score (0.0–1.0). @@ -20,6 +22,9 @@ pub struct ReRankConfig { impl Default for ReRankConfig { fn default() -> Self { Self { + // Relevance leads: the metadata signals break ties and nudge, they + // do not decide. See `BENCHMARKS.md`, "Recency discrimination". + relevance_weight: 1.0, temporal_weight: 0.3, authority_weight: 0.2, activation_weight: 0.5, @@ -41,6 +46,8 @@ pub struct ReRankResult { pub authority_score: f32, /// Normalised Hebbian activation score in [0, 1]. pub activation_score: f32, + /// The retrieval score carried through from the input. + pub relevance_score: f32, } /// Compute an exponential decay temporal score. @@ -105,6 +112,15 @@ pub struct RerankInput { pub source_channel: String, /// Raw Hebbian activation weight for this entry. pub raw_activation: f32, + /// The retrieval score that put this entry in the candidate list. + /// + /// Re-ranking is meant to *adjust* the retriever's ordering with signals + /// it does not have, not to replace it. Without this the combined score + /// was made of recency, authority and activation alone, so a candidate + /// pool came back ordered by age with its relevance ordering discarded. + /// Callers with no meaningful score can pass the same value for every + /// entry, which reduces to the old behaviour. + pub relevance: f32, } /// Re-rank a list of retrieval results using multi-factor scoring. @@ -138,7 +154,8 @@ pub fn rerank( let auth = source_authority_score(&inp.source_channel); let act = activation_score(inp.raw_activation); - let combined = config.temporal_weight * ts + let combined = config.relevance_weight * inp.relevance + + config.temporal_weight * ts + config.authority_weight * auth + config.activation_weight * act; @@ -148,6 +165,7 @@ pub fn rerank( temporal_score: ts, authority_score: auth, activation_score: act, + relevance_score: inp.relevance, } }) .collect(); @@ -253,22 +271,51 @@ mod tests { timestamp: 0.0, // very old source_channel: "other".to_string(), raw_activation: 0.1, + relevance: 0.0, }, RerankInput { index: 1, timestamp: 86_400.0, // one day ago source_channel: "conversation".to_string(), raw_activation: 0.5, + relevance: 0.0, }, RerankInput { index: 2, timestamp: 172_800.0, // "now" source_channel: "user_correction".to_string(), raw_activation: 1.0, + relevance: 0.0, }, ] } + #[test] + fn relevance_leads_but_recency_breaks_near_ties() { + let entry = |index, timestamp, relevance| RerankInput { + index, + timestamp, + source_channel: "conversation".to_string(), + raw_activation: 1.0, + relevance, + }; + let now = 10.0 * 86_400.0; + let config = ReRankConfig::default(); + + // A clearly better match wins despite being much older. Before + // `relevance` existed the combined score ignored it entirely, so this + // returned the newer, irrelevant entry. + let ranked = rerank(&[entry(0, 0.0, 1.0), entry(1, now, 0.1)], &config, now); + assert_eq!(ranked[0].index, 0, "{ranked:?}"); + + // Between near-equal matches, the newer one wins. + let ranked = rerank(&[entry(0, 0.0, 0.80), entry(1, now, 0.79)], &config, now); + assert_eq!(ranked[0].index, 1, "{ranked:?}"); + + // The breakdown carries the relevance through. + assert_eq!(ranked[0].relevance_score, 0.79); + } + #[test] fn rerank_returns_all_entries() { let inputs = make_inputs(); @@ -302,6 +349,7 @@ mod tests { #[test] fn rerank_score_breakdown_matches_manual_calculation() { let config = ReRankConfig { + relevance_weight: 0.0, temporal_weight: 1.0, authority_weight: 0.0, activation_weight: 0.0, @@ -312,6 +360,7 @@ mod tests { timestamp: 0.0, source_channel: "other".to_string(), raw_activation: 0.5, + relevance: 0.0, }]; let now = 3600.0_f64; // exactly one half-life later let results = rerank(&inputs, &config, now); diff --git a/crates/clawhdf5-bench/src/bin/longmemeval_bench.rs b/crates/clawhdf5-bench/src/bin/longmemeval_bench.rs index b508bab..7d4858c 100644 --- a/crates/clawhdf5-bench/src/bin/longmemeval_bench.rs +++ b/crates/clawhdf5-bench/src/bin/longmemeval_bench.rs @@ -107,6 +107,7 @@ impl Mode { } } + #[cfg_attr(not(feature = "embeddings"), allow(dead_code))] fn reranked(mut self, label: &'static str, rerank: ReRankConfig) -> Self { self.label = label; self.rerank = Some(rerank); @@ -929,6 +930,7 @@ fn main() { let mut limit: Option = None; let mut weights_dir: Option = None; let mut sweep = false; + #[cfg_attr(not(feature = "embeddings"), allow(unused_mut, unused_variables))] let mut rerank_sweep = false; let mut args = std::env::args().skip(1); while let Some(arg) = args.next() { @@ -938,7 +940,16 @@ fn main() { limit = Some(v.parse().expect("--limit must be a positive integer")); } "--sweep" => sweep = true, - "--rerank-sweep" => rerank_sweep = true, + "--rerank-sweep" => { + // Re-ranking needs the vector stage to have candidates worth + // reordering, so this is an embeddings-only comparison. + #[cfg(feature = "embeddings")] + { + rerank_sweep = true; + } + #[cfg(not(feature = "embeddings"))] + eprintln!("warning: --rerank-sweep needs --features embeddings; ignoring"); + } "--embeddings" => { weights_dir = Some(args.next().expect("--embeddings needs a directory")); }