feat(agent): optional keyword stemming, measured and left off by default

The keyword stage had no stemming, so "training" and "trains" were unrelated
terms. bm25::TokenFilter::Stemmed strips common English inflections (plurals,
-ing/-ed, with consonant un-doubling) from documents and queries alike;
BM25Index::build_with and HDF5Memory::set_token_filter select it, and the
index records which filter built it so a stale one is rebuilt rather than
mixed.

Measured over the full LongMemEval haystack (500 questions, real MiniLM
embeddings) rather than adopted on principle — and it is a trade, not a win:

  BM25 only         Hit@1 53.8%  Hit@5 75.0%  Hit@10 81.6%  MRR 0.6320
  BM25 stemmed      Hit@1 52.0%  Hit@5 77.8%  Hit@10 84.0%  MRR 0.6320
  Hybrid 0.4/0.6    Hit@1 51.6%  Hit@5 81.4%  Hit@10 87.8%  MRR 0.6430
  Hybrid stemmed    Hit@1 50.2%  Hit@5 81.4%  Hit@10 88.2%  MRR 0.6394

Conflation buys depth and costs the top rank: on BM25 alone MRR is unchanged
to four decimal places, the deeper gains exactly offsetting the rank-1 loss.
On the shipping hybrid configuration the vector stage already supplies most of
that recall, so the trade is narrower and slightly negative. Default stays
Plain; Stemmed is there for callers who want Hit@5/@10 over rank-1 precision.

The stemmer is deliberately conservative — it only strips inflections, and
only when the stem stays long enough to be meaningful, since an aggressive one
also conflates unrelated words. Tests pin both the pairs that must meet and
the pairs that must not.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>

crates/clawhdf5-bench/src/bin/longmemeval_bench.rs

@@ -55,6 +55,7 @@ use std::time::{Duration, Instant};
 #[path = "longmemeval_bench/embedder.rs"]
 mod embedder;
 
+use clawhdf5_agent::bm25::TokenFilter;
 use clawhdf5_agent::hybrid::Fusion;
 use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry};
 use serde::Deserialize;
@@ -64,9 +65,13 @@ const EMBEDDING_DIM: usize = 384;
 
 /// A mode's fusion, as one short string for the reports.
 fn describe(mode: Mode) -> String {
-    match mode.fusion {
+    let fusion = match mode.fusion {
         Fusion::Weighted { vector, keyword } => format!("vector_{vector:.1}_keyword_{keyword:.1}"),
         Fusion::Rrf { k } => format!("rrf_k{k:.0}"),
+    };
+    match mode.tokens {
+        TokenFilter::Plain => fusion,
+        TokenFilter::Stemmed => format!("{fusion}_stemmed"),
     }
 }
 
@@ -76,6 +81,8 @@ struct Mode {
     label: &'static str,
     /// How the two retrieval stages are combined into one ranking.
     fusion: Fusion,
+    /// How keyword tokens are normalised before indexing and querying.
+    tokens: TokenFilter,
 }
 
 impl Mode {
@@ -83,8 +90,15 @@ impl Mode {
         Self {
             label,
             fusion: Fusion::Weighted { vector, keyword },
+            tokens: TokenFilter::Plain,
         }
     }
+
+    const fn stemmed(mut self, label: &'static str) -> Self {
+        self.label = label;
+        self.tokens = TokenFilter::Stemmed;
+        self
+    }
 }
 
 /// The only mode available without real embeddings. Passing zero vectors with
@@ -106,8 +120,15 @@ const HYBRID: Mode = Mode::weighted("Hybrid (0.4 vector / 0.6 BM25, tuned)", 0.4
 const RRF: Mode = Mode {
     label: "Hybrid (reciprocal rank fusion, k=60)",
     fusion: Fusion::Rrf { k: 60.0 },
+    tokens: TokenFilter::Plain,
 };
 
+/// The same two configurations with stemmed keyword tokens, so the tokenizer's
+/// effect is isolated from everything else.
+const BM25_STEMMED: Mode = BM25_ONLY.stemmed("BM25 only, stemmed tokens");
+#[cfg(feature = "embeddings")]
+const HYBRID_STEMMED: Mode = HYBRID.stemmed("Hybrid 0.4/0.6, stemmed tokens");
+
 /// Every 0.1 step of vector weight, keyword weight taking the remainder.
 ///
 /// Labels are leaked to `&'static str` because `Mode::label` is a `&'static
@@ -291,6 +312,7 @@ fn evaluate_question(
     config.compact_threshold = 0.0;
 
     let mut memory = HDF5Memory::create(config).expect("failed to create HDF5Memory");
+    memory.set_token_filter(mode.tokens);
 
     // Build MemoryEntry list from all haystack sessions
     let mut entries: Vec<MemoryEntry> = Vec::new();
@@ -771,18 +793,27 @@ fn main() {
             if sweep {
                 sweep_modes()
             } else {
-                vec![BM25_ONLY, VECTOR_ONLY, HYBRID, RRF]
+                vec![
+                    BM25_ONLY,
+                    VECTOR_ONLY,
+                    HYBRID,
+                    RRF,
+                    BM25_STEMMED,
+                    HYBRID_STEMMED,
+                ]
             }
         }
         #[cfg(not(feature = "embeddings"))]
         {
-            vec![BM25_ONLY]
+            vec![BM25_ONLY, BM25_STEMMED]
         }
     } else {
         if sweep {
             eprintln!("warning: --sweep needs --embeddings; running BM25 only");
         }
-        vec![BM25_ONLY]
+        // Stemming is a property of the keyword stage, so it can be compared
+        // without a model.
+        vec![BM25_ONLY, BM25_STEMMED]
     };
 
     for (mode_idx, mode) in modes.iter().enumerate() {