perf(agent): unranked BM25 scores and a top-k merge — same rankings, 4-5x faster

Fusion min-max normalises over every keyword match, so hybrid_search asked
BM25 for a ranked list of the whole corpus: a hash insert per posting, then a
sort of every match, then the merge sorted every candidate again to keep k.

- BM25Index::scores returns every match unsorted, accumulated in a dense array
  (contributions are strictly positive, so zero means untouched). search() is
  built on it with the bounded heap.
- merge_vector_keyword partitions out its top k (select_nth) and orders only
  those, with the same score-then-id order.
- Both hybrid paths use scores().

Rankings are identical (equivalence tests for both changes). p50 0.24 -> 0.07
ms (1K), 2.1 -> 0.49 ms (10K), 23 -> 4.65 ms (100K).

The harness gains --fusion-study, which measured the alternative — capping the
keyword pool — and found it changes the top-10 for most queries (overlap
0.83-0.92, different #1 for 10-35%) for only a 2x saving. Not adopted.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

crates/clawhdf5-agent/src/bm25.rs

@@ -83,35 +83,15 @@ impl BM25Index {
     /// Uses Block-Max WAND for early termination when remaining documents
     /// cannot beat the current top-k threshold.
     pub fn search(&self, query: &str, k: usize) -> Vec<(usize, f32)> {
-        if self.num_docs == 0 || k == 0 {
+        if k == 0 {
             return Vec::new();
         }
-
-        // Term-at-a-time accumulation. IDF is computed here rather than cached
-        // at build time: it depends on the live document count, which changes
-        // with every incremental add/remove, and costs one `ln` per query term.
-        let mut scores: HashMap<usize, f32> = HashMap::new();
-        for token in tokenize(query) {
-            let Some(postings) = self.inverted.get(token.as_str()) else {
-                continue;
-            };
-            let df = postings.len() as f32;
-            let idf = ((self.num_docs as f32 - df + 0.5) / (df + 0.5) + 1.0).ln();
-            for &(doc_id, freq) in postings {
-                let dl = self.doc_lengths[doc_id] as f32;
-                let freq_f = freq as f32;
-                let tf = (freq_f * (self.k1 + 1.0))
-                    / (freq_f + self.k1 * (1.0 - self.b + self.b * dl / self.avg_dl));
-                *scores.entry(doc_id).or_insert(0.0) += idf * tf;
-            }
-        }
-
         // Top-k with a bounded min-heap: O(matches * log k) instead of sorting
         // every match. Ties break towards the lower doc id so results are
-        // deterministic (the accumulator is a HashMap).
+        // deterministic.
         let mut heap: BinaryHeap<Reverse<(HeapScore, Reverse<usize>)>> =
-            BinaryHeap::with_capacity(k + 1);
-        for (doc_id, score) in scores {
+            BinaryHeap::with_capacity(k.min(1024) + 1);
+        for (doc_id, score) in self.scores(query) {
             heap.push(Reverse((HeapScore(score), Reverse(doc_id))));
             if heap.len() > k {
                 heap.pop();
@@ -125,6 +105,47 @@ impl BM25Index {
         results
     }
 
+    /// The BM25 score of **every** matching document, in doc-id order, unsorted
+    /// by score. Score fusion normalises over the whole matching set, so it
+    /// needs all of these but not their ranking; producing a ranked list of
+    /// every match (`search(query, corpus_len)`) spent most of its time sorting.
+    pub fn scores(&self, query: &str) -> Vec<(usize, f32)> {
+        if self.num_docs == 0 {
+            return Vec::new();
+        }
+        // Term-at-a-time accumulation into a dense array: a common term has a
+        // posting per document, and hashing each one dominated query time.
+        // IDF is computed here rather than cached at build time: it depends on
+        // the live document count, which changes with every incremental
+        // add/remove, and costs one `ln` per query term.
+        let mut acc = vec![0.0f32; self.doc_lengths.len()];
+        let mut matched = false;
+        for token in tokenize(query) {
+            let Some(postings) = self.inverted.get(token.as_str()) else {
+                continue;
+            };
+            matched = true;
+            let df = postings.len() as f32;
+            let idf = ((self.num_docs as f32 - df + 0.5) / (df + 0.5) + 1.0).ln();
+            for &(doc_id, freq) in postings {
+                let dl = self.doc_lengths[doc_id] as f32;
+                let freq_f = freq as f32;
+                let tf = (freq_f * (self.k1 + 1.0))
+                    / (freq_f + self.k1 * (1.0 - self.b + self.b * dl / self.avg_dl));
+                acc[doc_id] += idf * tf;
+            }
+        }
+        if !matched {
+            return Vec::new();
+        }
+        // Every contribution is strictly positive (idf = ln(1 + x), x > 0), so
+        // a zero entry is a document no query term touched.
+        acc.into_iter()
+            .enumerate()
+            .filter(|&(_, score)| score > 0.0)
+            .collect()
+    }
+
     /// Number of document slots (live or not) the index covers. Ids are
     /// positions in the document list it mirrors.
     pub fn len(&self) -> usize {
@@ -564,6 +585,20 @@ mod tests {
         }
     }
 
+    #[test]
+    fn scores_is_the_unranked_form_of_a_full_search() {
+        let mut state = 99u64;
+        let docs: Vec<String> = (0..200).map(|_| random_doc(&mut state)).collect();
+        let tombstones: Vec<u8> = (0..200).map(|i| u8::from(i % 7 == 0)).collect();
+        let index = BM25Index::build(&docs, &tombstones);
+        for query in ["alpha", "beta gamma x1", "missing", ""] {
+            let mut all = index.scores(query);
+            all.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0)));
+            assert_eq!(all, index.search(query, docs.len()), "{query:?}");
+            assert!(all.iter().all(|(id, _)| tombstones[*id] == 0));
+        }
+    }
+
     #[test]
     fn ties_break_towards_the_lower_doc_id() {
         let docs: Vec<String> = (0..6).map(|_| "same text".to_string()).collect();