fix(ann): deletions near the query no longer shrink search results

search() collected ef candidates, then filtered out soft-deleted nodes, then
took k. When the records nearest a query had been deleted, every candidate was
a tombstone and the search returned fewer than k results — 39 of 40 queries in
the new test, which deletes each query's 40 nearest neighbours.

search_layer takes an optional skip mask: a skipped node is still pushed onto
the candidate queue (a tombstone is a valid waypoint) but never into the
result heap, so the ef result slots hold live nodes only. Build and insert
pass no mask. Recall and speed without deletions are unchanged.

Co-Authored-By: Claude Fable 5.1 <[email protected]>

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## Unreleased
+
+### Search
+- `clawhdf5-ann`: `HnswIndex::search` returned fewer than `k` results — often
+  none — when the records nearest the query had been deleted: it collected `ef`
+  candidates, *then* dropped the deleted ones, *then* took `k`. Deleted nodes
+  are now traversed as waypoints but never occupy a result slot, so a search
+  returns the `k` nearest live records. Matters for any store that deletes or
+  supersedes memories without compacting straight away.
+
 ## v2.4.0 (2026-09-19)
 
 ### Upgrade Notes

crates/clawhdf5-ann/src/hnsw.rs

@@ -275,6 +275,7 @@ impl HnswIndex {
                     ep,
                     ef_construction,
                     metric,
+                    None,
                 );
 
                 let scored: Vec<(usize, f32)> =
@@ -408,6 +409,7 @@ impl HnswIndex {
                 ep,
                 self.ef_construction,
                 self.metric,
+                None,
             );
             let scored: Vec<(usize, f32)> = neighbors.iter().map(|c| (c.id, c.distance)).collect();
             let selected = select_neighbors(&self.vectors, &scored, max_conn, self.metric);
@@ -517,13 +519,21 @@ impl HnswIndex {
             ep = greedy_closest(&self.vectors, &self.graph[layer], query, ep, self.metric);
         }
 
-        // Search layer 0 with ef candidates. Deleted nodes are still traversed
+        // Search layer 0 for the ef nearest *live* nodes. Deleted nodes are
-        // (they remain valid graph waypoints) but are filtered from the result.
+        // still traversed (they remain valid graph waypoints) but take no
-        let candidates = search_layer(&self.vectors, &self.graph[0], query, ep, ef, self.metric);
+        // result slot, so deletions near the query don't shrink the answer.
+        let candidates = search_layer(
+            &self.vectors,
+            &self.graph[0],
+            query,
+            ep,
+            ef,
+            self.metric,
+            Some(&self.deleted),
+        );
 
         candidates
             .into_iter()
-            .filter(|c| !self.deleted[c.id])
             .take(k)
             .map(|c| (c.id, c.distance))
             .collect()
@@ -940,6 +950,14 @@ fn greedy_closest(
 }
 
 /// Search a single layer for the ef closest nodes to `query`.
+/// Best-first search of one layer, returning up to `ef` nodes by ascending
+/// distance.
+///
+/// `skip` marks nodes that must not be *returned* (soft-deleted ones). They
+/// are still traversed — a tombstone is a perfectly good waypoint — but they
+/// never occupy one of the `ef` result slots. Filtering them out afterwards
+/// instead meant a query whose neighbourhood had been deleted got back fewer
+/// than `k` results, or none, however many live records were nearby.
 fn search_layer(
     vectors: &[Vec<f32>],
     layer: &[Vec<usize>],
@@ -947,6 +965,7 @@ fn search_layer(
     ep: usize,
     ef: usize,
     metric: DistanceMetric,
+    skip: Option<&[bool]>,
 ) -> Vec<Candidate> {
     let ep_dist = compute_distance(query, &vectors[ep], metric);
 
@@ -959,16 +978,18 @@ fn search_layer(
 
     // Max-heap of current results (furthest first)
     let mut results = BinaryHeap::new();
+    if !skip.is_some_and(|s| s[ep]) {
         results.push(FarCandidate {
             id: ep,
             distance: ep_dist,
         });
+    }
 
     VISITED.with_borrow_mut(|visited| {
         visited.begin(vectors.len());
         visited.insert(ep);
         search_layer_visit(
-            vectors, layer, query, ef, metric, visited, candidates, results,
+            vectors, layer, query, ef, metric, skip, visited, candidates, results,
         )
     })
 }
@@ -1016,6 +1037,7 @@ fn search_layer_visit(
     query: &[f32],
     ef: usize,
     metric: DistanceMetric,
+    skip: Option<&[bool]>,
     visited: &mut Visited,
     mut candidates: BinaryHeap<Candidate>,
     mut results: BinaryHeap<FarCandidate>,
@@ -1039,6 +1061,9 @@ fn search_layer_visit(
                     id: neighbor,
                     distance: d,
                 });
+                if skip.is_some_and(|s| s[neighbor]) {
+                    continue; // explore through it, but never return it
+                }
                 results.push(FarCandidate {
                     id: neighbor,
                     distance: d,
@@ -1384,6 +1409,39 @@ mod tests {
         assert!(recall >= 0.95, "incremental recall@10 = {recall}");
     }
 
+    #[test]
+    fn deletions_near_the_query_do_not_shrink_or_degrade_results() {
+        let mut vectors = clustered(2040, 16, 20, 11);
+        let queries = vectors.split_off(2000);
+        let mut index = HnswIndex::build_with_metric(&vectors, 8, 40, DistanceMetric::L2);
+
+        let mut short = 0;
+        let mut hits = 0;
+        for q in &queries {
+            // Delete this query's 40 nearest neighbours: more than ef, so every
+            // candidate a plain search collects is a tombstone.
+            let mut exact: Vec<(usize, f32)> = vectors
+                .iter()
+                .enumerate()
+                .filter(|(i, _)| !index.is_deleted(*i))
+                .map(|(i, v)| (i, compute_distance(q, v, DistanceMetric::L2)))
+                .collect();
+            exact.sort_by(|a, b| a.1.total_cmp(&b.1));
+            for &(id, _) in &exact[..40] {
+                index.mark_deleted(id);
+            }
+            let want: Vec<usize> = exact[40..50].iter().map(|e| e.0).collect();
+
+            let got = index.search(q, 10, 32);
+            assert!(got.iter().all(|(id, _)| !index.is_deleted(*id)));
+            short += usize::from(got.len() < 10);
+            hits += got.iter().filter(|(id, _)| want.contains(id)).count();
+        }
+        assert_eq!(short, 0, "searches returned fewer than k live results");
+        let recall = hits as f64 / (10 * queries.len()) as f64;
+        assert!(recall >= 0.9, "recall@10 among live records = {recall}");
+    }
+
     #[test]
     fn graph_bytes_round_trip_gives_identical_searches() {
         let mut vectors = clustered(1260, 16, 12, 9);