diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f54dbf..2a8f8c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## Unreleased + +### Search +- `clawhdf5-ann`: `HnswIndex::search` returned fewer than `k` results — often + none — when the records nearest the query had been deleted: it collected `ef` + candidates, *then* dropped the deleted ones, *then* took `k`. Deleted nodes + are now traversed as waypoints but never occupy a result slot, so a search + returns the `k` nearest live records. Matters for any store that deletes or + supersedes memories without compacting straight away. + ## v2.4.0 (2026-09-19) ### Upgrade Notes diff --git a/crates/clawhdf5-ann/src/hnsw.rs b/crates/clawhdf5-ann/src/hnsw.rs index b9e7ce3..3fc6f00 100644 --- a/crates/clawhdf5-ann/src/hnsw.rs +++ b/crates/clawhdf5-ann/src/hnsw.rs @@ -275,6 +275,7 @@ impl HnswIndex { ep, ef_construction, metric, + None, ); let scored: Vec<(usize, f32)> = @@ -408,6 +409,7 @@ impl HnswIndex { ep, self.ef_construction, self.metric, + None, ); let scored: Vec<(usize, f32)> = neighbors.iter().map(|c| (c.id, c.distance)).collect(); let selected = select_neighbors(&self.vectors, &scored, max_conn, self.metric); @@ -517,13 +519,21 @@ impl HnswIndex { ep = greedy_closest(&self.vectors, &self.graph[layer], query, ep, self.metric); } - // Search layer 0 with ef candidates. Deleted nodes are still traversed - // (they remain valid graph waypoints) but are filtered from the result. - let candidates = search_layer(&self.vectors, &self.graph[0], query, ep, ef, self.metric); + // Search layer 0 for the ef nearest *live* nodes. Deleted nodes are + // still traversed (they remain valid graph waypoints) but take no + // result slot, so deletions near the query don't shrink the answer. + let candidates = search_layer( + &self.vectors, + &self.graph[0], + query, + ep, + ef, + self.metric, + Some(&self.deleted), + ); candidates .into_iter() - .filter(|c| !self.deleted[c.id]) .take(k) .map(|c| (c.id, c.distance)) .collect() @@ -940,6 +950,14 @@ fn greedy_closest( } /// Search a single layer for the ef closest nodes to `query`. +/// Best-first search of one layer, returning up to `ef` nodes by ascending +/// distance. +/// +/// `skip` marks nodes that must not be *returned* (soft-deleted ones). They +/// are still traversed — a tombstone is a perfectly good waypoint — but they +/// never occupy one of the `ef` result slots. Filtering them out afterwards +/// instead meant a query whose neighbourhood had been deleted got back fewer +/// than `k` results, or none, however many live records were nearby. fn search_layer( vectors: &[Vec], layer: &[Vec], @@ -947,6 +965,7 @@ fn search_layer( ep: usize, ef: usize, metric: DistanceMetric, + skip: Option<&[bool]>, ) -> Vec { let ep_dist = compute_distance(query, &vectors[ep], metric); @@ -959,16 +978,18 @@ fn search_layer( // Max-heap of current results (furthest first) let mut results = BinaryHeap::new(); - results.push(FarCandidate { - id: ep, - distance: ep_dist, - }); + if !skip.is_some_and(|s| s[ep]) { + results.push(FarCandidate { + id: ep, + distance: ep_dist, + }); + } VISITED.with_borrow_mut(|visited| { visited.begin(vectors.len()); visited.insert(ep); search_layer_visit( - vectors, layer, query, ef, metric, visited, candidates, results, + vectors, layer, query, ef, metric, skip, visited, candidates, results, ) }) } @@ -1016,6 +1037,7 @@ fn search_layer_visit( query: &[f32], ef: usize, metric: DistanceMetric, + skip: Option<&[bool]>, visited: &mut Visited, mut candidates: BinaryHeap, mut results: BinaryHeap, @@ -1039,6 +1061,9 @@ fn search_layer_visit( id: neighbor, distance: d, }); + if skip.is_some_and(|s| s[neighbor]) { + continue; // explore through it, but never return it + } results.push(FarCandidate { id: neighbor, distance: d, @@ -1384,6 +1409,39 @@ mod tests { assert!(recall >= 0.95, "incremental recall@10 = {recall}"); } + #[test] + fn deletions_near_the_query_do_not_shrink_or_degrade_results() { + let mut vectors = clustered(2040, 16, 20, 11); + let queries = vectors.split_off(2000); + let mut index = HnswIndex::build_with_metric(&vectors, 8, 40, DistanceMetric::L2); + + let mut short = 0; + let mut hits = 0; + for q in &queries { + // Delete this query's 40 nearest neighbours: more than ef, so every + // candidate a plain search collects is a tombstone. + let mut exact: Vec<(usize, f32)> = vectors + .iter() + .enumerate() + .filter(|(i, _)| !index.is_deleted(*i)) + .map(|(i, v)| (i, compute_distance(q, v, DistanceMetric::L2))) + .collect(); + exact.sort_by(|a, b| a.1.total_cmp(&b.1)); + for &(id, _) in &exact[..40] { + index.mark_deleted(id); + } + let want: Vec = exact[40..50].iter().map(|e| e.0).collect(); + + let got = index.search(q, 10, 32); + assert!(got.iter().all(|(id, _)| !index.is_deleted(*id))); + short += usize::from(got.len() < 10); + hits += got.iter().filter(|(id, _)| want.contains(id)).count(); + } + assert_eq!(short, 0, "searches returned fewer than k live results"); + let recall = hits as f64 / (10 * queries.len()) as f64; + assert!(recall >= 0.9, "recall@10 among live records = {recall}"); + } + #[test] fn graph_bytes_round_trip_gives_identical_searches() { let mut vectors = clustered(1260, 16, 12, 9);