From f507803ec135497a7e58a795f3e487662d29493c Mon Sep 17 00:00:00 2001 From: osobh Date: Sat, 19 Sep 2026 13:52:23 -0700 Subject: [PATCH] feat(agent): build the vector index in parallel by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `parallel` joins the agent's default features, so the HNSW bulk build uses the thread pool: cold index build at 10K records 1152 -> ~380 ms in a same-moment A/B (the graph is identical either way). Nothing else on the measured paths changes — ingest, checkpoint, open and steady-state query times are the same with the feature on or off. Adds rayon to the default dependency set; opt out with `--no-default-features --features float16,hnsw`. Harness: `--e2e-only` runs the end-to-end section without the index benchmarks. Note for anyone comparing numbers: this machine's absolute timings drifted ~1.5x over a long session, so only same-moment A/B runs are comparable. Co-Authored-By: Claude Fable 5.1 --- CHANGELOG.md | 4 +++- CLAUDE.md | 2 ++ crates/clawhdf5-agent/Cargo.toml | 2 +- crates/clawhdf5-bench/src/bin/search_harness.rs | 8 ++++++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8564e22..0c75b95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,9 @@ With the `parallel` feature, planning and pruning run on a thread pool (10K: 388 ms, 100K: ~21 s -> 5.9 s on 16 cores). The graph is deterministic and identical with or without the feature. `clawhdf5-agent`'s `parallel` feature - now enables it for the agent's index. + enables it for the agent's index and is now **on by default** (adds `rayon` + to the default dependency set; build with `--no-default-features --features + float16,hnsw` to opt out). - `clawhdf5-ann`: `HnswIndex::search` returned fewer than `k` results — often none — when the records nearest the query had been deleted: it collected `ef` candidates, *then* dropped the deleted ones, *then* took `k`. Deleted nodes diff --git a/CLAUDE.md b/CLAUDE.md index 2a47ad5..2dfc918 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -33,6 +33,8 @@ Cargo workspace with 16 crates under `crates/` (plus `libaec-sys`, an internal F the approximate `clawhdf5-ann` index for the vector stage (the index mirrors the cache and self-heals on drift). Build the agent with `--no-default-features --features float16` to force the exact linear cosine scan. + The agent's `parallel` feature (also default) builds the index on a thread + pool; the graph is identical with or without it. The index uses the HNSW paper's diversity heuristic for neighbour selection (plain closest-M capped recall on clustered data: 0.31 recall@10 at 100K). Its graph is saved to `.h5.ann` at each checkpoint and reloaded by `open()` diff --git a/crates/clawhdf5-agent/Cargo.toml b/crates/clawhdf5-agent/Cargo.toml index a029583..0d44038 100644 --- a/crates/clawhdf5-agent/Cargo.toml +++ b/crates/clawhdf5-agent/Cargo.toml @@ -45,7 +45,7 @@ name = "memory_bench" harness = false [features] -default = ["float16", "hnsw"] +default = ["float16", "hnsw", "parallel"] float16 = ["half"] # Rayon-parallel brute-force search strategies, and a parallel bulk build of # the HNSW index (same graph, several times faster on a multi-core machine). diff --git a/crates/clawhdf5-bench/src/bin/search_harness.rs b/crates/clawhdf5-bench/src/bin/search_harness.rs index 3c43af7..d058a60 100644 --- a/crates/clawhdf5-bench/src/bin/search_harness.rs +++ b/crates/clawhdf5-bench/src/bin/search_harness.rs @@ -485,8 +485,12 @@ fn main() { let mut json = Vec::new(); println!("## Search harness"); - for &n in sizes { - bench_ann(n, &mut json); + // `--e2e-only` skips the index benchmarks, so the end-to-end section runs + // in a process that has not already spun up a thread pool. + if !args.iter().any(|a| a == "--e2e-only") { + for &n in sizes { + bench_ann(n, &mut json); + } } if ann_only {