diff --git a/BENCHMARKS.md b/BENCHMARKS.md index d7a0ed9..1f6b21e 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -28,6 +28,72 @@ --- +## Search harness baseline (v2.3.0) + +Produced by `cargo run --release -p clawhdf5-bench --bin search_harness -- --full` +on deterministic **clustered** synthetic data (384-dim, unit-normalised; points = +cluster centre + noise — uniform random vectors are nearly equidistant in high +dimension and say nothing about embeddings). Recall is measured against an exact +brute-force scan, 200 queries. This is the *before* picture for the search +hot-path work; every change to that path should be justified by a re-run. + +Two things stand out: + +* **HNSW recall does not respond to `ef`** and degrades sharply with size + (0.87 → 0.67 → 0.31 recall@10 at 1K / 10K / 100K). Latency plateaus at the same + point, i.e. the search exhausts the nodes it can reach: on clustered data the + graph is poorly connected. The index selects neighbours by plain top-M + distance rather than the HNSW paper's diversity heuristic. +* **End-to-end `hybrid_search` is ~1000x slower than its vector stage** (49 ms + vs ~0.03 ms at 10K; 884 ms at 100K). Each query rebuilds the BM25 index from + scratch and rewrites the whole `.h5` file. The first query after `open()` + additionally rebuilds the HNSW index (10.5 s at 100K). + +### HNSW, N = 1000, dim = 384, M = 16, ef_construction = 64 + +build: 72.4 ms (13818 vectors/s) · exact scan: 3854 QPS, p50 258 µs + +| ef | recall@10 | QPS | p50 µs | p99 µs | +|---:|---:|---:|---:|---:| +| 16 | 0.8710 | 59484 | 16 | 31 | +| 32 | 0.8730 | 46302 | 21 | 25 | +| 64 | 0.8730 | 31683 | 31 | 44 | +| 128 | 0.8730 | 24715 | 40 | 49 | +| 256 | 0.8730 | 24788 | 40 | 50 | + +### HNSW, N = 10000, dim = 384, M = 16, ef_construction = 64 + +build: 802.5 ms (12461 vectors/s) · exact scan: 418 QPS, p50 2363 µs + +| ef | recall@10 | QPS | p50 µs | p99 µs | +|---:|---:|---:|---:|---:| +| 16 | 0.6695 | 44031 | 19 | 51 | +| 32 | 0.6705 | 45066 | 22 | 30 | +| 64 | 0.6705 | 32746 | 30 | 41 | +| 128 | 0.6705 | 27542 | 36 | 51 | +| 256 | 0.6705 | 27754 | 36 | 49 | + +### HNSW, N = 100000, dim = 384, M = 16, ef_construction = 64 + +build: 9752.6 ms (10254 vectors/s) · exact scan: 40 QPS, p50 24648 µs + +| ef | recall@10 | QPS | p50 µs | p99 µs | +|---:|---:|---:|---:|---:| +| 16 | 0.3085 | 18046 | 57 | 84 | +| 32 | 0.3110 | 21621 | 43 | 75 | +| 64 | 0.3130 | 20015 | 49 | 70 | +| 128 | 0.3135 | 15822 | 63 | 99 | +| 256 | 0.3135 | 15308 | 66 | 124 | + +### End to end: `HDF5Memory::hybrid_search` (k = 10, weights 0.7 / 0.3) + +| N | ingest ms | checkpoint ms | open ms | first query ms | p50 ms | p99 ms | QPS | +|---:|---:|---:|---:|---:|---:|---:|---:| +| 1000 | 11 | 3.9 | 0.9 | 68.1 | 5.48 | 5.57 | 182.5 | +| 10000 | 114 | 32.2 | 10.9 | 845.0 | 48.56 | 78.65 | 19.8 | +| 100000 | 1486 | 713.0 | 354.5 | 10486.5 | 883.51 | 975.23 | 1.1 | +wrote /tmp/claude-1000/-home-osobh-projects-clawhdf5/422f755e-dd25-4c35-8613-5439087e3aaa/scratchpad/baseline_full.json + ## Vector Search Latency Brute-force cosine similarity over 384-dimensional embeddings (OpenAI text-embedding-3-small size). diff --git a/crates/clawhdf5-bench/Cargo.toml b/crates/clawhdf5-bench/Cargo.toml index 5cf059d..7f8ffbf 100644 --- a/crates/clawhdf5-bench/Cargo.toml +++ b/crates/clawhdf5-bench/Cargo.toml @@ -13,6 +13,10 @@ path = "src/bin/longmemeval_bench.rs" name = "memory_arena" path = "src/bin/memory_arena.rs" +[[bin]] +name = "search_harness" +path = "src/bin/search_harness.rs" + [[bin]] name = "footprint_bench" path = "src/bin/footprint_bench.rs" @@ -48,6 +52,7 @@ harness = false [dependencies] clawhdf5-agent = { path = "../clawhdf5-agent" } +clawhdf5-ann = { path = "../clawhdf5-ann" } clawhdf5-io = { path = "../clawhdf5-io" } mpi = { version = "0.8", optional = true } serde = { workspace = true } diff --git a/crates/clawhdf5-bench/src/bin/search_harness.rs b/crates/clawhdf5-bench/src/bin/search_harness.rs new file mode 100644 index 0000000..0db6931 --- /dev/null +++ b/crates/clawhdf5-bench/src/bin/search_harness.rs @@ -0,0 +1,381 @@ +//! Search measurement harness: recall vs. speed for the HNSW index, and +//! end-to-end `hybrid_search` latency as the store grows. +//! +//! Every search-path change should be justified by a before/after run of this +//! binary. It reports, for deterministic synthetic data: +//! +//! * **ANN** — index build time, and for each `ef`: recall@10 against an exact +//! brute-force scan, queries/second, and p50/p99 latency. +//! * **End to end** — `HDF5Memory`: ingest time, checkpoint time, `open()` +//! time, the first query after open (which pays for any index rebuild), and +//! steady-state `hybrid_search` p50/p99 at each store size. +//! +//! Data is *clustered* (points = cluster centre + noise, unit-normalised), not +//! uniform: uniform random high-dimensional vectors are nearly equidistant, +//! which makes recall numbers meaningless and is nothing like embeddings. +//! +//! ```text +//! cargo run --release -p clawhdf5-bench --bin search_harness # 1K, 10K +//! cargo run --release -p clawhdf5-bench --bin search_harness -- --full # + 100K +//! cargo run --release -p clawhdf5-bench --bin search_harness -- --json out.json +//! ``` + +use std::time::{Duration, Instant}; + +use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry}; +use clawhdf5_ann::{DistanceMetric, HnswIndex}; + +const DIM: usize = 384; +const K: usize = 10; +const N_QUERIES: usize = 200; +const HNSW_M: usize = 16; +const HNSW_EF_CONSTRUCTION: usize = 64; +const EF_VALUES: [usize; 5] = [16, 32, 64, 128, 256]; + +// --------------------------------------------------------------------------- +// Deterministic data +// --------------------------------------------------------------------------- + +struct Rng(u64); + +impl Rng { + fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + /// Uniform in [0, 1). + fn unit(&mut self) -> f32 { + (self.next_u64() >> 40) as f32 / (1u64 << 24) as f32 + } + + /// Approximately standard normal (sum of uniforms). + fn gauss(&mut self) -> f32 { + let sum: f32 = (0..6).map(|_| self.unit()).sum(); + (sum - 3.0) * std::f32::consts::SQRT_2 + } + + fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n as u64) as usize + } +} + +fn normalize(v: &mut [f32]) { + let norm = v.iter().map(|x| x * x).sum::().sqrt(); + if norm > 0.0 { + v.iter_mut().for_each(|x| *x /= norm); + } +} + +struct Dataset { + vectors: Vec>, + queries: Vec>, + /// Cluster id of each vector (used to give records topical text). + cluster_of: Vec, + query_cluster: Vec, +} + +fn make_dataset(n: usize, seed: u64) -> Dataset { + let mut rng = Rng(seed); + let n_clusters = (n / 100).clamp(8, 512); + let centres: Vec> = (0..n_clusters) + .map(|_| { + let mut c: Vec = (0..DIM).map(|_| rng.gauss()).collect(); + normalize(&mut c); + c + }) + .collect(); + let point = |rng: &mut Rng, cluster: usize| { + // Noise comparable to the centre's per-dimension magnitude, so + // clusters overlap and the nearest neighbours are non-trivial. + let scale = 0.6 / (DIM as f32).sqrt(); + let mut v: Vec = centres[cluster] + .iter() + .map(|c| c + rng.gauss() * scale) + .collect(); + normalize(&mut v); + v + }; + let mut vectors = Vec::with_capacity(n); + let mut cluster_of = Vec::with_capacity(n); + for _ in 0..n { + let c = rng.below(n_clusters); + vectors.push(point(&mut rng, c)); + cluster_of.push(c); + } + let mut queries = Vec::with_capacity(N_QUERIES); + let mut query_cluster = Vec::with_capacity(N_QUERIES); + for _ in 0..N_QUERIES { + let c = rng.below(n_clusters); + queries.push(point(&mut rng, c)); + query_cluster.push(c); + } + Dataset { + vectors, + queries, + cluster_of, + query_cluster, + } +} + +const WORDS: &[&str] = &[ + "deploy", "latency", "cache", "schema", "index", "vector", "memory", "agent", "kernel", + "buffer", "socket", "thread", "tensor", "gradient", "ledger", "invoice", "meeting", "roadmap", + "customer", "contract", "sensor", "orbit", "protein", "genome", "harbor", "bridge", "engine", + "battery", "harvest", "weather", "museum", "recipe", +]; + +/// Text whose vocabulary is biased by cluster, so keyword and vector signals +/// agree the way they do for real embedded text. +fn text_for(cluster: usize, i: usize, rng: &mut Rng) -> String { + let topic = [ + WORDS[cluster % WORDS.len()], + WORDS[(cluster / 7 + 3) % WORDS.len()], + ]; + let mut words = Vec::with_capacity(14); + for j in 0..14 { + if j % 3 == 0 { + words.push(topic[j / 3 % 2]); + } else { + words.push(WORDS[rng.below(WORDS.len())]); + } + } + format!("record {i}: {}", words.join(" ")) +} + +// --------------------------------------------------------------------------- +// Measurement helpers +// --------------------------------------------------------------------------- + +fn exact_top_k(vectors: &[Vec], query: &[f32], k: usize) -> Vec { + // Vectors are unit length, so cosine order == dot-product order. + let mut scored: Vec<(usize, f32)> = vectors + .iter() + .enumerate() + .map(|(i, v)| (i, v.iter().zip(query).map(|(a, b)| a * b).sum())) + .collect(); + scored.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0))); + scored.truncate(k); + scored.into_iter().map(|(i, _)| i).collect() +} + +struct Latency { + p50: Duration, + p99: Duration, + qps: f64, +} + +fn summarize(mut samples: Vec) -> Latency { + samples.sort(); + let total: Duration = samples.iter().sum(); + let at = |q: f64| samples[((samples.len() - 1) as f64 * q).round() as usize]; + Latency { + p50: at(0.50), + p99: at(0.99), + qps: samples.len() as f64 / total.as_secs_f64(), + } +} + +fn micros(d: Duration) -> f64 { + d.as_secs_f64() * 1e6 +} + +fn millis(d: Duration) -> f64 { + d.as_secs_f64() * 1e3 +} + +// --------------------------------------------------------------------------- +// ANN: recall vs speed +// --------------------------------------------------------------------------- + +fn bench_ann(n: usize, json: &mut Vec) { + let data = make_dataset(n, 0xA11CE ^ n as u64); + let truth: Vec> = data + .queries + .iter() + .map(|q| exact_top_k(&data.vectors, q, K)) + .collect(); + + let started = Instant::now(); + let index = HnswIndex::build_with_metric( + &data.vectors, + HNSW_M, + HNSW_EF_CONSTRUCTION, + DistanceMetric::Cosine, + ); + let build = started.elapsed(); + + // Exact scan baseline, for scale. + let exact = summarize( + data.queries + .iter() + .map(|q| { + let t = Instant::now(); + std::hint::black_box(exact_top_k(&data.vectors, q, K)); + t.elapsed() + }) + .collect(), + ); + + println!( + "\n### HNSW, N = {n}, dim = {DIM}, M = {HNSW_M}, ef_construction = {HNSW_EF_CONSTRUCTION}\n" + ); + println!( + "build: {:.1} ms ({:.0} vectors/s) · exact scan: {:.0} QPS, p50 {:.0} µs\n", + millis(build), + n as f64 / build.as_secs_f64(), + exact.qps, + micros(exact.p50) + ); + println!("| ef | recall@{K} | QPS | p50 µs | p99 µs |"); + println!("|---:|---:|---:|---:|---:|"); + for ef in EF_VALUES { + let mut hits = 0usize; + let mut samples = Vec::with_capacity(data.queries.len()); + for (q, want) in data.queries.iter().zip(&truth) { + let t = Instant::now(); + let got = index.search(q, K, ef); + samples.push(t.elapsed()); + hits += got.iter().filter(|(id, _)| want.contains(id)).count(); + } + let recall = hits as f64 / (K * data.queries.len()) as f64; + let lat = summarize(samples); + println!( + "| {ef} | {recall:.4} | {:.0} | {:.0} | {:.0} |", + lat.qps, + micros(lat.p50), + micros(lat.p99) + ); + json.push(serde_json::json!({ + "bench": "hnsw", "n": n, "ef": ef, "recall_at_10": recall, + "qps": lat.qps, "p50_us": micros(lat.p50), "p99_us": micros(lat.p99), + "build_ms": millis(build), + })); + } +} + +// --------------------------------------------------------------------------- +// End to end: HDF5Memory::hybrid_search +// --------------------------------------------------------------------------- + +fn bench_end_to_end(n: usize, json: &mut Vec) { + let data = make_dataset(n, 0xE2E ^ n as u64); + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("store.h5"); + let mut rng = Rng(7); + + let entries: Vec = data + .vectors + .iter() + .enumerate() + .map(|(i, v)| MemoryEntry { + chunk: text_for(data.cluster_of[i], i, &mut rng), + embedding: v.clone(), + source_channel: "bench".into(), + timestamp: i as f64, + session_id: format!("s{}", i % 50), + tags: format!("t{i}"), + }) + .collect(); + let query_texts: Vec = data + .query_cluster + .iter() + .enumerate() + .map(|(i, c)| text_for(*c, i, &mut rng)) + .collect(); + + let mut mem = HDF5Memory::create(MemoryConfig::new(path.clone(), "bench", DIM)).unwrap(); + let t = Instant::now(); + mem.save_batch(entries).unwrap(); + let ingest = t.elapsed(); + let t = Instant::now(); + mem.flush_wal().unwrap(); + let checkpoint = t.elapsed(); + drop(mem); + + let t = Instant::now(); + let mut mem = HDF5Memory::open(&path).unwrap(); + let open = t.elapsed(); + + // The first query after open pays for whatever is rebuilt lazily. + let t = Instant::now(); + std::hint::black_box(mem.hybrid_search(&data.queries[0], &query_texts[0], 0.7, 0.3, K)); + let first_query = t.elapsed(); + + // Fewer steady-state samples at large N: each query is currently O(N). + let samples_wanted = if n >= 100_000 { 20 } else { N_QUERIES.min(100) }; + let steady = summarize( + (0..samples_wanted) + .map(|i| { + let t = Instant::now(); + std::hint::black_box(mem.hybrid_search( + &data.queries[i % N_QUERIES], + &query_texts[i % N_QUERIES], + 0.7, + 0.3, + K, + )); + t.elapsed() + }) + .collect(), + ); + + println!( + "| {n} | {:.0} | {:.1} | {:.1} | {:.1} | {:.2} | {:.2} | {:.1} |", + millis(ingest), + millis(checkpoint), + millis(open), + millis(first_query), + millis(steady.p50), + millis(steady.p99), + steady.qps + ); + json.push(serde_json::json!({ + "bench": "hybrid_search", "n": n, + "ingest_ms": millis(ingest), "checkpoint_ms": millis(checkpoint), + "open_ms": millis(open), "first_query_ms": millis(first_query), + "p50_ms": millis(steady.p50), "p99_ms": millis(steady.p99), "qps": steady.qps, + })); +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let full = args.iter().any(|a| a == "--full"); + let json_path = args + .iter() + .position(|a| a == "--json") + .and_then(|i| args.get(i + 1)) + .cloned(); + let sizes: &[usize] = if full { + &[1_000, 10_000, 100_000] + } else { + &[1_000, 10_000] + }; + + if cfg!(debug_assertions) { + eprintln!("warning: debug build — numbers are meaningless. Use --release."); + } + + let mut json = Vec::new(); + println!("## Search harness"); + for &n in sizes { + bench_ann(n, &mut json); + } + + println!("\n### End to end: `HDF5Memory::hybrid_search` (k = {K}, weights 0.7 / 0.3)\n"); + println!( + "| N | ingest ms | checkpoint ms | open ms | first query ms | p50 ms | p99 ms | QPS |" + ); + println!("|---:|---:|---:|---:|---:|---:|---:|---:|"); + for &n in sizes { + bench_end_to_end(n, &mut json); + } + + if let Some(path) = json_path { + std::fs::write(&path, serde_json::to_string_pretty(&json).unwrap()).unwrap(); + eprintln!("wrote {path}"); + } +}