bench: re-run every stale BENCHMARKS.md section, dated and traced

Every undated or pre-September section re-run on one machine on one day (tank, AMD Ryzen 7 7800X3D, 2026-09-24, commit 5c8323c), 24 commands run serially with the load average checked before each, with the command recorded for each section. A separate check traced every changed number back to the raw output; its corrections are applied (e.g. the on-disk ~820 B/record is float16 plus always-deflated text on a synthetic corpus of 40 distinct texts, not float16 alone). Two apparent regressions were isolated rather than published: - knowledge-graph traversal: a real bug, fixed in the previous commit; - the write path: v2.3.0 built and run on the same machine measures the same as today, so the old 18 us / 6.17 ms figures (undated, other hardware) are not reproducible; float16 adds ~2 us per save and the int8 index nothing (both isolated by switching the bench's config). Also: - new multimodal_bench: cross-modal search at 1K/10K records, which the README claimed but nothing measured; - footprint_bench reports whether it built float16 or f32 stores and takes --f32 (it kept printing "f32" after the default changed); - README: performance tables, the "Why" table figures and the SQLite migration section (from the previous migrate commit); - CHANGELOG for this branch. Not re-run: consolidation_efficiency's 100K row and its memory-reduction part (stopped for time), and cross_platform.sh. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
2026-09-24 23:45:15 -05:00
parent 1b3bbb054a
commit dce5559ff2
6 changed files with 880 additions and 175 deletions
@@ -45,6 +45,10 @@ harness = false
 name = "memory_bench"
 harness = false

+[[bench]]
+name = "multimodal_bench"
+harness = false
+
 [features]
 default = ["float16", "hnsw", "parallel"]
 float16 = ["half"]
@@ -0,0 +1,107 @@
+//! Multi-modal memory search benchmarks (`clawhdf5_agent::multimodal`).
+//!
+//! Covers `MultiModalStore::search_cross_modal` (every embedding of every
+//! record, whatever its modality) and, for comparison,
+//! `MultiModalStore::search_by_modality` restricted to one modality.
+//!
+//! Corpus: N records (1K and 10K), each carrying two 384-dim embeddings —
+//! a text embedding of its caption plus one embedding of its primary modality,
+//! cycling Image / Audio / Video — so a cross-modal query scores 2N vectors.
+//! All data comes from a fixed-seed LCG, so every run sees the same corpus.
+//!
+//! Run: `cargo bench -p clawhdf5-agent --bench multimodal_bench`
+
+use std::collections::HashMap;
+
+use clawhdf5_agent::multimodal::{
+    MediaRef, ModalEmbedding, Modality, MultiModalRecord, MultiModalStore,
+};
+use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
+
+// ---------------------------------------------------------------------------
+// Simple deterministic PRNG (LCG), same as the other agent benches
+// ---------------------------------------------------------------------------
+
+struct Rng(u32);
+
+impl Rng {
+    fn new(seed: u32) -> Self {
+        Self(seed)
+    }
+    fn next_u32(&mut self) -> u32 {
+        self.0 = self.0.wrapping_mul(1103515245).wrapping_add(12345);
+        self.0 >> 16
+    }
+    fn next_f32(&mut self) -> f32 {
+        self.next_u32() as f32 / 65536.0 - 0.5
+    }
+}
+
+fn make_vec(rng: &mut Rng, dim: usize) -> Vec<f32> {
+    (0..dim).map(|_| rng.next_f32()).collect()
+}
+
+// ---------------------------------------------------------------------------
+// Corpus
+// ---------------------------------------------------------------------------
+
+const DIM: usize = 384;
+const K: usize = 10;
+
+const MEDIA: [(Modality, &str, &str); 3] = [
+    (Modality::Image, "image/png", "clip-vit-base"),
+    (Modality::Audio, "audio/wav", "clap-base"),
+    (Modality::Video, "video/mp4", "xclip-base"),
+];
+
+fn build_store(n: usize, seed: u32) -> MultiModalStore {
+    let mut rng = Rng::new(seed);
+    let mut store = MultiModalStore::new();
+    for i in 0..n {
+        let (modality, mime, model) = &MEDIA[i % MEDIA.len()];
+        let embeddings = vec![
+            ModalEmbedding::new(Modality::Text, make_vec(&mut rng, DIM), "minilm-l6"),
+            ModalEmbedding::new(modality.clone(), make_vec(&mut rng, DIM), *model),
+        ];
+        store.add_record(MultiModalRecord {
+            id: 0,
+            primary_modality: modality.clone(),
+            text_content: Some(format!("{modality} memory {i}")),
+            media_ref: Some(MediaRef::path(format!("/media/{i}"), *mime)),
+            embeddings,
+            observation: None,
+            timestamp: 1_700_000_000.0 + i as f64,
+            metadata: HashMap::new(),
+        });
+    }
+    store
+}
+
+// ---------------------------------------------------------------------------
+// Benchmarks
+// ---------------------------------------------------------------------------
+
+fn multimodal_search_benches(c: &mut Criterion) {
+    let query = make_vec(&mut Rng::new(99), DIM);
+
+    let mut group = c.benchmark_group("multimodal_search");
+    group.sample_size(50);
+
+    for (label, n) in [("1k", 1_000usize), ("10k", 10_000)] {
+        let store = build_store(n, 42);
+        assert_eq!(store.count(), n);
+
+        group.bench_with_input(BenchmarkId::new("cross_modal", label), &n, |b, _| {
+            b.iter(|| store.search_cross_modal(&query, K));
+        });
+
+        group.bench_with_input(BenchmarkId::new("by_modality_image", label), &n, |b, _| {
+            b.iter(|| store.search_by_modality(&Modality::Image, &query, K));
+        });
+    }
+
+    group.finish();
+}
+
+criterion_group!(multimodal_benches, multimodal_search_benches);
+criterion_main!(multimodal_benches);
@@ -11,12 +11,14 @@
 //!
 //! Configuration matrix:
 //!   - Text lengths: short (50 chars), medium (200 chars), long (1000 chars)
-//!   - Embedding: 384-dim f32 (1536 bytes raw per record)
+//!   - Embedding: 384-dim, stored as float16 (the default for new stores) or
+//!     f32 with `--f32`; "raw" bytes are counted as f32 input either way
 //!   - WAL: enabled and disabled
 //!
 //! # Usage
 //! ```
-//! cargo run --release --bin footprint_bench
+//! cargo run --release --bin footprint_bench           # float16 stores
+//! cargo run --release --bin footprint_bench -- --f32  # f32 stores
 //! ```

 use std::time::Instant;
@@ -24,6 +26,9 @@ use std::time::Instant;
 use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry};
 use tempfile::TempDir;

+/// `--f32`: build f32 stores instead of the library's float16 default.
+static F32: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
+
 const EMBEDDING_DIM: usize = 384;

 // Raw bytes per record: 384 f32 embeddings + median text + overhead
@@ -152,6 +157,9 @@ fn measure_footprint(
    config.compression = compression;
    config.compression_level = if compression { 6 } else { 0 };
    config.compact_threshold = 0.0;
+    if F32.load(std::sync::atomic::Ordering::Relaxed) {
+        config.float16 = false;
+    }

    let mut memory = HDF5Memory::create(config).expect("HDF5Memory::create failed");

@@ -241,11 +249,19 @@ fn fmt_n(n: usize) -> String {
 // ---------------------------------------------------------------------------

 fn main() {
+    if std::env::args().skip(1).any(|a| a == "--f32") {
+        F32.store(true, std::sync::atomic::Ordering::Relaxed);
+    }
+    let stored = if F32.load(std::sync::atomic::Ordering::Relaxed) {
+        "f32 (1,536 bytes per record)"
+    } else {
+        "float16 (768 bytes per record; the default for new stores)"
+    };
    println!("=================================================================");
    println!("  ClawhDF5 Memory Footprint Benchmark");
    println!("=================================================================");
    println!();
-    println!("Embedding: 384-dim f32 = 1,536 bytes raw per record");
+    println!("Embedding: 384-dim, stored as {stored}; raw input counted as f32");
    println!("Text lengths: short=50 chars, medium=200 chars, long=1000 chars");
    println!();