osobhandClaude Opus 5.5
dce5559ff2
Every undated or pre-September section re-run on one machine on one day
(tank, AMD Ryzen 7 7800X3D, 2026-09-24, commit 5c8323c), 24 commands run
serially with the load average checked before each, with the command
recorded for each section. A separate check traced every changed number
back to the raw output; its corrections are applied (e.g. the on-disk
~820 B/record is float16 plus always-deflated text on a synthetic corpus
of 40 distinct texts, not float16 alone).
Two apparent regressions were isolated rather than published:
- knowledge-graph traversal: a real bug, fixed in the previous commit;
- the write path: v2.3.0 built and run on the same machine measures the
same as today, so the old 18 us / 6.17 ms figures (undated, other
hardware) are not reproducible; float16 adds ~2 us per save and the
int8 index nothing (both isolated by switching the bench's config).
Also:
- new multimodal_bench: cross-modal search at 1K/10K records, which the
README claimed but nothing measured;
- footprint_bench reports whether it built float16 or f32 stores and
takes --f32 (it kept printing "f32" after the default changed);
- README: performance tables, the "Why" table figures and the SQLite
migration section (from the previous migrate commit);
- CHANGELOG for this branch.
Not re-run: consolidation_efficiency's 100K row and its memory-reduction
part (stopped for time), and cross_platform.sh.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
108 lines
3.7 KiB
Rust
108 lines
3.7 KiB
Rust
//! Multi-modal memory search benchmarks (`clawhdf5_agent::multimodal`).
|
|
//!
|
|
//! Covers `MultiModalStore::search_cross_modal` (every embedding of every
|
|
//! record, whatever its modality) and, for comparison,
|
|
//! `MultiModalStore::search_by_modality` restricted to one modality.
|
|
//!
|
|
//! Corpus: N records (1K and 10K), each carrying two 384-dim embeddings —
|
|
//! a text embedding of its caption plus one embedding of its primary modality,
|
|
//! cycling Image / Audio / Video — so a cross-modal query scores 2N vectors.
|
|
//! All data comes from a fixed-seed LCG, so every run sees the same corpus.
|
|
//!
|
|
//! Run: `cargo bench -p clawhdf5-agent --bench multimodal_bench`
|
|
|
|
use std::collections::HashMap;
|
|
|
|
use clawhdf5_agent::multimodal::{
|
|
MediaRef, ModalEmbedding, Modality, MultiModalRecord, MultiModalStore,
|
|
};
|
|
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Simple deterministic PRNG (LCG), same as the other agent benches
|
|
// ---------------------------------------------------------------------------
|
|
|
|
struct Rng(u32);
|
|
|
|
impl Rng {
|
|
fn new(seed: u32) -> Self {
|
|
Self(seed)
|
|
}
|
|
fn next_u32(&mut self) -> u32 {
|
|
self.0 = self.0.wrapping_mul(1103515245).wrapping_add(12345);
|
|
self.0 >> 16
|
|
}
|
|
fn next_f32(&mut self) -> f32 {
|
|
self.next_u32() as f32 / 65536.0 - 0.5
|
|
}
|
|
}
|
|
|
|
fn make_vec(rng: &mut Rng, dim: usize) -> Vec<f32> {
|
|
(0..dim).map(|_| rng.next_f32()).collect()
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Corpus
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const DIM: usize = 384;
|
|
const K: usize = 10;
|
|
|
|
const MEDIA: [(Modality, &str, &str); 3] = [
|
|
(Modality::Image, "image/png", "clip-vit-base"),
|
|
(Modality::Audio, "audio/wav", "clap-base"),
|
|
(Modality::Video, "video/mp4", "xclip-base"),
|
|
];
|
|
|
|
fn build_store(n: usize, seed: u32) -> MultiModalStore {
|
|
let mut rng = Rng::new(seed);
|
|
let mut store = MultiModalStore::new();
|
|
for i in 0..n {
|
|
let (modality, mime, model) = &MEDIA[i % MEDIA.len()];
|
|
let embeddings = vec![
|
|
ModalEmbedding::new(Modality::Text, make_vec(&mut rng, DIM), "minilm-l6"),
|
|
ModalEmbedding::new(modality.clone(), make_vec(&mut rng, DIM), *model),
|
|
];
|
|
store.add_record(MultiModalRecord {
|
|
id: 0,
|
|
primary_modality: modality.clone(),
|
|
text_content: Some(format!("{modality} memory {i}")),
|
|
media_ref: Some(MediaRef::path(format!("/media/{i}"), *mime)),
|
|
embeddings,
|
|
observation: None,
|
|
timestamp: 1_700_000_000.0 + i as f64,
|
|
metadata: HashMap::new(),
|
|
});
|
|
}
|
|
store
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Benchmarks
|
|
// ---------------------------------------------------------------------------
|
|
|
|
fn multimodal_search_benches(c: &mut Criterion) {
|
|
let query = make_vec(&mut Rng::new(99), DIM);
|
|
|
|
let mut group = c.benchmark_group("multimodal_search");
|
|
group.sample_size(50);
|
|
|
|
for (label, n) in [("1k", 1_000usize), ("10k", 10_000)] {
|
|
let store = build_store(n, 42);
|
|
assert_eq!(store.count(), n);
|
|
|
|
group.bench_with_input(BenchmarkId::new("cross_modal", label), &n, |b, _| {
|
|
b.iter(|| store.search_cross_modal(&query, K));
|
|
});
|
|
|
|
group.bench_with_input(BenchmarkId::new("by_modality_image", label), &n, |b, _| {
|
|
b.iter(|| store.search_by_modality(&Modality::Image, &query, K));
|
|
});
|
|
}
|
|
|
|
group.finish();
|
|
}
|
|
|
|
criterion_group!(multimodal_benches, multimodal_search_benches);
|
|
criterion_main!(multimodal_benches);
|