bench: re-run every stale BENCHMARKS.md section, dated and traced
Every undated or pre-September section re-run on one machine on one day
(tank, AMD Ryzen 7 7800X3D, 2026-09-24, commit 5c8323c), 24 commands run
serially with the load average checked before each, with the command
recorded for each section. A separate check traced every changed number
back to the raw output; its corrections are applied (e.g. the on-disk
~820 B/record is float16 plus always-deflated text on a synthetic corpus
of 40 distinct texts, not float16 alone).
Two apparent regressions were isolated rather than published:
- knowledge-graph traversal: a real bug, fixed in the previous commit;
- the write path: v2.3.0 built and run on the same machine measures the
same as today, so the old 18 us / 6.17 ms figures (undated, other
hardware) are not reproducible; float16 adds ~2 us per save and the
int8 index nothing (both isolated by switching the bench's config).
Also:
- new multimodal_bench: cross-modal search at 1K/10K records, which the
README claimed but nothing measured;
- footprint_bench reports whether it built float16 or f32 stores and
takes --f32 (it kept printing "f32" after the default changed);
- README: performance tables, the "Why" table figures and the SQLite
migration section (from the previous migrate commit);
- CHANGELOG for this branch.
Not re-run: consolidation_efficiency's 100K row and its memory-reduction
part (stopped for time), and cross_platform.sh.
Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -11,12 +11,14 @@
|
||||
//!
|
||||
//! Configuration matrix:
|
||||
//! - Text lengths: short (50 chars), medium (200 chars), long (1000 chars)
|
||||
//! - Embedding: 384-dim f32 (1536 bytes raw per record)
|
||||
//! - Embedding: 384-dim, stored as float16 (the default for new stores) or
|
||||
//! f32 with `--f32`; "raw" bytes are counted as f32 input either way
|
||||
//! - WAL: enabled and disabled
|
||||
//!
|
||||
//! # Usage
|
||||
//! ```
|
||||
//! cargo run --release --bin footprint_bench
|
||||
//! cargo run --release --bin footprint_bench # float16 stores
|
||||
//! cargo run --release --bin footprint_bench -- --f32 # f32 stores
|
||||
//! ```
|
||||
|
||||
use std::time::Instant;
|
||||
@@ -24,6 +26,9 @@ use std::time::Instant;
|
||||
use clawhdf5_agent::{AgentMemory, HDF5Memory, MemoryConfig, MemoryEntry};
|
||||
use tempfile::TempDir;
|
||||
|
||||
/// `--f32`: build f32 stores instead of the library's float16 default.
|
||||
static F32: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
|
||||
|
||||
const EMBEDDING_DIM: usize = 384;
|
||||
|
||||
// Raw bytes per record: 384 f32 embeddings + median text + overhead
|
||||
@@ -152,6 +157,9 @@ fn measure_footprint(
|
||||
config.compression = compression;
|
||||
config.compression_level = if compression { 6 } else { 0 };
|
||||
config.compact_threshold = 0.0;
|
||||
if F32.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
config.float16 = false;
|
||||
}
|
||||
|
||||
let mut memory = HDF5Memory::create(config).expect("HDF5Memory::create failed");
|
||||
|
||||
@@ -241,11 +249,19 @@ fn fmt_n(n: usize) -> String {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn main() {
|
||||
if std::env::args().skip(1).any(|a| a == "--f32") {
|
||||
F32.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
let stored = if F32.load(std::sync::atomic::Ordering::Relaxed) {
|
||||
"f32 (1,536 bytes per record)"
|
||||
} else {
|
||||
"float16 (768 bytes per record; the default for new stores)"
|
||||
};
|
||||
println!("=================================================================");
|
||||
println!(" ClawhDF5 Memory Footprint Benchmark");
|
||||
println!("=================================================================");
|
||||
println!();
|
||||
println!("Embedding: 384-dim f32 = 1,536 bytes raw per record");
|
||||
println!("Embedding: 384-dim, stored as {stored}; raw input counted as f32");
|
||||
println!("Text lengths: short=50 chars, medium=200 chars, long=1000 chars");
|
||||
println!();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user