bench(longmemeval): --float16, and float16 measured on real embeddings
`longmemeval_bench --float16` builds every per-question store with MemoryConfig::float16, so the vector stage searches half-rounded embeddings exactly as such a store holds them. Full longmemeval_s (500 questions, ~494 turns each) with real all-MiniLM-L6-v2 embeddings, f32 vs float16, on tank (CUDA): identical at every Hit@k and MRR, turn and session level, in all eight modes — bar RRF session MRR 0.9253 vs 0.9254 and one or two flips out of ~320 in which gold session ranks first. The f32 run reproduces the published hybrid numbers exactly. The earlier float16 evidence was synthetic clustered data only; this is the real-embedding check. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
@@ -64,6 +64,11 @@ use tempfile::TempDir;
|
||||
|
||||
const EMBEDDING_DIM: usize = 384;
|
||||
|
||||
/// `--float16`: build every per-question store with `MemoryConfig::float16`,
|
||||
/// so embeddings are rounded to half precision as they are saved — exactly
|
||||
/// what such a store searches over.
|
||||
static FLOAT16: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
|
||||
|
||||
/// A mode's fusion, as one short string for the reports.
|
||||
fn describe(mode: Mode) -> String {
|
||||
let fusion = match mode.fusion {
|
||||
@@ -431,6 +436,7 @@ fn evaluate_question(
|
||||
let mut config = MemoryConfig::new(dir.path().join("lme.h5"), "lme-bench", EMBEDDING_DIM);
|
||||
config.wal_enabled = false;
|
||||
config.compact_threshold = 0.0;
|
||||
config.float16 = FLOAT16.load(std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let mut memory = HDF5Memory::create(config).expect("failed to create HDF5Memory");
|
||||
memory.set_token_filter(mode.tokens);
|
||||
@@ -940,6 +946,10 @@ fn main() {
|
||||
limit = Some(v.parse().expect("--limit must be a positive integer"));
|
||||
}
|
||||
"--sweep" => sweep = true,
|
||||
"--float16" => {
|
||||
FLOAT16.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
eprintln!("Stores use MemoryConfig::float16 (half-precision embeddings)");
|
||||
}
|
||||
"--rerank-sweep" => {
|
||||
// Re-ranking needs the vector stage to have candidates worth
|
||||
// reordering, so this is an embeddings-only comparison.
|
||||
@@ -971,6 +981,9 @@ fn main() {
|
||||
--rerank-sweep\n\
|
||||
compare re-ranking off, metadata-only (the old\n\
|
||||
behaviour) and blended at several half-lives.\n\
|
||||
--float16\n\
|
||||
build each store with MemoryConfig::float16, to\n\
|
||||
compare retrieval on half-precision embeddings.\n\
|
||||
--sweep instead of the three named modes, sweep vector_weight\n\
|
||||
from 0.0 to 1.0 in 0.1 steps. The 0.7/0.3 default was\n\
|
||||
never searched; this is what searches it."
|
||||
|
||||
Reference in New Issue
Block a user