quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 1 Wiki Activity

feat(agent): HDF5Memory::search with source filters, re-ranking, confidence

Browse Source 
`HDF5Memory::search(query_embedding, query_text, &SearchOptions)` is the
store's full search path. `SearchOptions::new(k)` is plain hybrid search
with the tuned default fusion; each further stage is opt-in:

- `with_sources([..])`: only records from these source channels. The
  filter applies before ranking, so a filtered search still returns up
  to k results, normalised over what it can return. The HNSW pool is
  over-fetched in proportion to what the filter removes, and the allowed
  records are scanned exactly whenever that costs fewer distance
  evaluations than the index would (~pool x M) — and as the fallback if
  the pool comes back short. Keyword matches are filtered too.
- `with_rerank(ReRankConfig)` re-ranks a max(3k, 10) candidate pool by
  relevance, recency, source authority and activation;
  `with_confidence(ConfidenceConfig)` drops low-confidence results;
  `at_time(now)` pins the recency clock.

These were reachable only through the OpenClaw backend, which is now
`search` with both on. Its Hebbian boost now goes to the k results it
returns rather than the whole 3k candidate pool. `hybrid_search` and
`hybrid_search_with` are wrappers and unchanged (tested bit for bit).

Measured on tank (search_harness --options-study --full, 3 runs): at
100K every filter — 50%, 10%, 1% of the store, and records far from the
query — returns the exact filtered top 10, and none is slower than an
unfiltered search (1%: 2.3 ms vs 4.6 ms). Re-rank + confidence costs
about 3%. A first version decided between index and exact scan by pool
size vs store size; it measured 0.976 recall at 12.3 ms on the
far-from-query filter, which is why the rule compares costs instead.

Tests: tests/search_options.rs (filter correctness and full pages via
both paths, far-from-query fallback, edge cases, equality with
hybrid_search_with, re-rank recency, confidence, boost scope).

Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
osobh
2026-09-24 16:35:42 -05:00
co-authored by Claude Opus 5.5
parent d0db83812b
commit c470244a6f
10 changed files with 951 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-bench/src/bin/search_harness.rs
+185
View File
@@ -20,6 +20,7 @@
//! cargo run --release -p clawhdf5-bench --bin search_harness -- --json out.json
//! cargo run --release -p clawhdf5-bench --bin search_harness -- --ann-only --uniform
//! cargo run --release -p clawhdf5-bench --bin search_harness -- --float16-study --full
//! cargo run --release -p clawhdf5-bench --bin search_harness -- --options-study --full
//! ```
use std::time::{Duration, Instant};
@@ -487,6 +488,177 @@ fn bench_end_to_end(n: usize, json: &mut Vec<serde_json::Value>) {
}));
}
// ---------------------------------------------------------------------------
// Search options study: source filters, re-ranking, confidence rejection
// ---------------------------------------------------------------------------
/// `--options-study`: what `HDF5Memory::search`'s options cost and whether a
/// filtered search finds the right records. Filters keep 50%, 10% or 1% of
/// the store at random, or two whole clusters away from the query (the case
/// the index cannot serve, which falls back to an exact scan). Recall is
/// vector-only against an exact scan of the allowed records; latency is full
/// hybrid search. Hebbian boosting is off.
fn options_study(n: usize) {
use clawhdf5_agent::SearchOptions;
use clawhdf5_agent::confidence::ConfidenceConfig;
use clawhdf5_agent::hybrid::Fusion;
use clawhdf5_agent::reranker::ReRankConfig;
let data = make_dataset(n, 0x0B7 ^ n as u64);
let n_clusters = data.cluster_of.iter().max().map_or(1, |m| m + 1);
let mut rng = Rng(5);
let bucket_of: Vec<usize> = (0..n).map(|_| rng.below(100)).collect();
let bucket = &bucket_of;
let query_texts: Vec<String> = data
.query_cluster
.iter()
.enumerate()
.map(|(i, c)| text_for(*c, i, &mut rng))
.collect();
let exact_top = |q: &[f32], allowed: &dyn Fn(usize) -> bool| -> Vec<usize> {
let mut s: Vec<(usize, f32)> = (0..n)
.filter(|&i| allowed(i))
.map(|i| (i, data.vectors[i].iter().zip(q).map(|(a, b)| a * b).sum()))
.collect();
s.sort_by(|a, b| b.1.total_cmp(&a.1).then(a.0.cmp(&b.0)));
s.into_iter().take(K).map(|(i, _)| i).collect()
};
// Two stores: channel = random bucket, and channel = cluster.
let dir = tempfile::tempdir().unwrap();
let mut stores = Vec::new();
for by_cluster in [false, true] {
let mut rng = Rng(3);
let entries: Vec<MemoryEntry> = data
.vectors
.iter()
.enumerate()
.map(|(i, v)| MemoryEntry {
chunk: text_for(data.cluster_of[i], i, &mut rng),
embedding: v.clone(),
source_channel: if by_cluster {
format!("c{}", data.cluster_of[i])
} else {
format!("b{}", bucket[i])
},
timestamp: i as f64,
session_id: format!("s{}", i % 50),
tags: format!("t{i}"),
})
.collect();
let mut config = MemoryConfig::new(
dir.path().join(format!("opt_{by_cluster}.h5")),
"bench",
DIM,
);
config.hebbian_boost = 0.0;
let mut mem = HDF5Memory::create(config).unwrap();
mem.save_batch(entries).unwrap();
std::hint::black_box(mem.search(&data.queries[0], "", &SearchOptions::new(K)));
stores.push(mem);
}
let vector_only = SearchOptions::new(K).with_fusion(Fusion::Weighted {
vector: 1.0,
keyword: 0.0,
});
// (label, store, channels for query i, allowed(i, record))
type Case<'a> = (
String,
usize,
Box<dyn Fn(usize) -> Option<Vec<String>> + 'a>,
Box<dyn Fn(usize, usize) -> bool + 'a>,
);
let mut cases: Vec<Case> = vec![(
"no filter".into(),
0,
Box::new(|_| None),
Box::new(|_, _| true),
)];
for pct in [50usize, 10, 1] {
cases.push((
format!("random {pct}%"),
0,
Box::new(move |_| Some((0..pct).map(|b| format!("b{b}")).collect())),
Box::new(move |_, i| bucket[i] < pct),
));
}
let d = &data;
let away = move |qi: usize| {
let qc = d.query_cluster[qi];
[
(qc + n_clusters / 3) % n_clusters,
(qc + 2 * n_clusters / 3) % n_clusters,
]
};
cases.push((
"2 clusters away from the query".into(),
1,
Box::new(move |qi| Some(away(qi).iter().map(|c| format!("c{c}")).collect())),
Box::new(move |qi, i| away(qi).contains(&d.cluster_of[i])),
));
for (label, store, channels, allowed) in &cases {
let mem = &mut stores[*store];
let mut hits = 0;
let mut kept = 0;
for (qi, q) in data.queries.iter().enumerate() {
let mut opts = vector_only.clone();
opts.source_channels = channels(qi);
let got = mem.search(q, "", &opts);
let want = exact_top(q, &|i| allowed(qi, i));
kept += want.len();
hits += got.iter().filter(|r| want.contains(&r.index)).count();
}
let latency = summarize(
(0..N_QUERIES)
.map(|qi| {
let mut opts = SearchOptions::new(K);
opts.source_channels = channels(qi);
let t = Instant::now();
std::hint::black_box(mem.search(&data.queries[qi], &query_texts[qi], &opts));
t.elapsed()
})
.collect(),
);
println!(
"| {n} | {label} | {:.4} | {:.3} | {:.3} |",
hits as f64 / kept.max(1) as f64,
millis(latency.p50),
millis(latency.p99),
);
}
let mem = &mut stores[0];
for (label, opts) in [
(
"re-rank",
SearchOptions::new(K).with_rerank(ReRankConfig::default()),
),
(
"re-rank + confidence",
SearchOptions::new(K)
.with_rerank(ReRankConfig::default())
.with_confidence(ConfidenceConfig::default()),
),
] {
let latency = summarize(
(0..N_QUERIES)
.map(|qi| {
let t = Instant::now();
std::hint::black_box(mem.search(&data.queries[qi], &query_texts[qi], &opts));
t.elapsed()
})
.collect(),
);
println!(
"| {n} | {label} | — | {:.3} | {:.3} |",
millis(latency.p50),
millis(latency.p99)
);
}
}
// ---------------------------------------------------------------------------
// float16 study: what does half-precision embedding storage cost?
// ---------------------------------------------------------------------------
@@ -788,6 +960,19 @@ fn main() {
}
return;
}
if args.iter().any(|a| a == "--options-study") {
println!("## Search options ({DIM}-dim, k = {K}, Hebbian boost off)\n");
println!("| N | options | filtered recall@10 | p50 ms | p99 ms |");
println!("|---:|---|---:|---:|---:|");
for &n in if full {
&[10_000, 100_000][..]
} else {
&[10_000][..]
} {
options_study(n);
}
return;
}
if args.iter().any(|a| a == "--f16-first") {
F16_FIRST.store(true, std::sync::atomic::Ordering::Relaxed);
}