feat(decide): memory rerank and paper triage on the decision tier
mission_memory::recall is two stages when a key is set: BM25 proposes 8
candidates, one Noul per candidate ('this earlier verdict is relevant to
the task') reorders them and drops those under 0.3, three are kept. BM25
measures keyword overlap, and a verdict about MICROVM.md shares words
with every task that names a file. Without a key the BM25 order stands.
continuous_research: topic_tags was written as [] on every manifest line
since the manifest existed. triage_papers asks, per harvested paper, a
Choice over the mission's topics (readable names, the arXiv query as the
description, 'none' offered) and a four-level relevance Score; the tags
(every topic ≥ 0.3) and {score, confidence} land on the line the agents
already read. Probe on PORTICO's abstract: relevance 3.0 at 1.0; topic
'none' 0.59 / verification 0.41 — true, the topic list has no
authority/sandboxing entry. Untriaged papers write the old empty line.
Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5
parent
2656d73def
commit
33560c7fe6
@@ -27,6 +27,15 @@ use uuid::Uuid;
|
||||
/// tried" without becoming the prompt.
|
||||
pub const RECALL_K: usize = 3;
|
||||
|
||||
/// How many BM25 candidates the reranker sees. Wider than `RECALL_K` so a
|
||||
/// relevant verdict that keyword overlap ranked fourth can still make the
|
||||
/// brief; narrow enough that one call stays one call.
|
||||
pub const CANDIDATES: usize = 8;
|
||||
|
||||
/// Below this a candidate is dropped even if fewer than `RECALL_K` remain:
|
||||
/// a brief that carries an irrelevant verdict is worse than a shorter one.
|
||||
pub const RELEVANT_AT: f64 = 0.3;
|
||||
|
||||
/// The heading the recalled lines go under. Named here because the scorer and
|
||||
/// the prompt-order tests read it back.
|
||||
pub const SECTION_HEADING: &str = "# What past missions on this repository learned";
|
||||
@@ -112,18 +121,88 @@ fn remember_in(
|
||||
|
||||
/// The past verdicts most relevant to `query` (the phase's task text).
|
||||
/// Empty when the repo has no brain yet, which is every repo's first mission.
|
||||
pub fn recall(repo_id: Uuid, query: &str) -> Vec<String> {
|
||||
recall_in(&cm_runtime::brain::brain_dir(), repo_id, query)
|
||||
///
|
||||
/// Two stages when a decision model is configured: BM25 proposes
|
||||
/// `CANDIDATES`, one Noul per candidate — "is this past verdict relevant
|
||||
/// to the task?" — reorders them and drops the ones below `RELEVANT_AT`.
|
||||
/// Keyword overlap is what BM25 measures, and a verdict about MICROVM.md
|
||||
/// shares words with every task that mentions a file; the rerank is the
|
||||
/// vendor's own pattern and costs one ~200 ms call. Without a key the
|
||||
/// BM25 order stands, as before.
|
||||
pub async fn recall(repo_id: Uuid, query: &str) -> Vec<String> {
|
||||
let candidates = recall_in(&cm_runtime::brain::brain_dir(), repo_id, query, CANDIDATES);
|
||||
match cm_decide::jev::Jev::from_env() {
|
||||
Some(jev) if candidates.len() > 1 => rerank(&jev, query, candidates).await,
|
||||
_ => candidates.into_iter().take(RECALL_K).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn recall_in(dir: &Path, repo_id: Uuid, query: &str) -> Vec<String> {
|
||||
async fn rerank(jev: &cm_decide::jev::Jev, query: &str, candidates: Vec<String>) -> Vec<String> {
|
||||
use cm_decide::{Answer, Decider as _, Question};
|
||||
let questions: std::collections::BTreeMap<String, Question> = candidates
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, c)| {
|
||||
(
|
||||
format!("c{i}"),
|
||||
Question::noul(format!(
|
||||
"This earlier judge verdict is relevant to the task and would help an \
|
||||
agent doing it: {c}"
|
||||
)),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let decided = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
jev.decide(query, &questions),
|
||||
)
|
||||
.await;
|
||||
let decision = match decided {
|
||||
Ok(Ok(d)) => d,
|
||||
Ok(Err(e)) => {
|
||||
eprintln!("mission_memory: rerank failed ({e}); keeping the BM25 order");
|
||||
return candidates.into_iter().take(RECALL_K).collect();
|
||||
}
|
||||
Err(_) => {
|
||||
eprintln!("mission_memory: rerank timed out; keeping the BM25 order");
|
||||
return candidates.into_iter().take(RECALL_K).collect();
|
||||
}
|
||||
};
|
||||
let scored: Vec<(String, f64)> = candidates
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, c)| {
|
||||
let p = match decision.answers.get(&format!("c{i}")) {
|
||||
Some(Answer::Noul { noul }) => *noul,
|
||||
_ => 0.0,
|
||||
};
|
||||
(c, p)
|
||||
})
|
||||
.collect();
|
||||
let kept: Vec<String> = cm_decide::patterns::rerank(scored)
|
||||
.into_iter()
|
||||
.filter(|(_, p)| *p >= RELEVANT_AT)
|
||||
.take(RECALL_K)
|
||||
.map(|(c, _)| c)
|
||||
.collect();
|
||||
eprintln!(
|
||||
"mission_memory: reranked {} candidate(s) with {}, kept {} ({} ms)",
|
||||
questions.len(),
|
||||
decision.model,
|
||||
kept.len(),
|
||||
decision.latency.as_millis()
|
||||
);
|
||||
kept
|
||||
}
|
||||
|
||||
fn recall_in(dir: &Path, repo_id: Uuid, query: &str, k: usize) -> Vec<String> {
|
||||
let path = brain_path(dir, repo_id);
|
||||
if !path.exists() {
|
||||
return Vec::new();
|
||||
}
|
||||
match ClawBrain::open_or_create(&path, &format!("repo_{repo_id}")) {
|
||||
Ok(brain) => brain
|
||||
.recall(query, RECALL_K)
|
||||
.recall(query, k)
|
||||
.into_iter()
|
||||
// `remember` stores "role: text"; the role is ours and not a lesson.
|
||||
.map(|m| m.strip_prefix("judge: ").map(str::to_string).unwrap_or(m))
|
||||
@@ -227,13 +306,13 @@ mod tests {
|
||||
None,
|
||||
);
|
||||
remember_in(&dir, repo, Uuid::now_v7(), "benchmark", "a baseline is recorded", &v);
|
||||
let got = recall_in(&dir, repo, "record a performance baseline for the hot path");
|
||||
let got = recall_in(&dir, repo, "record a performance baseline for the hot path", RECALL_K);
|
||||
assert_eq!(got.len(), 1, "{got:?}");
|
||||
assert!(got[0].starts_with("MET — benchmark phase"), "{}", got[0]);
|
||||
assert!(!got[0].starts_with("judge: "));
|
||||
// A repo with no history recalls nothing and creates no file.
|
||||
let other = Uuid::now_v7();
|
||||
assert!(recall_in(&dir, other, "anything").is_empty());
|
||||
assert!(recall_in(&dir, other, "anything", RECALL_K).is_empty());
|
||||
assert!(!brain_path(&dir, other).exists());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user