feat(memory): missions remember their verdicts, per repository
Until now missions wrote no memory. The chat path records every turn into the claw's .brain, but a mission's crew is minted per mission, so a brain keyed by agent would be written once and never read. What persists across missions is the repository: mission_memory keeps one .brain per repo_id, writes each judge verdict into it (reason when met, sanitized guidance when not — the operator reason may quote the acceptance text), and recalls against the next phase's task text into the brief, under a heading all three executors carry because it rides on the task. Recall is BM25 over the keyword index, no embedder; the harness asserts the brief carries the section once the repo has one judged mission behind it, and says 'first mission' rather than failing before that. OpenClaw's flush-before-compaction was the other half of this item and is moot here: the chat loop has no compaction and already remembers both halves of every turn. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01WZb5A2kfVfjpdwSochkuHz
This commit is contained in:
Omar Sobh
co-authored by
Claude Opus 5
Claude Opus 5
parent
76ac3714f1
commit
13f7fb3aff
@@ -0,0 +1,240 @@
|
||||
//! Project memory: what past missions on a repository learned.
|
||||
//!
|
||||
//! Until 2026-09-20 missions wrote no memory at all. The chat path records
|
||||
//! every turn into the claw's `.brain`, but a mission's crew is minted per
|
||||
//! mission (`per-mission-crews`: reuse is OFF by operator decision), so a
|
||||
//! brain keyed by agent would be written once and never read. What persists
|
||||
//! across missions is the repository. So the memory is keyed by `repo_id`:
|
||||
//! one `.brain` per repo, holding the judge's verdicts, recalled by the next
|
||||
//! mission's task text and placed in its brief.
|
||||
//!
|
||||
//! What is remembered is the verdict, not the work: for a met phase the
|
||||
//! judge's `reason` (what it found), for an unmet one its `guidance` — the
|
||||
//! agent-facing half, already stripped of acceptance literals by
|
||||
//! `evaluator::sanitize_guidance`, because a verdict quoted verbatim into
|
||||
//! the next brief is how the 2026-08-01 Goodhart incident happened.
|
||||
//!
|
||||
//! Recall is BM25 over the keyword index (`cm_brain::ClawBrain::recall`);
|
||||
//! there is no embedder. Measured before anything richer is built: the test
|
||||
//! is a second mission on the same repo recalling the first's verdict.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use cm_brain::ClawBrain;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// How many past verdicts a brief carries. Three is enough to say "this was
|
||||
/// tried" without becoming the prompt.
|
||||
pub const RECALL_K: usize = 3;
|
||||
|
||||
/// The heading the recalled lines go under. Named here because the scorer and
|
||||
/// the prompt-order tests read it back.
|
||||
pub const SECTION_HEADING: &str = "# What past missions on this repository learned";
|
||||
|
||||
fn brain_path(dir: &Path, repo_id: Uuid) -> PathBuf {
|
||||
dir.join(format!("repo_{repo_id}.h5"))
|
||||
}
|
||||
|
||||
/// One line of memory from a verdict. Pure, so the shape is testable without
|
||||
/// a brain file.
|
||||
pub fn verdict_line(
|
||||
mission_id: Uuid,
|
||||
phase_kind: &str,
|
||||
condition: &str,
|
||||
verdict: &crate::evaluator::Verdict,
|
||||
) -> Option<String> {
|
||||
// A judge that could not be reached has not judged; there is no lesson.
|
||||
if verdict.error.is_some() {
|
||||
return None;
|
||||
}
|
||||
let outcome = if verdict.met { "MET" } else { "UNMET" };
|
||||
let finding = if verdict.met {
|
||||
verdict.reason.trim()
|
||||
} else {
|
||||
verdict.guidance.trim()
|
||||
};
|
||||
if finding.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let short = mission_id.simple().to_string();
|
||||
Some(format!(
|
||||
"{outcome} — {phase_kind} phase of mission {} — condition: {} — judge: {}",
|
||||
&short[..8],
|
||||
head(condition, 200),
|
||||
head(finding, 400),
|
||||
))
|
||||
}
|
||||
|
||||
/// Record a verdict in the repo's brain. Best-effort and loud on failure:
|
||||
/// memory must never fail a phase, and a brain that silently stopped
|
||||
/// recording is the kind of thing that stays broken for a month.
|
||||
pub fn remember_verdict(
|
||||
repo_id: Uuid,
|
||||
mission_id: Uuid,
|
||||
phase_kind: &str,
|
||||
condition: &str,
|
||||
verdict: &crate::evaluator::Verdict,
|
||||
) {
|
||||
remember_in(
|
||||
&cm_runtime::brain::brain_dir(),
|
||||
repo_id,
|
||||
mission_id,
|
||||
phase_kind,
|
||||
condition,
|
||||
verdict,
|
||||
)
|
||||
}
|
||||
|
||||
fn remember_in(
|
||||
dir: &Path,
|
||||
repo_id: Uuid,
|
||||
mission_id: Uuid,
|
||||
phase_kind: &str,
|
||||
condition: &str,
|
||||
verdict: &crate::evaluator::Verdict,
|
||||
) {
|
||||
let Some(line) = verdict_line(mission_id, phase_kind, condition, verdict) else {
|
||||
return;
|
||||
};
|
||||
let path = brain_path(dir, repo_id);
|
||||
if let Some(dir) = path.parent() {
|
||||
let _ = std::fs::create_dir_all(dir);
|
||||
}
|
||||
match ClawBrain::open_or_create(&path, &format!("repo_{repo_id}")) {
|
||||
Ok(mut brain) => {
|
||||
if let Err(e) = brain.remember("judge", &line, &mission_id.to_string()) {
|
||||
eprintln!("mission_memory: could not record verdict for repo {repo_id}: {e}");
|
||||
}
|
||||
}
|
||||
Err(e) => eprintln!("mission_memory: could not open brain for repo {repo_id}: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The past verdicts most relevant to `query` (the phase's task text).
|
||||
/// Empty when the repo has no brain yet, which is every repo's first mission.
|
||||
pub fn recall(repo_id: Uuid, query: &str) -> Vec<String> {
|
||||
recall_in(&cm_runtime::brain::brain_dir(), repo_id, query)
|
||||
}
|
||||
|
||||
fn recall_in(dir: &Path, repo_id: Uuid, query: &str) -> Vec<String> {
|
||||
let path = brain_path(dir, repo_id);
|
||||
if !path.exists() {
|
||||
return Vec::new();
|
||||
}
|
||||
match ClawBrain::open_or_create(&path, &format!("repo_{repo_id}")) {
|
||||
Ok(brain) => brain
|
||||
.recall(query, RECALL_K)
|
||||
.into_iter()
|
||||
// `remember` stores "role: text"; the role is ours and not a lesson.
|
||||
.map(|m| m.strip_prefix("judge: ").map(str::to_string).unwrap_or(m))
|
||||
.collect(),
|
||||
Err(e) => {
|
||||
eprintln!("mission_memory: could not open brain for repo {repo_id}: {e}");
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The section a brief carries, or nothing when there is nothing to say —
|
||||
/// an empty heading tells the agent there is history and then shows none.
|
||||
pub fn section(recalled: &[String]) -> Option<String> {
|
||||
if recalled.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut out = String::from(SECTION_HEADING);
|
||||
out.push_str(
|
||||
"\n\nJudge verdicts from earlier missions here, most relevant first. \
|
||||
They say what was checked and what was found; they are not the task.\n",
|
||||
);
|
||||
for line in recalled {
|
||||
out.push_str("- ");
|
||||
out.push_str(line);
|
||||
out.push('\n');
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
fn head(s: &str, n: usize) -> String {
|
||||
match s.char_indices().nth(n) {
|
||||
Some((i, _)) => format!("{}…", &s[..i]),
|
||||
None => s.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::evaluator::{Usage, Verdict};
|
||||
|
||||
fn verdict(met: bool, reason: &str, guidance: &str, error: Option<&str>) -> Verdict {
|
||||
Verdict {
|
||||
met,
|
||||
reason: reason.into(),
|
||||
guidance: guidance.into(),
|
||||
model: "m".into(),
|
||||
error: error.map(str::to_string),
|
||||
checks: Vec::new(),
|
||||
independent: true,
|
||||
usage: Usage::default(),
|
||||
expectation: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Unmet carries the sanitized guidance, never the operator reason —
|
||||
/// the reason may quote the acceptance text the next mission must earn.
|
||||
#[test]
|
||||
fn unmet_remembers_guidance_not_reason() {
|
||||
let v = verdict(false, "token ZZQX-9 is absent", "the required marker is absent", None);
|
||||
let line = verdict_line(Uuid::nil(), "coding", "cond", &v).unwrap();
|
||||
assert!(line.starts_with("UNMET — coding phase"));
|
||||
assert!(line.contains("the required marker is absent"));
|
||||
assert!(!line.contains("ZZQX-9"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn met_remembers_what_the_judge_found() {
|
||||
let v = verdict(true, "MICROVM.md holds both lines", "", None);
|
||||
let line = verdict_line(Uuid::nil(), "coding", "cond", &v).unwrap();
|
||||
assert!(line.starts_with("MET — "));
|
||||
assert!(line.contains("MICROVM.md holds both lines"));
|
||||
}
|
||||
|
||||
/// No judgement, no lesson.
|
||||
#[test]
|
||||
fn an_unreachable_judge_leaves_no_memory() {
|
||||
let v = verdict(false, "could not evaluate", "could not evaluate", Some("429"));
|
||||
assert!(verdict_line(Uuid::nil(), "coding", "cond", &v).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn section_is_absent_when_nothing_was_recalled() {
|
||||
assert!(section(&[]).is_none());
|
||||
let s = section(&["MET — x".into()]).unwrap();
|
||||
assert!(s.starts_with(SECTION_HEADING));
|
||||
assert!(s.contains("- MET — x\n"));
|
||||
}
|
||||
|
||||
/// Round trip through a real brain file: what one mission's verdict
|
||||
/// wrote, a query shaped like the next mission's task recalls.
|
||||
#[test]
|
||||
fn a_second_mission_recalls_the_first_verdict() {
|
||||
let dir = std::env::temp_dir().join(format!("cm-mission-memory-{}", Uuid::now_v7()));
|
||||
let repo = Uuid::now_v7();
|
||||
let v = verdict(
|
||||
true,
|
||||
"BASELINE.md records 0.689 ns/iter from benches/add_bench.rs",
|
||||
"",
|
||||
None,
|
||||
);
|
||||
remember_in(&dir, repo, Uuid::now_v7(), "benchmark", "a baseline is recorded", &v);
|
||||
let got = recall_in(&dir, repo, "record a performance baseline for the hot path");
|
||||
assert_eq!(got.len(), 1, "{got:?}");
|
||||
assert!(got[0].starts_with("MET — benchmark phase"), "{}", got[0]);
|
||||
assert!(!got[0].starts_with("judge: "));
|
||||
// A repo with no history recalls nothing and creates no file.
|
||||
let other = Uuid::now_v7();
|
||||
assert!(recall_in(&dir, other, "anything").is_empty());
|
||||
assert!(!brain_path(&dir, other).exists());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user