diff --git a/crates/cm-api/src/lib.rs b/crates/cm-api/src/lib.rs index 91de312..8baebc7 100644 --- a/crates/cm-api/src/lib.rs +++ b/crates/cm-api/src/lib.rs @@ -29,6 +29,7 @@ pub mod mission_delivery; pub mod podcast; pub mod mission_events; pub mod mission_fs; +pub mod mission_memory; pub mod mission_gc; pub mod mission_orchestrator; pub mod mission_schedule; diff --git a/crates/cm-api/src/mission_memory.rs b/crates/cm-api/src/mission_memory.rs new file mode 100644 index 0000000..774e20d --- /dev/null +++ b/crates/cm-api/src/mission_memory.rs @@ -0,0 +1,240 @@ +//! Project memory: what past missions on a repository learned. +//! +//! Until 2026-09-20 missions wrote no memory at all. The chat path records +//! every turn into the claw's `.brain`, but a mission's crew is minted per +//! mission (`per-mission-crews`: reuse is OFF by operator decision), so a +//! brain keyed by agent would be written once and never read. What persists +//! across missions is the repository. So the memory is keyed by `repo_id`: +//! one `.brain` per repo, holding the judge's verdicts, recalled by the next +//! mission's task text and placed in its brief. +//! +//! What is remembered is the verdict, not the work: for a met phase the +//! judge's `reason` (what it found), for an unmet one its `guidance` — the +//! agent-facing half, already stripped of acceptance literals by +//! `evaluator::sanitize_guidance`, because a verdict quoted verbatim into +//! the next brief is how the 2026-08-01 Goodhart incident happened. +//! +//! Recall is BM25 over the keyword index (`cm_brain::ClawBrain::recall`); +//! there is no embedder. Measured before anything richer is built: the test +//! is a second mission on the same repo recalling the first's verdict. + +use std::path::{Path, PathBuf}; + +use cm_brain::ClawBrain; +use uuid::Uuid; + +/// How many past verdicts a brief carries. Three is enough to say "this was +/// tried" without becoming the prompt. +pub const RECALL_K: usize = 3; + +/// The heading the recalled lines go under. Named here because the scorer and +/// the prompt-order tests read it back. +pub const SECTION_HEADING: &str = "# What past missions on this repository learned"; + +fn brain_path(dir: &Path, repo_id: Uuid) -> PathBuf { + dir.join(format!("repo_{repo_id}.h5")) +} + +/// One line of memory from a verdict. Pure, so the shape is testable without +/// a brain file. +pub fn verdict_line( + mission_id: Uuid, + phase_kind: &str, + condition: &str, + verdict: &crate::evaluator::Verdict, +) -> Option { + // A judge that could not be reached has not judged; there is no lesson. + if verdict.error.is_some() { + return None; + } + let outcome = if verdict.met { "MET" } else { "UNMET" }; + let finding = if verdict.met { + verdict.reason.trim() + } else { + verdict.guidance.trim() + }; + if finding.is_empty() { + return None; + } + let short = mission_id.simple().to_string(); + Some(format!( + "{outcome} — {phase_kind} phase of mission {} — condition: {} — judge: {}", + &short[..8], + head(condition, 200), + head(finding, 400), + )) +} + +/// Record a verdict in the repo's brain. Best-effort and loud on failure: +/// memory must never fail a phase, and a brain that silently stopped +/// recording is the kind of thing that stays broken for a month. +pub fn remember_verdict( + repo_id: Uuid, + mission_id: Uuid, + phase_kind: &str, + condition: &str, + verdict: &crate::evaluator::Verdict, +) { + remember_in( + &cm_runtime::brain::brain_dir(), + repo_id, + mission_id, + phase_kind, + condition, + verdict, + ) +} + +fn remember_in( + dir: &Path, + repo_id: Uuid, + mission_id: Uuid, + phase_kind: &str, + condition: &str, + verdict: &crate::evaluator::Verdict, +) { + let Some(line) = verdict_line(mission_id, phase_kind, condition, verdict) else { + return; + }; + let path = brain_path(dir, repo_id); + if let Some(dir) = path.parent() { + let _ = std::fs::create_dir_all(dir); + } + match ClawBrain::open_or_create(&path, &format!("repo_{repo_id}")) { + Ok(mut brain) => { + if let Err(e) = brain.remember("judge", &line, &mission_id.to_string()) { + eprintln!("mission_memory: could not record verdict for repo {repo_id}: {e}"); + } + } + Err(e) => eprintln!("mission_memory: could not open brain for repo {repo_id}: {e}"), + } +} + +/// The past verdicts most relevant to `query` (the phase's task text). +/// Empty when the repo has no brain yet, which is every repo's first mission. +pub fn recall(repo_id: Uuid, query: &str) -> Vec { + recall_in(&cm_runtime::brain::brain_dir(), repo_id, query) +} + +fn recall_in(dir: &Path, repo_id: Uuid, query: &str) -> Vec { + let path = brain_path(dir, repo_id); + if !path.exists() { + return Vec::new(); + } + match ClawBrain::open_or_create(&path, &format!("repo_{repo_id}")) { + Ok(brain) => brain + .recall(query, RECALL_K) + .into_iter() + // `remember` stores "role: text"; the role is ours and not a lesson. + .map(|m| m.strip_prefix("judge: ").map(str::to_string).unwrap_or(m)) + .collect(), + Err(e) => { + eprintln!("mission_memory: could not open brain for repo {repo_id}: {e}"); + Vec::new() + } + } +} + +/// The section a brief carries, or nothing when there is nothing to say — +/// an empty heading tells the agent there is history and then shows none. +pub fn section(recalled: &[String]) -> Option { + if recalled.is_empty() { + return None; + } + let mut out = String::from(SECTION_HEADING); + out.push_str( + "\n\nJudge verdicts from earlier missions here, most relevant first. \ + They say what was checked and what was found; they are not the task.\n", + ); + for line in recalled { + out.push_str("- "); + out.push_str(line); + out.push('\n'); + } + Some(out) +} + +fn head(s: &str, n: usize) -> String { + match s.char_indices().nth(n) { + Some((i, _)) => format!("{}…", &s[..i]), + None => s.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::evaluator::{Usage, Verdict}; + + fn verdict(met: bool, reason: &str, guidance: &str, error: Option<&str>) -> Verdict { + Verdict { + met, + reason: reason.into(), + guidance: guidance.into(), + model: "m".into(), + error: error.map(str::to_string), + checks: Vec::new(), + independent: true, + usage: Usage::default(), + expectation: None, + } + } + + /// Unmet carries the sanitized guidance, never the operator reason — + /// the reason may quote the acceptance text the next mission must earn. + #[test] + fn unmet_remembers_guidance_not_reason() { + let v = verdict(false, "token ZZQX-9 is absent", "the required marker is absent", None); + let line = verdict_line(Uuid::nil(), "coding", "cond", &v).unwrap(); + assert!(line.starts_with("UNMET — coding phase")); + assert!(line.contains("the required marker is absent")); + assert!(!line.contains("ZZQX-9")); + } + + #[test] + fn met_remembers_what_the_judge_found() { + let v = verdict(true, "MICROVM.md holds both lines", "", None); + let line = verdict_line(Uuid::nil(), "coding", "cond", &v).unwrap(); + assert!(line.starts_with("MET — ")); + assert!(line.contains("MICROVM.md holds both lines")); + } + + /// No judgement, no lesson. + #[test] + fn an_unreachable_judge_leaves_no_memory() { + let v = verdict(false, "could not evaluate", "could not evaluate", Some("429")); + assert!(verdict_line(Uuid::nil(), "coding", "cond", &v).is_none()); + } + + #[test] + fn section_is_absent_when_nothing_was_recalled() { + assert!(section(&[]).is_none()); + let s = section(&["MET — x".into()]).unwrap(); + assert!(s.starts_with(SECTION_HEADING)); + assert!(s.contains("- MET — x\n")); + } + + /// Round trip through a real brain file: what one mission's verdict + /// wrote, a query shaped like the next mission's task recalls. + #[test] + fn a_second_mission_recalls_the_first_verdict() { + let dir = std::env::temp_dir().join(format!("cm-mission-memory-{}", Uuid::now_v7())); + let repo = Uuid::now_v7(); + let v = verdict( + true, + "BASELINE.md records 0.689 ns/iter from benches/add_bench.rs", + "", + None, + ); + remember_in(&dir, repo, Uuid::now_v7(), "benchmark", "a baseline is recorded", &v); + let got = recall_in(&dir, repo, "record a performance baseline for the hot path"); + assert_eq!(got.len(), 1, "{got:?}"); + assert!(got[0].starts_with("MET — benchmark phase"), "{}", got[0]); + assert!(!got[0].starts_with("judge: ")); + // A repo with no history recalls nothing and creates no file. + let other = Uuid::now_v7(); + assert!(recall_in(&dir, other, "anything").is_empty()); + assert!(!brain_path(&dir, other).exists()); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/cm-api/src/phase_runner.rs b/crates/cm-api/src/phase_runner.rs index 7150763..8fdb1f8 100644 --- a/crates/cm-api/src/phase_runner.rs +++ b/crates/cm-api/src/phase_runner.rs @@ -858,7 +858,9 @@ async fn start_pending_phases( m.runtime_kind, m.backend, m.target_node_id, m.team_engine, -- Whether a checkout exists at all. A repo-less mission's -- /mission/repo is scratch space, and the task text must say so. - (m.repo_id IS NOT NULL) AS has_repo + (m.repo_id IS NOT NULL) AS has_repo, + -- The key of the project memory (`mission_memory`). + m.repo_id FROM mission_phases mp JOIN missions m ON m.id = mp.mission_id WHERE mp.status = 'pending' @@ -890,6 +892,7 @@ async fn start_pending_phases( let target_node_id: Option = row.get("target_node_id"); let team_engine: Option = row.get("team_engine"); let has_repo: bool = row.get("has_repo"); + let repo_id: Option = row.get("repo_id"); if let Err(e) = launch_phase( pool, @@ -909,6 +912,7 @@ async fn start_pending_phases( target_node_id, team_engine: team_engine.as_deref(), has_repo, + repo_id, }, ) .await @@ -955,6 +959,8 @@ struct PhaseLaunch<'a> { /// `/mission/repo` is a git checkout or a scratch workspace whose contents /// are captured as artifacts. has_repo: bool, + /// `missions.repo_id`, the key of the project memory a brief recalls from. + repo_id: Option, } /// Which team purposes execute a phase of this kind. @@ -995,6 +1001,7 @@ async fn launch_phase( backend: _, target_node_id: _, team_engine: _, + repo_id: _, } = p; // Which team purposes should execute this phase. let purposes: &[&str] = purposes_for(kind); @@ -1220,6 +1227,26 @@ async fn launch_phase( _ => task, }; + // What earlier missions on this repository learned, by the judge's own + // account, recalled against this phase's task. Every mission's crew is + // new, so this is the only memory a mission has of the ones before it. + // Appended to the task rather than the identity so all three executors + // carry it: the task text is the one thing they share. + let task = match p.repo_id { + Some(repo) => match crate::mission_memory::section(&crate::mission_memory::recall( + repo, &task, + )) { + Some(memory) => { + eprintln!( + "phase_runner: phase {phase_id} brief carries project memory for repo {repo}" + ); + format!("{task}\n\n{memory}") + } + None => task, + }, + None => task, + }; + // The container tier is deliberately NOT given this: it injects per-turn in // `topology_exec`, with the running node's own role, and appending here too // would put every crew member's skills in every turn twice. @@ -2537,7 +2564,7 @@ async fn evaluate_finished_phases( ) -> Result<(), String> { let rows = sqlx::query( "SELECT mp.id, mp.mission_id, mp.kind, mp.done_when, mp.max_iterations, mp.iteration, - m.runtime_kind + m.runtime_kind, m.repo_id FROM mission_phases mp JOIN missions m ON m.id = mp.mission_id WHERE mp.status = 'evaluating' AND m.status = 'running' @@ -2558,6 +2585,7 @@ async fn evaluate_finished_phases( let max_iterations: i32 = row.get("max_iterations"); let iteration: i32 = row.get("iteration"); let runtime_kind: String = row.get("runtime_kind"); + let repo_id: Option = row.get("repo_id"); // Pull the agent's work onto the host BEFORE judging it. // @@ -2603,6 +2631,11 @@ async fn evaluate_finished_phases( { eprintln!("phase_runner: recording evaluation for {phase_id} failed: {e}"); } + // The project remembers the verdict. Only a repo-backed mission has a + // project to remember into; a repo-less one leaves no trace here. + if let Some(repo) = repo_id { + crate::mission_memory::remember_verdict(repo, mission_id, &kind, &condition, &verdict); + } // A judge that could not be REACHED has not judged. `Verdict.error` is // set only when the evaluator itself failed — "could not judge" as diff --git a/crates/cm-runtime/src/brain.rs b/crates/cm-runtime/src/brain.rs index 0f65e45..225d7f9 100644 --- a/crates/cm-runtime/src/brain.rs +++ b/crates/cm-runtime/src/brain.rs @@ -15,7 +15,9 @@ use std::path::PathBuf; use cm_brain::ClawBrain; -fn brain_dir() -> PathBuf { +/// Where the working `.brain` files live. Shared with `cm_api::mission_memory`, +/// which keeps the per-repository brains beside the per-claw ones. +pub fn brain_dir() -> PathBuf { std::env::var("CLAWMATES_BRAIN_DIR") .map(PathBuf::from) .unwrap_or_else(|_| std::env::temp_dir().join("clawmates-brains")) diff --git a/crates/cm-runtime/src/lib.rs b/crates/cm-runtime/src/lib.rs index 35cdc06..9e714c6 100644 --- a/crates/cm-runtime/src/lib.rs +++ b/crates/cm-runtime/src/lib.rs @@ -2,7 +2,7 @@ //! executes tools, and journals every event before any observer sees it //! (the gateway streams exactly this journal, live or replayed). -mod brain; +pub mod brain; mod events; pub mod outbox; mod runtime; diff --git a/scripts/verify-mission-delivery.sh b/scripts/verify-mission-delivery.sh index 07c186b..5a0b587 100755 --- a/scripts/verify-mission-delivery.sh +++ b/scripts/verify-mission-delivery.sh @@ -355,6 +355,9 @@ assert_microvm() { # # calls at all, and none of them may have been a write. assert_verifier_read_only "$mission" microvm + # Project memory: the brief carries earlier verdicts on this repository. + assert_project_memory "$mission" microvm + # And the verdict: judged, and by whom. `independent` is only true when the # judge came from a different provider family than the agent. indep=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \ @@ -900,6 +903,34 @@ assert_verifier_read_only() { #