Omar SobhandClaude Opus 5.5
9ca71e5fa4
The first self_audit run on a planted brief/condition mismatch found the pattern and quoted it, then diagnosed "the agent skipped the section": the record held the condition but not the brief, and working agents never see the condition. It also read two missions as one — a UUIDv7's first 8 chars are a timestamp, and missions 34 s apart both rendered as 01a0cb38. - verdict_line records the phase brief (config.task) beside the condition - the short mission id is the uuid tail - self_audit copies the record into the checkout (the judge cannot read /mission/memory) and compares brief with condition Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
453 lines
18 KiB
Rust
453 lines
18 KiB
Rust
//! Project memory: what past missions on a repository learned.
|
|
//!
|
|
//! Until 2026-09-20 missions wrote no memory at all. The chat path records
|
|
//! every turn into the claw's `.brain`, but a mission's crew is minted per
|
|
//! mission (`per-mission-crews`: reuse is OFF by operator decision), so a
|
|
//! brain keyed by agent would be written once and never read. What persists
|
|
//! across missions is the repository. So the memory is keyed by `repo_id`:
|
|
//! one `.brain` per repo, holding the judge's verdicts, recalled by the next
|
|
//! mission's task text and placed in its brief.
|
|
//!
|
|
//! What is remembered is the verdict, not the work: for a met phase the
|
|
//! judge's `reason` (what it found), for an unmet one its `guidance` — the
|
|
//! agent-facing half, already stripped of acceptance literals by
|
|
//! `evaluator::sanitize_guidance`, because a verdict quoted verbatim into
|
|
//! the next brief is how the 2026-08-01 Goodhart incident happened.
|
|
//!
|
|
//! Recall is BM25 over the keyword index (`cm_brain::ClawBrain::recall`);
|
|
//! there is no embedder. Measured before anything richer is built: the test
|
|
//! is a second mission on the same repo recalling the first's verdict.
|
|
|
|
use std::path::{Path, PathBuf};
|
|
|
|
use cm_brain::ClawBrain;
|
|
use uuid::Uuid;
|
|
|
|
/// How many past verdicts a brief carries. Three is enough to say "this was
|
|
/// tried" without becoming the prompt.
|
|
pub const RECALL_K: usize = 3;
|
|
|
|
/// How many BM25 candidates the reranker sees. Wider than `RECALL_K` so a
|
|
/// relevant verdict that keyword overlap ranked fourth can still make the
|
|
/// brief; narrow enough that one call stays one call.
|
|
pub const CANDIDATES: usize = 8;
|
|
|
|
/// Below this a candidate is dropped even if fewer than `RECALL_K` remain:
|
|
/// a brief that carries an irrelevant verdict is worse than a shorter one.
|
|
pub const RELEVANT_AT: f64 = 0.3;
|
|
|
|
/// The heading the recalled lines go under. Named here because the scorer and
|
|
/// the prompt-order tests read it back.
|
|
pub const SECTION_HEADING: &str = "# What past missions on this repository learned";
|
|
|
|
fn brain_path(dir: &Path, repo_id: Uuid) -> PathBuf {
|
|
dir.join(format!("repo_{repo_id}.h5"))
|
|
}
|
|
|
|
/// One line of memory from a verdict. Pure, so the shape is testable without
|
|
/// a brain file.
|
|
pub fn verdict_line(
|
|
mission_id: Uuid,
|
|
phase_kind: &str,
|
|
brief: &str,
|
|
condition: &str,
|
|
verdict: &crate::evaluator::Verdict,
|
|
) -> Option<String> {
|
|
// A judge that could not be reached has not judged; there is no lesson.
|
|
if verdict.error.is_some() {
|
|
return None;
|
|
}
|
|
let outcome = if verdict.met { "MET" } else { "UNMET" };
|
|
let finding = if verdict.met {
|
|
verdict.reason.trim()
|
|
} else {
|
|
verdict.guidance.trim()
|
|
};
|
|
if finding.is_empty() {
|
|
return None;
|
|
}
|
|
// The TAIL. A UUIDv7 leads with its timestamp, so two missions launched
|
|
// seconds apart share their first eight characters — measured: two planted
|
|
// missions 34 s apart both rendered as `01a0cb38`, and the self-audit read
|
|
// them as one mission failing twice.
|
|
let short = mission_id.simple().to_string();
|
|
let short = &short[short.len() - 8..];
|
|
// The brief is what the agent was TOLD; the condition is what it was
|
|
// judged against, and working agents are not shown it. Without the brief a
|
|
// reader of the record cannot tell "the agent skipped a requirement" from
|
|
// "nobody asked for it" — the first self-audit on a planted brief/condition
|
|
// mismatch diagnosed the former and proposed a fix that would not have
|
|
// helped.
|
|
let brief = brief.trim();
|
|
let told = if brief.is_empty() {
|
|
String::new()
|
|
} else {
|
|
format!(" — brief: {}", head(&brief.split_whitespace().collect::<Vec<_>>().join(" "), 200))
|
|
};
|
|
Some(format!(
|
|
"{outcome} — {phase_kind} phase of mission {short}{told} — condition: {} — judge: {}",
|
|
head(condition, 200),
|
|
head(finding, 400),
|
|
))
|
|
}
|
|
|
|
/// Record a verdict in the repo's brain. Best-effort and loud on failure:
|
|
/// memory must never fail a phase, and a brain that silently stopped
|
|
/// recording is the kind of thing that stays broken for a month.
|
|
pub fn remember_verdict(
|
|
repo_id: Uuid,
|
|
mission_id: Uuid,
|
|
phase_kind: &str,
|
|
brief: &str,
|
|
condition: &str,
|
|
verdict: &crate::evaluator::Verdict,
|
|
) {
|
|
remember_in(
|
|
&cm_runtime::brain::brain_dir(),
|
|
repo_id,
|
|
mission_id,
|
|
phase_kind,
|
|
brief,
|
|
condition,
|
|
verdict,
|
|
)
|
|
}
|
|
|
|
fn remember_in(
|
|
dir: &Path,
|
|
repo_id: Uuid,
|
|
mission_id: Uuid,
|
|
phase_kind: &str,
|
|
brief: &str,
|
|
condition: &str,
|
|
verdict: &crate::evaluator::Verdict,
|
|
) {
|
|
let Some(line) = verdict_line(mission_id, phase_kind, brief, condition, verdict) else {
|
|
return;
|
|
};
|
|
let path = brain_path(dir, repo_id);
|
|
if let Some(dir) = path.parent() {
|
|
let _ = std::fs::create_dir_all(dir);
|
|
}
|
|
match ClawBrain::open_or_create(&path, &format!("repo_{repo_id}")) {
|
|
Ok(mut brain) => {
|
|
if let Err(e) = brain.remember("judge", &line, &mission_id.to_string()) {
|
|
eprintln!("mission_memory: could not record verdict for repo {repo_id}: {e}");
|
|
}
|
|
}
|
|
Err(e) => eprintln!("mission_memory: could not open brain for repo {repo_id}: {e}"),
|
|
}
|
|
}
|
|
|
|
/// The past verdicts most relevant to `query` (the phase's task text).
|
|
/// Empty when the repo has no brain yet, which is every repo's first mission.
|
|
///
|
|
/// Two stages when a decision model is configured: BM25 proposes
|
|
/// `CANDIDATES`, one Noul per candidate — "is this past verdict relevant
|
|
/// to the task?" — reorders them and drops the ones below `RELEVANT_AT`.
|
|
/// Keyword overlap is what BM25 measures, and a verdict about MICROVM.md
|
|
/// shares words with every task that mentions a file; the rerank is the
|
|
/// vendor's own pattern and costs one ~200 ms call. Without a key the
|
|
/// BM25 order stands, as before.
|
|
pub async fn recall(repo_id: Uuid, query: &str) -> Vec<String> {
|
|
let candidates = recall_in(&cm_runtime::brain::brain_dir(), repo_id, query, CANDIDATES);
|
|
match cm_decide::jev::Jev::from_env() {
|
|
Some(jev) if candidates.len() > 1 => rerank(&jev, query, candidates).await,
|
|
_ => candidates.into_iter().take(RECALL_K).collect(),
|
|
}
|
|
}
|
|
|
|
async fn rerank(jev: &cm_decide::jev::Jev, query: &str, candidates: Vec<String>) -> Vec<String> {
|
|
use cm_decide::{Answer, Decider as _, Question};
|
|
let questions: std::collections::BTreeMap<String, Question> = candidates
|
|
.iter()
|
|
.enumerate()
|
|
.map(|(i, c)| {
|
|
(
|
|
format!("c{i}"),
|
|
Question::noul(format!(
|
|
"This earlier judge verdict is relevant to the task and would help an \
|
|
agent doing it: {c}"
|
|
)),
|
|
)
|
|
})
|
|
.collect();
|
|
let decided = tokio::time::timeout(
|
|
std::time::Duration::from_secs(10),
|
|
jev.decide(query, &questions),
|
|
)
|
|
.await;
|
|
let decision = match decided {
|
|
Ok(Ok(d)) => d,
|
|
Ok(Err(e)) => {
|
|
eprintln!("mission_memory: rerank failed ({e}); keeping the BM25 order");
|
|
return candidates.into_iter().take(RECALL_K).collect();
|
|
}
|
|
Err(_) => {
|
|
eprintln!("mission_memory: rerank timed out; keeping the BM25 order");
|
|
return candidates.into_iter().take(RECALL_K).collect();
|
|
}
|
|
};
|
|
let scored: Vec<(String, f64)> = candidates
|
|
.into_iter()
|
|
.enumerate()
|
|
.map(|(i, c)| {
|
|
let p = match decision.answers.get(&format!("c{i}")) {
|
|
Some(Answer::Noul { noul }) => *noul,
|
|
_ => 0.0,
|
|
};
|
|
(c, p)
|
|
})
|
|
.collect();
|
|
let kept: Vec<String> = cm_decide::patterns::rerank(scored)
|
|
.into_iter()
|
|
.filter(|(_, p)| *p >= RELEVANT_AT)
|
|
.take(RECALL_K)
|
|
.map(|(c, _)| c)
|
|
.collect();
|
|
eprintln!(
|
|
"mission_memory: reranked {} candidate(s) with {}, kept {} ({} ms)",
|
|
questions.len(),
|
|
decision.model,
|
|
kept.len(),
|
|
decision.latency.as_millis()
|
|
);
|
|
kept
|
|
}
|
|
|
|
fn recall_in(dir: &Path, repo_id: Uuid, query: &str, k: usize) -> Vec<String> {
|
|
let path = brain_path(dir, repo_id);
|
|
if !path.exists() {
|
|
return Vec::new();
|
|
}
|
|
match ClawBrain::open_or_create(&path, &format!("repo_{repo_id}")) {
|
|
Ok(brain) => brain
|
|
.recall(query, k)
|
|
.into_iter()
|
|
// `remember` stores "role: text"; the role is ours and not a lesson.
|
|
.map(|m| m.strip_prefix("judge: ").map(str::to_string).unwrap_or(m))
|
|
.collect(),
|
|
Err(e) => {
|
|
eprintln!("mission_memory: could not open brain for repo {repo_id}: {e}");
|
|
Vec::new()
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Where a mission finds the whole of its repository's memory, readable.
|
|
///
|
|
/// Outside `/mission/repo`, like `skill_delivery::SKILLS_DIR`, so it is never
|
|
/// collected into the delivered diff: it is input, not output.
|
|
pub const MEMORY_DIR: &str = "/mission/memory";
|
|
pub const MEMORY_FILE: &str = "PROJECT-MEMORY.md";
|
|
|
|
/// Most entries an export carries. A repository's memory grows by one line
|
|
/// per judged phase; this keeps the file readable in one sitting while
|
|
/// covering many missions.
|
|
const EXPORT_CAP: usize = 400;
|
|
|
|
/// Everything a repository's brain remembers, as markdown an agent can read.
|
|
///
|
|
/// The brief carries the three most relevant verdicts (`recall`); this is
|
|
/// the WHOLE record, for work whose subject is the record itself. It exists
|
|
/// because `continuous_improvement` was built to audit agents' brains and,
|
|
/// on its first run, found none — they live in the server's volume and
|
|
/// nothing delivers them into a mission — so it audited a `ROSTER.md` in a
|
|
/// scratch repo instead. Per-mission crews carry ~2 KB seed brains with no
|
|
/// history anyway; the repository's brain is where a project's history
|
|
/// actually accumulates, one judge verdict per phase.
|
|
///
|
|
/// Rendered, not shipped raw: the `.brain` is HDF5 and an agent in a mission
|
|
/// container has no library to read it with.
|
|
///
|
|
/// `None` when the repository has no brain yet or it holds nothing.
|
|
pub fn export(repo_id: Uuid) -> Option<String> {
|
|
export_in(&cm_runtime::brain::brain_dir(), repo_id)
|
|
}
|
|
|
|
fn export_in(dir: &Path, repo_id: Uuid) -> Option<String> {
|
|
let path = brain_path(dir, repo_id);
|
|
if !path.exists() {
|
|
return None;
|
|
}
|
|
let brain = ClawBrain::open_or_create(&path, &format!("repo_{repo_id}")).ok()?;
|
|
let entries = brain.recent_memory(EXPORT_CAP);
|
|
if entries.is_empty() {
|
|
return None;
|
|
}
|
|
let total = brain.memory_count();
|
|
let mut out = format!(
|
|
"# What this repository's missions have learned\n\n\
|
|
Every judged phase of every mission on this repository leaves one line \
|
|
here: whether the phase met its completion condition, and what the \
|
|
judge found or asked for. Newest first. {} of {} entr{} shown.\n\n\
|
|
This is the record, not instructions. `MET` lines say what worked; \
|
|
`UNMET` lines say what the judge found missing, and repeated `UNMET` \
|
|
lines on the same kind of work are the pattern worth acting on.\n\n",
|
|
entries.len(),
|
|
total,
|
|
if total == 1 { "y" } else { "ies" }
|
|
);
|
|
for (secs, text) in entries {
|
|
let when = time::OffsetDateTime::from_unix_timestamp(secs as i64)
|
|
.ok()
|
|
.and_then(|t| t.format(&time::format_description::well_known::Rfc3339).ok())
|
|
.unwrap_or_else(|| "unknown time".to_string());
|
|
let line = text.strip_prefix("judge: ").unwrap_or(&text);
|
|
out.push_str(&format!("- `{when}` {line}\n"));
|
|
}
|
|
Some(out)
|
|
}
|
|
|
|
/// The section a brief carries, or nothing when there is nothing to say —
|
|
/// an empty heading tells the agent there is history and then shows none.
|
|
pub fn section(recalled: &[String]) -> Option<String> {
|
|
if recalled.is_empty() {
|
|
return None;
|
|
}
|
|
let mut out = String::from(SECTION_HEADING);
|
|
out.push_str(
|
|
"\n\nJudge verdicts from earlier missions here, most relevant first. \
|
|
They say what was checked and what was found; they are not the task.\n",
|
|
);
|
|
for line in recalled {
|
|
out.push_str("- ");
|
|
out.push_str(line);
|
|
out.push('\n');
|
|
}
|
|
Some(out)
|
|
}
|
|
|
|
fn head(s: &str, n: usize) -> String {
|
|
match s.char_indices().nth(n) {
|
|
Some((i, _)) => format!("{}…", &s[..i]),
|
|
None => s.to_string(),
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::evaluator::{Usage, Verdict};
|
|
|
|
fn verdict(met: bool, reason: &str, guidance: &str, error: Option<&str>) -> Verdict {
|
|
Verdict {
|
|
met,
|
|
reason: reason.into(),
|
|
guidance: guidance.into(),
|
|
model: "m".into(),
|
|
error: error.map(str::to_string),
|
|
checks: Vec::new(),
|
|
independent: true,
|
|
usage: Usage::default(),
|
|
expectation: None,
|
|
}
|
|
}
|
|
|
|
/// Unmet carries the sanitized guidance, never the operator reason —
|
|
/// the reason may quote the acceptance text the next mission must earn.
|
|
#[test]
|
|
fn unmet_remembers_guidance_not_reason() {
|
|
let v = verdict(false, "token ZZQX-9 is absent", "the required marker is absent", None);
|
|
let line = verdict_line(Uuid::nil(), "coding", "", "cond", &v).unwrap();
|
|
assert!(line.starts_with("UNMET — coding phase"));
|
|
assert!(line.contains("the required marker is absent"));
|
|
assert!(!line.contains("ZZQX-9"));
|
|
}
|
|
|
|
#[test]
|
|
fn met_remembers_what_the_judge_found() {
|
|
let v = verdict(true, "MICROVM.md holds both lines", "", None);
|
|
let line = verdict_line(Uuid::nil(), "coding", "", "cond", &v).unwrap();
|
|
assert!(line.starts_with("MET — "));
|
|
assert!(line.contains("MICROVM.md holds both lines"));
|
|
}
|
|
|
|
/// What the agent was told sits beside what it was judged against, and two
|
|
/// missions launched back to back stay two missions. Both were missing when
|
|
/// the first self-audit read a planted brief/condition mismatch as "the
|
|
/// agent skipped the section" and two missions as one.
|
|
#[test]
|
|
fn a_line_carries_the_brief_and_a_distinguishing_mission_id() {
|
|
let v = verdict(false, "r", "add a Limitations section", None);
|
|
let a = Uuid::now_v7();
|
|
let b = Uuid::now_v7();
|
|
let brief = "Write NOTES.md:\n five bullet points";
|
|
let la = verdict_line(a, "research", brief, "ends with Limitations", &v).unwrap();
|
|
let lb = verdict_line(b, "research", brief, "ends with Limitations", &v).unwrap();
|
|
assert!(la.contains(" — brief: Write NOTES.md: five bullet points — condition: "), "{la}");
|
|
assert_ne!(la, lb, "same-second UUIDv7s must not render as one mission");
|
|
let tail = a.simple().to_string();
|
|
assert!(la.contains(&format!("mission {}", &tail[tail.len() - 8..])), "{la}");
|
|
let none = verdict_line(a, "research", " ", "c", &v).unwrap();
|
|
assert!(!none.contains("brief:"), "an empty brief adds no segment: {none}");
|
|
}
|
|
|
|
/// No judgement, no lesson.
|
|
#[test]
|
|
fn an_unreachable_judge_leaves_no_memory() {
|
|
let v = verdict(false, "could not evaluate", "could not evaluate", Some("429"));
|
|
assert!(verdict_line(Uuid::nil(), "coding", "", "cond", &v).is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn section_is_absent_when_nothing_was_recalled() {
|
|
assert!(section(&[]).is_none());
|
|
let s = section(&["MET — x".into()]).unwrap();
|
|
assert!(s.starts_with(SECTION_HEADING));
|
|
assert!(s.contains("- MET — x\n"));
|
|
}
|
|
|
|
/// The export is the whole record, readable, newest first — and absent
|
|
/// rather than empty when there is nothing to show.
|
|
#[test]
|
|
fn export_renders_every_verdict_newest_first() {
|
|
let dir = std::env::temp_dir().join(format!("cm-mission-export-{}", Uuid::now_v7()));
|
|
let repo = Uuid::now_v7();
|
|
assert!(export_in(&dir, repo).is_none(), "no brain, no export");
|
|
|
|
remember_in(&dir, repo, Uuid::now_v7(), "coding", "", "first",
|
|
&verdict(false, "r", "the tests do not cover the empty case", None));
|
|
std::thread::sleep(std::time::Duration::from_millis(5));
|
|
remember_in(&dir, repo, Uuid::now_v7(), "coding", "", "second",
|
|
&verdict(true, "all three tests pass", "", None));
|
|
|
|
let md = export_in(&dir, repo).expect("two verdicts, so an export");
|
|
assert!(md.starts_with("# What this repository's missions have learned"));
|
|
assert!(md.contains("2 of 2 entries shown"), "{md}");
|
|
let met = md.find("MET — coding").unwrap();
|
|
let unmet = md.find("UNMET — coding").unwrap();
|
|
assert!(met < unmet, "newest (MET) must come first:\n{md}");
|
|
assert!(md.contains("the tests do not cover the empty case"));
|
|
// `remember` stores "judge: <line>"; that ROLE prefix must not follow
|
|
// the timestamp. (The line itself legitimately says "— judge: …".)
|
|
assert!(!md.contains("` judge: "), "the storage prefix leaked:\n{md}");
|
|
assert!(md.contains("` MET — coding"), "{md}");
|
|
let _ = std::fs::remove_dir_all(&dir);
|
|
}
|
|
|
|
/// Round trip through a real brain file: what one mission's verdict
|
|
/// wrote, a query shaped like the next mission's task recalls.
|
|
#[test]
|
|
fn a_second_mission_recalls_the_first_verdict() {
|
|
let dir = std::env::temp_dir().join(format!("cm-mission-memory-{}", Uuid::now_v7()));
|
|
let repo = Uuid::now_v7();
|
|
let v = verdict(
|
|
true,
|
|
"BASELINE.md records 0.689 ns/iter from benches/add_bench.rs",
|
|
"",
|
|
None,
|
|
);
|
|
remember_in(&dir, repo, Uuid::now_v7(), "benchmark", "", "a baseline is recorded", &v);
|
|
let got = recall_in(&dir, repo, "record a performance baseline for the hot path", RECALL_K);
|
|
assert_eq!(got.len(), 1, "{got:?}");
|
|
assert!(got[0].starts_with("MET — benchmark phase"), "{}", got[0]);
|
|
assert!(!got[0].starts_with("judge: "));
|
|
// A repo with no history recalls nothing and creates no file.
|
|
let other = Uuid::now_v7();
|
|
assert!(recall_in(&dir, other, "anything", RECALL_K).is_empty());
|
|
assert!(!brain_path(&dir, other).exists());
|
|
let _ = std::fs::remove_dir_all(&dir);
|
|
}
|
|
}
|