fix(memory): verdict lines carry the brief and a distinguishing mission id
The first self_audit run on a planted brief/condition mismatch found the pattern and quoted it, then diagnosed "the agent skipped the section": the record held the condition but not the brief, and working agents never see the condition. It also read two missions as one — a UUIDv7's first 8 chars are a timestamp, and missions 34 s apart both rendered as 01a0cb38. - verdict_line records the phase brief (config.task) beside the condition - the short mission id is the uuid tail - self_audit copies the record into the checkout (the judge cannot read /mission/memory) and compares brief with condition Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
fca828b5a1
commit
9ca71e5fa4
@@ -49,6 +49,7 @@ fn brain_path(dir: &Path, repo_id: Uuid) -> PathBuf {
|
||||
pub fn verdict_line(
|
||||
mission_id: Uuid,
|
||||
phase_kind: &str,
|
||||
brief: &str,
|
||||
condition: &str,
|
||||
verdict: &crate::evaluator::Verdict,
|
||||
) -> Option<String> {
|
||||
@@ -65,10 +66,26 @@ pub fn verdict_line(
|
||||
if finding.is_empty() {
|
||||
return None;
|
||||
}
|
||||
// The TAIL. A UUIDv7 leads with its timestamp, so two missions launched
|
||||
// seconds apart share their first eight characters — measured: two planted
|
||||
// missions 34 s apart both rendered as `01a0cb38`, and the self-audit read
|
||||
// them as one mission failing twice.
|
||||
let short = mission_id.simple().to_string();
|
||||
let short = &short[short.len() - 8..];
|
||||
// The brief is what the agent was TOLD; the condition is what it was
|
||||
// judged against, and working agents are not shown it. Without the brief a
|
||||
// reader of the record cannot tell "the agent skipped a requirement" from
|
||||
// "nobody asked for it" — the first self-audit on a planted brief/condition
|
||||
// mismatch diagnosed the former and proposed a fix that would not have
|
||||
// helped.
|
||||
let brief = brief.trim();
|
||||
let told = if brief.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(" — brief: {}", head(&brief.split_whitespace().collect::<Vec<_>>().join(" "), 200))
|
||||
};
|
||||
Some(format!(
|
||||
"{outcome} — {phase_kind} phase of mission {} — condition: {} — judge: {}",
|
||||
&short[..8],
|
||||
"{outcome} — {phase_kind} phase of mission {short}{told} — condition: {} — judge: {}",
|
||||
head(condition, 200),
|
||||
head(finding, 400),
|
||||
))
|
||||
@@ -81,6 +98,7 @@ pub fn remember_verdict(
|
||||
repo_id: Uuid,
|
||||
mission_id: Uuid,
|
||||
phase_kind: &str,
|
||||
brief: &str,
|
||||
condition: &str,
|
||||
verdict: &crate::evaluator::Verdict,
|
||||
) {
|
||||
@@ -89,6 +107,7 @@ pub fn remember_verdict(
|
||||
repo_id,
|
||||
mission_id,
|
||||
phase_kind,
|
||||
brief,
|
||||
condition,
|
||||
verdict,
|
||||
)
|
||||
@@ -99,10 +118,11 @@ fn remember_in(
|
||||
repo_id: Uuid,
|
||||
mission_id: Uuid,
|
||||
phase_kind: &str,
|
||||
brief: &str,
|
||||
condition: &str,
|
||||
verdict: &crate::evaluator::Verdict,
|
||||
) {
|
||||
let Some(line) = verdict_line(mission_id, phase_kind, condition, verdict) else {
|
||||
let Some(line) = verdict_line(mission_id, phase_kind, brief, condition, verdict) else {
|
||||
return;
|
||||
};
|
||||
let path = brain_path(dir, repo_id);
|
||||
@@ -329,7 +349,7 @@ mod tests {
|
||||
#[test]
|
||||
fn unmet_remembers_guidance_not_reason() {
|
||||
let v = verdict(false, "token ZZQX-9 is absent", "the required marker is absent", None);
|
||||
let line = verdict_line(Uuid::nil(), "coding", "cond", &v).unwrap();
|
||||
let line = verdict_line(Uuid::nil(), "coding", "", "cond", &v).unwrap();
|
||||
assert!(line.starts_with("UNMET — coding phase"));
|
||||
assert!(line.contains("the required marker is absent"));
|
||||
assert!(!line.contains("ZZQX-9"));
|
||||
@@ -338,16 +358,36 @@ mod tests {
|
||||
#[test]
|
||||
fn met_remembers_what_the_judge_found() {
|
||||
let v = verdict(true, "MICROVM.md holds both lines", "", None);
|
||||
let line = verdict_line(Uuid::nil(), "coding", "cond", &v).unwrap();
|
||||
let line = verdict_line(Uuid::nil(), "coding", "", "cond", &v).unwrap();
|
||||
assert!(line.starts_with("MET — "));
|
||||
assert!(line.contains("MICROVM.md holds both lines"));
|
||||
}
|
||||
|
||||
/// What the agent was told sits beside what it was judged against, and two
|
||||
/// missions launched back to back stay two missions. Both were missing when
|
||||
/// the first self-audit read a planted brief/condition mismatch as "the
|
||||
/// agent skipped the section" and two missions as one.
|
||||
#[test]
|
||||
fn a_line_carries_the_brief_and_a_distinguishing_mission_id() {
|
||||
let v = verdict(false, "r", "add a Limitations section", None);
|
||||
let a = Uuid::now_v7();
|
||||
let b = Uuid::now_v7();
|
||||
let brief = "Write NOTES.md:\n five bullet points";
|
||||
let la = verdict_line(a, "research", brief, "ends with Limitations", &v).unwrap();
|
||||
let lb = verdict_line(b, "research", brief, "ends with Limitations", &v).unwrap();
|
||||
assert!(la.contains(" — brief: Write NOTES.md: five bullet points — condition: "), "{la}");
|
||||
assert_ne!(la, lb, "same-second UUIDv7s must not render as one mission");
|
||||
let tail = a.simple().to_string();
|
||||
assert!(la.contains(&format!("mission {}", &tail[tail.len() - 8..])), "{la}");
|
||||
let none = verdict_line(a, "research", " ", "c", &v).unwrap();
|
||||
assert!(!none.contains("brief:"), "an empty brief adds no segment: {none}");
|
||||
}
|
||||
|
||||
/// No judgement, no lesson.
|
||||
#[test]
|
||||
fn an_unreachable_judge_leaves_no_memory() {
|
||||
let v = verdict(false, "could not evaluate", "could not evaluate", Some("429"));
|
||||
assert!(verdict_line(Uuid::nil(), "coding", "cond", &v).is_none());
|
||||
assert!(verdict_line(Uuid::nil(), "coding", "", "cond", &v).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -366,10 +406,10 @@ mod tests {
|
||||
let repo = Uuid::now_v7();
|
||||
assert!(export_in(&dir, repo).is_none(), "no brain, no export");
|
||||
|
||||
remember_in(&dir, repo, Uuid::now_v7(), "coding", "first",
|
||||
remember_in(&dir, repo, Uuid::now_v7(), "coding", "", "first",
|
||||
&verdict(false, "r", "the tests do not cover the empty case", None));
|
||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||
remember_in(&dir, repo, Uuid::now_v7(), "coding", "second",
|
||||
remember_in(&dir, repo, Uuid::now_v7(), "coding", "", "second",
|
||||
&verdict(true, "all three tests pass", "", None));
|
||||
|
||||
let md = export_in(&dir, repo).expect("two verdicts, so an export");
|
||||
@@ -398,7 +438,7 @@ mod tests {
|
||||
"",
|
||||
None,
|
||||
);
|
||||
remember_in(&dir, repo, Uuid::now_v7(), "benchmark", "a baseline is recorded", &v);
|
||||
remember_in(&dir, repo, Uuid::now_v7(), "benchmark", "", "a baseline is recorded", &v);
|
||||
let got = recall_in(&dir, repo, "record a performance baseline for the hot path", RECALL_K);
|
||||
assert_eq!(got.len(), 1, "{got:?}");
|
||||
assert!(got[0].starts_with("MET — benchmark phase"), "{}", got[0]);
|
||||
|
||||
@@ -2622,7 +2622,7 @@ async fn evaluate_finished_phases(
|
||||
) -> Result<(), String> {
|
||||
let rows = sqlx::query(
|
||||
"SELECT mp.id, mp.mission_id, mp.kind, mp.done_when, mp.max_iterations, mp.iteration,
|
||||
m.runtime_kind, m.repo_id
|
||||
mp.config->>'task' AS task, m.runtime_kind, m.repo_id
|
||||
FROM mission_phases mp
|
||||
JOIN missions m ON m.id = mp.mission_id
|
||||
WHERE mp.status = 'evaluating' AND m.status = 'running'
|
||||
@@ -2692,7 +2692,10 @@ async fn evaluate_finished_phases(
|
||||
// The project remembers the verdict. Only a repo-backed mission has a
|
||||
// project to remember into; a repo-less one leaves no trace here.
|
||||
if let Some(repo) = repo_id {
|
||||
crate::mission_memory::remember_verdict(repo, mission_id, &kind, &condition, &verdict);
|
||||
let brief = row.get::<Option<String>, _>("task").unwrap_or_default();
|
||||
crate::mission_memory::remember_verdict(
|
||||
repo, mission_id, &kind, &brief, &condition, &verdict,
|
||||
);
|
||||
}
|
||||
|
||||
// A judge that could not be REACHED has not judged. `Verdict.error` is
|
||||
|
||||
Reference in New Issue
Block a user