diff --git a/crates/cm-api/src/evaluator.rs b/crates/cm-api/src/evaluator.rs index 2a52fe4..920effe 100644 --- a/crates/cm-api/src/evaluator.rs +++ b/crates/cm-api/src/evaluator.rs @@ -85,6 +85,11 @@ pub struct Verdict { /// What this attempt cost. Recorded to `usage_events` by [`record`]. #[serde(default)] pub usage: Usage, + /// The judge's verification plan, committed from the condition alone + /// BEFORE it read the evidence. See [`commit_expectation`]. `None` when + /// the judge had no commit round (fallback paths, or the round failed). + #[serde(default)] + pub expectation: Option, } impl Verdict { @@ -112,6 +117,7 @@ impl Verdict { error, checks: Vec::new(), usage: Usage::default(), + expectation: None, } } } @@ -240,6 +246,110 @@ what you need to verify first, then read each file once. Outputs from earlier \ rounds are shortened to their opening lines to save space, so read a file in \ the round you intend to check it."; +/// Prompt for the commit round: the judge writes its verification plan from +/// the condition alone, before it has seen a word of what the agents claim. +/// +/// Self-Play Reward Hacking of Reference-Free Judges (arXiv 2607.05904) +/// measured a judge's pass rate climbing 0.72 → 0.94 across rounds while the +/// answers stayed 0.20 correct: a judge that reads the candidate first is +/// argued into the candidate's framing. Cross-family judges and three-judge +/// ensembles did not help. The one mitigation that did was making the judge +/// commit to its own answer first (false-positive rate 0.719 → 0.012). +/// +/// The commitment here is a plan, not an answer — a judge that "commits" to +/// an expected VALUE re-derives a measurement the condition may describe for +/// a different machine, which `EVAL_SYSTEM_VERIFYING` already forbids. What +/// it commits to is which files, strings and tests would show MET, and which +/// commands would show it. The verifying prompt then holds it to that. +const EVAL_SYSTEM_COMMIT: &str = "\ +You are about to judge whether a phase of automated work is complete. You \ +have NOT yet seen what the agents produced, and you must not guess at it. + +From the COMPLETION CONDITION alone, write down what MET would look like: + +- each requirement the condition ACTUALLY STATES, one per line — do not add \ +requirements it does not state, and do not re-derive expected values; +- for each, the concrete evidence that would show it: which file, which \ +string or symbol, which test name, which command output; +- the commands you intend to run to check it, fewest first — a `git diff` \ +or `rg` that settles several requirements at once beats one command each. + +Plain text, at most 20 lines. No verdict yet."; + +/// The prompt the verifying judge reads, with its own commitment placed +/// between the condition and the evidence so it meets the agents' claims +/// already knowing what it is looking for. +fn judge_user(condition: &str, evidence: &str, expectation: Option<&str>) -> String { + match expectation { + Some(plan) => format!( + "COMPLETION CONDITION:\n{condition}\n\n\ + YOUR VERIFICATION PLAN (you wrote this before seeing the evidence — \ + check what it names, and if the evidence pulls you toward a different \ + reading of the condition, say so in `reason` rather than silently \ + adopting it):\n{plan}\n\n\ + EVIDENCE (agent claims — verify them):\n{evidence}" + ), + None => format!( + "COMPLETION CONDITION:\n{condition}\n\nEVIDENCE (agent claims — verify them):\n{evidence}" + ), + } +} + +/// One tool-free request on the condition alone. Counted in `usage` like any +/// other; a failure here is logged and the verdict proceeds without a plan, +/// because the round exists to make the judge harder to argue with, and a +/// judge that cannot be reached at all fails on the next request anyway. +async fn commit_expectation( + provider: &dyn cm_llm::LlmProvider, + condition: &str, + model: &str, + usage: &mut Usage, +) -> Option { + use cm_llm::{ChatMessage, ChatRequest, ChatRole, ContentPart, LlmEvent}; + use futures::StreamExt as _; + let request = ChatRequest { + system: EVAL_SYSTEM_COMMIT.to_string(), + model: model.to_string(), + messages: vec![ChatMessage { + role: ChatRole::User, + parts: vec![ContentPart::text(format!("COMPLETION CONDITION:\n{condition}"))], + }], + tools: vec![], + // A reasoning model thinks before the 20 lines; see the verdict + // request for why the budget is generous and only emitted tokens bill. + max_tokens: 8192, + web_search: false, + }; + usage.requests += 1; + let mut stream = match provider.stream(request).await { + Ok(s) => s, + Err(e) => { + eprintln!("evaluator: commit round failed for {model}, judging without a plan: {e}"); + return None; + } + }; + let mut text = String::new(); + while let Some(event) = stream.next().await { + match event { + Ok(LlmEvent::TextDelta(t)) => text.push_str(&t), + Ok(LlmEvent::Usage { input_tokens, output_tokens }) => { + usage.tokens_in += u64::from(input_tokens); + usage.tokens_out += u64::from(output_tokens); + } + Ok(_) => {} + Err(e) => { + eprintln!("evaluator: commit round failed for {model}, judging without a plan: {e}"); + return None; + } + } + } + let text = text.trim(); + if text.is_empty() { + return None; + } + Some(head(text, 4000)) +} + /// Which provider family a model spec belongs to. /// /// `"glm:glm-4.7"` → `glm`, `"kimi:k2"` → `kimi`, `"claude-opus-4-8"` → `anthropic`. @@ -492,9 +602,6 @@ pub async fn evaluate( condition: &str, evidence: &str, ) -> Verdict { - let user = format!( - "COMPLETION CONDITION:\n{condition}\n\nEVIDENCE (agent claims — verify them):\n{evidence}" - ); let sandbox = crate::evaluator_tools::Sandbox::for_mission(mission_id); // Purged explicitly at every exit below: `Drop` runs as uid 65532 and cannot // delete the root-owned `target/` the judge's own `cargo test` leaves behind. @@ -520,6 +627,9 @@ pub async fn evaluate( provider_family(&model) ); let mut usage = Usage::default(); + let expectation = + commit_expectation(provider.as_ref(), condition, &model, &mut usage).await; + let user = judge_user(condition, evidence, expectation.as_deref()); match judge_with_tools( provider.as_ref(), &system, @@ -536,6 +646,7 @@ pub async fn evaluate( v.checks = checks; v.independent = true; v.usage = usage; + v.expectation = expectation; return v; } // Deliberately NOT a silent fall-through to the house judge. An @@ -553,6 +664,7 @@ pub async fn evaluate( Some(e), ); v.usage = usage; + v.expectation = expectation; return v; } } @@ -567,6 +679,8 @@ pub async fn evaluate( None => format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}"), }; let mut usage = Usage::default(); + let expectation = commit_expectation(&provider, condition, &model, &mut usage).await; + let user = judge_user(condition, evidence, expectation.as_deref()); let outcome = judge_with_tools(&provider, &system, &user, &model, sandbox.as_ref(), &mut usage) .await; @@ -588,12 +702,13 @@ pub async fn evaluate( }; v.usage = usage; v.independent = implementer != "anthropic"; + v.expectation = expectation; return v; } // Fallback paths have no tool loop, so they judge claims only and must say so. let system = format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}"); - let (eval_system, user) = (system.as_str(), user); + let (eval_system, user) = (system.as_str(), judge_user(condition, evidence, None)); let model = evaluator_model(); // Same routing as the door governor (mcp_door.rs): `runtime:` goes @@ -917,6 +1032,7 @@ fn parse_verdict(model: &str, text: &str) -> Verdict { error: None, checks: Vec::new(), usage: Usage::default(), + expectation: None, } } @@ -940,13 +1056,14 @@ pub async fn record( sqlx::query( "INSERT INTO mission_phase_evaluations (id, mission_id, phase_id, iteration, met, reason, guidance, model, error, - checks, independent) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + checks, independent, expectation) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (phase_id, iteration) DO UPDATE SET met = EXCLUDED.met, reason = EXCLUDED.reason, guidance = EXCLUDED.guidance, model = EXCLUDED.model, error = EXCLUDED.error, checks = EXCLUDED.checks, - independent = EXCLUDED.independent", + independent = EXCLUDED.independent, + expectation = EXCLUDED.expectation", ) .bind(Uuid::now_v7()) .bind(mission_id) @@ -959,6 +1076,7 @@ pub async fn record( .bind(v.error.as_deref()) .bind(serde_json::json!(v.checks)) .bind(v.independent) + .bind(v.expectation.as_deref()) .execute(pool) .await?; @@ -1291,6 +1409,35 @@ mod cross_provider_tests { mod tests { use super::*; + /// Commit-first: the plan sits between the condition and the evidence, + /// so the judge meets the claims already knowing what it is looking for. + /// Without a plan the prompt is byte-for-byte what it was before the + /// round existed — older recorded prompts stay comparable. + #[test] + fn plan_sits_between_condition_and_evidence() { + let with = judge_user("COND", "EVID", Some("PLAN")); + let c = with.find("COND").unwrap(); + let p = with.find("PLAN").unwrap(); + let e = with.find("EVID").unwrap(); + assert!(c < p && p < e, "{with}"); + assert!(with.contains("before seeing the evidence")); + + let without = judge_user("COND", "EVID", None); + assert_eq!( + without, + "COMPLETION CONDITION:\nCOND\n\nEVIDENCE (agent claims — verify them):\nEVID" + ); + } + + /// The commit prompt must not invite the judge to add requirements or + /// re-derive values — the two failure modes `done-when-wording` measured. + #[test] + fn commit_prompt_forbids_invented_requirements() { + assert!(EVAL_SYSTEM_COMMIT.contains("do not add")); + assert!(EVAL_SYSTEM_COMMIT.contains("do not re-derive")); + assert!(EVAL_SYSTEM_COMMIT.contains("No verdict yet")); + } + #[test] fn parses_a_well_formed_verdict() { let v = parse_verdict("m", r#"{"met": true, "reason": "tests pass"}"#); diff --git a/crates/cm-api/src/routes/missions.rs b/crates/cm-api/src/routes/missions.rs index 3b10915..e0aa209 100644 --- a/crates/cm-api/src/routes/missions.rs +++ b/crates/cm-api/src/routes/missions.rs @@ -1336,7 +1336,7 @@ pub async fn list_phase_evaluations( .ok_or(ApiError::NotFound)?; use sqlx::Row; let rows = sqlx::query( - "SELECT iteration, met, reason, model, error, created_at, checks + "SELECT iteration, met, reason, model, error, created_at, checks, expectation FROM mission_phase_evaluations WHERE mission_id = $1 AND phase_id = $2 ORDER BY iteration DESC", @@ -1359,6 +1359,10 @@ pub async fn list_phase_evaluations( // empty list means the verdict rests on agent claims // alone, which an operator should be able to see. "checks": r.get::("checks"), + // What the judge said it would check BEFORE it read the + // evidence; set beside `checks` so an operator can see + // whether it kept to its plan. + "expectation": r.get::, _>("expectation"), "created_at": created_at .format(&time::format_description::well_known::Rfc3339) .unwrap_or_default(), diff --git a/crates/cm-api/tests/phase_conditions.rs b/crates/cm-api/tests/phase_conditions.rs index c28b4e0..f936a71 100644 --- a/crates/cm-api/tests/phase_conditions.rs +++ b/crates/cm-api/tests/phase_conditions.rs @@ -280,6 +280,7 @@ async fn evaluations_are_unique_per_iteration_and_upsert() { checks: Vec::new(), independent: false, usage: Default::default(), + expectation: None, }; cm_api::evaluator::record(&pool, mission, phase, 0, &first) .await @@ -300,6 +301,7 @@ async fn evaluations_are_unique_per_iteration_and_upsert() { }], independent: false, usage: Default::default(), + expectation: Some("rg 'brief' docs/".into()), }; cm_api::evaluator::record(&pool, mission, phase, 0, &second) .await @@ -330,6 +332,15 @@ async fn evaluations_are_unique_per_iteration_and_upsert() { .unwrap() .get("reason"); assert_eq!(reason, "brief written"); + // The commit-first plan is stored beside the verdict it governed. + let expectation: Option = + sqlx::query("SELECT expectation FROM mission_phase_evaluations WHERE phase_id = $1") + .bind(phase) + .fetch_one(&pool) + .await + .unwrap() + .get("expectation"); + assert_eq!(expectation.as_deref(), Some("rg 'brief' docs/")); } /// `latest` must return the newest pass, which is what feeds guidance into the @@ -359,6 +370,7 @@ async fn latest_returns_the_most_recent_iteration() { checks: Vec::new(), independent: false, usage: Default::default(), + expectation: None, }, ) .await diff --git a/migrations/0086_evaluation_expectation.sql b/migrations/0086_evaluation_expectation.sql new file mode 100644 index 0000000..9a4eece --- /dev/null +++ b/migrations/0086_evaluation_expectation.sql @@ -0,0 +1,19 @@ +-- What the judge expected to find, written down BEFORE it saw the agents' output. +-- +-- Self-Play Reward Hacking of Reference-Free Judges (arXiv 2607.05904) measured +-- a judge's pass rate climbing 0.72 -> 0.94 across rounds while the answers +-- stayed 0.20 correct: a judge that reads the candidate first is argued into +-- the candidate's framing. Cross-family judges and ensembles did not help. +-- The one mitigation that did was making the judge commit to its own answer +-- before seeing the candidate (false-positive rate 0.719 -> 0.012). +-- +-- Our analogue: one tool-free round on the condition alone, producing a +-- verification plan — which files, strings, tests would show MET — that the +-- verifying prompt then holds the judge to. Stored beside the verdict so an +-- operator can see whether the checks the judge ran are the ones it said it +-- would run, and NULL on every verdict written before the round existed. +ALTER TABLE mission_phase_evaluations + ADD COLUMN IF NOT EXISTS expectation TEXT; + +COMMENT ON COLUMN mission_phase_evaluations.expectation IS + 'The judge''s verification plan, committed from the condition alone before it read the evidence; NULL when the judge had no commit round.';