From 1a44405308cbe61c543157ec83a0c0a1b2f463d0 Mon Sep 17 00:00:00 2001 From: Omar Sobh Date: Sat, 19 Sep 2026 00:13:31 -0500 Subject: [PATCH] perf(judge): a read lives for two rounds, and the judge is told it is complete MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 64 KB window did its part: mission 01a0b803's judge read REPORT.md in full, 18,698 B untruncated, in one command. It then ran wc, head -120, tail -116, sed 1,120p and two greps against the same file — six commands re-reading content it had been given. Two causes, one of them mine. compact_earlier_results shrank that read to 800 bytes as soon as the next round's results arrived, so by the time the judge went to check a claim against the report, the report was gone from its context. The most recent round's results now stay whole, so a read survives the call that receives it and the one after; only older rounds compact. The quadratic term stays bounded — it was the SUM over rounds, and one extra whole round is linear. The system prompt never mentioned the budget. It now says a cat comes back complete unless the output says otherwise, not to re-read with head/tail/ sed/grep, to decide what to verify before reading, and that earlier rounds are shortened — so read in the round you intend to check. Still 13 checks / 9 requests on that mission; the measurement is the next one. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WZb5A2kfVfjpdwSochkuHz --- crates/cm-api/src/evaluator.rs | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/crates/cm-api/src/evaluator.rs b/crates/cm-api/src/evaluator.rs index 16c7fc4..2a52fe4 100644 --- a/crates/cm-api/src/evaluator.rs +++ b/crates/cm-api/src/evaluator.rs @@ -232,7 +232,13 @@ require content the condition does not ask for. Judge the condition AS WRITTEN. Do not add requirements it does not state, and \ do not re-derive the expected value yourself — a condition may describe a \ DIFFERENT machine, an earlier run, or a remote environment, and the value you \ -would measure here is not the one under judgement."; +would measure here is not the one under judgement. +You have a limited number of commands. A file you `cat` comes back COMPLETE \ +unless the output says bytes were omitted; do not read it again with head, \ +tail, sed or grep — check the claim against the text you already have. Decide \ +what you need to verify first, then read each file once. Outputs from earlier \ +rounds are shortened to their opening lines to save space, so read a file in \ +the round you intend to check it."; /// Which provider family a model spec belongs to. /// @@ -834,7 +840,19 @@ const KEPT_OF_EARLIER_RESULT: usize = 800; fn compact_earlier_results(messages: &mut [cm_llm::ChatMessage]) { use cm_llm::ContentPart; const MARKER: &str = "\n[… output elided here — it was shown in full when this check ran]"; - for m in messages.iter_mut() { + // The most recent round's results stay whole, so a read lives through + // TWO model calls — the one that receives it and the one after. With + // everything compacted at once, mission 01a0b803 read REPORT.md in full + // (18,698 B, the 64 KB window working) and then, the round after, ran + // wc/head/tail/sed/grep against it anyway: the file was already down to + // 800 bytes by the time the judge went to check a claim against it. + let keep = messages + .iter() + .rposition(|m| m.parts.iter().any(|p| matches!(p, ContentPart::ToolResult { .. }))); + for (i, m) in messages.iter_mut().enumerate() { + if Some(i) == keep { + continue; + } for part in m.parts.iter_mut() { if let ContentPart::ToolResult { content, .. } = part { if let Some(text) = content.as_str() { @@ -1016,6 +1034,12 @@ mod cross_provider_tests { ], }, ]; + // A second, later round of results: the compaction must leave THIS one + // whole and shrink only the round before it. + messages.push(ChatMessage { + role: ChatRole::User, + parts: vec![ContentPart::ToolResult { tool_use_id: "c".into(), content: Value::String(big.clone()) }], + }); compact_earlier_results(&mut messages); for part in &messages[1].parts { let ContentPart::ToolResult { content, .. } = part else { panic!() }; @@ -1024,11 +1048,16 @@ mod cross_provider_tests { assert!(s.starts_with("line of output"), "the head survives"); assert!(s.contains("elided"), "and says so"); } + let ContentPart::ToolResult { content, .. } = &messages[2].parts[0] else { panic!() }; + assert_eq!(content.as_str().unwrap().len(), big.len(), "the latest round stays whole"); // Idempotent: a second pass must not shrink the reminder further. let once: Vec = messages[1].parts.iter().map(|p| serde_json::to_string(p).unwrap()).collect(); compact_earlier_results(&mut messages); let twice: Vec = messages[1].parts.iter().map(|p| serde_json::to_string(p).unwrap()).collect(); assert_eq!(once, twice); + // And the latest round is still whole after the second pass. + let ContentPart::ToolResult { content, .. } = &messages[2].parts[0] else { panic!() }; + assert_eq!(content.as_str().unwrap().len(), big.len()); // Plain text parts are untouched. assert!(matches!(&messages[0].parts[0], ContentPart::Text { text } if text == "judge this")); }