diff --git a/crates/cm-api/src/evaluator.rs b/crates/cm-api/src/evaluator.rs index 16c7fc4..2a52fe4 100644 --- a/crates/cm-api/src/evaluator.rs +++ b/crates/cm-api/src/evaluator.rs @@ -232,7 +232,13 @@ require content the condition does not ask for. Judge the condition AS WRITTEN. Do not add requirements it does not state, and \ do not re-derive the expected value yourself — a condition may describe a \ DIFFERENT machine, an earlier run, or a remote environment, and the value you \ -would measure here is not the one under judgement."; +would measure here is not the one under judgement. +You have a limited number of commands. A file you `cat` comes back COMPLETE \ +unless the output says bytes were omitted; do not read it again with head, \ +tail, sed or grep — check the claim against the text you already have. Decide \ +what you need to verify first, then read each file once. Outputs from earlier \ +rounds are shortened to their opening lines to save space, so read a file in \ +the round you intend to check it."; /// Which provider family a model spec belongs to. /// @@ -834,7 +840,19 @@ const KEPT_OF_EARLIER_RESULT: usize = 800; fn compact_earlier_results(messages: &mut [cm_llm::ChatMessage]) { use cm_llm::ContentPart; const MARKER: &str = "\n[… output elided here — it was shown in full when this check ran]"; - for m in messages.iter_mut() { + // The most recent round's results stay whole, so a read lives through + // TWO model calls — the one that receives it and the one after. With + // everything compacted at once, mission 01a0b803 read REPORT.md in full + // (18,698 B, the 64 KB window working) and then, the round after, ran + // wc/head/tail/sed/grep against it anyway: the file was already down to + // 800 bytes by the time the judge went to check a claim against it. + let keep = messages + .iter() + .rposition(|m| m.parts.iter().any(|p| matches!(p, ContentPart::ToolResult { .. }))); + for (i, m) in messages.iter_mut().enumerate() { + if Some(i) == keep { + continue; + } for part in m.parts.iter_mut() { if let ContentPart::ToolResult { content, .. } = part { if let Some(text) = content.as_str() { @@ -1016,6 +1034,12 @@ mod cross_provider_tests { ], }, ]; + // A second, later round of results: the compaction must leave THIS one + // whole and shrink only the round before it. + messages.push(ChatMessage { + role: ChatRole::User, + parts: vec![ContentPart::ToolResult { tool_use_id: "c".into(), content: Value::String(big.clone()) }], + }); compact_earlier_results(&mut messages); for part in &messages[1].parts { let ContentPart::ToolResult { content, .. } = part else { panic!() }; @@ -1024,11 +1048,16 @@ mod cross_provider_tests { assert!(s.starts_with("line of output"), "the head survives"); assert!(s.contains("elided"), "and says so"); } + let ContentPart::ToolResult { content, .. } = &messages[2].parts[0] else { panic!() }; + assert_eq!(content.as_str().unwrap().len(), big.len(), "the latest round stays whole"); // Idempotent: a second pass must not shrink the reminder further. let once: Vec = messages[1].parts.iter().map(|p| serde_json::to_string(p).unwrap()).collect(); compact_earlier_results(&mut messages); let twice: Vec = messages[1].parts.iter().map(|p| serde_json::to_string(p).unwrap()).collect(); assert_eq!(once, twice); + // And the latest round is still whole after the second pass. + let ContentPart::ToolResult { content, .. } = &messages[2].parts[0] else { panic!() }; + assert_eq!(content.as_str().unwrap().len(), big.len()); // Plain text parts are untouched. assert!(matches!(&messages[0].parts[0], ContentPart::Text { text } if text == "judge this")); }