feat(skill-use): red-first is observable from what the RUNS reported

The open item said this needed the repository diff rather than tool
order. That was wrong, and the skill says why: "Commit the RED-to-GREEN
pair as one commit." The failing test and its fix land together by
instruction, so the diff and the commit history are as blind as the tool
ordering already was — in Rust one `Edit` adds the implementation and its
`#[cfg(test)] mod tests` in the same call.

The only remaining witness is what each test run itself printed, and the
tap was throwing it away. Claude Code's PostToolUse payload carries
`tool_response` — verified against the real binary, keys
stdout/stderr/interrupted, plus `duration_ms` and `tool_use_id`.

So `Observed.response` now keeps it, for COMMANDS only: a `Read`'s
response is the file it just read and a `Write`'s restates its own
argument — both already knowable, both large, and storing them would
double the biggest write path in the system for nothing.

`bounded_response` keeps the **end** of the output, which is the opposite
of `bounded_input` and deliberately so. An argument's meaning is its verb,
at the start. A command's meaning is its verdict, at the end: `cargo test`
prints hundreds of lines and then `test result: ok` or `FAILED`. A
head-biased truncation would keep the noise and discard the only thing
being stored for — negative-controlled with a 400-line fixture.

`red_before_green` now falls through to the run outcomes:

  failing run, then a passing one  → Pass, red then green observed
  every run failed                 → Fail, the loop ends on green
  every run passed                 → NotObservable, and the reason says
                                     why: a test that never failed is
                                     equally what a correct implementation
                                     written first looks like
  no outputs recorded              → NotObservable (pre-capture missions)

Read from the runner's verdict line, not an exit code — the payload
carries none.

Co-Authored-By: Claude Opus 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_018i9Ten1LU4jUr5d7TAWda9

crates/cm-api/src/mission_events.rs

@@ -196,6 +196,12 @@ pub struct ToolEvidence {
     pub path: Option<String>,
     /// The tool's arguments, bounded by `vm_tool_tap::bounded_input`.
     pub input: Value,
+    /// What a command produced, bounded by `vm_tool_tap::bounded_response`.
+    ///
+    /// Null for every tool that is not a command. This is where a failing test
+    /// run is visible, and it is the only place it is — the recorded stream has
+    /// no exit codes.
+    pub response: Value,
 }
 
 impl ToolEvidence {
@@ -241,6 +247,7 @@ pub async fn tool_evidence_for_mission(
                 .and_then(Value::as_str)
                 .map(str::to_string),
             input: detail.get("input").cloned().unwrap_or(Value::Null),
+            response: detail.get("response").cloned().unwrap_or(Value::Null),
         })
         .collect())
 }

crates/cm-api/src/phase_runner.rs

@@ -321,6 +321,7 @@ mod attribution_tests {
             path: None,
             session: session.map(str::to_string),
             input: serde_json::json!({"command": "ls"}),
+            response: serde_json::Value::Null,
         }
     }
 
@@ -2234,7 +2235,13 @@ pub(crate) async fn record_vm_tools(
             // and was handed a null on every container-tier call; `input`
             // because the tool name alone cannot answer a single behavioural
             // question about the phase.
-            detail: serde_json::json!({ "path": t.path, "input": t.input }),
+            detail: serde_json::json!({
+                "path": t.path,
+                "input": t.input,
+                // Only commands carry one; `bounded_response` returns null for
+                // everything else, and a null key here is noise.
+                "response": t.response,
+            }),
         });
         if let Some(path) = &t.path {
             events.push(crate::mission_events::MissionEvent {

crates/cm-api/src/skill_use.rs

@@ -136,6 +136,18 @@ impl<'a> Evidence<'a> {
         self.tools.iter().filter_map(|t| t.command())
     }
 
+    /// Test runs, in order, each with whether it went red or green.
+    ///
+    /// `None` where the outcome was not recorded — a mission whose events
+    /// predate the response capture, which must not read as either.
+    fn test_runs(&self) -> Vec<Option<bool>> {
+        self.tools
+            .iter()
+            .filter(|t| t.command().is_some_and(is_test_command))
+            .map(|t| test_passed(&t.response))
+            .collect()
+    }
+
     /// Every file-modifying call that named a path, in order.
     fn writes(&self) -> impl Iterator<Item = &crate::mission_events::ToolEvidence> {
         self.tools
@@ -463,6 +475,41 @@ fn strip_int_prefix(subject: &str) -> &str {
 ///   under test, so "wrote the file, then ran the test" is what writing the
 ///   failing test first looks like from here. Scoring that as a failure would
 ///   fail the agents who followed the skill most precisely.
+/// Did this test run pass?
+///
+/// Read from the runner's own verdict line, not from an exit code — Claude
+/// Code's `tool_response` carries `stdout`/`stderr`/`interrupted` and no
+/// status. That is why `bounded_response` keeps the END of the output: every
+/// one of these markers is printed last.
+fn test_passed(response: &serde_json::Value) -> Option<bool> {
+    let text = format!(
+        "{}\n{}",
+        response.get("stdout").and_then(|v| v.as_str()).unwrap_or(""),
+        response.get("stderr").and_then(|v| v.as_str()).unwrap_or("")
+    );
+    if text.trim().is_empty() {
+        return None;
+    }
+    const FAILED: [&str; 6] = [
+        "test result: FAILED",
+        "FAILED (",
+        "error: test failed",
+        "Tests:  ",
+        "failures:",
+        "error[E",
+    ];
+    const PASSED: [&str; 4] = ["test result: ok", "ok. ", "passed", "PASS"];
+    // Failure first: a run that reports both a passing binary and a failing one
+    // is a failing run, and the summary line of the failure is what matters.
+    if FAILED.iter().any(|m| text.contains(m)) {
+        return Some(false);
+    }
+    if PASSED.iter().any(|m| text.contains(m)) {
+        return Some(true);
+    }
+    None
+}
+
 fn red_before_green(ev: &Evidence<'_>) -> Verdict {
     if ev.tools.is_empty() {
         return Verdict::NotObservable(
@@ -490,12 +537,38 @@ fn red_before_green(ev: &Evidence<'_>) -> Verdict {
                 .count()
         )),
         (Some(w), Some(t)) if t < w => Verdict::Pass,
-        _ => Verdict::NotObservable(
+        _ => {
-            "a test ran, but after the first write — in Rust the unit test \
+            // Ordering is exhausted here, and so is the repository: the skill
-             lives in the file under test, so this ordering cannot separate \
+            // says to "commit the RED-to-GREEN pair as one commit", so the
-             red-first from tests-added-after"
+            // failing test and its fix land together by instruction. The only
+            // remaining witness is what the runs THEMSELVES reported.
+            let runs = ev.test_runs();
+            let went_red = runs.iter().position(|r| *r == Some(false));
+            let went_green = runs.iter().rposition(|r| *r == Some(true));
+            match (went_red, went_green) {
+                (Some(red), Some(green)) if red < green => Verdict::Pass,
+                (Some(_), None) => Verdict::Fail(
+                    "every recorded test run failed — the loop ends on green, \
+                     and this phase never got there"
                         .into(),
                 ),
+                _ if runs.iter().all(Option::is_none) => Verdict::NotObservable(
+                    "a test ran after the first write and no run recorded its \
+                     output — in Rust the unit test lives in the file under \
+                     test, so ordering cannot separate red-first from \
+                     tests-added-after, and without the run's own verdict there \
+                     is nothing else to read"
+                        .into(),
+                ),
+                _ => Verdict::NotObservable(
+                    "every recorded test run passed — a test that never failed \
+                     is a test that was never red, but it is equally what a \
+                     correct implementation written first looks like, and these \
+                     runs cannot tell them apart"
+                        .into(),
+                ),
+            }
+        }
     }
 }
 
@@ -756,10 +829,21 @@ mod tests {
                 tool: (*tool).to_string(),
                 path: path.map(str::to_string),
                 input: input.clone(),
+                response: serde_json::Value::Null,
             })
             .collect()
     }
 
+    /// A `Bash` call that ran a test and reported an outcome.
+    fn test_run(cmd: &str, stdout: &str) -> ToolEvidence {
+        ToolEvidence {
+            tool: "Bash".into(),
+            path: None,
+            input: json!({ "command": cmd }),
+            response: json!({ "stdout": stdout, "stderr": "" }),
+        }
+    }
+
     /// One `Bash` call.
     fn ran(cmd: &str) -> (&'static str, Option<&'static str>, serde_json::Value) {
         ("Bash", None, serde_json::json!({ "command": cmd }))
@@ -1167,6 +1251,115 @@ mod tests {
         );
     }
 
+    /// Red then green, read from what the runs themselves reported.
+    ///
+    /// This is the case ordering cannot decide and the repository cannot
+    /// either: one `Edit` adds the implementation and its `#[cfg(test)] mod
+    /// tests` together, and `tdd-red-green-refactor` says in so many words to
+    /// "commit the RED-to-GREEN pair as one commit". The run outputs are the
+    /// only witness left.
+    #[test]
+    fn a_failing_run_followed_by_a_passing_one_is_red_then_green() {
+        let prompt = rendered(&[("tdd-red-green-refactor", "Red, green, refactor.")]);
+        let mut tools = acted(&[(
+            "Write",
+            Some("/mission/repo/src/a.rs"),
+            json!({"file_path": "/mission/repo/src/a.rs"}),
+        )]);
+        tools.push(test_run(
+            "cargo test",
+            "running 1 test\ntest slugs ... FAILED\n\ntest result: FAILED. 0 passed; 1 failed",
+        ));
+        tools.push(test_run(
+            "cargo test",
+            "running 1 test\ntest slugs ... ok\n\ntest result: ok. 1 passed; 0 failed",
+        ));
+        assert_eq!(
+            score(&prompt, &Evidence::new("", &tools), &builtin)[0].compliance,
+            Verdict::Pass
+        );
+    }
+
+    /// Every run green is NOT evidence of red-first.
+    #[test]
+    fn runs_that_only_ever_passed_cannot_prove_the_loop() {
+        let prompt = rendered(&[("tdd-red-green-refactor", "Red, green, refactor.")]);
+        let mut tools = acted(&[(
+            "Write",
+            Some("/mission/repo/src/a.rs"),
+            json!({"file_path": "/mission/repo/src/a.rs"}),
+        )]);
+        tools.push(test_run("cargo test", "test result: ok. 3 passed; 0 failed"));
+        match &score(&prompt, &Evidence::new("", &tools), &builtin)[0].compliance {
+            Verdict::NotObservable(why) => assert!(
+                why.contains("never failed"),
+                "a test that never failed is equally what a correct \
+                 implementation written first looks like: {why}"
+            ),
+            other => panic!("got {other:?}"),
+        }
+    }
+
+    /// Ending red is a failure of the loop, and a different one from never
+    /// running a test at all.
+    #[test]
+    fn a_phase_that_never_reached_green_fails() {
+        let prompt = rendered(&[("tdd-red-green-refactor", "Red, green, refactor.")]);
+        let mut tools = acted(&[(
+            "Write",
+            Some("/mission/repo/src/a.rs"),
+            json!({"file_path": "/mission/repo/src/a.rs"}),
+        )]);
+        tools.push(test_run("cargo test", "test result: FAILED. 0 passed; 2 failed"));
+        match &score(&prompt, &Evidence::new("", &tools), &builtin)[0].compliance {
+            Verdict::Fail(why) => assert!(why.contains("never got there"), "{why}"),
+            other => panic!("got {other:?}"),
+        }
+    }
+
+    /// A mission recorded before responses were captured must not read as
+    /// either verdict.
+    #[test]
+    fn runs_with_no_recorded_output_stay_unobservable() {
+        let prompt = rendered(&[("tdd-red-green-refactor", "Red, green, refactor.")]);
+        let tools = acted(&[
+            ("Write", Some("/mission/repo/src/a.rs"), json!({"file_path": "/mission/repo/src/a.rs"})),
+            ("Bash", None, json!({"command": "cargo test"})),
+        ]);
+        assert!(matches!(
+            score(&prompt, &Evidence::new("", &tools), &builtin)[0].compliance,
+            Verdict::NotObservable(_)
+        ));
+    }
+
+    /// The verdict line is at the END of a test run, which is why the response
+    /// bound keeps the tail.
+    #[test]
+    fn a_truncated_response_still_carries_the_verdict() {
+        let long = "compiling…\n".repeat(400) + "test result: FAILED. 0 passed; 1 failed";
+        let bounded = crate::vm_tool_tap::bounded_response(
+            "Bash",
+            &json!({ "stdout": long, "stderr": "" }),
+        );
+        let kept = bounded["stdout"].as_str().expect("stdout kept");
+        assert!(
+            kept.contains("test result: FAILED"),
+            "a head-biased truncation would keep the noise and drop the verdict, \
+             which is the only reason the output is stored"
+        );
+        assert!(kept.starts_with("[truncated]"), "{kept}");
+    }
+
+    /// A `Read`'s response is the file it just read — already knowable, and
+    /// large.
+    #[test]
+    fn only_commands_carry_a_response() {
+        assert_eq!(
+            crate::vm_tool_tap::bounded_response("Read", &json!({"stdout": "x"})),
+            serde_json::Value::Null
+        );
+    }
+
     /// A phase that wrote nothing had no implementation to test-drive.
     /// The verdict the first live run got wrong.
     ///

crates/cm-api/src/vm_tool_tap.rs

@@ -79,6 +79,20 @@ pub struct Observed {
     /// is per-CONTAINER and every role in a phase shares one, so without this
     /// the whole phase arrives as an undifferentiated stream.
     pub session: Option<String>,
+    /// What a **command** produced, bounded by [`bounded_response`].
+    ///
+    /// Only for tools that run something. `Read`'s response is the file it just
+    /// read and `Write`'s is a restatement of what was written — both are
+    /// already knowable from the arguments and the delivered diff, and storing
+    /// them would double the largest write path in the system for nothing.
+    ///
+    /// A command's OUTCOME is different: it is the only place a failing test
+    /// run is visible. Without it "did this phase go red before it went green"
+    /// cannot be answered from anything — not from tool order (in Rust the
+    /// unit test lives in the file under test, so one `Edit` adds both), and
+    /// not from the repository either, because `tdd-red-green-refactor` says
+    /// in so many words to "commit the RED-to-GREEN pair as one commit".
+    pub response: Value,
     /// The tool's arguments, bounded by [`bounded_input`].
     ///
     /// Kept because the tool NAME alone answers almost nothing. A phase that
@@ -106,6 +120,46 @@ const MAX_ARG_LEN: usize = 512;
 /// authority on what was written anyway.
 const BODY_KEYS: [&str; 4] = ["content", "new_string", "old_string", "edits"];
 
+/// Tools whose response is an outcome rather than a restatement.
+const RESPONSE_TOOLS: [&str; 1] = ["Bash"];
+
+/// How much of a command's output to keep.
+const MAX_OUTPUT_LEN: usize = 600;
+
+/// Shrink a command's response, keeping the **end** of its output.
+///
+/// The opposite of [`bounded_input`], and deliberately so. An argument's
+/// meaning is at the start — the verb of the command. A command's meaning is at
+/// the END: `cargo test` prints hundreds of lines and then `test result: ok` or
+/// `test result: FAILED`, and a head-biased truncation would keep the noise and
+/// throw away the verdict, which is the one thing being stored for.
+pub fn bounded_response(tool: &str, response: &Value) -> Value {
+    if !RESPONSE_TOOLS.contains(&tool) {
+        return Value::Null;
+    }
+    let Some(obj) = response.as_object() else {
+        return Value::Null;
+    };
+    let mut out = serde_json::Map::new();
+    for key in ["stdout", "stderr", "interrupted"] {
+        match obj.get(key) {
+            Some(Value::String(s)) if s.len() > MAX_OUTPUT_LEN => {
+                let start = s
+                    .char_indices()
+                    .map(|(i, _)| i)
+                    .find(|i| *i >= s.len().saturating_sub(MAX_OUTPUT_LEN))
+                    .unwrap_or(0);
+                out.insert(key.into(), Value::String(format!("[truncated]…{}", &s[start..])));
+            }
+            Some(v) => {
+                out.insert(key.into(), v.clone());
+            }
+            None => {}
+        }
+    }
+    Value::Object(out)
+}
+
 /// Shrink a tool's arguments to something safe to store on every call.
 ///
 /// Bounded rather than whitelisted on purpose. A whitelist of "interesting"
@@ -250,9 +304,15 @@ pub fn parse(raw: &str) -> Vec<Observed> {
                 .or_else(|| v.get("toolInput"))
                 .cloned()
                 .unwrap_or(Value::Null);
+            let response = v
+                .get("tool_response")
+                .or_else(|| v.get("toolResponse"))
+                .cloned()
+                .unwrap_or(Value::Null);
             Some(Observed {
                 path: crate::mission_events::tool_path(&input),
                 input: bounded_input(&input),
+                response: bounded_response(&tool, &response),
                 session: v
                     .get("session_id")
                     .or_else(|| v.get("sessionId"))
@@ -351,12 +411,14 @@ mod tests {
                     path: Some("/mission/repo/src/a.rs".into()),
                     input: json!({"file_path": "/mission/repo/src/a.rs"}),
                     session: None,
+                    response: Value::Null,
                 },
                 Observed {
                     tool: "Bash".into(),
                     path: None,
                     input: json!({"command": "ls"}),
                     session: None,
+                    response: Value::Null,
                 },
             ]
         );