test(gate): tolerate EPIPE in the no-node test; harness asserts skill.triage
The no-node test runs the hook with an empty PATH, so cat is missing too and the script exits before reading stdin; on Linux the test's write can lose that race (CI run 6483). The child exiting unread is the no-node path working. Harness: assert_skill_triage on chain and microvm — the event must exist; agreement with what the agent read is reported. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01WZb5A2kfVfjpdwSochkuHz
This commit is contained in:
co-authored by
Claude Opus 5
parent
0a2bd6f868
commit
37eb6bcad1
@@ -890,14 +890,15 @@ mod shell_tests {
|
|||||||
.stderr(Stdio::piped())
|
.stderr(Stdio::piped())
|
||||||
.spawn()
|
.spawn()
|
||||||
.expect("spawn sh");
|
.expect("spawn sh");
|
||||||
child
|
// With an empty PATH `cat` is missing too, so the script never reads
|
||||||
.stdin
|
// its stdin and may exit before this write lands. EPIPE here IS the
|
||||||
.as_mut()
|
// no-node path succeeding, not a failure: CI run 6483 lost that race
|
||||||
.unwrap()
|
// on Linux after every earlier run had won it.
|
||||||
.write_all(
|
if let Err(e) = child.stdin.as_mut().unwrap().write_all(
|
||||||
br#"{"tool_name":"Bash","tool_input":{"command":"git push --force origin main"}}"#,
|
br#"{"tool_name":"Bash","tool_input":{"command":"git push --force origin main"}}"#,
|
||||||
)
|
) {
|
||||||
.unwrap();
|
assert_eq!(e.kind(), std::io::ErrorKind::BrokenPipe, "{e}");
|
||||||
|
}
|
||||||
let out = child.wait_with_output().expect("wait");
|
let out = child.wait_with_output().expect("wait");
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
@@ -357,6 +357,7 @@ assert_microvm() { # <token> <mission> <report>
|
|||||||
|
|
||||||
# Project memory: the brief carries earlier verdicts on this repository.
|
# Project memory: the brief carries earlier verdicts on this repository.
|
||||||
assert_project_memory "$mission" microvm
|
assert_project_memory "$mission" microvm
|
||||||
|
assert_skill_triage "$token" "$mission" microvm
|
||||||
|
|
||||||
# And the verdict: judged, and by whom. `independent` is only true when the
|
# And the verdict: judged, and by whom. `independent` is only true when the
|
||||||
# judge came from a different provider family than the agent.
|
# judge came from a different provider family than the agent.
|
||||||
@@ -1106,6 +1107,7 @@ assert_chain() { # <token> <mission> <report>
|
|||||||
[ "$pushed" = "True" ] || fail "chain: phase $idx not pushed (commit_error=$cerr push_error=$perr)"
|
[ "$pushed" = "True" ] || fail "chain: phase $idx not pushed (commit_error=$cerr push_error=$perr)"
|
||||||
case "$files" in 0|-) fail "chain: phase $idx delivered no files" ;; esac
|
case "$files" in 0|-) fail "chain: phase $idx delivered no files" ;; esac
|
||||||
done <<<"$report"
|
done <<<"$report"
|
||||||
|
assert_skill_triage "$token" "$mission" chain
|
||||||
delivered=$(fetch_delivered "$token" "$mission" CHAIN.md) \
|
delivered=$(fetch_delivered "$token" "$mission" CHAIN.md) \
|
||||||
|| { fail "chain: could not read CHAIN.md from the pushed branch"; return 1; }
|
|| { fail "chain: could not read CHAIN.md from the pushed branch"; return 1; }
|
||||||
case "$delivered" in
|
case "$delivered" in
|
||||||
@@ -1222,6 +1224,40 @@ assert_credentials_revoked() { # <mission> <label>
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Shadow skill triage: one skill.triage event per phase, and — where the
|
||||||
|
# delivery arm makes retrieval observable — how the oracle's "applies" set
|
||||||
|
# compares with what the agent actually read. Informational on agreement
|
||||||
|
# (shadow mode measures; it does not select), PASS/FAIL on the event being
|
||||||
|
# there at all: a triage that silently stopped firing is the state to catch.
|
||||||
|
assert_skill_triage() { # <token> <mission> <label>
|
||||||
|
local token="$1" mission="$2" label="$3" ev agree
|
||||||
|
ev=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
||||||
|
\"select count(*)::text || ' ' || coalesce(max(detail->>'model'),'-') || ' ' || \
|
||||||
|
coalesce(max((detail->>'latency_ms')::int),0)::text || ' ' || \
|
||||||
|
coalesce(max((select count(*) from jsonb_object_keys(detail->'skills'))),0)::text \
|
||||||
|
from mission_events where mission_id='$mission' and kind='skill.triage';\"" | head -1 | tr -d '\r')
|
||||||
|
set -- $ev
|
||||||
|
case "${1:-0}" in
|
||||||
|
0) fail "$label-triage: no skill.triage event — TYPESAFE_API_KEY unset on the server, or the call failed (see the server log)" ;;
|
||||||
|
*) pass "$label-triage: $1 event(s) by $2 over $4 skills, ${3} ms" ;;
|
||||||
|
esac
|
||||||
|
# Oracle vs agent, from the Skill-Use report: skills the oracle said apply
|
||||||
|
# (triage_p >= 0.5) that the agent read, and ones it read the oracle did
|
||||||
|
# not expect. Only meaningful when the arm is a retrieval arm.
|
||||||
|
agree=$(api "$token" GET "/api/missions/$mission/skill-use" | python3 -c '
|
||||||
|
import json,sys
|
||||||
|
d=json.load(sys.stdin); sk=d.get("skills") or []
|
||||||
|
if not sk or all(s.get("triage_p") is None for s in sk): print("n/a"); sys.exit()
|
||||||
|
def read(s): return s["trigger"].get("verdict") in ("pass",) if isinstance(s.get("trigger"),dict) else str(s.get("trigger")).startswith("pass")
|
||||||
|
applies=[s for s in sk if (s.get("triage_p") or 0)>=0.5]
|
||||||
|
hit=sum(1 for s in applies if read(s)); read_unexpected=sum(1 for s in sk if read(s) and (s.get("triage_p") or 0)<0.5)
|
||||||
|
print(f"oracle says {len(applies)} of {len(sk)} delivered skills apply; agent read {hit} of those and {read_unexpected} it did not expect")' 2>/dev/null)
|
||||||
|
case "$agree" in
|
||||||
|
""|n/a) pass "$label-triage: agreement n/a on this arm (no retrieval to compare against)" ;;
|
||||||
|
*) pass "$label-triage: $agree" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
# ── Scenario: multi-role with a real test suite ──────────────────
|
# ── Scenario: multi-role with a real test suite ──────────────────
|
||||||
#
|
#
|
||||||
# The workload that failed with `COMMIT_EDITMSG: Permission denied` under the
|
# The workload that failed with `COMMIT_EDITMSG: Permission denied` under the
|
||||||
|
|||||||
Reference in New Issue
Block a user