test(gate): tolerate EPIPE in the no-node test; harness asserts skill.triage
The no-node test runs the hook with an empty PATH, so cat is missing too and the script exits before reading stdin; on Linux the test's write can lose that race (CI run 6483). The child exiting unread is the no-node path working. Harness: assert_skill_triage on chain and microvm — the event must exist; agreement with what the agent read is reported. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01WZb5A2kfVfjpdwSochkuHz
This commit is contained in:
co-authored by
Claude Opus 5
parent
0a2bd6f868
commit
37eb6bcad1
@@ -357,6 +357,7 @@ assert_microvm() { # <token> <mission> <report>
|
||||
|
||||
# Project memory: the brief carries earlier verdicts on this repository.
|
||||
assert_project_memory "$mission" microvm
|
||||
assert_skill_triage "$token" "$mission" microvm
|
||||
|
||||
# And the verdict: judged, and by whom. `independent` is only true when the
|
||||
# judge came from a different provider family than the agent.
|
||||
@@ -1106,6 +1107,7 @@ assert_chain() { # <token> <mission> <report>
|
||||
[ "$pushed" = "True" ] || fail "chain: phase $idx not pushed (commit_error=$cerr push_error=$perr)"
|
||||
case "$files" in 0|-) fail "chain: phase $idx delivered no files" ;; esac
|
||||
done <<<"$report"
|
||||
assert_skill_triage "$token" "$mission" chain
|
||||
delivered=$(fetch_delivered "$token" "$mission" CHAIN.md) \
|
||||
|| { fail "chain: could not read CHAIN.md from the pushed branch"; return 1; }
|
||||
case "$delivered" in
|
||||
@@ -1222,6 +1224,40 @@ assert_credentials_revoked() { # <mission> <label>
|
||||
fi
|
||||
}
|
||||
|
||||
# Shadow skill triage: one skill.triage event per phase, and — where the
|
||||
# delivery arm makes retrieval observable — how the oracle's "applies" set
|
||||
# compares with what the agent actually read. Informational on agreement
|
||||
# (shadow mode measures; it does not select), PASS/FAIL on the event being
|
||||
# there at all: a triage that silently stopped firing is the state to catch.
|
||||
assert_skill_triage() { # <token> <mission> <label>
|
||||
local token="$1" mission="$2" label="$3" ev agree
|
||||
ev=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
||||
\"select count(*)::text || ' ' || coalesce(max(detail->>'model'),'-') || ' ' || \
|
||||
coalesce(max((detail->>'latency_ms')::int),0)::text || ' ' || \
|
||||
coalesce(max((select count(*) from jsonb_object_keys(detail->'skills'))),0)::text \
|
||||
from mission_events where mission_id='$mission' and kind='skill.triage';\"" | head -1 | tr -d '\r')
|
||||
set -- $ev
|
||||
case "${1:-0}" in
|
||||
0) fail "$label-triage: no skill.triage event — TYPESAFE_API_KEY unset on the server, or the call failed (see the server log)" ;;
|
||||
*) pass "$label-triage: $1 event(s) by $2 over $4 skills, ${3} ms" ;;
|
||||
esac
|
||||
# Oracle vs agent, from the Skill-Use report: skills the oracle said apply
|
||||
# (triage_p >= 0.5) that the agent read, and ones it read the oracle did
|
||||
# not expect. Only meaningful when the arm is a retrieval arm.
|
||||
agree=$(api "$token" GET "/api/missions/$mission/skill-use" | python3 -c '
|
||||
import json,sys
|
||||
d=json.load(sys.stdin); sk=d.get("skills") or []
|
||||
if not sk or all(s.get("triage_p") is None for s in sk): print("n/a"); sys.exit()
|
||||
def read(s): return s["trigger"].get("verdict") in ("pass",) if isinstance(s.get("trigger"),dict) else str(s.get("trigger")).startswith("pass")
|
||||
applies=[s for s in sk if (s.get("triage_p") or 0)>=0.5]
|
||||
hit=sum(1 for s in applies if read(s)); read_unexpected=sum(1 for s in sk if read(s) and (s.get("triage_p") or 0)<0.5)
|
||||
print(f"oracle says {len(applies)} of {len(sk)} delivered skills apply; agent read {hit} of those and {read_unexpected} it did not expect")' 2>/dev/null)
|
||||
case "$agree" in
|
||||
""|n/a) pass "$label-triage: agreement n/a on this arm (no retrieval to compare against)" ;;
|
||||
*) pass "$label-triage: $agree" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ── Scenario: multi-role with a real test suite ──────────────────
|
||||
#
|
||||
# The workload that failed with `COMMIT_EDITMSG: Permission denied` under the
|
||||
|
||||
Reference in New Issue
Block a user