test(harness): cover security_hardening — 4 of 5 recipes now run end to end
The third recipe whose defining phase is not `coding`, and so the third that nothing could fail before `PRODUCING_KINDS` widened: a `security_scan` phase that ran no scanner and wrote nothing reported success. One phase, not the recipe's full scan->research->code chain — what is under test is the phase KIND, and the other two kinds are already covered. Two assertions, because the first alone is weak. "Delivered a file" is satisfied by an agent that writes "I scanned it, all clear" and runs nothing — the letter-not-purpose shape this codebase keeps paying for. So the delivered patch must also carry the scanner's OWN output. Verified against the real run: the agent produced gitleaks' banner, INF/ERR lines, byte counts and exit code, not a claim about them. Only `refactor` is now uncovered, and deliberately: its single phase is `coding`, structurally identical to chain/multirole/microvm/noop. It would add runtime and no new signal. security 4/4 against the live fleet.
This commit is contained in:
@@ -531,6 +531,54 @@ assert_benchmark() { # <token> <mission> <report>
|
|||||||
done <<<"$report"
|
done <<<"$report"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ── Scenario: security_hardening — the scan phase must deliver ────
|
||||||
|
#
|
||||||
|
# The third recipe whose defining phase is not `coding`, and so the third that
|
||||||
|
# nothing in the platform could fail until `PRODUCING_KINDS` widened. A
|
||||||
|
# `security_scan` phase that ran no scanner and wrote nothing reported success.
|
||||||
|
#
|
||||||
|
# One phase, not the recipe's full scan->research->code chain: what is under test
|
||||||
|
# is the phase KIND, and the later two are kinds the suite already covers. The
|
||||||
|
# scanners are real (`cargo-audit`, `gitleaks`, `trivy`, `semgrep` are all
|
||||||
|
# present in agent-claude:dev — verified in the image, not assumed from the
|
||||||
|
# directive that names them).
|
||||||
|
SECURITY_BODY=$(cat <<JSON
|
||||||
|
{"title":"verify: a security scan phase delivers findings",
|
||||||
|
"template_kind":"security_hardening",
|
||||||
|
"team_template_id":"$TEAM_TEMPLATE",
|
||||||
|
"repo_id":"$REPO_ID",
|
||||||
|
"description":"Prove the delivery guard covers a security_scan phase.",
|
||||||
|
"phases":[
|
||||||
|
{"kind":"security_scan","order_idx":0,"config":{"commit_policy":"always","max_iterations":1,
|
||||||
|
"task":"Run gitleaks against this repository (it is installed). Write SECURITY.md at the repository root recording the exact command you ran and what it reported — including 'no findings' if that is the result. Create no other files."}}
|
||||||
|
]}
|
||||||
|
JSON
|
||||||
|
)
|
||||||
|
|
||||||
|
assert_security() { # <token> <mission> <report>
|
||||||
|
local mission="$2" report="$3" evidence
|
||||||
|
while read -r idx status files _p _b cerr perr; do
|
||||||
|
[ "$status" = "completed" ] \
|
||||||
|
&& pass "security: phase $idx completed" \
|
||||||
|
|| fail "security: phase $idx status=$status (commit_error=$cerr push_error=$perr)"
|
||||||
|
case "$files" in
|
||||||
|
0|-) fail "security: phase $idx delivered no files — a scan that records nothing is not a scan" ;;
|
||||||
|
*) pass "security: phase $idx delivered $files file(s), and a security_scan phase is now held to it" ;;
|
||||||
|
esac
|
||||||
|
done <<<"$report"
|
||||||
|
|
||||||
|
# Delivering A FILE is not the same as running the scanner. An agent that
|
||||||
|
# wrote "I scanned it, all clear" satisfies the guard above while doing
|
||||||
|
# nothing — the exact letter-not-purpose shape the evaluator exists for.
|
||||||
|
# The tool's own output is the evidence, so require it by name.
|
||||||
|
evidence=$(ssh "$HOST" "docker exec clawmates_server_1 sh -c 'find \
|
||||||
|
/var/lib/clawmates-missions/_outputs/$mission -name \"*.patch\" \
|
||||||
|
-exec grep -lc gitleaks {} \; 2>/dev/null | head -1'" | tr -d '[:space:]')
|
||||||
|
[ -n "$evidence" ] \
|
||||||
|
&& pass "security: the delivered file carries the scanner's own output" \
|
||||||
|
|| fail "security: nothing delivered mentions gitleaks — the scan may not have run"
|
||||||
|
}
|
||||||
|
|
||||||
# ── Scenario: the two engines composed ───────────────────────────
|
# ── Scenario: the two engines composed ───────────────────────────
|
||||||
#
|
#
|
||||||
# A `team_engine=composed` mission is a durable ZeroClaw graph whose every node
|
# A `team_engine=composed` mission is a durable ZeroClaw graph whose every node
|
||||||
@@ -926,6 +974,9 @@ case "${1:-all}" in
|
|||||||
benchmark)
|
benchmark)
|
||||||
run_scenario benchmark "$(echo "$BENCHMARK_BODY" | tr -d '\n')" assert_benchmark
|
run_scenario benchmark "$(echo "$BENCHMARK_BODY" | tr -d '\n')" assert_benchmark
|
||||||
;;
|
;;
|
||||||
|
security)
|
||||||
|
run_scenario security "$(echo "$SECURITY_BODY" | tr -d '\n')" assert_security
|
||||||
|
;;
|
||||||
composed)
|
composed)
|
||||||
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
|
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
|
||||||
;;
|
;;
|
||||||
@@ -943,11 +994,12 @@ case "${1:-all}" in
|
|||||||
run_scenario gatecap "$(echo "$GATECAP_BODY" | tr -d '\n')" assert_gate_cap
|
run_scenario gatecap "$(echo "$GATECAP_BODY" | tr -d '\n')" assert_gate_cap
|
||||||
run_scenario research-only "$(echo "$RESEARCH_ONLY_BODY" | tr -d '\n')" assert_research_only no-checkout
|
run_scenario research-only "$(echo "$RESEARCH_ONLY_BODY" | tr -d '\n')" assert_research_only no-checkout
|
||||||
run_scenario benchmark "$(echo "$BENCHMARK_BODY" | tr -d '\n')" assert_benchmark
|
run_scenario benchmark "$(echo "$BENCHMARK_BODY" | tr -d '\n')" assert_benchmark
|
||||||
|
run_scenario security "$(echo "$SECURITY_BODY" | tr -d '\n')" assert_security
|
||||||
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
|
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
|
||||||
scenario_roster
|
scenario_roster
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|gatecap|research-only|benchmark|composed|roster|all)"
|
die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|gatecap|research-only|benchmark|security|composed|roster|all)"
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user