test(harness): cover security_hardening — 4 of 5 recipes now run end to end

The third recipe whose defining phase is not `coding`, and so the third that
nothing could fail before `PRODUCING_KINDS` widened: a `security_scan` phase
that ran no scanner and wrote nothing reported success.

One phase, not the recipe's full scan->research->code chain — what is under test
is the phase KIND, and the other two kinds are already covered.

Two assertions, because the first alone is weak. "Delivered a file" is satisfied
by an agent that writes "I scanned it, all clear" and runs nothing — the
letter-not-purpose shape this codebase keeps paying for. So the delivered patch
must also carry the scanner's OWN output. Verified against the real run: the
agent produced gitleaks' banner, INF/ERR lines, byte counts and exit code, not a
claim about them.

Only `refactor` is now uncovered, and deliberately: its single phase is `coding`,
structurally identical to chain/multirole/microvm/noop. It would add runtime and
no new signal.

security 4/4 against the live fleet.
This commit is contained in:
Omar Sobh
2026-08-07 15:38:50 -07:00
parent 6dd7937ece
commit 2a9a62c784
+53 -1
View File
@@ -531,6 +531,54 @@ assert_benchmark() { # <token> <mission> <report>
done <<<"$report" done <<<"$report"
} }
# ── Scenario: security_hardening — the scan phase must deliver ────
#
# The third recipe whose defining phase is not `coding`, and so the third that
# nothing in the platform could fail until `PRODUCING_KINDS` widened. A
# `security_scan` phase that ran no scanner and wrote nothing reported success.
#
# One phase, not the recipe's full scan->research->code chain: what is under test
# is the phase KIND, and the later two are kinds the suite already covers. The
# scanners are real (`cargo-audit`, `gitleaks`, `trivy`, `semgrep` are all
# present in agent-claude:dev — verified in the image, not assumed from the
# directive that names them).
SECURITY_BODY=$(cat <<JSON
{"title":"verify: a security scan phase delivers findings",
"template_kind":"security_hardening",
"team_template_id":"$TEAM_TEMPLATE",
"repo_id":"$REPO_ID",
"description":"Prove the delivery guard covers a security_scan phase.",
"phases":[
{"kind":"security_scan","order_idx":0,"config":{"commit_policy":"always","max_iterations":1,
"task":"Run gitleaks against this repository (it is installed). Write SECURITY.md at the repository root recording the exact command you ran and what it reported — including 'no findings' if that is the result. Create no other files."}}
]}
JSON
)
assert_security() { # <token> <mission> <report>
local mission="$2" report="$3" evidence
while read -r idx status files _p _b cerr perr; do
[ "$status" = "completed" ] \
&& pass "security: phase $idx completed" \
|| fail "security: phase $idx status=$status (commit_error=$cerr push_error=$perr)"
case "$files" in
0|-) fail "security: phase $idx delivered no files — a scan that records nothing is not a scan" ;;
*) pass "security: phase $idx delivered $files file(s), and a security_scan phase is now held to it" ;;
esac
done <<<"$report"
# Delivering A FILE is not the same as running the scanner. An agent that
# wrote "I scanned it, all clear" satisfies the guard above while doing
# nothing — the exact letter-not-purpose shape the evaluator exists for.
# The tool's own output is the evidence, so require it by name.
evidence=$(ssh "$HOST" "docker exec clawmates_server_1 sh -c 'find \
/var/lib/clawmates-missions/_outputs/$mission -name \"*.patch\" \
-exec grep -lc gitleaks {} \; 2>/dev/null | head -1'" | tr -d '[:space:]')
[ -n "$evidence" ] \
&& pass "security: the delivered file carries the scanner's own output" \
|| fail "security: nothing delivered mentions gitleaks — the scan may not have run"
}
# ── Scenario: the two engines composed ─────────────────────────── # ── Scenario: the two engines composed ───────────────────────────
# #
# A `team_engine=composed` mission is a durable ZeroClaw graph whose every node # A `team_engine=composed` mission is a durable ZeroClaw graph whose every node
@@ -926,6 +974,9 @@ case "${1:-all}" in
benchmark) benchmark)
run_scenario benchmark "$(echo "$BENCHMARK_BODY" | tr -d '\n')" assert_benchmark run_scenario benchmark "$(echo "$BENCHMARK_BODY" | tr -d '\n')" assert_benchmark
;; ;;
security)
run_scenario security "$(echo "$SECURITY_BODY" | tr -d '\n')" assert_security
;;
composed) composed)
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
;; ;;
@@ -943,11 +994,12 @@ case "${1:-all}" in
run_scenario gatecap "$(echo "$GATECAP_BODY" | tr -d '\n')" assert_gate_cap run_scenario gatecap "$(echo "$GATECAP_BODY" | tr -d '\n')" assert_gate_cap
run_scenario research-only "$(echo "$RESEARCH_ONLY_BODY" | tr -d '\n')" assert_research_only no-checkout run_scenario research-only "$(echo "$RESEARCH_ONLY_BODY" | tr -d '\n')" assert_research_only no-checkout
run_scenario benchmark "$(echo "$BENCHMARK_BODY" | tr -d '\n')" assert_benchmark run_scenario benchmark "$(echo "$BENCHMARK_BODY" | tr -d '\n')" assert_benchmark
run_scenario security "$(echo "$SECURITY_BODY" | tr -d '\n')" assert_security
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
scenario_roster scenario_roster
;; ;;
*) *)
die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|gatecap|research-only|benchmark|composed|roster|all)" die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|gatecap|research-only|benchmark|security|composed|roster|all)"
;; ;;
esac esac