test(harness): cover the two recipes that had none — research_only and benchmark

The portal offers five workflow recipes. Every one of the harness's seven
fixtures was `research_and_code`, so four recipes had never run end to end —
and that is not a theoretical gap. `research_only` DESTROYED its output for as
long as it existed: `requires_repo = false`, so the capture query's
`AND m.repo_id IS NOT NULL` skipped it, the container was reaped unread, and
eight ClawHDF5 research documents were lost while the mission reported
`completed`. Nothing in 550+ tests could see it, because nothing ran the recipe.

`research-only` asserts the whole chain the loss ran through, not just the
happy end of it:
  - the phase completes
  - document artifacts exist AT ALL (the missing thing)
  - the agent's seven identity files (SOUL.md, MEMORY.md, …) are NOT published
    — the first live capture published all seven, because `.git/info/exclude`
    cannot protect a mission with no `.git`
  - the captured text reads back through the content endpoint, since an
    artifact row pointing at nothing is a 404 with no explanation

`benchmark` covers the other half: a benchmark mission is ONE benchmark phase,
and while `empty_delivery_is_a_failure` tested `kind == "coding"` that phase was
exempt — nothing in the platform could fail it. The scenario asserts it both
completes AND delivers files.

Also: `run_scenario` takes an optional `no-checkout`. The single-writer uid probe
is a property OF A CHECKOUT, and a repo-less mission has none by design, so
probing reports a platform fault that is really a category error. It is declared
per scenario rather than inferred from a missing directory — that inference would
silently excuse a repo-BACKED mission whose checkout was reaped early, which is
the exact condition the probe exists to catch.

research-only 4/4, benchmark 3/3 against the live fleet.
This commit is contained in:
Omar Sobh
2026-08-07 15:13:40 -07:00
parent a20702d55b
commit 6dd7937ece
+128 -4
View File
@@ -426,6 +426,111 @@ assert_gate_cap() { # <token> <mission> <report>
esac esac
} }
# ── Scenario: research_only — a repo-less mission keeps its work ──
#
# The portal offers five recipes; until now the harness exercised ONE
# (research_and_code). `research_only` is the recipe that lost real work: it sets
# `requires_repo = false`, so `capture_finished_coding_phases` (which selects
# `AND m.repo_id IS NOT NULL`) never looked at it, the container was reaped
# unread, and eight ClawHDF5 research documents were destroyed while the mission
# reported `completed`.
#
# Nothing in the suite would have caught that, because nothing ever ran this
# recipe. That is the gap this closes.
RESEARCH_ONLY_BODY=$(cat <<JSON
{"title":"verify: a repo-less research mission keeps its output",
"template_kind":"research_only",
"team_template_id":"$TEAM_TEMPLATE",
"description":"Prove mission_outputs captures a phase with no git checkout.",
"phases":[
{"kind":"research","order_idx":0,"config":{"max_iterations":1,
"task":"Write exactly two markdown files under /mission/repo/research/: 01_findings.md and 02_notes.md. Each 5-10 lines about Rust error handling. Create no other files."}}
]}
JSON
)
assert_research_only() { # <token> <mission> <report>
local token="$1" mission="$2" report="$3" docs names scaffold
while read -r idx status _f _p _b _c _e; do
[ "$status" = "completed" ] \
&& pass "research-only: phase $idx completed" \
|| fail "research-only: phase $idx status=$status"
done <<<"$report"
# The artifacts are the whole point: without them the mission is the silent
# loss this scenario exists to detect.
docs=$(api "$token" GET "/api/missions/$mission" | python3 -c '
import json,sys
d=json.load(sys.stdin)
print(sum(1 for a in (d.get("artifacts") or []) if a.get("kind")=="document"))')
case "$docs" in
""|0) fail "research-only: NO documents captured — a repo-less phase lost its work" ;;
*) pass "research-only: $docs document artifact(s) captured from a mission with no repo" ;;
esac
# The agent runtime seeds SOUL.md/MEMORY.md/etc into the workspace root. In a
# repo-backed mission `.git/info/exclude` hides them; a repo-less mission has
# no `.git`, and the first live run published all seven as artifacts.
scaffold=$(api "$token" GET "/api/missions/$mission" | python3 -c '
import json,sys
SEED={"AGENTS.md","HEARTBEAT.md","IDENTITY.md","MEMORY.md","SOUL.md","TOOLS.md","USER.md"}
d=json.load(sys.stdin)
print(sum(1 for a in (d.get("artifacts") or []) if (a.get("title") or "") in SEED))')
[ "${scaffold:-0}" = "0" ] \
&& pass "research-only: the agent's own identity files were not published" \
|| fail "research-only: $scaffold agent scaffolding file(s) leaked into artifacts"
# And the text must actually be readable — an artifact row pointing at nothing
# is a 404 with no explanation, which is how a reader experiences lost work.
names=$(api "$token" GET "/api/missions/$mission" | python3 -c '
import json,sys
d=json.load(sys.stdin)
a=[x for x in (d.get("artifacts") or []) if x.get("kind")=="document"]
print(a[0]["id"] if a else "")')
if [ -n "$names" ]; then
local body
body=$(api "$token" GET "/api/missions/$mission/artifacts/$names/content" | python3 -c '
import json,sys
try: print(len(json.load(sys.stdin).get("content") or ""))
except Exception: print(0)')
[ "${body:-0}" -gt 0 ] \
&& pass "research-only: the captured document reads back ($body chars)" \
|| fail "research-only: the artifact exists but its content is unreadable"
fi
}
# ── Scenario: benchmark — a non-coding phase must still deliver ───
#
# A `benchmark` mission is ONE benchmark phase. `empty_delivery_is_a_failure`
# tested `kind == "coding"`, so that phase was exempt and NOTHING in the platform
# could fail it: an agent that produced no benchmark at all reported success.
# This runs the recipe the guard now covers.
BENCHMARK_BODY=$(cat <<JSON
{"title":"verify: a benchmark phase delivers files",
"template_kind":"benchmark",
"team_template_id":"$TEAM_TEMPLATE",
"repo_id":"$REPO_ID",
"description":"Prove the delivery guard covers a non-coding producing phase.",
"phases":[
{"kind":"benchmark","order_idx":0,"config":{"commit_policy":"always","max_iterations":1,
"task":"Create BENCH.md at the repository root recording a simple timing measurement you actually ran (loop a cheap operation and time it). Create no other files."}}
]}
JSON
)
assert_benchmark() { # <token> <mission> <report>
local report="$3"
while read -r idx status files _p _b cerr perr; do
[ "$status" = "completed" ] \
&& pass "benchmark: phase $idx completed" \
|| fail "benchmark: phase $idx status=$status (commit_error=$cerr push_error=$perr)"
case "$files" in
0|-) fail "benchmark: phase $idx delivered no files — the widened guard did not fire" ;;
*) pass "benchmark: phase $idx delivered $files file(s), and a non-coding phase is now held to it" ;;
esac
done <<<"$report"
}
# ── Scenario: the two engines composed ─────────────────────────── # ── Scenario: the two engines composed ───────────────────────────
# #
# A `team_engine=composed` mission is a durable ZeroClaw graph whose every node # A `team_engine=composed` mission is a durable ZeroClaw graph whose every node
@@ -615,8 +720,8 @@ if d: print(d[0]["id"], len(d[0]["roster"]["members"]))
check_single_uid "$mission" roster check_single_uid "$mission" roster
} }
run_scenario() { # run_scenario <label> <json> <assert-fn> run_scenario() { # run_scenario <label> <json> <assert-fn> [no-checkout]
local label="$1" body="$2" assert_fn="$3" token mission status local label="$1" body="$2" assert_fn="$3" checkout="${4:-checkout}" token mission status
# Every one of these MUST go through fail()/norun(). The first version of # Every one of these MUST go through fail()/norun(). The first version of
# this function called a `die` that lived inside `$(...)` — which exits the # this function called a `die` that lived inside `$(...)` — which exits the
# command substitution's subshell, not the script — so a run where the # command substitution's subshell, not the script — so a run where the
@@ -639,7 +744,18 @@ run_scenario() { # run_scenario <label> <json> <assert-fn>
fi fi
printf '%s\n' "$report" | sed 's/^/ phase /' printf '%s\n' "$report" | sed 's/^/ phase /'
"$assert_fn" "$token" "$mission" "$report" "$assert_fn" "$token" "$mission" "$report"
check_single_uid "$mission" "$label" # The single-writer invariant is a property OF A CHECKOUT. A repo-less mission
# has none by design (`ensure_checkout` returns Ok(None)), so probing for one
# reports a platform fault that is really a category error.
#
# Declared per scenario, never inferred from "the directory is missing": that
# inference would silently excuse a repo-BACKED mission whose checkout was
# reaped early — which is exactly the condition this probe exists to catch.
if [ "$checkout" = "no-checkout" ]; then
info "$label: no checkout to probe (repo-less mission) — single-writer check n/a"
else
check_single_uid "$mission" "$label"
fi
} }
# ── Scenario: phase continuity ─────────────────────────────────── # ── Scenario: phase continuity ───────────────────────────────────
@@ -804,6 +920,12 @@ case "${1:-all}" in
gatecap) gatecap)
run_scenario gatecap "$(echo "$GATECAP_BODY" | tr -d '\n')" assert_gate_cap run_scenario gatecap "$(echo "$GATECAP_BODY" | tr -d '\n')" assert_gate_cap
;; ;;
research-only)
run_scenario research-only "$(echo "$RESEARCH_ONLY_BODY" | tr -d '\n')" assert_research_only no-checkout
;;
benchmark)
run_scenario benchmark "$(echo "$BENCHMARK_BODY" | tr -d '\n')" assert_benchmark
;;
composed) composed)
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
;; ;;
@@ -819,11 +941,13 @@ case "${1:-all}" in
run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
scenario_microvm_unavailable_backend scenario_microvm_unavailable_backend
run_scenario gatecap "$(echo "$GATECAP_BODY" | tr -d '\n')" assert_gate_cap run_scenario gatecap "$(echo "$GATECAP_BODY" | tr -d '\n')" assert_gate_cap
run_scenario research-only "$(echo "$RESEARCH_ONLY_BODY" | tr -d '\n')" assert_research_only no-checkout
run_scenario benchmark "$(echo "$BENCHMARK_BODY" | tr -d '\n')" assert_benchmark
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
scenario_roster scenario_roster
;; ;;
*) *)
die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|gatecap|composed|roster|all)" die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|gatecap|research-only|benchmark|composed|roster|all)"
;; ;;
esac esac