test(harness): cover the two recipes that had none — research_only and benchmark
The portal offers five workflow recipes. Every one of the harness's seven
fixtures was `research_and_code`, so four recipes had never run end to end —
and that is not a theoretical gap. `research_only` DESTROYED its output for as
long as it existed: `requires_repo = false`, so the capture query's
`AND m.repo_id IS NOT NULL` skipped it, the container was reaped unread, and
eight ClawHDF5 research documents were lost while the mission reported
`completed`. Nothing in 550+ tests could see it, because nothing ran the recipe.
`research-only` asserts the whole chain the loss ran through, not just the
happy end of it:
- the phase completes
- document artifacts exist AT ALL (the missing thing)
- the agent's seven identity files (SOUL.md, MEMORY.md, …) are NOT published
— the first live capture published all seven, because `.git/info/exclude`
cannot protect a mission with no `.git`
- the captured text reads back through the content endpoint, since an
artifact row pointing at nothing is a 404 with no explanation
`benchmark` covers the other half: a benchmark mission is ONE benchmark phase,
and while `empty_delivery_is_a_failure` tested `kind == "coding"` that phase was
exempt — nothing in the platform could fail it. The scenario asserts it both
completes AND delivers files.
Also: `run_scenario` takes an optional `no-checkout`. The single-writer uid probe
is a property OF A CHECKOUT, and a repo-less mission has none by design, so
probing reports a platform fault that is really a category error. It is declared
per scenario rather than inferred from a missing directory — that inference would
silently excuse a repo-BACKED mission whose checkout was reaped early, which is
the exact condition the probe exists to catch.
research-only 4/4, benchmark 3/3 against the live fleet.
This commit is contained in:
@@ -426,6 +426,111 @@ assert_gate_cap() { # <token> <mission> <report>
|
|||||||
esac
|
esac
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ── Scenario: research_only — a repo-less mission keeps its work ──
|
||||||
|
#
|
||||||
|
# The portal offers five recipes; until now the harness exercised ONE
|
||||||
|
# (research_and_code). `research_only` is the recipe that lost real work: it sets
|
||||||
|
# `requires_repo = false`, so `capture_finished_coding_phases` (which selects
|
||||||
|
# `AND m.repo_id IS NOT NULL`) never looked at it, the container was reaped
|
||||||
|
# unread, and eight ClawHDF5 research documents were destroyed while the mission
|
||||||
|
# reported `completed`.
|
||||||
|
#
|
||||||
|
# Nothing in the suite would have caught that, because nothing ever ran this
|
||||||
|
# recipe. That is the gap this closes.
|
||||||
|
RESEARCH_ONLY_BODY=$(cat <<JSON
|
||||||
|
{"title":"verify: a repo-less research mission keeps its output",
|
||||||
|
"template_kind":"research_only",
|
||||||
|
"team_template_id":"$TEAM_TEMPLATE",
|
||||||
|
"description":"Prove mission_outputs captures a phase with no git checkout.",
|
||||||
|
"phases":[
|
||||||
|
{"kind":"research","order_idx":0,"config":{"max_iterations":1,
|
||||||
|
"task":"Write exactly two markdown files under /mission/repo/research/: 01_findings.md and 02_notes.md. Each 5-10 lines about Rust error handling. Create no other files."}}
|
||||||
|
]}
|
||||||
|
JSON
|
||||||
|
)
|
||||||
|
|
||||||
|
assert_research_only() { # <token> <mission> <report>
|
||||||
|
local token="$1" mission="$2" report="$3" docs names scaffold
|
||||||
|
while read -r idx status _f _p _b _c _e; do
|
||||||
|
[ "$status" = "completed" ] \
|
||||||
|
&& pass "research-only: phase $idx completed" \
|
||||||
|
|| fail "research-only: phase $idx status=$status"
|
||||||
|
done <<<"$report"
|
||||||
|
|
||||||
|
# The artifacts are the whole point: without them the mission is the silent
|
||||||
|
# loss this scenario exists to detect.
|
||||||
|
docs=$(api "$token" GET "/api/missions/$mission" | python3 -c '
|
||||||
|
import json,sys
|
||||||
|
d=json.load(sys.stdin)
|
||||||
|
print(sum(1 for a in (d.get("artifacts") or []) if a.get("kind")=="document"))')
|
||||||
|
case "$docs" in
|
||||||
|
""|0) fail "research-only: NO documents captured — a repo-less phase lost its work" ;;
|
||||||
|
*) pass "research-only: $docs document artifact(s) captured from a mission with no repo" ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
# The agent runtime seeds SOUL.md/MEMORY.md/etc into the workspace root. In a
|
||||||
|
# repo-backed mission `.git/info/exclude` hides them; a repo-less mission has
|
||||||
|
# no `.git`, and the first live run published all seven as artifacts.
|
||||||
|
scaffold=$(api "$token" GET "/api/missions/$mission" | python3 -c '
|
||||||
|
import json,sys
|
||||||
|
SEED={"AGENTS.md","HEARTBEAT.md","IDENTITY.md","MEMORY.md","SOUL.md","TOOLS.md","USER.md"}
|
||||||
|
d=json.load(sys.stdin)
|
||||||
|
print(sum(1 for a in (d.get("artifacts") or []) if (a.get("title") or "") in SEED))')
|
||||||
|
[ "${scaffold:-0}" = "0" ] \
|
||||||
|
&& pass "research-only: the agent's own identity files were not published" \
|
||||||
|
|| fail "research-only: $scaffold agent scaffolding file(s) leaked into artifacts"
|
||||||
|
|
||||||
|
# And the text must actually be readable — an artifact row pointing at nothing
|
||||||
|
# is a 404 with no explanation, which is how a reader experiences lost work.
|
||||||
|
names=$(api "$token" GET "/api/missions/$mission" | python3 -c '
|
||||||
|
import json,sys
|
||||||
|
d=json.load(sys.stdin)
|
||||||
|
a=[x for x in (d.get("artifacts") or []) if x.get("kind")=="document"]
|
||||||
|
print(a[0]["id"] if a else "")')
|
||||||
|
if [ -n "$names" ]; then
|
||||||
|
local body
|
||||||
|
body=$(api "$token" GET "/api/missions/$mission/artifacts/$names/content" | python3 -c '
|
||||||
|
import json,sys
|
||||||
|
try: print(len(json.load(sys.stdin).get("content") or ""))
|
||||||
|
except Exception: print(0)')
|
||||||
|
[ "${body:-0}" -gt 0 ] \
|
||||||
|
&& pass "research-only: the captured document reads back ($body chars)" \
|
||||||
|
|| fail "research-only: the artifact exists but its content is unreadable"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# ── Scenario: benchmark — a non-coding phase must still deliver ───
|
||||||
|
#
|
||||||
|
# A `benchmark` mission is ONE benchmark phase. `empty_delivery_is_a_failure`
|
||||||
|
# tested `kind == "coding"`, so that phase was exempt and NOTHING in the platform
|
||||||
|
# could fail it: an agent that produced no benchmark at all reported success.
|
||||||
|
# This runs the recipe the guard now covers.
|
||||||
|
BENCHMARK_BODY=$(cat <<JSON
|
||||||
|
{"title":"verify: a benchmark phase delivers files",
|
||||||
|
"template_kind":"benchmark",
|
||||||
|
"team_template_id":"$TEAM_TEMPLATE",
|
||||||
|
"repo_id":"$REPO_ID",
|
||||||
|
"description":"Prove the delivery guard covers a non-coding producing phase.",
|
||||||
|
"phases":[
|
||||||
|
{"kind":"benchmark","order_idx":0,"config":{"commit_policy":"always","max_iterations":1,
|
||||||
|
"task":"Create BENCH.md at the repository root recording a simple timing measurement you actually ran (loop a cheap operation and time it). Create no other files."}}
|
||||||
|
]}
|
||||||
|
JSON
|
||||||
|
)
|
||||||
|
|
||||||
|
assert_benchmark() { # <token> <mission> <report>
|
||||||
|
local report="$3"
|
||||||
|
while read -r idx status files _p _b cerr perr; do
|
||||||
|
[ "$status" = "completed" ] \
|
||||||
|
&& pass "benchmark: phase $idx completed" \
|
||||||
|
|| fail "benchmark: phase $idx status=$status (commit_error=$cerr push_error=$perr)"
|
||||||
|
case "$files" in
|
||||||
|
0|-) fail "benchmark: phase $idx delivered no files — the widened guard did not fire" ;;
|
||||||
|
*) pass "benchmark: phase $idx delivered $files file(s), and a non-coding phase is now held to it" ;;
|
||||||
|
esac
|
||||||
|
done <<<"$report"
|
||||||
|
}
|
||||||
|
|
||||||
# ── Scenario: the two engines composed ───────────────────────────
|
# ── Scenario: the two engines composed ───────────────────────────
|
||||||
#
|
#
|
||||||
# A `team_engine=composed` mission is a durable ZeroClaw graph whose every node
|
# A `team_engine=composed` mission is a durable ZeroClaw graph whose every node
|
||||||
@@ -615,8 +720,8 @@ if d: print(d[0]["id"], len(d[0]["roster"]["members"]))
|
|||||||
check_single_uid "$mission" roster
|
check_single_uid "$mission" roster
|
||||||
}
|
}
|
||||||
|
|
||||||
run_scenario() { # run_scenario <label> <json> <assert-fn>
|
run_scenario() { # run_scenario <label> <json> <assert-fn> [no-checkout]
|
||||||
local label="$1" body="$2" assert_fn="$3" token mission status
|
local label="$1" body="$2" assert_fn="$3" checkout="${4:-checkout}" token mission status
|
||||||
# Every one of these MUST go through fail()/norun(). The first version of
|
# Every one of these MUST go through fail()/norun(). The first version of
|
||||||
# this function called a `die` that lived inside `$(...)` — which exits the
|
# this function called a `die` that lived inside `$(...)` — which exits the
|
||||||
# command substitution's subshell, not the script — so a run where the
|
# command substitution's subshell, not the script — so a run where the
|
||||||
@@ -639,7 +744,18 @@ run_scenario() { # run_scenario <label> <json> <assert-fn>
|
|||||||
fi
|
fi
|
||||||
printf '%s\n' "$report" | sed 's/^/ phase /'
|
printf '%s\n' "$report" | sed 's/^/ phase /'
|
||||||
"$assert_fn" "$token" "$mission" "$report"
|
"$assert_fn" "$token" "$mission" "$report"
|
||||||
check_single_uid "$mission" "$label"
|
# The single-writer invariant is a property OF A CHECKOUT. A repo-less mission
|
||||||
|
# has none by design (`ensure_checkout` returns Ok(None)), so probing for one
|
||||||
|
# reports a platform fault that is really a category error.
|
||||||
|
#
|
||||||
|
# Declared per scenario, never inferred from "the directory is missing": that
|
||||||
|
# inference would silently excuse a repo-BACKED mission whose checkout was
|
||||||
|
# reaped early — which is exactly the condition this probe exists to catch.
|
||||||
|
if [ "$checkout" = "no-checkout" ]; then
|
||||||
|
info "$label: no checkout to probe (repo-less mission) — single-writer check n/a"
|
||||||
|
else
|
||||||
|
check_single_uid "$mission" "$label"
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# ── Scenario: phase continuity ───────────────────────────────────
|
# ── Scenario: phase continuity ───────────────────────────────────
|
||||||
@@ -804,6 +920,12 @@ case "${1:-all}" in
|
|||||||
gatecap)
|
gatecap)
|
||||||
run_scenario gatecap "$(echo "$GATECAP_BODY" | tr -d '\n')" assert_gate_cap
|
run_scenario gatecap "$(echo "$GATECAP_BODY" | tr -d '\n')" assert_gate_cap
|
||||||
;;
|
;;
|
||||||
|
research-only)
|
||||||
|
run_scenario research-only "$(echo "$RESEARCH_ONLY_BODY" | tr -d '\n')" assert_research_only no-checkout
|
||||||
|
;;
|
||||||
|
benchmark)
|
||||||
|
run_scenario benchmark "$(echo "$BENCHMARK_BODY" | tr -d '\n')" assert_benchmark
|
||||||
|
;;
|
||||||
composed)
|
composed)
|
||||||
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
|
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
|
||||||
;;
|
;;
|
||||||
@@ -819,11 +941,13 @@ case "${1:-all}" in
|
|||||||
run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
|
run_scenario microvm "$(echo "$MICROVM_BODY" | tr -d '\n')" assert_microvm
|
||||||
scenario_microvm_unavailable_backend
|
scenario_microvm_unavailable_backend
|
||||||
run_scenario gatecap "$(echo "$GATECAP_BODY" | tr -d '\n')" assert_gate_cap
|
run_scenario gatecap "$(echo "$GATECAP_BODY" | tr -d '\n')" assert_gate_cap
|
||||||
|
run_scenario research-only "$(echo "$RESEARCH_ONLY_BODY" | tr -d '\n')" assert_research_only no-checkout
|
||||||
|
run_scenario benchmark "$(echo "$BENCHMARK_BODY" | tr -d '\n')" assert_benchmark
|
||||||
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
|
run_scenario composed "$(echo "$COMPOSED_BODY" | tr -d '\n')" assert_composed
|
||||||
scenario_roster
|
scenario_roster
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|gatecap|composed|roster|all)"
|
die "unknown scenario: $1 (selftest|uids|chain|multirole|noop|microvm|gatecap|research-only|benchmark|composed|roster|all)"
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user