fix(harness): count what the roster run ADDED, not what the file holds

Forcing the planner onto the local link produced a green chain and a red
assertion:

  roster: the planner sized this mission at 1 member(s)          PASS
  roster: ROSTER.md has 3 line(s) for a 1-member roster          FAIL

The model was right and the check was wrong. ROSTER.md does not start empty —
the auto-merge work put an earlier run's two lines onto main — so a 1-member
roster that correctly appended one line delivered three, and the scenario
reported a model that had ignored its own proposal.

It now measures the DELTA against main. Any assertion against a scratch repo
that accumulates has to, or it decays into a test of how many times it has been
run before.

Proven on the local model end to end: opus 429 -> local:ornith-fleet:9b
answered -> `mission_roster: ... local:ornith-fleet:9b proposed 1 member(s)` ->
the composed graph ran -> the branch added exactly one line. 5/5.

CLAWMATES_MODEL_FALLBACK is removed from gw-04's .env again; it was set only to
force the last link for this test, and the deployed default is the full chain.

Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
Omar Sobh
2026-08-09 14:24:46 -07:00
co-authored by Claude Opus 5
parent b18e62041b
commit 5afcf63324
+26 -3
View File
@@ -252,6 +252,20 @@ for a in d.get("artifacts") or []:
'https://git.redclaw.dev/api/v1/repos/$repo/raw/$3?ref=$enc'" 'https://git.redclaw.dev/api/v1/repos/$repo/raw/$3?ref=$enc'"
} }
# Read a file from the repo's DEFAULT branch, so a scenario can measure what a
# run ADDED rather than what the file happens to contain. Prints nothing and
# succeeds when the file is absent — an empty baseline is a real baseline.
fetch_main() { # fetch_main <path>
local token repo
token=$(ssh "$HOST" 'docker exec clawmates_server_1 printenv GITEA_TOKEN' | tr -d '\r')
repo=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
\"select owner || '/' || name from repos where id='$REPO_ID';\"" \
| head -1 | tr -d '[:space:]')
[ -n "$token" ] && [ -n "$repo" ] || return 0
ssh "$HOST" "curl -sf -H 'Authorization: token $token' \
'https://git.redclaw.dev/api/v1/repos/$repo/raw/$1?ref=main'" 2>/dev/null || true
}
# ── Scenario: a phase runs inside a microVM, and fans out ───────── # ── Scenario: a phase runs inside a microVM, and fans out ─────────
# #
# Guards everything the microVM track proved by hand: that the agent ran in a # Guards everything the microVM track proved by hand: that the agent ran in a
@@ -816,10 +830,19 @@ if d: print(d[0]["id"], len(d[0]["roster"]["members"]))
delivered=$(fetch_delivered "$token" "$mission" ROSTER.md) \ delivered=$(fetch_delivered "$token" "$mission" ROSTER.md) \
|| { fail "roster: could not read ROSTER.md from the pushed branch"; return 1; } || { fail "roster: could not read ROSTER.md from the pushed branch"; return 1; }
lines=$(printf '%s\n' "$delivered" | grep -c '[^[:space:]]') lines=$(printf '%s\n' "$delivered" | grep -c '[^[:space:]]')
if [ "${lines:-0}" = "$members" ]; then # Counted as a DELTA against main, not as a total.
pass "roster: ROSTER.md has one line per proposed member ($lines)" #
# The absolute form assumed ROSTER.md starts empty, and it does not: the
# auto-merge work put an earlier run's two lines onto main, so the next
# 1-member roster delivered three lines and was reported as a model that
# ignored its own proposal. It had done exactly the right thing. Any check
# against a scratch repo that ACCUMULATES has to measure what this run added.
base=$(fetch_main "ROSTER.md" | grep -c '[^[:space:]]')
local added=$((lines - base))
if [ "$added" = "$members" ]; then
pass "roster: this run added one line per proposed member ($added, on top of $base already on main)"
else else
fail "roster: ROSTER.md has $lines line(s) for a $members-member roster: $(printf '%s' "$delivered" | tr '\n' '|')" fail "roster: this run added $added line(s) for a $members-member roster (branch=$lines main=$base): $(printf '%s' "$delivered" | tr '\n' '|')"
fi fi
check_single_uid "$mission" roster check_single_uid "$mission" roster
} }