fix(harness): count what the roster run ADDED, not what the file holds
Forcing the planner onto the local link produced a green chain and a red assertion: roster: the planner sized this mission at 1 member(s) PASS roster: ROSTER.md has 3 line(s) for a 1-member roster FAIL The model was right and the check was wrong. ROSTER.md does not start empty — the auto-merge work put an earlier run's two lines onto main — so a 1-member roster that correctly appended one line delivered three, and the scenario reported a model that had ignored its own proposal. It now measures the DELTA against main. Any assertion against a scratch repo that accumulates has to, or it decays into a test of how many times it has been run before. Proven on the local model end to end: opus 429 -> local:ornith-fleet:9b answered -> `mission_roster: ... local:ornith-fleet:9b proposed 1 member(s)` -> the composed graph ran -> the branch added exactly one line. 5/5. CLAWMATES_MODEL_FALLBACK is removed from gw-04's .env again; it was set only to force the last link for this test, and the deployed default is the full chain. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5
parent
b18e62041b
commit
5afcf63324
@@ -252,6 +252,20 @@ for a in d.get("artifacts") or []:
|
|||||||
'https://git.redclaw.dev/api/v1/repos/$repo/raw/$3?ref=$enc'"
|
'https://git.redclaw.dev/api/v1/repos/$repo/raw/$3?ref=$enc'"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Read a file from the repo's DEFAULT branch, so a scenario can measure what a
|
||||||
|
# run ADDED rather than what the file happens to contain. Prints nothing and
|
||||||
|
# succeeds when the file is absent — an empty baseline is a real baseline.
|
||||||
|
fetch_main() { # fetch_main <path>
|
||||||
|
local token repo
|
||||||
|
token=$(ssh "$HOST" 'docker exec clawmates_server_1 printenv GITEA_TOKEN' | tr -d '\r')
|
||||||
|
repo=$(ssh "$HOST" "docker exec clawmates_postgres_1 psql -U postgres -d clawmates -tAc \
|
||||||
|
\"select owner || '/' || name from repos where id='$REPO_ID';\"" \
|
||||||
|
| head -1 | tr -d '[:space:]')
|
||||||
|
[ -n "$token" ] && [ -n "$repo" ] || return 0
|
||||||
|
ssh "$HOST" "curl -sf -H 'Authorization: token $token' \
|
||||||
|
'https://git.redclaw.dev/api/v1/repos/$repo/raw/$1?ref=main'" 2>/dev/null || true
|
||||||
|
}
|
||||||
|
|
||||||
# ── Scenario: a phase runs inside a microVM, and fans out ─────────
|
# ── Scenario: a phase runs inside a microVM, and fans out ─────────
|
||||||
#
|
#
|
||||||
# Guards everything the microVM track proved by hand: that the agent ran in a
|
# Guards everything the microVM track proved by hand: that the agent ran in a
|
||||||
@@ -816,10 +830,19 @@ if d: print(d[0]["id"], len(d[0]["roster"]["members"]))
|
|||||||
delivered=$(fetch_delivered "$token" "$mission" ROSTER.md) \
|
delivered=$(fetch_delivered "$token" "$mission" ROSTER.md) \
|
||||||
|| { fail "roster: could not read ROSTER.md from the pushed branch"; return 1; }
|
|| { fail "roster: could not read ROSTER.md from the pushed branch"; return 1; }
|
||||||
lines=$(printf '%s\n' "$delivered" | grep -c '[^[:space:]]')
|
lines=$(printf '%s\n' "$delivered" | grep -c '[^[:space:]]')
|
||||||
if [ "${lines:-0}" = "$members" ]; then
|
# Counted as a DELTA against main, not as a total.
|
||||||
pass "roster: ROSTER.md has one line per proposed member ($lines)"
|
#
|
||||||
|
# The absolute form assumed ROSTER.md starts empty, and it does not: the
|
||||||
|
# auto-merge work put an earlier run's two lines onto main, so the next
|
||||||
|
# 1-member roster delivered three lines and was reported as a model that
|
||||||
|
# ignored its own proposal. It had done exactly the right thing. Any check
|
||||||
|
# against a scratch repo that ACCUMULATES has to measure what this run added.
|
||||||
|
base=$(fetch_main "ROSTER.md" | grep -c '[^[:space:]]')
|
||||||
|
local added=$((lines - base))
|
||||||
|
if [ "$added" = "$members" ]; then
|
||||||
|
pass "roster: this run added one line per proposed member ($added, on top of $base already on main)"
|
||||||
else
|
else
|
||||||
fail "roster: ROSTER.md has $lines line(s) for a $members-member roster: $(printf '%s' "$delivered" | tr '\n' '|')"
|
fail "roster: this run added $added line(s) for a $members-member roster (branch=$lines main=$base): $(printf '%s' "$delivered" | tr '\n' '|')"
|
||||||
fi
|
fi
|
||||||
check_single_uid "$mission" roster
|
check_single_uid "$mission" roster
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user