test(skill-use): keep the harness that runs the measurement
The first baseline was produced by a throwaway script that no longer exists, so the second measurement could not be run the same way as the first — which is most of what makes two numbers comparable. Local stack only, because production auth is Clerk and a mission cannot be launched from a terminal there. `--score <id>` re-scores a finished run without spending another one, and every run is held for 90 days so it stays re-scorable when the scorer changes again. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_018i9Ten1LU4jUr5d7TAWda9
This commit is contained in:
Omar Sobh
co-authored by
Claude Opus 5
Claude Opus 5
parent
d0b657a24b
commit
4a6d0dfe01
Executable
+131
@@ -0,0 +1,131 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Run one mission on the LOCAL stack and score its Skill-Use.
|
||||||
|
#
|
||||||
|
# The first Skill-Use baseline (docs/SKILL-USE-BASELINE.md, 2026-08-19) was
|
||||||
|
# produced by a throwaway script that no longer exists, so the second
|
||||||
|
# measurement could not be run the same way as the first — which is most of
|
||||||
|
# what makes two numbers comparable. This file is that script, kept.
|
||||||
|
#
|
||||||
|
# It talks to the local stack only. Production auth is Clerk and a mission
|
||||||
|
# cannot be launched from a terminal there.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# scripts/skill-use-run.sh "<title>" "<task description>"
|
||||||
|
# scripts/skill-use-run.sh --score <mission-id> # re-score, no new run
|
||||||
|
#
|
||||||
|
# Environment:
|
||||||
|
# API local server (default http://127.0.0.1:8080)
|
||||||
|
# PG local postgres ctr (default clawmates-postgres-1)
|
||||||
|
# OWNER account to mint for (default [email protected])
|
||||||
|
# TEMPLATE workflow recipe (default research_only)
|
||||||
|
# TIMEOUT seconds to wait (default 1800)
|
||||||
|
# RETAIN_DAYS hold events this long so the run stays re-scorable (default 90)
|
||||||
|
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
API="${API:-http://127.0.0.1:8080}"
|
||||||
|
PG="${PG:-clawmates-postgres-1}"
|
||||||
|
OWNER="${OWNER:-om[email protected]}"
|
||||||
|
TEMPLATE="${TEMPLATE:-research_only}"
|
||||||
|
TIMEOUT="${TIMEOUT:-1800}"
|
||||||
|
RETAIN_DAYS="${RETAIN_DAYS:-90}"
|
||||||
|
|
||||||
|
psql_() { docker exec "$PG" psql -U postgres -d clawmates -tAc "$1"; }
|
||||||
|
|
||||||
|
# A session minted straight into the table, as scripts/verify-mission-delivery.sh
|
||||||
|
# does. Not a shortcut around auth: it is the same row `POST /api/auth/login`
|
||||||
|
# writes, and it avoids putting the owner's password in a process list.
|
||||||
|
mint_session() {
|
||||||
|
local secret hash rows
|
||||||
|
secret="skilluse-$(openssl rand -hex 16)"
|
||||||
|
hash=$(printf '%s' "$secret" | openssl dgst -sha256 -binary \
|
||||||
|
| openssl base64 -A | tr '+/' '-_' | tr -d '=')
|
||||||
|
rows=$(psql_ "insert into auth_sessions (user_id, token_hash, expires_at)
|
||||||
|
select id, '$hash', now() + interval '120 minutes'
|
||||||
|
from users where email='$OWNER' limit 1 returning 1;" \
|
||||||
|
2>/dev/null | head -1 | tr -d '[:space:]')
|
||||||
|
[ "$rows" = "1" ] || { echo "no session for $OWNER (no such user?)" >&2; return 1; }
|
||||||
|
printf '%s' "$secret"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Body on STDIN, never interpolated into the command: a task description
|
||||||
|
# containing an apostrophe is the normal case, not the edge case.
|
||||||
|
api() { # api <token> <METHOD> <path> [json]
|
||||||
|
local t="$1" m="$2" p="$3" b="${4:-}"
|
||||||
|
if [ -n "$b" ]; then
|
||||||
|
printf '%s' "$b" | curl -s -X "$m" \
|
||||||
|
-H "Authorization: Bearer $t" -H 'Content-Type: application/json' \
|
||||||
|
-d @- "$API$p"
|
||||||
|
else
|
||||||
|
curl -s -X "$m" -H "Authorization: Bearer $t" "$API$p"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
jqv() { python3 -c "import sys,json;d=json.load(sys.stdin);print(d$1)"; }
|
||||||
|
|
||||||
|
score() { # score <token> <mission-id>
|
||||||
|
local token="$1" id="$2"
|
||||||
|
echo
|
||||||
|
echo "── what the agents DID ─────────────────────────────────────"
|
||||||
|
psql_ "select kind || ' ' || coalesce(target,'') ||
|
||||||
|
coalesce(' ' || left(detail->>'input', 160), '')
|
||||||
|
from mission_events
|
||||||
|
where mission_id='$id' and kind in ('tool.call','file.touch')
|
||||||
|
order by id;"
|
||||||
|
echo
|
||||||
|
echo "── Skill-Use ───────────────────────────────────────────────"
|
||||||
|
api "$token" GET "/api/missions/$id/skill-use" | python3 -m json.tool
|
||||||
|
}
|
||||||
|
|
||||||
|
token=$(mint_session) || exit 1
|
||||||
|
|
||||||
|
if [ "${1:-}" = "--score" ]; then
|
||||||
|
score "$token" "${2:?mission id}"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
TITLE="${1:?title}"
|
||||||
|
TASK="${2:?task description}"
|
||||||
|
|
||||||
|
body=$(python3 - "$TITLE" "$TASK" "$TEMPLATE" <<'PY'
|
||||||
|
import json, sys
|
||||||
|
print(json.dumps({
|
||||||
|
"title": sys.argv[1],
|
||||||
|
"description": sys.argv[2],
|
||||||
|
"template_kind": sys.argv[3],
|
||||||
|
}))
|
||||||
|
PY
|
||||||
|
)
|
||||||
|
|
||||||
|
created=$(api "$token" POST /api/missions "$body")
|
||||||
|
id=$(printf '%s' "$created" | jqv "['id']" 2>/dev/null)
|
||||||
|
[ -n "${id:-}" ] || { echo "create failed: $created" >&2; exit 1; }
|
||||||
|
echo "mission $id ($TITLE)"
|
||||||
|
|
||||||
|
# Hold the evidence before the run starts. Mission events are reaped after 7
|
||||||
|
# days, and a measurement whose evidence expires cannot be re-scored when the
|
||||||
|
# scorer changes — which it just did.
|
||||||
|
psql_ "update missions set retain_events_until = now() + interval '$RETAIN_DAYS days'
|
||||||
|
where id='$id';" >/dev/null
|
||||||
|
|
||||||
|
launched=$(api "$token" PATCH "/api/missions/$id/status" '{"status":"running"}')
|
||||||
|
printf '%s' "$launched" | grep -q '"status"' \
|
||||||
|
|| { echo "launch failed: $launched" >&2; exit 1; }
|
||||||
|
|
||||||
|
deadline=$(( $(date +%s) + TIMEOUT ))
|
||||||
|
status=running
|
||||||
|
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||||
|
status=$(psql_ "select status from missions where id='$id';" | tr -d '[:space:]')
|
||||||
|
case "$status" in
|
||||||
|
completed|failed|cancelled) break ;;
|
||||||
|
esac
|
||||||
|
printf '\r %s %ss elapsed ' "$status" "$(( $(date +%s) - (deadline - TIMEOUT) ))"
|
||||||
|
sleep 15
|
||||||
|
done
|
||||||
|
echo
|
||||||
|
echo "mission finished: $status"
|
||||||
|
|
||||||
|
# The container tier's tap is drained by phase_runner on a tick AFTER the phase
|
||||||
|
# completes. Give it one.
|
||||||
|
sleep 30
|
||||||
|
score "$token" "$id"
|
||||||
Reference in New Issue
Block a user