Omar SobhandClaude Opus 5
7dd3aa0965
The A/B has its answer. Across four matched production runs — same recipe, same task, same three offered skills — the MCP-door arm retrieved 1 in 9 and the file arm retrieved 3 of 3, with the judge loop closing on the same run (01a098dd). A signal, not a rate; but 0, 1, 0 → 3 on an otherwise identical task is not noise, and the mechanism is explained rather than guessed: the door is a deferred tool the agents never load, and Read is not. A code default and not CLAWMATES_SKILL_DELIVERY on one server, for the reason always_inject moved into the skill files: a setting that exists only in one deployment is a setting nobody can find. The env var still overrides, and `index` and `inline` stay selectable per mission so the comparison remains runnable against one binary. Garbage in the env var still falls to `inline`, not to the default — an unreadable value must not silently select an arm that needs something installed. A test pins the default so the next change to it is a decision made with the numbers in front of you, not a slip. Co-Authored-By: Claude Opus 5 <[email protected]> Claude-Session: https://claude.ai/code/session_01WZb5A2kfVfjpdwSochkuHz
157 lines
6.5 KiB
Bash
Executable File
157 lines
6.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Run one mission on the LOCAL stack and score its Skill-Use.
|
|
#
|
|
# The first Skill-Use baseline (docs/SKILL-USE-BASELINE.md, 2026-08-19) was
|
|
# produced by a throwaway script that no longer exists, so the second
|
|
# measurement could not be run the same way as the first — which is most of
|
|
# what makes two numbers comparable. This file is that script, kept.
|
|
#
|
|
# It talks to the local stack only. Production auth is Clerk and a mission
|
|
# cannot be launched from a terminal there.
|
|
#
|
|
# Usage:
|
|
# scripts/skill-use-run.sh "<title>" "<task description>"
|
|
# scripts/skill-use-run.sh --score <mission-id> # re-score, no new run
|
|
#
|
|
# Environment:
|
|
# API local server (default http://127.0.0.1:8080)
|
|
# PG local postgres ctr (default clawmates-postgres-1)
|
|
# OWNER account to mint for (default [email protected])
|
|
# TEMPLATE workflow recipe (default research_only)
|
|
# REPO_ID repository to check out; required by the coding recipes, and
|
|
# the only way the TDD and commit checks can ever fire — a
|
|
# repo-less run writes markdown and commits nothing
|
|
# DELIVERY skill delivery arm `files` (default since 2026-09-13), `index`, or `inline`
|
|
# `index` sends name + when_to_use + a uri and makes the agent
|
|
# fetch bodies through the skills door (a DEFERRED MCP tool —
|
|
# 1 retrieval in 9 across three matched runs). `files` sends
|
|
# the same entry with a path under /mission/skills and the
|
|
# agent Reads it. Trigger is a question only under these two.
|
|
# Set per mission, so every arm runs against ONE server process
|
|
# — restarting between arms would put a confound in the
|
|
# comparison that the numbers do not show.
|
|
# TIMEOUT seconds to wait (default 1800)
|
|
# RETAIN_DAYS hold events this long so the run stays re-scorable (default 90)
|
|
|
|
set -uo pipefail
|
|
|
|
API="${API:-http://127.0.0.1:8080}"
|
|
PG="${PG:-clawmates-postgres-1}"
|
|
OWNER="${OWNER:-om[email protected]}"
|
|
TEMPLATE="${TEMPLATE:-research_only}"
|
|
DELIVERY="${DELIVERY:-}"
|
|
TIMEOUT="${TIMEOUT:-1800}"
|
|
RETAIN_DAYS="${RETAIN_DAYS:-90}"
|
|
|
|
psql_() { docker exec "$PG" psql -U postgres -d clawmates -tAc "$1"; }
|
|
|
|
# A session minted straight into the table, as scripts/verify-mission-delivery.sh
|
|
# does. Not a shortcut around auth: it is the same row `POST /api/auth/login`
|
|
# writes, and it avoids putting the owner's password in a process list.
|
|
mint_session() {
|
|
local secret hash rows
|
|
secret="skilluse-$(openssl rand -hex 16)"
|
|
hash=$(printf '%s' "$secret" | openssl dgst -sha256 -binary \
|
|
| openssl base64 -A | tr '+/' '-_' | tr -d '=')
|
|
rows=$(psql_ "insert into auth_sessions (user_id, token_hash, expires_at)
|
|
select id, '$hash', now() + interval '120 minutes'
|
|
from users where email='$OWNER' limit 1 returning 1;" \
|
|
2>/dev/null | head -1 | tr -d '[:space:]')
|
|
[ "$rows" = "1" ] || { echo "no session for $OWNER (no such user?)" >&2; return 1; }
|
|
printf '%s' "$secret"
|
|
}
|
|
|
|
# Body on STDIN, never interpolated into the command: a task description
|
|
# containing an apostrophe is the normal case, not the edge case.
|
|
api() { # api <token> <METHOD> <path> [json]
|
|
local t="$1" m="$2" p="$3" b="${4:-}"
|
|
if [ -n "$b" ]; then
|
|
printf '%s' "$b" | curl -s -X "$m" \
|
|
-H "Authorization: Bearer $t" -H 'Content-Type: application/json' \
|
|
-d @- "$API$p"
|
|
else
|
|
curl -s -X "$m" -H "Authorization: Bearer $t" "$API$p"
|
|
fi
|
|
}
|
|
|
|
jqv() { python3 -c "import sys,json;d=json.load(sys.stdin);print(d$1)"; }
|
|
|
|
score() { # score <token> <mission-id>
|
|
local token="$1" id="$2" arm
|
|
echo
|
|
echo "── what the agents DID ─────────────────────────────────────"
|
|
psql_ "select kind || ' ' || coalesce(target,'') ||
|
|
coalesce(' ' || left(detail->>'input', 160), '')
|
|
from mission_events
|
|
where mission_id='$id' and kind in ('tool.call','file.touch')
|
|
order by id;"
|
|
echo
|
|
echo "── Skill-Use ───────────────────────────────────────────────"
|
|
# Which arm ran, printed next to the scores it produced. A Trigger column of
|
|
# "not observable" means something different under each arm, and a report
|
|
# that does not say which one is not comparable to the next one.
|
|
arm=$(psql_ "select coalesce(skill_delivery, 'inline (unrecorded)') from missions where id='$id';" | tr -d '[:space:]')
|
|
echo "delivery arm: $arm"
|
|
api "$token" GET "/api/missions/$id/skill-use" | python3 -m json.tool
|
|
}
|
|
|
|
token=$(mint_session) || exit 1
|
|
|
|
if [ "${1:-}" = "--score" ]; then
|
|
score "$token" "${2:?mission id}"
|
|
exit 0
|
|
fi
|
|
|
|
TITLE="${1:?title}"
|
|
TASK="${2:?task description}"
|
|
|
|
body=$(python3 - "$TITLE" "$TASK" "$TEMPLATE" "${REPO_ID:-}" "$DELIVERY" <<'PY'
|
|
import json, sys
|
|
req = {
|
|
"title": sys.argv[1],
|
|
"description": sys.argv[2],
|
|
"template_kind": sys.argv[3],
|
|
}
|
|
if sys.argv[4]:
|
|
req["repo_id"] = sys.argv[4]
|
|
if sys.argv[5]:
|
|
# The recipe merges `phase_teams` INTO this object rather than replacing
|
|
# it, so an arm set here survives staffing.
|
|
req["config"] = {"skill_delivery": sys.argv[5]}
|
|
print(json.dumps(req))
|
|
PY
|
|
)
|
|
|
|
created=$(api "$token" POST /api/missions "$body")
|
|
id=$(printf '%s' "$created" | jqv "['id']" 2>/dev/null)
|
|
[ -n "${id:-}" ] || { echo "create failed: $created" >&2; exit 1; }
|
|
echo "mission $id ($TITLE)"
|
|
|
|
# Hold the evidence before the run starts. Mission events are reaped after 7
|
|
# days, and a measurement whose evidence expires cannot be re-scored when the
|
|
# scorer changes — which it just did.
|
|
psql_ "update missions set retain_events_until = now() + interval '$RETAIN_DAYS days'
|
|
where id='$id';" >/dev/null
|
|
|
|
launched=$(api "$token" PATCH "/api/missions/$id/status" '{"status":"running"}')
|
|
printf '%s' "$launched" | grep -q '"status"' \
|
|
|| { echo "launch failed: $launched" >&2; exit 1; }
|
|
|
|
deadline=$(( $(date +%s) + TIMEOUT ))
|
|
status=running
|
|
while [ "$(date +%s)" -lt "$deadline" ]; do
|
|
status=$(psql_ "select status from missions where id='$id';" | tr -d '[:space:]')
|
|
case "$status" in
|
|
completed|failed|cancelled) break ;;
|
|
esac
|
|
printf '\r %s %ss elapsed ' "$status" "$(( $(date +%s) - (deadline - TIMEOUT) ))"
|
|
sleep 15
|
|
done
|
|
echo
|
|
echo "mission finished: $status"
|
|
|
|
# The container tier's tap is drained by phase_runner on a tick AFTER the phase
|
|
# completes. Give it one.
|
|
sleep 30
|
|
score "$token" "$id"
|