Compare commits
218
Commits
af89020dfd
..
main
+108
-10
@@ -21,6 +21,19 @@ on:
|
||||
# Lets you re-run a deploy without an empty commit.
|
||||
workflow_dispatch:
|
||||
|
||||
# Two pushes close together used to STOMP each other. Runs 490 and 491 started
|
||||
# 16 minutes apart, a full suite takes longer than that, and the first thing a
|
||||
# run does is `docker rm -fv` the shared test Postgres — so the newer run
|
||||
# deleted the older run's database mid-suite and both failed. Nothing in the
|
||||
# code was wrong; the logs blamed the tests.
|
||||
#
|
||||
# `cancel-in-progress` because a superseded run is testing a commit that is no
|
||||
# longer the tip: finishing it costs 20 minutes to learn something that no
|
||||
# longer matters.
|
||||
concurrency:
|
||||
group: deploy-${{ gitea.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
REGISTRY: 100.94.185.103:5000
|
||||
NAMESPACE: clawmates
|
||||
@@ -28,6 +41,9 @@ env:
|
||||
jobs:
|
||||
test:
|
||||
runs-on: gw04
|
||||
env:
|
||||
# Shared by the start and stop steps.
|
||||
PG: cm-ci-pg-${{ gitea.run_id }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -37,15 +53,37 @@ jobs:
|
||||
# only means "no database here".
|
||||
- name: Start test Postgres
|
||||
run: |
|
||||
docker rm -f cm-ci-pg 2>/dev/null || true
|
||||
docker run -d --name cm-ci-pg \
|
||||
# Where every step leaves its full output, on the HOST, so a failed
|
||||
# run can be read afterwards without the actions-log API.
|
||||
#
|
||||
# STEP is a breadcrumb: each step overwrites it on entry, so the last
|
||||
# value names the step that died. Two steps used to create this
|
||||
# directory, which made "the directory exists" ambiguous about how far
|
||||
# the job got — and that ambiguity cost a whole debugging cycle.
|
||||
mkdir -p /tmp/ci-logs && rm -f /tmp/ci-logs/*.log /tmp/ci-logs/STEP
|
||||
echo "1-start-postgres" > /tmp/ci-logs/STEP
|
||||
set -x
|
||||
# Run-scoped name. `cm-ci-pg` was shared by every run, so a second
|
||||
# run removed the first one's database while it was still being used.
|
||||
# The concurrency group above should prevent overlap; this makes the
|
||||
# failure impossible rather than merely unlikely.
|
||||
docker rm -fv "$PG" 2>/dev/null || true
|
||||
# --shm-size: Docker defaults /dev/shm to 64MB. cm-testkit creates a
|
||||
# database per test and the suite runs many at once, so Postgres
|
||||
# exhausts its parallel-query segments mid-run. It surfaces as
|
||||
# `could not resize shared memory segment ... No space left on device`
|
||||
# during MIGRATIONS, which reads like a schema fault and is not one.
|
||||
# Hit locally on 2026-08-19; scripts/test-server.sh carries the same
|
||||
# flag for the same reason.
|
||||
docker run -d --name "$PG" \
|
||||
--shm-size=1g \
|
||||
-e POSTGRES_PASSWORD=postgres -e POSTGRES_USER=postgres \
|
||||
-p 127.0.0.1:55432:5432 postgres:16-alpine
|
||||
for i in $(seq 1 30); do
|
||||
docker exec cm-ci-pg pg_isready -U postgres >/dev/null 2>&1 && break
|
||||
docker exec "$PG" pg_isready -U postgres >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
docker exec cm-ci-pg pg_isready -U postgres
|
||||
docker exec "$PG" pg_isready -U postgres
|
||||
|
||||
# Rust lives in a container because gw-04 has no cargo. The named volumes
|
||||
# are the whole reason this is not painfully slow: without them every run
|
||||
@@ -75,6 +113,12 @@ jobs:
|
||||
# The token is a repo secret, so it is masked in logs and never in git.
|
||||
- name: Rust tests
|
||||
run: |
|
||||
echo "2-rust" > /tmp/ci-logs/STEP
|
||||
# The DOCKER RUN's own output, on the host. cargo's log only exists
|
||||
# if cargo runs; run 494 died in this step with no rust.log at all,
|
||||
# which means apt-get, git config or docker itself failed and the
|
||||
# message went only to the job log we cannot read.
|
||||
set +e
|
||||
docker run --rm --network host \
|
||||
-v "$PWD":/w -w /w \
|
||||
-v cm-ci-cargo-registry:/usr/local/cargo/registry \
|
||||
@@ -86,24 +130,66 @@ jobs:
|
||||
-e CARGO_NET_GIT_FETCH_WITH_CLI=true \
|
||||
-e FORGE_TOKEN='${{ secrets.FORGE_TOKEN }}' \
|
||||
-e CM_TEST_DATABASE_URL=postgres://postgres:[email protected]:55432/postgres \
|
||||
-v /tmp/ci-logs:/cilog \
|
||||
rust:1.96-slim \
|
||||
sh -c 'set -e
|
||||
# NO APOSTROPHES BELOW THIS LINE. Everything here is inside a
|
||||
# single-quoted sh -c, so one apostrophe in a COMMENT closes the
|
||||
# quote and the step dies with "unexpected EOF while looking for
|
||||
# matching quote" — before running anything, which is why no log
|
||||
# ever appeared. Runs 491 through 496 failed on the word
|
||||
# "cm-api" followed by an apostrophe-s.
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq pkg-config libssl-dev cmake git >/dev/null
|
||||
# nodejs: the vm_tool_gate shell tests in cm-api EXECUTE the generated
|
||||
# PreToolUse hook, which parses its JSON payload with node (no jq
|
||||
# in the runtime image; node is guaranteed there because Claude
|
||||
# Code is a node program). Without it the hook takes its
|
||||
# allow-and-record-inert path and the two "blocks" tests fail —
|
||||
# which is how this was found, on the first push that carried them.
|
||||
apt-get install -y -qq pkg-config libssl-dev cmake git nodejs >/dev/null
|
||||
git config --global url."https://oauth2:[email protected]/".insteadOf "https://git.redclaw.dev/"
|
||||
cargo test --workspace'
|
||||
# Full output to a host-mounted file, then the tail, then exit
|
||||
# with the cargo status. Piping cargo into `tail` would report
|
||||
# the exit code of tail — a green job over a red suite. The log
|
||||
# survives the container so a failure is diagnosable at all:
|
||||
# the Gitea actions-log API returns 403 for our token, and three
|
||||
# failed runs were debugged blind before this existed.
|
||||
set +e
|
||||
cargo test --workspace > /cilog/rust.log 2>&1
|
||||
rc=$?
|
||||
set -e
|
||||
grep -nE "test result: FAILED|^error(\[|:)|panicked at" /cilog/rust.log | head -40 || true
|
||||
tail -40 /cilog/rust.log
|
||||
exit $rc' > /tmp/ci-logs/rust-step.log 2>&1
|
||||
rc=$?
|
||||
set -e
|
||||
tail -60 /tmp/ci-logs/rust-step.log
|
||||
exit $rc
|
||||
|
||||
# -v, not just -f. The postgres image declares a VOLUME, so removing the
|
||||
# container without it orphans an anonymous data directory EVERY run.
|
||||
# cm-testkit creates a database per test, so those grew to 2.8 GB each —
|
||||
# 38 GB of leaked volumes before anyone noticed.
|
||||
- name: Stop test Postgres
|
||||
if: always()
|
||||
run: docker rm -f cm-ci-pg 2>/dev/null || true
|
||||
run: |
|
||||
echo "3-stop-postgres" >> /tmp/ci-logs/STEP
|
||||
docker rm -fv "$PG" 2>/dev/null || true
|
||||
|
||||
# node 22 is on the host, so these run directly.
|
||||
- name: Frontend checks
|
||||
working-directory: frontend
|
||||
run: |
|
||||
npm ci --no-audit --no-fund
|
||||
npm run typecheck
|
||||
npm run test
|
||||
echo "4-frontend" >> /tmp/ci-logs/STEP
|
||||
set +e
|
||||
npm ci --no-audit --no-fund > /tmp/ci-logs/npm-ci.log 2>&1; ci=$?
|
||||
npm run typecheck > /tmp/ci-logs/typecheck.log 2>&1; tc=$?
|
||||
npm run test > /tmp/ci-logs/vitest.log 2>&1; vt=$?
|
||||
set -e
|
||||
for f in npm-ci typecheck vitest; do
|
||||
printf '=== %s ===\n' "$f"; tail -25 "/tmp/ci-logs/$f.log" || true
|
||||
done
|
||||
[ "$ci" -eq 0 ] && [ "$tc" -eq 0 ] && [ "$vt" -eq 0 ]
|
||||
# Lint is advisory: the repo currently has pre-existing max-lines and
|
||||
# set-state-in-effect errors that predate this pipeline. Failing the
|
||||
# deploy on them would mean nothing could ship until they are cleared.
|
||||
@@ -117,6 +203,15 @@ jobs:
|
||||
|
||||
- name: Build + push images
|
||||
run: |
|
||||
# Same host-log treatment as the test job. The build job failed four
|
||||
# runs in a row with nothing readable: the actions-log API returns
|
||||
# 403 for our token, so "failure" was the entire message. It turned
|
||||
# out to be transient disk pressure — a runtime image being built on
|
||||
# this same host at the same time — and a docs-only commit was the
|
||||
# first casualty, which made it look like a code regression.
|
||||
mkdir -p /tmp/ci-logs
|
||||
echo "5-build" > /tmp/ci-logs/STEP
|
||||
df -h / > /tmp/ci-logs/build-disk.log 2>&1
|
||||
set -eu
|
||||
SHA=$(git rev-parse --short HEAD)
|
||||
echo "SHA=$SHA" >> "$GITHUB_ENV"
|
||||
@@ -145,6 +240,7 @@ jobs:
|
||||
-t "$REGISTRY/$NAMESPACE/$svc:latest" .
|
||||
docker push "$REGISTRY/$NAMESPACE/$svc:main-$SHA"
|
||||
docker push "$REGISTRY/$NAMESPACE/$svc:latest"
|
||||
echo "$svc built+pushed" >> /tmp/ci-logs/build-progress.log
|
||||
done
|
||||
|
||||
# `docker push :latest` does NOT reliably move the tag on this registry:
|
||||
@@ -156,6 +252,7 @@ jobs:
|
||||
# previously leave prod on a stale image.
|
||||
- name: Repoint :latest
|
||||
run: |
|
||||
echo "6-repoint" > /tmp/ci-logs/STEP
|
||||
set -eu
|
||||
for svc in server frontend broker; do
|
||||
ct=$(curl -s -o /tmp/m.json -D- \
|
||||
@@ -174,6 +271,7 @@ jobs:
|
||||
# pipeline exists to prevent.
|
||||
- name: Wait for the rolling deploy
|
||||
run: |
|
||||
echo "7-wait-deploy" > /tmp/ci-logs/STEP
|
||||
set -eu
|
||||
want=$(docker image inspect -f '{{.Id}}' "$REGISTRY/$NAMESPACE/server:latest")
|
||||
for i in $(seq 1 30); do
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
# Release: build the images both deploy targets share, assemble the SIGNED
|
||||
# air-gapped bundle, verify it offline, rehearse the customer's install, and
|
||||
# attach everything to the Gitea release for the tag.
|
||||
#
|
||||
# Moved from .github/workflows/ and rewritten for this forge. The old copy could
|
||||
# never have run: `runs-on: ubuntu-latest` matches no runner here, and
|
||||
# `softprops/action-gh-release` talks to GitHub's API, not Gitea's.
|
||||
#
|
||||
# The signing key is a repo secret (BUNDLE_SIGNING_KEY, hex ed25519 from
|
||||
# `clawmates-bundler keygen`). The matching PUBLIC key is published out of band
|
||||
# so customers can verify a bundle before `docker load`.
|
||||
name: release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ["v*"]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
bundle:
|
||||
runs-on: gw04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Version from tag
|
||||
run: |
|
||||
# workflow_dispatch has no tag; fall back to the short sha so a manual
|
||||
# run produces a clearly-not-a-release version rather than an empty one.
|
||||
if [ "${GITHUB_REF_TYPE:-}" = "tag" ]; then
|
||||
echo "VERSION=${GITHUB_REF_NAME#v}" >> "$GITHUB_ENV"
|
||||
else
|
||||
echo "VERSION=0.0.0-$(git rev-parse --short HEAD)" >> "$GITHUB_ENV"
|
||||
fi
|
||||
|
||||
- name: Build images
|
||||
run: |
|
||||
set -eu
|
||||
docker build -t "clawmates/server:$VERSION" -f images/server.Dockerfile .
|
||||
docker build -t "clawmates/frontend:$VERSION" -f images/frontend.Dockerfile .
|
||||
docker build -t "clawmates/broker:$VERSION" -f images/broker.Dockerfile .
|
||||
docker build -t "clawmates/agent-base:$VERSION" images/agent-base
|
||||
docker build -t "clawmates/agent-browser:$VERSION" images/agent-browser
|
||||
docker pull -q postgres:16-alpine
|
||||
docker pull -q tecnativa/docker-socket-proxy:0.3
|
||||
|
||||
# syft goes in the workspace, NOT /usr/local/bin. The host executor runs
|
||||
# as root on the production gateway; a release should not leave binaries
|
||||
# behind on it.
|
||||
- name: SBOMs for every shipped image
|
||||
run: |
|
||||
set -eu
|
||||
mkdir -p dist/sboms .tools
|
||||
curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh \
|
||||
| sh -s -- -b .tools
|
||||
for image in server frontend broker agent-base agent-browser; do
|
||||
./.tools/syft "clawmates/$image:$VERSION" -o spdx-json \
|
||||
> "dist/sboms/$image.spdx.json"
|
||||
done
|
||||
|
||||
- name: Save image tarballs
|
||||
run: |
|
||||
set -eu
|
||||
mkdir -p dist/images
|
||||
docker save "clawmates/server:$VERSION" -o dist/images/server.tar
|
||||
docker save "clawmates/frontend:$VERSION" -o dist/images/frontend.tar
|
||||
docker save "clawmates/broker:$VERSION" -o dist/images/broker.tar
|
||||
docker save "clawmates/agent-base:$VERSION" -o dist/images/agent-base.tar
|
||||
docker save "clawmates/agent-browser:$VERSION" -o dist/images/agent-browser.tar
|
||||
docker save tecnativa/docker-socket-proxy:0.3 -o dist/images/socket-proxy.tar
|
||||
docker save postgres:16-alpine -o dist/images/postgres.tar
|
||||
du -sh dist/images
|
||||
|
||||
# gw-04 has no cargo, so the bundler builds in a container — same pattern
|
||||
# and same cache volumes as deploy.yml. The forge credential is here
|
||||
# because cargo resolves the whole workspace, which includes cm-brain's
|
||||
# private clawhdf5 git dependency.
|
||||
- name: Build bundler
|
||||
run: |
|
||||
docker run --rm \
|
||||
-v "$PWD":/w -w /w \
|
||||
-v cm-ci-cargo-registry:/usr/local/cargo/registry \
|
||||
-v cm-ci-cargo-git:/usr/local/cargo/git \
|
||||
-v cm-ci-target:/w/target \
|
||||
-e SQLX_OFFLINE=true -e CARGO_NET_GIT_FETCH_WITH_CLI=true \
|
||||
-e FORGE_TOKEN='${{ secrets.FORGE_TOKEN }}' \
|
||||
rust:1.96-slim \
|
||||
sh -c 'set -e
|
||||
apt-get update -qq
|
||||
apt-get install -y -qq pkg-config libssl-dev cmake git >/dev/null
|
||||
git config --global url."https://oauth2:[email protected]/".insteadOf "https://git.redclaw.dev/"
|
||||
cargo build --release -p clawmates-bundler
|
||||
# Copy the binary OUT of the target volume and into the workspace.
|
||||
# /w/target is a named docker volume, so anything left there is
|
||||
# invisible to later steps running on the host — which is exactly
|
||||
# how this failed the first time (exit 127, No such file).
|
||||
mkdir -p /w/.tools
|
||||
cp target/release/clawmates-bundler /w/.tools/clawmates-bundler'
|
||||
test -x .tools/clawmates-bundler || { echo "bundler did not land in the workspace"; exit 1; }
|
||||
|
||||
- name: Assemble and sign the bundle
|
||||
env:
|
||||
BUNDLE_SIGNING_KEY: ${{ secrets.BUNDLE_SIGNING_KEY }}
|
||||
run: |
|
||||
set -eu
|
||||
test -n "$BUNDLE_SIGNING_KEY" || { echo "BUNDLE_SIGNING_KEY is empty"; exit 1; }
|
||||
umask 077
|
||||
printf '%s' "$BUNDLE_SIGNING_KEY" > /tmp/release.key
|
||||
BUNDLER=.tools/clawmates-bundler
|
||||
ARTIFACTS=""
|
||||
for tar in dist/images/*.tar; do
|
||||
ARTIFACTS="$ARTIFACTS $tar=images/$(basename "$tar")"
|
||||
done
|
||||
for migration in migrations/*.sql; do
|
||||
ARTIFACTS="$ARTIFACTS $migration=migrations/$(basename "$migration")"
|
||||
done
|
||||
# shellcheck disable=SC2086
|
||||
"$BUNDLER" assemble dist/bundle "$VERSION" /tmp/release.key \
|
||||
deploy/compose/docker-compose.yml=compose/docker-compose.yml \
|
||||
deploy/compose/clawmates.toml=compose/clawmates.toml \
|
||||
deploy/compose/.env.example=compose/.env.example \
|
||||
deploy/e2e/scenarios.toml=compose/scenarios.toml \
|
||||
images/seccomp/agent-profile.json=seccomp/agent-profile.json \
|
||||
deploy/airgapped/install.sh=install.sh \
|
||||
"$BUNDLER"=bin/clawmates-bundler \
|
||||
dist/sboms/server.spdx.json=sboms/server.spdx.json \
|
||||
dist/sboms/frontend.spdx.json=sboms/frontend.spdx.json \
|
||||
dist/sboms/agent-base.spdx.json=sboms/agent-base.spdx.json \
|
||||
dist/sboms/agent-browser.spdx.json=sboms/agent-browser.spdx.json \
|
||||
$ARTIFACTS
|
||||
chmod +x dist/bundle/bin/clawmates-bundler dist/bundle/install.sh
|
||||
rm -f /tmp/release.key
|
||||
|
||||
- name: Verify the bundle offline (public key only)
|
||||
env:
|
||||
BUNDLE_SIGNING_KEY: ${{ secrets.BUNDLE_SIGNING_KEY }}
|
||||
run: |
|
||||
set -eu
|
||||
umask 077
|
||||
printf '%s' "$BUNDLE_SIGNING_KEY" > /tmp/release.key
|
||||
.tools/clawmates-bundler pubkey /tmp/release.key dist/release.pub
|
||||
rm -f /tmp/release.key
|
||||
# The customer's exact procedure: the public half only, inside a
|
||||
# NETWORK-DISABLED container, proving verification needs no internet.
|
||||
docker run --rm --network none \
|
||||
-v "$PWD/dist:/dist:ro" \
|
||||
ubuntu:24.04 \
|
||||
/dist/bundle/bin/clawmates-bundler verify /dist/bundle /dist/release.pub
|
||||
|
||||
- name: Tarball
|
||||
run: tar -C dist -czf "clawmates-bundle-$VERSION.tgz" bundle
|
||||
|
||||
# The clean-room install rehearsal is DELIBERATELY NOT RUN HERE.
|
||||
#
|
||||
# Every other step in this job is inert with respect to production: it
|
||||
# builds images, writes SBOMs, signs a bundle, and verifies it in a
|
||||
# network-isolated container. The rehearsal is the one step whose entire
|
||||
# purpose is to stand a full stack UP and then tear it down with
|
||||
# `down -v` — on the machine serving production.
|
||||
#
|
||||
# On 2026-08-13 it did exactly that: the bundled compose file declares
|
||||
# `name: clawmates`, which beat --project-directory, so the rehearsal
|
||||
# adopted the live stack and its teardown deleted clawmates_pgdata. The
|
||||
# database was lost and there were no backups.
|
||||
#
|
||||
# scripts/rehearse-install.sh is now isolated (`-p rehearse-$$` plus a
|
||||
# guard that refuses the production project name) and its health probe is
|
||||
# fixed, so it is safe to run — just not on this host. Run it on a build
|
||||
# box or throwaway VM:
|
||||
#
|
||||
# CLAWMATES_BUNDLER=… COMPOSE=/path/to/compose-v2 ./scripts/rehearse-install.sh
|
||||
#
|
||||
# Restore this step here only if the release ever moves off the gateway.
|
||||
|
||||
# Gitea's release API, not softprops/action-gh-release (GitHub-only).
|
||||
# Create-or-reuse, so a re-run of the same tag updates instead of 409ing.
|
||||
# Tag pushes only. On workflow_dispatch GITHUB_REF_NAME is the BRANCH, so
|
||||
# this step previously created a release — and a git tag — literally named
|
||||
# "main". A smoke-test run must not be able to mint a release.
|
||||
- name: Attach to the Gitea release
|
||||
if: github.ref_type == 'tag'
|
||||
env:
|
||||
FORGE_TOKEN: ${{ secrets.FORGE_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
API="https://git.redclaw.dev/api/v1/repos/$GITHUB_REPOSITORY/releases"
|
||||
TAG="${GITHUB_REF_NAME}"
|
||||
id=$(curl -sS -H "Authorization: token $FORGE_TOKEN" "$API/tags/$TAG" \
|
||||
| sed -n 's/.*"id":[ ]*\([0-9]\+\).*/\1/p' | head -1)
|
||||
if [ -z "$id" ]; then
|
||||
id=$(curl -sS -X POST -H "Authorization: token $FORGE_TOKEN" \
|
||||
-H 'content-type: application/json' \
|
||||
-d "{\"tag_name\":\"$TAG\",\"name\":\"$TAG\",\"body\":\"Air-gapped bundle for $TAG. Verify with the published public key before docker load.\"}" \
|
||||
"$API" | sed -n 's/.*"id":[ ]*\([0-9]\+\).*/\1/p' | head -1)
|
||||
fi
|
||||
test -n "$id" || { echo "could not create or find the release for $TAG"; exit 1; }
|
||||
for f in "clawmates-bundle-$VERSION.tgz" dist/release.pub; do
|
||||
code=$(curl -sS -o /dev/null -w '%{http_code}' -X POST \
|
||||
-H "Authorization: token $FORGE_TOKEN" \
|
||||
-F "attachment=@$f" \
|
||||
"$API/$id/assets?name=$(basename "$f")")
|
||||
echo " attached $(basename "$f") (HTTP $code)"
|
||||
case "$code" in 20*) ;; *) echo "attach failed"; exit 1 ;; esac
|
||||
done
|
||||
|
||||
# Release artifacts are GBs of image tarballs on the production gateway.
|
||||
# Never `docker image prune -a` here: clawmates/agent-*:dev exist in no
|
||||
# registry and are the source of the microVM rootfs files.
|
||||
- name: Reclaim disk
|
||||
if: always()
|
||||
run: |
|
||||
rm -rf dist .tools "clawmates-bundle-$VERSION.tgz" || true
|
||||
for i in server frontend broker agent-base agent-browser; do
|
||||
docker rmi "clawmates/$i:$VERSION" 2>/dev/null || true
|
||||
done
|
||||
df -h / | awk 'NR==2{print " disk free: "$4}'
|
||||
@@ -1,209 +0,0 @@
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
gates:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: File size budget (1500 lines)
|
||||
run: ./ci/check-loc.sh
|
||||
- name: No placeholder markers
|
||||
run: ./ci/check-no-placeholders.sh
|
||||
- name: Compose config validates
|
||||
run: POSTGRES_PASSWORD=ci docker compose -f deploy/compose/docker-compose.yml config -q
|
||||
|
||||
rust:
|
||||
runs-on: ubuntu-latest
|
||||
needs: gates
|
||||
# Compile sqlx query! macros against the committed .sqlx cache (no DB needed).
|
||||
# Tests need a live Postgres — locally cm-testkit reads CM_TEST_DATABASE_URL
|
||||
# from .cargo/config.toml pointing at scripts/test-server.sh's host container.
|
||||
# The fleet act_runner uses the `host` executor (jobs run on morpheus/tank/
|
||||
# architect natively, not inside a container), so we start a per-run postgres
|
||||
# container and reach it via its bridge IP. GITHUB_RUN_ID scopes the name so
|
||||
# concurrent jobs on the same runner don't collide.
|
||||
#
|
||||
# GIT_CONFIG_GLOBAL points at a per-job empty file so cargo's git fetches
|
||||
# bypass the runner's includeIf mapping of git.redclaw.dev → /slab/projects
|
||||
# (local mirror lags and misses recently-pinned commits like the clawverse
|
||||
# rev cm-brain depends on). clawverse is public; no auth needed.
|
||||
env:
|
||||
SQLX_OFFLINE: "true"
|
||||
GIT_CONFIG_GLOBAL: /tmp/ci-empty-gitconfig-${{ github.run_id }}
|
||||
steps:
|
||||
- name: Prepare empty gitconfig for cargo fetches
|
||||
run: touch "$GIT_CONFIG_GLOBAL"
|
||||
- uses: actions/checkout@v4
|
||||
- name: Start postgres sidecar
|
||||
run: |
|
||||
set -euo pipefail
|
||||
NAME="ci-pg-${GITHUB_RUN_ID}"
|
||||
docker rm -f "$NAME" >/dev/null 2>&1 || true
|
||||
docker run -d --name "$NAME" \
|
||||
-e POSTGRES_PASSWORD=postgres \
|
||||
-e POSTGRES_DB=postgres \
|
||||
postgres:16-alpine >/dev/null
|
||||
# `.NetworkSettings.IPAddress` is empty (and template-parse errors) on
|
||||
# modern Docker where the IP lives under `.Networks.<name>.IPAddress`.
|
||||
# The range form picks the first non-empty IP across whatever network
|
||||
# docker put the container on.
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$NAME")
|
||||
if [ -z "$PG_IP" ]; then
|
||||
echo "postgres has no reachable IP" >&2
|
||||
docker inspect "$NAME" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "PG_CONTAINER=$NAME" >> "$GITHUB_ENV"
|
||||
echo "CM_TEST_DATABASE_URL=postgres://postgres:postgres@${PG_IP}:5432/postgres" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 30); do
|
||||
if docker exec "$NAME" pg_isready -U postgres -q >/dev/null 2>&1; then
|
||||
echo "postgres ready at ${PG_IP} after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "postgres never became ready" >&2
|
||||
docker logs "$NAME" >&2 || true
|
||||
exit 1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: 1.96.0
|
||||
components: rustfmt, clippy
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Format
|
||||
run: cargo fmt --all --check
|
||||
- name: Clippy
|
||||
run: cargo clippy --workspace --all-targets -- -D warnings
|
||||
- name: Test
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Re-derive the postgres URL inline instead of trusting that
|
||||
# CM_TEST_DATABASE_URL propagated through $GITHUB_ENV — act_runner
|
||||
# v1.0.8 has been observed to swallow env-file writes here.
|
||||
IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG_CONTAINER")
|
||||
[ -n "$IP" ] || { echo "no PG IP" >&2; exit 1; }
|
||||
export CM_TEST_DATABASE_URL="postgres://postgres:postgres@${IP}:5432/postgres"
|
||||
echo "using $CM_TEST_DATABASE_URL"
|
||||
cargo test --workspace
|
||||
- name: Air-gapped installer verify path
|
||||
run: ./ci/test-install.sh
|
||||
- name: Cleanup postgres sidecar
|
||||
if: always()
|
||||
run: docker rm -f "${PG_CONTAINER:-}" >/dev/null 2>&1 || true
|
||||
|
||||
frontend:
|
||||
runs-on: ubuntu-latest
|
||||
needs: gates
|
||||
defaults:
|
||||
run:
|
||||
working-directory: frontend
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
- name: Install
|
||||
run: npm ci
|
||||
if: ${{ hashFiles('frontend/package-lock.json') != '' }}
|
||||
- name: Lint
|
||||
run: npm run lint
|
||||
if: ${{ hashFiles('frontend/package-lock.json') != '' }}
|
||||
- name: Typecheck
|
||||
run: npm run typecheck
|
||||
if: ${{ hashFiles('frontend/package-lock.json') != '' }}
|
||||
- name: Unit and component tests
|
||||
run: npm test
|
||||
if: ${{ hashFiles('frontend/package-lock.json') != '' }}
|
||||
|
||||
# e2e is intentionally disabled for now. The suite has real product/test
|
||||
# drift (locators pointing at older versions of pages) that would need a
|
||||
# dedicated pass to reconcile — see the earlier follow-up notes. Publish
|
||||
# doesn't depend on this job anyway, but keeping it enabled produced a
|
||||
# steady red on every push that wasn't actionable. Flip `if:` back to
|
||||
# `true` (or delete the guard) when the tests get realigned.
|
||||
e2e:
|
||||
if: false
|
||||
runs-on: ubuntu-latest
|
||||
needs: [rust, frontend]
|
||||
env:
|
||||
SQLX_OFFLINE: "true"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
toolchain: 1.96.0
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
- name: Install frontend dependencies
|
||||
working-directory: frontend
|
||||
run: npm ci
|
||||
- name: Install Playwright browsers
|
||||
working-directory: frontend
|
||||
run: npx playwright install --with-deps chromium
|
||||
- name: Run end-to-end journeys against the real backend
|
||||
working-directory: frontend
|
||||
run: npx playwright test --grep-invert "@visual"
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: failure()
|
||||
with:
|
||||
name: playwright-traces
|
||||
path: frontend/test-results/
|
||||
|
||||
# Rolling deploy: on green main only, build the three prod images, tag with
|
||||
# :main-<sha> + :latest, push to the fleet registry (redclaw-web-01:5000 via
|
||||
# its Tailscale IP — the fleet's daemons trust it in insecure-registries by
|
||||
# IP, not by hostname). GW-04's clawmates-deploy.timer rolls forward within
|
||||
# ~1 minute of the push. Skipped on PRs.
|
||||
#
|
||||
# `e2e` is intentionally NOT in `needs`: it launches its own postgres + dex
|
||||
# via `docker run` on the host and then reaches them via 127.0.0.1, which
|
||||
# fails from inside the act_runner container. Migrating e2e to a physical
|
||||
# build node is a separate task; until then e2e is signal-only, not gating.
|
||||
# `rust` was restored to `needs` once the flakes were rooted out (approvals
|
||||
# SSE race + warm_pool agent-seeding + a couple health-check ambiguities).
|
||||
publish:
|
||||
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
needs: [gates, rust, frontend]
|
||||
env:
|
||||
REGISTRY: 100.94.185.103:5000
|
||||
NAMESPACE: clawmates
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Resolve short SHA
|
||||
run: echo "SHA=${GITHUB_SHA::7}" >> "$GITHUB_ENV"
|
||||
- name: Build images
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for svc in broker server frontend; do
|
||||
docker build \
|
||||
-t "${REGISTRY}/${NAMESPACE}/${svc}:main-${SHA}" \
|
||||
-t "${REGISTRY}/${NAMESPACE}/${svc}:latest" \
|
||||
-f "images/${svc}.Dockerfile" .
|
||||
done
|
||||
- name: Push images
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for svc in broker server frontend; do
|
||||
docker push "${REGISTRY}/${NAMESPACE}/${svc}:main-${SHA}"
|
||||
docker push "${REGISTRY}/${NAMESPACE}/${svc}:latest"
|
||||
done
|
||||
- name: Summary
|
||||
run: |
|
||||
{
|
||||
echo "## Published images"
|
||||
echo ""
|
||||
for svc in broker server frontend; do
|
||||
echo "- \`${REGISTRY}/${NAMESPACE}/${svc}:main-${SHA}\`"
|
||||
echo "- \`${REGISTRY}/${NAMESPACE}/${svc}:latest\`"
|
||||
done
|
||||
echo ""
|
||||
echo "GW-04 timer picks these up within ~1 minute."
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -1,117 +0,0 @@
|
||||
# Release: build the images both deploy targets share, assemble the
|
||||
# SIGNED air-gapped bundle, verify it offline, and attach everything to
|
||||
# the tag. The signing key lives in repo secrets (BUNDLE_SIGNING_KEY,
|
||||
# hex ed25519 from `clawmates-bundler keygen`); the matching public key is
|
||||
# published out of band so customers can verify before docker load.
|
||||
name: release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ["v*"]
|
||||
|
||||
jobs:
|
||||
bundle:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
|
||||
- name: Version from tag
|
||||
run: echo "VERSION=${GITHUB_REF_NAME#v}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build images
|
||||
run: |
|
||||
docker build -t "clawmates/server:$VERSION" -f images/server.Dockerfile .
|
||||
docker build -t "clawmates/frontend:$VERSION" -f images/frontend.Dockerfile .
|
||||
docker build -t "clawmates/broker:$VERSION" -f images/broker.Dockerfile .
|
||||
docker build -t "clawmates/agent-base:$VERSION" images/agent-base
|
||||
docker build -t "clawmates/agent-browser:$VERSION" images/agent-browser
|
||||
docker pull postgres:16-alpine
|
||||
|
||||
- name: SBOMs for every shipped image
|
||||
run: |
|
||||
mkdir -p dist/sboms
|
||||
curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh \
|
||||
| sh -s -- -b /usr/local/bin
|
||||
for image in server frontend broker agent-base agent-browser; do
|
||||
syft "clawmates/$image:$VERSION" -o spdx-json \
|
||||
> "dist/sboms/$image.spdx.json"
|
||||
done
|
||||
|
||||
- name: Save image tarballs
|
||||
run: |
|
||||
mkdir -p dist/images
|
||||
docker save "clawmates/server:$VERSION" -o dist/images/server.tar
|
||||
docker save "clawmates/frontend:$VERSION" -o dist/images/frontend.tar
|
||||
docker save "clawmates/broker:$VERSION" -o dist/images/broker.tar
|
||||
docker pull tecnativa/docker-socket-proxy:0.3
|
||||
docker save tecnativa/docker-socket-proxy:0.3 -o dist/images/socket-proxy.tar
|
||||
docker save "clawmates/agent-base:$VERSION" -o dist/images/agent-base.tar
|
||||
docker save "clawmates/agent-browser:$VERSION" -o dist/images/agent-browser.tar
|
||||
docker save postgres:16-alpine -o dist/images/postgres.tar
|
||||
|
||||
- name: Build bundler
|
||||
run: cargo build --release -p clawmates-bundler
|
||||
|
||||
- name: Assemble and sign the bundle
|
||||
env:
|
||||
BUNDLE_SIGNING_KEY: ${{ secrets.BUNDLE_SIGNING_KEY }}
|
||||
run: |
|
||||
printf '%s' "$BUNDLE_SIGNING_KEY" > /tmp/release.key
|
||||
BUNDLER=target/release/clawmates-bundler
|
||||
ARTIFACTS=""
|
||||
for tar in dist/images/*.tar; do
|
||||
ARTIFACTS="$ARTIFACTS $tar=images/$(basename "$tar")"
|
||||
done
|
||||
for migration in migrations/*.sql; do
|
||||
ARTIFACTS="$ARTIFACTS $migration=migrations/$(basename "$migration")"
|
||||
done
|
||||
# shellcheck disable=SC2086
|
||||
"$BUNDLER" assemble dist/bundle "$VERSION" /tmp/release.key \
|
||||
deploy/compose/docker-compose.yml=compose/docker-compose.yml \
|
||||
deploy/compose/clawmates.toml=compose/clawmates.toml \
|
||||
deploy/compose/.env.example=compose/.env.example \
|
||||
deploy/e2e/scenarios.toml=compose/scenarios.toml \
|
||||
images/seccomp/agent-profile.json=seccomp/agent-profile.json \
|
||||
deploy/airgapped/install.sh=install.sh \
|
||||
"$BUNDLER"=bin/clawmates-bundler \
|
||||
dist/sboms/server.spdx.json=sboms/server.spdx.json \
|
||||
dist/sboms/frontend.spdx.json=sboms/frontend.spdx.json \
|
||||
dist/sboms/agent-base.spdx.json=sboms/agent-base.spdx.json \
|
||||
dist/sboms/agent-browser.spdx.json=sboms/agent-browser.spdx.json \
|
||||
$ARTIFACTS
|
||||
chmod +x dist/bundle/bin/clawmates-bundler dist/bundle/install.sh
|
||||
rm /tmp/release.key
|
||||
|
||||
- name: Verify the bundle offline (public key only)
|
||||
env:
|
||||
BUNDLE_SIGNING_KEY: ${{ secrets.BUNDLE_SIGNING_KEY }}
|
||||
run: |
|
||||
printf '%s' "$BUNDLE_SIGNING_KEY" > /tmp/release.key
|
||||
target/release/clawmates-bundler pubkey /tmp/release.key dist/release.pub
|
||||
rm /tmp/release.key
|
||||
# The customer's exact procedure: only the public half — and
|
||||
# inside a NETWORK-DISABLED container, proving verification
|
||||
# needs no internet (the air-gapped contract).
|
||||
docker run --rm --network none \
|
||||
-v "$PWD/dist:/dist:ro" \
|
||||
ubuntu:24.04 \
|
||||
/dist/bundle/bin/clawmates-bundler verify /dist/bundle /dist/release.pub
|
||||
|
||||
- name: Tarball
|
||||
run: tar -C dist -czf "clawmates-bundle-$VERSION.tgz" bundle
|
||||
|
||||
- name: Clean-room install rehearsal
|
||||
run: |
|
||||
docker tag "clawmates/server:$VERSION" clawmates/server:latest
|
||||
docker tag "clawmates/frontend:$VERSION" clawmates/frontend:latest
|
||||
docker tag "clawmates/broker:$VERSION" clawmates/broker:latest
|
||||
./scripts/rehearse-install.sh
|
||||
|
||||
- name: Attach to release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
clawmates-bundle-*.tgz
|
||||
dist/release.pub
|
||||
+4
-1
@@ -28,4 +28,7 @@ deploy/compose/.env.*
|
||||
# runtime at MacBook-specific paths. docker-compose picks this file up
|
||||
# automatically, so committing it would silently reconfigure anyone who runs
|
||||
# deploy/compose.
|
||||
deploy/compose/docker-compose.override.yml
|
||||
# deploy/compose/docker-compose.override.yml is TRACKED as of 2026-09-18: it
|
||||
# holds the fixes for the five local bring-up gaps and every credential in it is
|
||||
# a ${VAR:?} reference into .env. It lived only on one laptop until then.
|
||||
crates/cm-decide/eval/out/
|
||||
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO auth_sessions (token_hash, user_id, expires_at, scope)\n VALUES ($1, $2, $3, $4)",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Uuid",
|
||||
"Timestamptz",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "105f8cc147247c69b3c45e2e3eb27fc33b1976accdda66ec3ccc7c57afecc8b9"
|
||||
}
|
||||
-16
@@ -1,16 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE topology_runs\n SET checkpoint = $2, last_event_id = $3, updated_at = now()\n WHERE id = $1",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Jsonb",
|
||||
"Int8"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "5fcbd4d6adbf02489051e2fa63d1df670863bf90e55c1c0ac0ab011759cbd272"
|
||||
}
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE topology_runs\n SET status = 'queued', updated_at = now()\n WHERE status = 'running' AND updated_at < now() - make_interval(secs => $1)",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Float8"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "7298995b5b58aed46888bb9e5c8d331483aee162fc6bcf1e53232d2afc7c3e62"
|
||||
}
|
||||
-56
@@ -1,56 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE topology_runs\n SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()\n WHERE id = (\n SELECT id FROM topology_runs\n WHERE status = 'queued'\n ORDER BY created_at\n FOR UPDATE SKIP LOCKED\n LIMIT 1\n )\n RETURNING id, workspace_id, task, graph, checkpoint, last_event_id, tier",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "workspace_id",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "task",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "graph",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "checkpoint",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "last_event_id",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "tier",
|
||||
"type_info": "Text"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "9eae6ca16ffc9346456128ce676ef04f3478f873d6ac5f95154b797f454f44c0"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT a.id, a.name, a.accent,\n COALESCE(SUM(u.credits), 0)::BIGINT AS \"credits!\",\n COALESCE(SUM(u.tokens_in + u.tokens_out), 0)::BIGINT AS \"tokens!\",\n COUNT(u.id)::BIGINT AS \"runs!\"\n FROM agents a\n LEFT JOIN usage_events u ON u.agent_id = a.id\n WHERE a.workspace_id = $1\n GROUP BY a.id, a.name, a.accent\n ORDER BY \"credits!\" DESC, \"tokens!\" DESC, a.name",
|
||||
"query": "SELECT a.id, a.name, a.accent,\n COALESCE(SUM(u.credits), 0)::BIGINT AS \"credits!\",\n COALESCE(SUM(u.tokens_in + u.tokens_out), 0)::BIGINT AS \"tokens!\",\n COUNT(u.id)::BIGINT AS \"runs!\"\n FROM agents a\n LEFT JOIN usage_events u ON u.agent_id = a.id\n -- deleted_at: a soft-deleted agent is gone everywhere else, so\n -- listing it here made deletion look like a no-op — the operator\n -- deletes it, the board still shows it, and deleting again does\n -- nothing because the row is already marked.\n WHERE a.workspace_id = $1 AND a.deleted_at IS NULL\n GROUP BY a.id, a.name, a.accent\n ORDER BY \"credits!\" DESC, \"tokens!\" DESC, a.name",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -48,5 +48,5 @@
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "d4ef449c48b15519b7195be637dca3d456140ce477993d25a87e209174f79aba"
|
||||
"hash": "d5bc028ca030daed4e6111990945d8d7011414d830f7d6d0a04980efb79af2a6"
|
||||
}
|
||||
+8
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT u.id, u.workspace_id, u.role\n FROM auth_sessions s\n JOIN users u ON u.id = s.user_id\n WHERE s.token_hash = $1 AND s.expires_at > now()",
|
||||
"query": "SELECT u.id, u.workspace_id, u.role, s.scope\n FROM auth_sessions s\n JOIN users u ON u.id = s.user_id\n WHERE s.token_hash = $1 AND s.expires_at > now()",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -17,6 +17,11 @@
|
||||
"ordinal": 2,
|
||||
"name": "role",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "scope",
|
||||
"type_info": "Text"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
@@ -25,10 +30,11 @@
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "900827c5c8c24f4861120e98e3cc8a5b70f22e9f4b4168c9e8eb51c53d68bdae"
|
||||
"hash": "e8f7cb9c34be37fe16c5406e9263159693674dda567b60a1f87c6763ec448951"
|
||||
}
|
||||
Generated
+1682
-86
File diff suppressed because it is too large
Load Diff
@@ -7,6 +7,7 @@ members = [
|
||||
"crates/cm-config",
|
||||
"crates/cm-db",
|
||||
"crates/cm-llm",
|
||||
"crates/cm-decide",
|
||||
"crates/cm-runtime",
|
||||
"crates/cm-tools",
|
||||
"crates/cm-safety",
|
||||
|
||||
@@ -6,9 +6,9 @@ the organizational *topology* that fits it.**
|
||||
Clawmates is a multi-agent platform where every unit of work is a **topology**: a graph of role-slots
|
||||
bound to real AI agents ("claws"). The same model nests recursively — a team is a topology of claws, a
|
||||
company is a topology of teams, an org is a topology of companies — so you compose and run agentic
|
||||
systems from one agent up to an entire organization. A safety invariant runs through all of it:
|
||||
**authority is topology-invariant** — no choice of structure can let an agent exceed its sandbox (spec
|
||||
§15).
|
||||
systems from one agent up to an entire organization. The work itself runs as **missions**: recipes of
|
||||
phases, staffed by team templates, executed in isolated containers or Firecracker microVMs, and checked
|
||||
by an independent judge before anything is called done.
|
||||
|
||||
Live at **[clawmates.work](https://clawmates.work)**.
|
||||
|
||||
@@ -16,34 +16,52 @@ Live at **[clawmates.work](https://clawmates.work)**.
|
||||
|
||||
## Features
|
||||
|
||||
- **The deploy ladder — single → team → company → org.** Pick a scale; each rung instantiates a baseline
|
||||
topology and binds it to real, individually-chattable claws. Higher rungs *compose* the rung below:
|
||||
a company is staffed with teams, an org with companies.
|
||||
- **Missions.** The primary user-facing unit: a scoped multi-team workload with team templates, live
|
||||
progress, bulk operations, and a canvas view. Missions are hard-required to include a team template
|
||||
and can span research + development teams.
|
||||
- **Recursive execution.** Running a parent runs each child's whole sub-topology, all the way down to
|
||||
the leaf claws — on a **durable, crash-resumable** runner (checkpointed per step, with cancellation).
|
||||
- **INFRA tier via Herdr.** Every fleet node runs a persistent `clawmates-node` daemon (herdr) reachable
|
||||
from the platform: mission wizard picks a target runtime, `fleet_herdr` dispatches on launch, and
|
||||
a Live Pane surfaces each node's herdr TUI via xterm.js.
|
||||
- **12 organizational topologies.** Hierarchical, pipeline, swarm, mesh, debate, hub-spoke, star-MoE,
|
||||
market, ring, flat, holacratic, blackboard — over five execution patterns.
|
||||
- **Multi-topology comparison + evolution.** Run one task across many topologies and get a quality/cost
|
||||
**Pareto front**; a MAP-Elites search evolves better (kind × team-size) configurations using the
|
||||
comparison harness as fitness.
|
||||
- **Recursive zoom canvas.** One view for every tier: click a node to drill down (org→company→team→claw),
|
||||
breadcrumb to zoom back up.
|
||||
- **§15 safety by construction.** Agents run tool-free in network-isolated sandboxes; every
|
||||
sandbox-leaving action is a gated, human-approvable "door" tool. A secret broker holds credentials that
|
||||
never reach agent code, and an allow-listed Docker socket caps blast radius.
|
||||
- **Heterogeneous models.** Bind any node to a different backend; supported providers include Claude
|
||||
(default for refine), GLM, Kimi, Groq. Configured per-node in the wizard.
|
||||
- **Level-Up.** Per-claw and per-team improvement proposals with an inbox + review drawer.
|
||||
- **Beszel + Tailscale integration.** First-class routes to the fleet's monitoring hub and mesh.
|
||||
- **Self-hostable.** A single-node Docker Compose deployment runs the whole platform with the same
|
||||
network-segmented security model as the Kubernetes path; a separate rolling-deploy path serves
|
||||
clawmates.work from gw-04 against the fleet registry.
|
||||
- **One dashboard.** The workspace home is a single live dashboard with a tier rail — **World, Agents,
|
||||
Missions, Repos, Infra, Podcast**. World is a live Gource-style graph of what agents are doing; the
|
||||
org → company → team structure is one expandable graph; each agent opens a slide-out "computer"
|
||||
(chat, terminal, files, brain).
|
||||
- **Missions.** A mission is a **recipe** (`templates/workflows/*.toml`) of ordered phases — research,
|
||||
coding, security scan, benchmark — each with a task brief and a `done_when` completion condition, staffed
|
||||
by a **team template** (`templates/teams/*.toml`). Seven recipes ship: `research_only`,
|
||||
`research_and_code`, `security_hardening`, `refactor`, `benchmark`, `continuous_research`, `self_audit`.
|
||||
- **Two execution tiers.** *Container tier*: each mission gets its own ZeroClaw runtime container on the
|
||||
gateway, running Claude Code (`claude_cli`) with fallback to Kimi and GLM. *MicroVM tier*: phases run
|
||||
in Firecracker microVMs on fleet nodes (`clawmates-node` + the `fcagent` guest), with per-backend
|
||||
egress (Claude, GLM, Kimi, or a local model over vsock).
|
||||
- **An independent judge.** Every conditioned phase is judged. When a cross-provider judge is configured
|
||||
(prod: GLM, with Kimi as automatic fallback) it is a model from a *different* provider family than the
|
||||
agents; without one (the self-host default) the phase is judged by Claude and recorded as **not**
|
||||
independent. The judge runs its own allow-listed
|
||||
checks — tests, `rg`, git — against a copy of the work, commits to a verification plan before reading
|
||||
the evidence, and installs npm dependencies offline from the lockfile. A phase that fails is retried
|
||||
with the judge's guidance, up to `max_iterations`. Measured with `scripts/judge-eval.sh` (15 known-answer
|
||||
cases).
|
||||
- **Judge quota watchdog.** Polls the GLM and Kimi usage APIs, warns at 80% of any window, and switches
|
||||
judging to the fallback at 95% (`GET /api/judge/quota`).
|
||||
- **Delivery to git.** Each phase's work is committed and pushed to a mission branch on the repo's forge;
|
||||
`continuous_research` auto-merges additive-only changes into the vault.
|
||||
- **Tool gates on every agent call.** A `PreToolUse` gate (both tiers) refuses destructive and exfiltrating
|
||||
commands, protects its own hook files, enforces per-role policy (e.g. a read-only verifier), and records
|
||||
task-permission and argument-provenance ("taint") violations in shadow mode. A stop gate sends a
|
||||
microVM agent back to work while its `done_when_check` fails, up to 3 times.
|
||||
- **Skills.** A catalog of skills bound to team roles, delivered to mission agents through the MCP
|
||||
skills door; per-mission skill triage and skill-use measurement.
|
||||
- **Project memory.** Each repository keeps a `.brain` (ClawhDF5) of every judge verdict; missions recall
|
||||
relevant past verdicts, and the `self_audit` recipe audits the whole record for failure patterns.
|
||||
- **Continuous research + podcast.** Harvests new arXiv papers into an Obsidian vault, triages them,
|
||||
writes an analysis and a two-host script, and renders audio with ElevenLabs.
|
||||
- **Decision tier (`cm-decide`).** Cheap calibrated classifiers (Jev) for gut-check decisions: the §15
|
||||
door governor (allow / deny / hold for approval), skill triage, paper triage, memory rerank.
|
||||
- **Master Planner.** The "+" deploy is a chat that proposes and scaffolds a team for a goal
|
||||
(specialists, swarm, scheduled, triggered).
|
||||
- **12 organizational topologies + evolution.** Hierarchical, pipeline, swarm, mesh, debate, hub-spoke,
|
||||
star-MoE, market, ring, flat, holacratic, blackboard; multi-topology comparison with a quality/cost
|
||||
**Pareto front**, and MAP-Elites evolution over (kind × team size).
|
||||
- **Fleet.** Multi-user workspaces with quotas; a node pool (`clawmates-node` over Tailscale) with
|
||||
capacity-aware placement, drain, per-node tool versions and one-click updates, and Beszel metrics
|
||||
feeding a rules engine.
|
||||
- **Agent-to-agent comms.** Chat rooms, a gated delegation bridge, per-claw door identity and A2A ingress.
|
||||
- **Self-hostable.** A single-node Docker Compose deployment runs the whole platform.
|
||||
|
||||
---
|
||||
|
||||
@@ -58,12 +76,13 @@ A Rust workspace (the platform) + a Next.js app (the web UI).
|
||||
| `cm-domain` | Shared types: ids, roles, workspaces, users |
|
||||
| `cm-topology` | Topology data model: 12-kind taxonomy, graph, classifier, per-kind builders + heuristics |
|
||||
| `cm-orchestrator` | Execution engine: async control-flow over a generic `TurnExecutor`; planners, comparison harness, MAP-Elites evolution |
|
||||
| `cm-runtime` | The §15-safe per-tenant agent runtime |
|
||||
| `cm-brain` | Shared LLM planning + reasoning primitives used by orchestrator and refine |
|
||||
| `cm-api` | REST/SSE API + streaming gateway + recursive tier execution + the MCP "door" + missions + fleet_herdr |
|
||||
| `cm-runtime` | The §15-safe per-tenant agent runtime and the LLM provider registry |
|
||||
| `cm-brain` | Facade over the canonical `.brain` (ClawhDF5 brain-pack): one HDF5 file per agent or repo holding definition + memory |
|
||||
| `cm-decide` | Typed, calibrated decisions (Jev classifiers) for the platform's code to branch on |
|
||||
| `cm-api` | REST/SSE API, missions + phase runner, judge, tool gates, delivery, the MCP doors, fleet, podcast |
|
||||
| `cm-db` | Postgres persistence (sqlx, offline-checked) |
|
||||
| `cm-llm` | Provider abstraction over the model backends |
|
||||
| `cm-secrets` / `clawmates-broker` | The secret broker — credentials never leave it |
|
||||
| `cm-llm` | Provider abstraction (Anthropic-format and OpenAI-compatible backends) |
|
||||
| `cm-secrets` | Secret storage behind the broker |
|
||||
| `cm-sandbox` / `cm-safety` | Sandbox provisioning + the §15 approval/gating model |
|
||||
| `cm-tools` | Tool contract + registry surfaced through the door |
|
||||
| `cm-testkit` | Shared test utilities (scripted providers, fixture builders) |
|
||||
@@ -73,21 +92,26 @@ A Rust workspace (the platform) + a Next.js app (the web UI).
|
||||
|
||||
| Bin | Role |
|
||||
|---|---|
|
||||
| `clawmates-server` | The single server binary (API + gateway + runtime + scheduler) |
|
||||
| `clawmates-server` | The single server binary (API + gateway + runtime + background workers) |
|
||||
| `clawmates-broker` | Out-of-process secret broker over a private unix socket |
|
||||
| `clawmates-node` | The **herdr** daemon: runs on every fleet node, dispatches missions to that node, exposes a TUI streamed into the Live Pane |
|
||||
| `clawmates-node` | Fleet node daemon: registers with the server, runs microVMs, terminals and tool updates on its node |
|
||||
| `fcagent` | PID 1 inside each Firecracker microVM; answers the host over vsock |
|
||||
|
||||
### Frontend (`frontend/`)
|
||||
|
||||
Next.js 16, React 19, Tailwind v4. Two-tier rail (structure + context), the recursive zoom canvas, and
|
||||
the deploy wizards. Missions surface (canvas + list + wizard + live pane + team tab + live events),
|
||||
Herdr sessions UI, Level-Up inbox + review drawer. Talks to the backend through a same-origin `/api`
|
||||
proxy that swaps the session for a bearer token and streams SSE.
|
||||
Next.js 16, React 19, Tailwind v4. The dashboard at `/` (tier rail, World graph, agent computer),
|
||||
plus agent, team, skills, approvals and team-management pages. Talks to the backend through a
|
||||
same-origin `/api` proxy that swaps the session for a bearer token and streams SSE.
|
||||
|
||||
### Content (`templates/`)
|
||||
|
||||
`templates/teams/` (12 team templates: roles, prompts, brain seeds, skill bindings) and
|
||||
`templates/workflows/` (7 recipes), loaded at boot. How mature each one is — which have been run and
|
||||
what they delivered — is tracked in [`docs/TEMPLATE-MATURITY.md`](docs/TEMPLATE-MATURITY.md).
|
||||
|
||||
### Data plane
|
||||
|
||||
Postgres, with the server self-migrating on boot. Migration series `0001–0057+`; slice-9 cleanup
|
||||
(`0053`) retired the legacy research/loops path after missions replaced it.
|
||||
Postgres, with the server self-migrating on boot from `migrations/` (`0001`–`0087`).
|
||||
|
||||
---
|
||||
|
||||
@@ -111,7 +135,8 @@ Owner + workspace on first boot. Then:
|
||||
- **API health** → http://localhost:8080/healthz
|
||||
|
||||
Backends: `openai_compat` by default (point `[llm].base_url` at vLLM/Ollama/llama.cpp); set
|
||||
`provider = "anthropic"` + `ANTHROPIC_API_KEY` to use Claude. Auth is `local` by default or `clerk` at
|
||||
`provider = "anthropic"` + `ANTHROPIC_API_KEY` to use Claude. Extra named providers (GLM, Kimi, a local
|
||||
model) go in `[[llm.providers]]` and are selected as `name:model`. Auth is `local` by default or `clerk` at
|
||||
runtime. See [`deploy/compose/README.md`](deploy/compose/README.md) for all knobs and the broker
|
||||
master-key backup step.
|
||||
|
||||
@@ -142,40 +167,45 @@ Run the reproducible topology benchmark (offline-deterministic; real models with
|
||||
cargo run -p cm-orchestrator --example topology_bench --features provider
|
||||
```
|
||||
|
||||
**End-to-end verification against a deployment:**
|
||||
|
||||
```bash
|
||||
scripts/verify-mission-delivery.sh <scenario> # launches real missions, asserts delivery, gates, judge
|
||||
scripts/judge-eval.sh # the judge's 15 known-answer cases (JUDGE=kimi to compare)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Production deployment (clawmates.work on gw-04)
|
||||
|
||||
The public site runs a different path than the airgapped compose. Gitea Actions builds and pushes
|
||||
`broker`, `server`, and `frontend` images to the fleet registry at
|
||||
`100.94.185.103:5000/clawmates/<svc>:latest`. gw-04 runs a systemd-timer-driven rolling deploy:
|
||||
- **CI:** a push to `main` runs [`.gitea/workflows/deploy.yml`](.gitea/workflows/deploy.yml) on the
|
||||
gw-04 Gitea runner: `cargo test --workspace`, then builds the images and pushes
|
||||
`main-<sha>` and `:latest` to the fleet registry at `100.94.185.103:5000`. The workflow then waits
|
||||
up to 5 minutes for prod to report the new commit.
|
||||
- **Roll-out:** [`deploy/gw-04/clawmates-deploy.sh`](deploy/gw-04/clawmates-deploy.sh), run every minute
|
||||
by `clawmates-deploy.timer`, drift-checks each running image against `:latest` and recreates the
|
||||
service on drift. Logs: `/var/log/clawmates-deploy.log`.
|
||||
- **Host config** lives outside git on gw-04: `/opt/clawmates/.env` and `/opt/clawmates/clawmates.toml`
|
||||
(provider registry, including the judge's GLM and Kimi providers).
|
||||
|
||||
- **Deploy script:** [`deploy/gw-04/clawmates-deploy.sh`](deploy/gw-04/clawmates-deploy.sh) — polls the
|
||||
registry, drift-checks each service's running image ID against `:latest`, and calls
|
||||
`docker compose up -d <svc>` on drift. Portable across `docker compose` v2 and legacy `docker-compose` v1.
|
||||
- **Timer + unit:** installed alongside the script under `/etc/systemd/system/`.
|
||||
- **Logs:** `/var/log/clawmates-deploy.log`.
|
||||
- **Compose file:** references registry-prefixed images directly — no retag bridging.
|
||||
|
||||
**gw-04-specific gotchas (bit us on 2026-07-09 and 2026-07-12):**
|
||||
**gw-04-specific gotchas:**
|
||||
|
||||
- `clawmates-runtime` on gw-04 is **not compose-managed** — it's a standalone `docker run` invocation.
|
||||
Provider env (`ANTHROPIC_API_KEY`, `ZEROCLAW_providers__*`) must be set on that container.
|
||||
Provider env (`ZEROCLAW_providers__*`) must be set on that container.
|
||||
- The server container runs as **UID 65532** (distroless nonroot). Any bind-mount host path must be
|
||||
`chown 65532:65532` before boot or the server can't write.
|
||||
- Per-team ZeroClaw containers inherit `ZEROCLAW_providers__*` from the server; those envs must live
|
||||
on the compose `server` block, not just on shared runtime.
|
||||
- Per-mission ZeroClaw containers get their provider keys forwarded from the server; those envs must live
|
||||
on the compose `server` block, not just on the shared runtime.
|
||||
- The judge's container (`clawmates-runtime` on `clawmates_core`) has **no route to the internet** by
|
||||
design; dependencies it needs are installed offline.
|
||||
|
||||
---
|
||||
|
||||
## CI budgets
|
||||
## Code size budget
|
||||
|
||||
- **Hard limit:** 1500 lines per source file. CI fails.
|
||||
- **Soft limit:** 1100 lines. CI warns — split before it hurts.
|
||||
- Enforced by [`ci/check-loc.sh`](ci/check-loc.sh).
|
||||
|
||||
Common split pattern: extract sub-components (`MissionLivePane`, `AutoProvisionCard`) into their own
|
||||
file when the parent creeps past the soft limit.
|
||||
[`ci/check-loc.sh`](ci/check-loc.sh) defines a soft limit of 1100 lines and a hard limit of 1500 per
|
||||
source file. It is **not currently run in CI**, and 12 files exceed the hard limit (the largest,
|
||||
`crates/cm-api/src/phase_runner.rs`, is ~3,450 lines). Split files when you touch them.
|
||||
|
||||
---
|
||||
|
||||
@@ -183,32 +213,26 @@ file when the parent creeps past the soft limit.
|
||||
|
||||
**Shipped**
|
||||
- ✅ Pure-Rust topology engine — model → classify → build (all 12 kinds) → execute → compare (Pareto) →
|
||||
workflow → LLM-judge → evolve.
|
||||
- ✅ Topologies UI: catalog browser, builder/visualizer, multi-topology comparison with a Pareto scatter.
|
||||
- ✅ Durable topology runs — crash-resumable, checkpointed per step, cancellable, with live SSE.
|
||||
- ✅ **The full deploy ladder** — single → team → company → org, with recursive execution down to the
|
||||
leaf claws and a recursive zoom canvas + two-tier navigation.
|
||||
- ✅ **Missions (slices 1–9)** — multi-team model, hard-required team templates, canvas + wizard + list,
|
||||
auto-refresh + Team tab + Live events tab, add/edit/delete toolbar, bulk delete, security-scan +
|
||||
benchmark trigger buttons, per-mission repo checkout on launch, LevelUpInbox mounted.
|
||||
- ✅ **Herdr (phases 0–3)** — `clawmates-node` daemon (systemd/launchd persistence), `fleet_herdr`
|
||||
dispatch module + node daemon ops, missions `runtime_kind` + `target_node` schema, wizard runtime
|
||||
picker with `on_launch` auto-dispatch, Live Pane (xterm.js → node's herdr TUI), INFRA-tier Herdr
|
||||
sessions surface.
|
||||
- ✅ **Refine on Opus 4.8** — before/after diff view, accept/cancel/restore controls.
|
||||
- ✅ **Level-Up** — per-claw/per-team improvement proposals, inbox + review drawer.
|
||||
- ✅ §15 safety: tool-free sandboxes, the gated MCP "door" (with real email/Slack delivery), the secret
|
||||
broker, allow-listed Docker socket.
|
||||
- ✅ Self-host: single-node Docker Compose with full network segmentation.
|
||||
- ✅ Prod path on gw-04: registry-driven rolling deploy via systemd timer.
|
||||
evolve; durable, crash-resumable topology runs with live SSE.
|
||||
- ✅ The deploy ladder (single → team → company → org) and the single live dashboard.
|
||||
- ✅ Missions as recipes of judged phases, on container and microVM tiers, delivered to git.
|
||||
- ✅ Independent cross-provider judge with its own checks, verification plans, offline npm installs,
|
||||
Kimi fallback and a quota watchdog.
|
||||
- ✅ `PreToolUse` gate on both tiers (floor rules, role policy, protected hook files) with task-permission
|
||||
and argument-provenance (taint) in shadow; microVM stop gate.
|
||||
- ✅ Skills catalog + MCP skills door; skill triage and skill-use measurement.
|
||||
- ✅ Per-repo project memory (`.brain`) and the `self_audit` recipe over it.
|
||||
- ✅ Continuous research → vault → podcast, end to end.
|
||||
- ✅ Decision tier (`cm-decide`): calibrated door governor with a held band for approval.
|
||||
- ✅ Multi-user workspaces, node pool with capacity-aware placement, Beszel metrics + rules.
|
||||
- ✅ Self-host via Docker Compose; CI → registry → timer-driven roll-out on gw-04.
|
||||
|
||||
**Next**
|
||||
- Team / company templates as first-class saved catalogs (compose orgs from reusable building blocks).
|
||||
- Per-leaf nested checkpoint resume (today the recursive runner resumes at parent-node granularity).
|
||||
- Persona injection into runtime turns (beyond role-driven prompting).
|
||||
- Richer per-tier dashboards (company coordination, org portfolio/governance metrics).
|
||||
- Group lifecycle management (delete/edit a deployed team/company/org; deprovision its agents) —
|
||||
bulk delete shipped for missions, extending to teams/companies/orgs next.
|
||||
- Enforce task permission and argument provenance (both still in shadow, gathering evidence).
|
||||
- Evidence the remaining team templates (4 of 12 still need a target stack: mobile, gpu, threejs, and
|
||||
`insight_research`).
|
||||
- A dedicated judge key, so no other consumer of a shared provider plan can starve the judge.
|
||||
- Wire `ci/check-loc.sh` into CI once the oversized files are split.
|
||||
|
||||
**Research**
|
||||
- The accompanying paper, *Large Dynamic Agentic Topologies* (`papers/dynamic-agentic-topologies.md`):
|
||||
@@ -218,8 +242,27 @@ file when the parent creeps past the soft limit.
|
||||
|
||||
## Safety
|
||||
|
||||
The network segmentation **is** the security model. Agent sandboxes run with no network at all; only the
|
||||
browser container has egress. The secret broker is reachable only over a private socket and credentials
|
||||
never enter agent code. The server reaches Docker through an allow-listed socket proxy that can manage
|
||||
sandbox containers and nothing else. Every sandbox-leaving action is gated behind a human approval. No
|
||||
topology — and no switch between topologies — can bypass any of this.
|
||||
The design goal is **authority that no topology can widen** (spec §15): a structure, or a switch between
|
||||
structures, never gives an agent more reach than its sandbox. What that means today, per tier:
|
||||
|
||||
- **Chat / §15 agents** run tool-free; every action that leaves the sandbox goes through the MCP door,
|
||||
where a calibrated governor allows, denies, or **holds for human approval**.
|
||||
- **Mission agents** do run tools (Bash, file edits) inside their own container or microVM, behind the
|
||||
`PreToolUse` gate. MicroVMs have no NIC and reach only an allow-listed set of hosts through a proxy.
|
||||
**Container-tier missions currently have open public-internet egress** (tailnet and host SSH are
|
||||
blocked); see [`docs/MISSION-EGRESS.md`](docs/MISSION-EGRESS.md) and
|
||||
[`docs/TASK-PERMISSION-AND-TAINT.md`](docs/TASK-PERMISSION-AND-TAINT.md) for the measurements and the
|
||||
controls being built on top.
|
||||
- Platform credentials are held by the secret broker behind a private socket, and a mission container
|
||||
gets only a narrowly scoped skills token, never a ClawMates session. **Model-provider keys never enter
|
||||
a container-tier mission** when the LLM proxy is on (`CLAWMATES_LLM_PROXY=1`, as on prod): the
|
||||
container holds a per-mission token, Claude Code's base URL points at the server's proxy on an
|
||||
unpublished port, and the proxy adds the real credential — honouring the token only while its mission
|
||||
is running. Behind that, delivery refuses to push any change containing a server key, and every
|
||||
recorded event, judge verdict and judge input is redacted. **MicroVM guests hold no provider key
|
||||
either** (node daemon 0.5.0+): the guest's CLI talks to its own loopback, fcagent pipes that to the
|
||||
node, and the node relays it to the proxy on the server's tailnet-only port — no key on the node or in
|
||||
the guest. The server reaches Docker through an allow-listed socket proxy.
|
||||
- The gate is a guardrail against accidents and obvious exfiltration, not a boundary against a
|
||||
determined agent (indirection defeats string matching). The boundaries are the VM, the network policy
|
||||
and the broker.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "clawmates-node"
|
||||
version = "0.4.0"
|
||||
version = "0.5.0"
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
@@ -56,14 +56,42 @@ pub fn uses_local_model(backend: Option<&str>) -> bool {
|
||||
/// case. An error means it should have had one and could not — reported by the
|
||||
/// caller, never silently swallowed, because the symptom otherwise is an agent
|
||||
/// that hangs on its first turn.
|
||||
/// Where a VM's model pipe leads: the node's own model, or the server's LLM
|
||||
/// proxy for a backend whose credential the guest must not hold.
|
||||
///
|
||||
/// The relay is how a microVM reaches a hosted model WITHOUT a provider key in
|
||||
/// the guest. The guest's CLI points at its loopback (`127.0.0.1:11434`, which
|
||||
/// fcagent already pipes here for every backend), holds a per-mission token,
|
||||
/// and the server's proxy adds the real credential. This node copies bytes and
|
||||
/// never sees a key. Only a tailnet address is accepted, so no message from the
|
||||
/// server can point a node's pipe at the internet.
|
||||
pub fn target_for(backend: Option<&str>, relay: Option<&str>) -> Result<Option<String>, String> {
|
||||
if uses_local_model(backend) {
|
||||
return Ok(Some(OLLAMA_ADDR.to_string()));
|
||||
}
|
||||
let Some(r) = relay.map(str::trim).filter(|r| !r.is_empty()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let addr: std::net::SocketAddr = r
|
||||
.parse()
|
||||
.map_err(|e| format!("model relay {r:?} is not an ip:port ({e})"))?;
|
||||
match addr.ip() {
|
||||
std::net::IpAddr::V4(ip) if ip.octets()[0] == 100 && (64..128).contains(&ip.octets()[1]) => {
|
||||
Ok(Some(addr.to_string()))
|
||||
}
|
||||
_ => Err(format!("model relay {r} is not a tailnet (100.64.0.0/10) address — refused")),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start(
|
||||
uds: &Path,
|
||||
vm_id: &str,
|
||||
backend: Option<&str>,
|
||||
relay: Option<&str>,
|
||||
) -> Result<Option<(PathBuf, tokio::task::JoinHandle<()>)>, String> {
|
||||
if !uses_local_model(backend) {
|
||||
let Some(target) = target_for(backend, relay)? else {
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
let path = PathBuf::from(format!("{}_{}", uds.display(), MODEL_PORT));
|
||||
// Firecracker leaves these behind exactly as it does its own socket, and a
|
||||
// stale file makes bind fail with EADDRINUSE.
|
||||
@@ -72,7 +100,7 @@ pub fn start(
|
||||
UnixListener::bind(&path).map_err(|e| format!("bind {}: {e}", path.display()))?;
|
||||
|
||||
eprintln!(
|
||||
"microvm {vm_id}: local model socket on {} -> {OLLAMA_ADDR}",
|
||||
"microvm {vm_id}: model socket on {} -> {target}",
|
||||
path.display()
|
||||
);
|
||||
let vm = vm_id.to_string();
|
||||
@@ -81,8 +109,9 @@ pub fn start(
|
||||
match listener.accept().await {
|
||||
Ok((s, _)) => {
|
||||
let vm = vm.clone();
|
||||
let target = target.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = pipe(s).await {
|
||||
if let Err(e) = pipe(s, &target).await {
|
||||
// Loud, because the failure a mission sees is a turn
|
||||
// that never answers. A refused connection here means
|
||||
// the node's model server is down, and that is worth
|
||||
@@ -102,10 +131,10 @@ pub fn start(
|
||||
}
|
||||
|
||||
/// Splice one guest connection onto a fresh connection to the node's model.
|
||||
async fn pipe(mut guest: tokio::net::UnixStream) -> Result<(), String> {
|
||||
let mut model = TcpStream::connect(OLLAMA_ADDR)
|
||||
async fn pipe(mut guest: tokio::net::UnixStream, target: &str) -> Result<(), String> {
|
||||
let mut model = TcpStream::connect(target)
|
||||
.await
|
||||
.map_err(|e| format!("connect {OLLAMA_ADDR}: {e}"))?;
|
||||
.map_err(|e| format!("connect {target}: {e}"))?;
|
||||
tokio::io::copy_bidirectional(&mut guest, &mut model)
|
||||
.await
|
||||
.map(|_| ())
|
||||
@@ -116,6 +145,25 @@ async fn pipe(mut guest: tokio::net::UnixStream) -> Result<(), String> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The local backend still pipes to Ollama, whatever relay is offered.
|
||||
#[test]
|
||||
fn the_local_backend_always_gets_the_nodes_own_model() {
|
||||
assert_eq!(target_for(Some("local-ornith"), Some("100.102.112.85:8089")).unwrap().as_deref(), Some(OLLAMA_ADDR));
|
||||
}
|
||||
|
||||
/// A hosted backend relays only when told to, and only to the tailnet.
|
||||
#[test]
|
||||
fn a_hosted_backend_relays_only_to_a_tailnet_address() {
|
||||
assert_eq!(target_for(Some("claude"), None).unwrap(), None, "no relay offered: no pipe, as before");
|
||||
assert_eq!(
|
||||
target_for(Some("glm"), Some("100.102.112.85:8089")).unwrap().as_deref(),
|
||||
Some("100.102.112.85:8089")
|
||||
);
|
||||
for bad in ["8.8.8.8:443", "127.0.0.1:8089", "10.0.0.5:8089", "100.128.0.1:8089", "api.z.ai:443", "100.102.112.85"] {
|
||||
assert!(target_for(Some("claude"), Some(bad)).is_err(), "{bad} must be refused");
|
||||
}
|
||||
}
|
||||
|
||||
/// Only the backends that are meant to have a local model get one.
|
||||
///
|
||||
/// The negative half is the point: an unrecognised backend acquiring a route
|
||||
@@ -144,24 +192,31 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// The guest cannot name a destination, so there is nothing to validate.
|
||||
/// The guest still cannot name a destination.
|
||||
///
|
||||
/// This asserts the property that makes this module safe enough to skip the
|
||||
/// allow-list entirely: the upstream address is a constant. If it ever
|
||||
/// becomes a parameter, this file needs everything `egress` has.
|
||||
/// This module used to dial one constant, and that was what let it skip the
|
||||
/// allow-list. It now has two destinations — the node's own model, and the
|
||||
/// server's LLM proxy as a relay — but the property the constant protected
|
||||
/// holds: the pipe protocol carries no address, the destination is chosen
|
||||
/// by `target_for` from the SERVER's `vm_create` message, and a relay is
|
||||
/// accepted only on the tailnet (see `a_hosted_backend_relays_only_to_a_tailnet_address`).
|
||||
/// If the guest ever gets to supply a target, this file needs everything
|
||||
/// `egress` has.
|
||||
#[test]
|
||||
fn the_upstream_address_is_a_constant_not_an_input() {
|
||||
fn the_guest_never_chooses_where_the_pipe_goes() {
|
||||
let src = include_str!("local_model.rs");
|
||||
// Needles are split so they do not match themselves in this file.
|
||||
assert_eq!(
|
||||
src.matches(concat!("TcpStream", "::connect(")).count(),
|
||||
1,
|
||||
"exactly one dial site, and it must use the constant"
|
||||
"exactly one dial site"
|
||||
);
|
||||
assert!(src.contains(concat!("TcpStream", "::connect(OLLAMA_ADDR)")));
|
||||
assert!(src.contains(concat!("TcpStream", "::connect(target)")));
|
||||
// `target` reaches `pipe` only from `start`, which gets it only from `target_for`.
|
||||
assert_eq!(src.matches(concat!("target_for", "(backend, relay)")).count(), 1);
|
||||
assert!(
|
||||
OLLAMA_ADDR.starts_with("127.0.0.1:"),
|
||||
"the model server must be reached on loopback only"
|
||||
"the node's model server must be reached on loopback only"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -328,6 +328,10 @@ fn capabilities_from(kvm: bool, firecracker: Option<&str>, backends: &[String])
|
||||
// binary but no KVM is gw-04. Computed here rather than in the
|
||||
// scheduler so the rule sits next to the probe that feeds it.
|
||||
"microvm": kvm && firecracker.is_some(),
|
||||
// The VM model pipe can relay to the server's LLM proxy, so a guest on a
|
||||
// hosted backend needs no provider key (local_model::target_for). The
|
||||
// server relays only to nodes that say so; older nodes keep the key.
|
||||
"model_relay": true,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -328,6 +328,7 @@ pub async fn create(
|
||||
vcpus: u32,
|
||||
mem_mib: u32,
|
||||
backend: Option<&str>,
|
||||
model_relay: Option<&str>,
|
||||
) -> Result<Value, String> {
|
||||
check_id(vm_id)?;
|
||||
let golden = rootfs_for(backend)?;
|
||||
@@ -399,7 +400,7 @@ pub async fn create(
|
||||
// The node's own model, for a backend that has one. Bound before firecracker
|
||||
// starts for the same reason egress is: a guest that dials before the host
|
||||
// listens gets a refusal it will not retry.
|
||||
let (model_uds, model_task) = match crate::local_model::start(&uds, vm_id, backend) {
|
||||
let (model_uds, model_task) = match crate::local_model::start(&uds, vm_id, backend, model_relay) {
|
||||
Ok(Some((p, t))) => (Some(p), Some(t)),
|
||||
Ok(None) => (None, None),
|
||||
// This backend was supposed to have a local model and does not. Not
|
||||
@@ -678,6 +679,7 @@ pub async fn handle_op(op: &str, v: &Value, vms: &Vms) -> (bool, String) {
|
||||
u("vcpus", 2) as u32,
|
||||
u("mem_mib", 2048) as u32,
|
||||
v.get("backend").and_then(Value::as_str),
|
||||
v.get("model_relay").and_then(Value::as_str),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -747,7 +749,7 @@ pub async fn selftest() -> bool {
|
||||
// default — booting the wrong rootfs would report success for whatever came
|
||||
// out of it. Checked here so the guarantee is exercised on real hardware and
|
||||
// not only in a unit test with a temp dir.
|
||||
match create(&vms, "selftest-absent", 2, 512, Some("definitely-not-built")).await {
|
||||
match create(&vms, "selftest-absent", 2, 512, Some("definitely-not-built"), None).await {
|
||||
Err(e) if e.contains("rootfs-definitely-not-built.ext4") => {
|
||||
check(true, "an absent backend image fails by name", String::new())
|
||||
}
|
||||
@@ -763,7 +765,7 @@ pub async fn selftest() -> bool {
|
||||
}
|
||||
|
||||
let started = std::time::Instant::now();
|
||||
let created = match create(&vms, id, 2, 1024, backend.as_deref()).await {
|
||||
let created = match create(&vms, id, 2, 1024, backend.as_deref(), None).await {
|
||||
Ok(v) => {
|
||||
check(
|
||||
true,
|
||||
|
||||
@@ -341,6 +341,11 @@ async fn run() -> Result<(), String> {
|
||||
// for INT-XX markers in event payloads and upserts mission_tasks
|
||||
// rows so the canvas renders a live status timeline.
|
||||
cm_api::task_card_worker::spawn(pool.clone());
|
||||
// Agents apply their own skill drafts. Announced at boot by the spawner
|
||||
// itself, because this flips an approval gate that existed since the
|
||||
// feature shipped — and a safety gate whose state is invisible is one
|
||||
// nobody notices has changed.
|
||||
cm_api::skill_self_authoring::spawn(pool.clone());
|
||||
// Load the workflow recipes now rather than lazily on first mission
|
||||
// create, so a malformed TOML shows up in the boot log instead of
|
||||
// silently yielding a mission with no phase config.
|
||||
@@ -381,6 +386,31 @@ async fn run() -> Result<(), String> {
|
||||
// has to work, so it is checked on the days it does not.
|
||||
cm_api::subscription::report_at_boot(runtime.clone());
|
||||
cm_api::phase_runner::spawn(pool.clone(), runtime.clone(), node_hub.clone());
|
||||
// Scheduled missions. `missions.schedule` has collected a cron from the
|
||||
// wizard since 0047 and NOTHING read it back — every scheduled mission ever
|
||||
// created sat in `draft` forever while the UI said it was on a schedule.
|
||||
// 60s matches the finest cron granularity; the sweep claims atomically and
|
||||
// records each occurrence in `mission_fires`, so replicas and restarts
|
||||
// cannot double-launch a container.
|
||||
// Render finished Continuous Research missions into episodes. A sweep, not
|
||||
// a phase step: rendering is not the agents' work and must not be able to
|
||||
// fail a phase that succeeded, and a transient API error simply retries on
|
||||
// the next tick.
|
||||
// Every 2 minutes, NOT 5. The mission checkout that holds script.md is
|
||||
// deleted 30 minutes after the mission reaches a terminal state, so this
|
||||
// sweep is racing a reaper. Two minutes leaves ~15 attempts inside that
|
||||
// window; a slower sweep loses the episode permanently.
|
||||
cm_api::podcast::spawn(
|
||||
pool.clone(),
|
||||
Some(blob.clone()),
|
||||
std::time::Duration::from_secs(2 * 60),
|
||||
);
|
||||
cm_api::mission_schedule::spawn(
|
||||
pool.clone(),
|
||||
Some(node_hub.clone()),
|
||||
Some(blob.clone()),
|
||||
std::time::Duration::from_secs(60),
|
||||
);
|
||||
// Per-mission runtime container sweeper (C3): tears down mission
|
||||
// runtime containers 30 min after the mission reaches a terminal
|
||||
// state so operators have a window to pull final artifacts.
|
||||
@@ -401,6 +431,16 @@ async fn run() -> Result<(), String> {
|
||||
// row has never deleted a directory — which is why the gateway, the smallest
|
||||
// disk in the fleet, accumulates mission trees that nothing reclaims.
|
||||
cm_api::mission_gc::spawn(pool.clone(), std::time::Duration::from_secs(3600));
|
||||
// Agent lifecycle: reap crews whose missions finished (after a 24h grace so
|
||||
// the results view can still show who did the work) and crews left bound to
|
||||
// nothing. Never touches an agent without an `agent_template_link` row —
|
||||
// that is the operator's own staff, which looks identical to an orphan if
|
||||
// you judge by team membership alone.
|
||||
cm_api::agent_lifecycle::spawn(
|
||||
pool.clone(),
|
||||
runtime.clone(),
|
||||
std::time::Duration::from_secs(3600),
|
||||
);
|
||||
// Fleet backstop: a node whose heartbeats stop (without a clean channel
|
||||
// close) goes offline within ~28s even if its control channel hangs.
|
||||
cm_api::fleet::spawn_node_sweeper(pool.clone(), std::time::Duration::from_secs(8), 20);
|
||||
@@ -408,6 +448,13 @@ async fn run() -> Result<(), String> {
|
||||
cm_api::beszel::spawn_poller(pool.clone(), std::time::Duration::from_secs(15));
|
||||
// Fleet automation: evaluate metric-threshold rules → drain/undrain/alert.
|
||||
cm_api::node_rules::spawn_evaluator(pool.clone(), std::time::Duration::from_secs(20));
|
||||
// Judge providers' plan usage (z.ai, Kimi): warn at 80%, and let the
|
||||
// evaluator skip a judge at 95% for the fallback. Ten minutes: the windows
|
||||
// are hours and days long, and each poll is one tiny GET per provider.
|
||||
cm_api::judge_quota::spawn_poller(std::time::Duration::from_secs(600));
|
||||
// Mission containers reach their models through this, holding a
|
||||
// per-mission token instead of provider keys. Off unless configured.
|
||||
cm_api::llm_proxy::spawn(pool.clone());
|
||||
// Nightly: check upstream for newer dev-tool releases (claude/kimi/ollama).
|
||||
cm_api::tool_versions::spawn_latest_checker(
|
||||
pool.clone(),
|
||||
@@ -457,6 +504,10 @@ async fn run() -> Result<(), String> {
|
||||
// every consequence — an ungated test suite, a scan that scanned nothing —
|
||||
// looked like a normal result rather than a broken deployment.
|
||||
cm_api::runtime_preflight::report_at_boot();
|
||||
// And whether the gateway those missions drive is configured at all. Both
|
||||
// of its variables are read at FIRST USE, so a deployment missing them
|
||||
// boots clean and fails on the first phase someone runs.
|
||||
cm_api::gateway_preflight::report_at_boot();
|
||||
// Graceful shutdown: on SIGTERM/Ctrl-C, stop accepting, finish in-flight
|
||||
// requests, then DRAIN the sandbox managers so no container is left running.
|
||||
let shutdown = async move {
|
||||
|
||||
@@ -25,12 +25,13 @@ futures = "0.3"
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
sqlx = { workspace = true }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] }
|
||||
cm-auth = { path = "../cm-auth" }
|
||||
cm-billing = { path = "../cm-billing" }
|
||||
cm-brain = { path = "../cm-brain" }
|
||||
cm-config = { path = "../cm-config" }
|
||||
cm-db = { path = "../cm-db" }
|
||||
cm-decide = { path = "../cm-decide" }
|
||||
cm-domain = { path = "../cm-domain" }
|
||||
cm-files = { path = "../cm-files" }
|
||||
tar = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
//! Which agents are working, which are finished, and which are orphaned.
|
||||
//!
|
||||
//! A mission mints a crew, and until now the only thing that reaped that crew
|
||||
//! was deleting the mission. A mission that merely *completed* left its agents
|
||||
//! in the roster forever, and a crew whose reap was skipped or failed left
|
||||
//! agents bound to nothing at all — indistinguishable, in the UI, from the
|
||||
//! operator's own staff.
|
||||
//!
|
||||
//! The discriminator is `agent_template_link`. `mission_orchestrator` writes one
|
||||
//! row per claw it mints, recording the template and role slot it was minted
|
||||
//! for. An agent WITHOUT that row was created by a human (or the planner) and is
|
||||
//! part of the workforce: it is never touched here, whatever it is bound to.
|
||||
//! Verified against live data — the two hand-created agents on this deployment
|
||||
//! have no link row and no team membership, while every mission crew member has
|
||||
//! both.
|
||||
//!
|
||||
//! ```text
|
||||
//! owned no template link → the operator's own agent. KEEP.
|
||||
//! active on a running/draft mission → doing work right now. KEEP.
|
||||
//! completed every mission terminal → reapable once past the grace window.
|
||||
//! orphaned minted, bound to nothing → reap.
|
||||
//! ```
|
||||
//!
|
||||
//! `completed` waits out a grace window rather than reaping the moment a mission
|
||||
//! finishes: the results view, the World's 24h replay and "who did this work?"
|
||||
//! all read the crew AFTER the run ends. Reaping on the terminal transition
|
||||
//! would delete the answer at the moment the question gets asked.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use sqlx::{PgPool, Row};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// How long a finished crew is kept before it is reaped. Matches the World's
|
||||
/// 24h window for finished missions, so nothing the UI can still show is
|
||||
/// collected out from under it.
|
||||
pub const COMPLETED_GRACE_HOURS: i64 = 24;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AgentState {
|
||||
Owned,
|
||||
Active,
|
||||
Completed,
|
||||
Orphaned,
|
||||
/// Soft-deleted by an operator. The `agents` row and its history survive.
|
||||
Deleted,
|
||||
}
|
||||
|
||||
impl AgentState {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
AgentState::Owned => "owned",
|
||||
AgentState::Active => "active",
|
||||
AgentState::Completed => "completed",
|
||||
AgentState::Orphaned => "orphaned",
|
||||
AgentState::Deleted => "deleted",
|
||||
}
|
||||
}
|
||||
/// `owned` and `active` are NEVER collected, and that is the whole safety
|
||||
/// property of this module.
|
||||
pub fn reapable(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
AgentState::Completed | AgentState::Orphaned | AgentState::Deleted
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Classified {
|
||||
pub id: Uuid,
|
||||
pub name: String,
|
||||
pub state: AgentState,
|
||||
/// When the newest mission this agent served reached a terminal state.
|
||||
/// `None` for owned/active/orphaned.
|
||||
pub finished_hours_ago: Option<f64>,
|
||||
}
|
||||
|
||||
/// The classification, as one query.
|
||||
///
|
||||
/// Soft-deleted rows are INCLUDED, classified `deleted`, and collected: a soft
|
||||
/// delete marks the row and leaves it, so "remove" never became permanent and
|
||||
/// re-deleting did nothing. Purging takes `usage_events` with it — accepted
|
||||
/// deliberately, since the alternative is rows that outlive the decision to
|
||||
/// delete them.
|
||||
const CENSUS_SQL: &str = r#"
|
||||
SELECT a.id,
|
||||
a.name,
|
||||
CASE
|
||||
-- First, so a soft-deleted agent is never mistaken for live staff:
|
||||
-- these rows have no template link either, and would otherwise read
|
||||
-- as 'owned' and be kept forever.
|
||||
WHEN a.deleted_at IS NOT NULL THEN 'deleted'
|
||||
WHEN atl.agent_id IS NULL THEN 'owned'
|
||||
WHEN EXISTS (
|
||||
SELECT 1 FROM team_members tm
|
||||
JOIN mission_teams mt ON mt.team_id = tm.team_id
|
||||
JOIN missions m ON m.id = mt.mission_id
|
||||
WHERE tm.claw_id = a.id AND m.status IN ('running', 'draft')
|
||||
) THEN 'active'
|
||||
WHEN EXISTS (
|
||||
SELECT 1 FROM team_members tm
|
||||
JOIN mission_teams mt ON mt.team_id = tm.team_id
|
||||
WHERE tm.claw_id = a.id
|
||||
) THEN 'completed'
|
||||
ELSE 'orphaned'
|
||||
END AS state,
|
||||
(SELECT EXTRACT(EPOCH FROM (now() - MAX(COALESCE(m.completed_at, m.updated_at)))) / 3600.0
|
||||
FROM team_members tm
|
||||
JOIN mission_teams mt ON mt.team_id = tm.team_id
|
||||
JOIN missions m ON m.id = mt.mission_id
|
||||
WHERE tm.claw_id = a.id) AS finished_hours_ago
|
||||
FROM agents a
|
||||
LEFT JOIN agent_template_link atl ON atl.agent_id = a.id
|
||||
WHERE a.workspace_id = $1
|
||||
ORDER BY a.created_at, a.id
|
||||
"#;
|
||||
|
||||
pub async fn census(pool: &PgPool, workspace_id: Uuid) -> Result<Vec<Classified>, String> {
|
||||
let rows = sqlx::query(CENSUS_SQL)
|
||||
.bind(workspace_id)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| format!("agent census: {e}"))?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|r| {
|
||||
let state = match r.get::<String, _>("state").as_str() {
|
||||
"owned" => AgentState::Owned,
|
||||
"active" => AgentState::Active,
|
||||
"completed" => AgentState::Completed,
|
||||
"deleted" => AgentState::Deleted,
|
||||
_ => AgentState::Orphaned,
|
||||
};
|
||||
Classified {
|
||||
id: r.get("id"),
|
||||
name: r.get("name"),
|
||||
state,
|
||||
finished_hours_ago: r.get::<Option<f64>, _>("finished_hours_ago"),
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// What one sweep did.
|
||||
#[derive(Debug, Default, PartialEq, Eq)]
|
||||
pub struct Swept {
|
||||
pub reaped: usize,
|
||||
pub failed: usize,
|
||||
pub kept_in_grace: usize,
|
||||
}
|
||||
|
||||
/// Decide, without touching the database, whether a classified agent should be
|
||||
/// collected on this pass. Split out so the policy is testable on its own —
|
||||
/// the expensive half is the purge, and the half that can silently delete a
|
||||
/// workforce is this one.
|
||||
pub fn should_reap(c: &Classified, grace_hours: i64) -> bool {
|
||||
match c.state {
|
||||
AgentState::Owned | AgentState::Active => false,
|
||||
// No grace: a human already decided. The soft delete IS the decision,
|
||||
// and these rows have sat for months waiting for something to honour it.
|
||||
AgentState::Deleted => true,
|
||||
AgentState::Orphaned => true,
|
||||
AgentState::Completed => c
|
||||
.finished_hours_ago
|
||||
// No timestamp means we cannot prove the grace has elapsed, so keep
|
||||
// it. A missing date must never read as "old enough to delete".
|
||||
.is_some_and(|h| h >= grace_hours as f64),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reap finished and orphaned crews across every workspace.
|
||||
pub async fn sweep(
|
||||
pool: &PgPool,
|
||||
runtime: &cm_runtime::Runtime,
|
||||
grace_hours: i64,
|
||||
) -> Result<Swept, String> {
|
||||
let workspaces: Vec<Uuid> = sqlx::query_scalar("SELECT id FROM workspaces")
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| format!("list workspaces: {e}"))?;
|
||||
|
||||
let provisioner = crate::runtime_provision::RuntimeProvisioner::from_env();
|
||||
let mut out = Swept::default();
|
||||
for ws in workspaces {
|
||||
for c in census(pool, ws).await? {
|
||||
if !c.state.reapable() {
|
||||
continue;
|
||||
}
|
||||
if !should_reap(&c, grace_hours) {
|
||||
out.kept_in_grace += 1;
|
||||
continue;
|
||||
}
|
||||
let report = crate::routes::claws::purge_agent(
|
||||
pool,
|
||||
runtime,
|
||||
provisioner.as_ref(),
|
||||
cm_domain::AgentId::from(c.id),
|
||||
)
|
||||
.await;
|
||||
match report.counts {
|
||||
Ok(_) => {
|
||||
out.reaped += 1;
|
||||
eprintln!(
|
||||
"agent_lifecycle: reaped {} claw {} ({})",
|
||||
c.state.as_str(),
|
||||
c.name,
|
||||
c.id
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
out.failed += 1;
|
||||
eprintln!("agent_lifecycle: purge {} failed (continuing): {e}", c.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Spawn the sweeper.
|
||||
pub fn spawn(pool: PgPool, runtime: cm_runtime::Runtime, interval: Duration) {
|
||||
tokio::spawn(async move {
|
||||
let mut tick = tokio::time::interval(interval);
|
||||
// The first tick fires immediately; skip it so a restart loop cannot
|
||||
// turn into a reap loop.
|
||||
tick.tick().await;
|
||||
loop {
|
||||
tick.tick().await;
|
||||
match sweep(&pool, &runtime, COMPLETED_GRACE_HOURS).await {
|
||||
Ok(s) if s.reaped > 0 || s.failed > 0 => eprintln!(
|
||||
"agent_lifecycle: swept — {} reaped, {} failed, {} still in grace",
|
||||
s.reaped, s.failed, s.kept_in_grace
|
||||
),
|
||||
Ok(_) => {}
|
||||
Err(e) => eprintln!("agent_lifecycle: sweep failed: {e}"),
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn c(state: AgentState, hours: Option<f64>) -> Classified {
|
||||
Classified {
|
||||
id: Uuid::now_v7(),
|
||||
name: "x".into(),
|
||||
state,
|
||||
finished_hours_ago: hours,
|
||||
}
|
||||
}
|
||||
|
||||
/// The property that matters most: this sweeper must never be able to
|
||||
/// delete the operator's own staff, no matter what it is bound to.
|
||||
#[test]
|
||||
fn owned_and_active_are_never_reaped() {
|
||||
for hours in [None, Some(0.0), Some(1_000_000.0)] {
|
||||
assert!(!should_reap(&c(AgentState::Owned, hours), 24));
|
||||
assert!(!should_reap(&c(AgentState::Active, hours), 24));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orphans_go_immediately() {
|
||||
assert!(should_reap(&c(AgentState::Orphaned, None), 24));
|
||||
}
|
||||
|
||||
/// A soft delete is a decision that was never honoured — the row stayed,
|
||||
/// the agent kept appearing, and deleting it again did nothing. Collect it
|
||||
/// without a grace window: the human already waited.
|
||||
#[test]
|
||||
fn soft_deleted_agents_are_purged_without_a_grace_window() {
|
||||
assert!(should_reap(&c(AgentState::Deleted, None), 24));
|
||||
assert!(should_reap(&c(AgentState::Deleted, Some(0.0)), 24));
|
||||
}
|
||||
|
||||
/// The safety property restated against the new state: `deleted` must not
|
||||
/// widen into anything that can take live staff with it.
|
||||
#[test]
|
||||
fn adding_deleted_did_not_make_owned_reapable() {
|
||||
assert!(!AgentState::Owned.reapable());
|
||||
assert!(!AgentState::Active.reapable());
|
||||
assert!(AgentState::Deleted.reapable());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_finished_crew_waits_out_the_grace_window() {
|
||||
assert!(!should_reap(&c(AgentState::Completed, Some(1.0)), 24));
|
||||
assert!(!should_reap(&c(AgentState::Completed, Some(23.9)), 24));
|
||||
assert!(should_reap(&c(AgentState::Completed, Some(24.0)), 24));
|
||||
}
|
||||
|
||||
/// A completed crew with no usable timestamp must be KEPT. Treating a
|
||||
/// missing date as "old" is how a sweeper deletes something it was never
|
||||
/// able to prove was finished.
|
||||
#[test]
|
||||
fn a_missing_finish_time_is_not_treated_as_old() {
|
||||
assert!(!should_reap(&c(AgentState::Completed, None), 24));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,551 @@
|
||||
//! Gate and observe the tools a **container-tier** mission agent runs.
|
||||
//!
|
||||
//! The container tier is the one that actually runs missions in production,
|
||||
//! and until now it had neither. Both gaps have the same cause: `claude_cli`
|
||||
//! runs claude as a subprocess, claude runs its tools inside that subprocess,
|
||||
//! and so those calls never pass through ZeroClaw's executor — which is the
|
||||
//! only thing that emits `TurnEvent::ToolCall`, and therefore the only thing
|
||||
//! the gateway turns into a frame ClawMates can see. Recovering the calls from
|
||||
//! the CLI's own `stream-json` output does not help either: the transport was
|
||||
//! never the problem, and a mission proved it by producing zero `tool.call`
|
||||
//! events with the parser working perfectly.
|
||||
//!
|
||||
//! Hooks are the way in, and they are already proven. Claude Code reads
|
||||
//! `hooks.PreToolUse` / `PostToolUse` from the document passed to `--settings`
|
||||
//! and honours them under `-p` — measured against the real binary, where a
|
||||
//! `PreToolUse` hook blocked a `Bash` call, recorded the payload, and got its
|
||||
//! refusal reason back to the model.
|
||||
//!
|
||||
//! So this module writes the same hook scripts the microVM tier already uses
|
||||
//! into the mission's container, and the provider is pointed at the settings
|
||||
//! document. One mechanism, two tiers.
|
||||
//!
|
||||
//! # Everything here degrades to "no hooks", never to a failed mission
|
||||
//!
|
||||
//! A phase that runs unobserved still delivers. A phase that fails to start
|
||||
//! because telemetry could not be installed delivers nothing, which is a worse
|
||||
//! trade — the same stance `microvm_executor` takes for the same reason.
|
||||
|
||||
use bollard::Docker;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Where the hooks live inside the mission container.
|
||||
///
|
||||
/// Under `/root`, never under `/mission/repo`: anything written into the
|
||||
/// checkout would show up in the diff the mission delivers.
|
||||
pub const HOOK_DIR: &str = "/root/toolhooks";
|
||||
/// The settings document `claude -p --settings` is pointed at.
|
||||
pub const SETTINGS_PATH: &str = "/root/toolhooks/settings.json";
|
||||
/// Where the `PostToolUse` tap appends, inside the container.
|
||||
pub const TAP_DIR: &str = "/root/toolhooks/tap";
|
||||
|
||||
pub const INSTALL_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Install the pre-execution gate and the tool tap into a mission container.
|
||||
///
|
||||
/// Returns the settings path on success. `None` means the container runs
|
||||
/// without hooks — logged, never fatal.
|
||||
pub async fn install(docker: &Docker, container: &str) -> Option<String> {
|
||||
install_with(docker, container, None).await
|
||||
}
|
||||
|
||||
/// As [`install`], carrying a phase's task policy into the gate.
|
||||
pub async fn install_with(
|
||||
docker: &Docker,
|
||||
container: &str,
|
||||
task: Option<&crate::vm_tool_gate::TaskPolicy>,
|
||||
) -> Option<String> {
|
||||
let script = build_install_script(task);
|
||||
let argv = vec!["sh".to_string(), "-lc".to_string(), script];
|
||||
match crate::container_exec::exec_as_root(docker, container, None, &argv, INSTALL_TIMEOUT).await
|
||||
{
|
||||
Ok(out) if out.exit_code == Some(0) => Some(SETTINGS_PATH.to_string()),
|
||||
other => {
|
||||
eprintln!(
|
||||
"container_tool_hooks: could not install hooks in {container} ({other:?}) — \
|
||||
this mission's tool calls will run unchecked and unrecorded"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One shell script that lays down both hooks and the settings document.
|
||||
///
|
||||
/// Composed here rather than by each hook module writing its own file: two
|
||||
/// writers of one `settings.json` is a silent clobber, and the microVM tier
|
||||
/// already learned that the expensive way.
|
||||
fn build_install_script(task: Option<&crate::vm_tool_gate::TaskPolicy>) -> String {
|
||||
let settings = crate::vm_tool_tap::guest_settings(
|
||||
None,
|
||||
Some(TAP_DIR),
|
||||
Some(HOOK_DIR),
|
||||
);
|
||||
format!(
|
||||
"set -e\n\
|
||||
mkdir -p {hooks} {tap}\n\
|
||||
cat > {hooks}/tool-gate.sh <<'CM_GATE_EOF'\n{gate}\nCM_GATE_EOF\n\
|
||||
chmod +x {hooks}/tool-gate.sh\n\
|
||||
cat > {tap}/tap.sh <<'CM_TAP_EOF'\n{tap_script}\nCM_TAP_EOF\n\
|
||||
chmod +x {tap}/tap.sh\n\
|
||||
cat > {settings_path} <<'CM_SETTINGS_EOF'\n{settings}\nCM_SETTINGS_EOF\n",
|
||||
hooks = HOOK_DIR,
|
||||
tap = TAP_DIR,
|
||||
gate = crate::vm_tool_gate::hook_script_with(HOOK_DIR, task),
|
||||
tap_script = crate::vm_tool_tap::hook_script(TAP_DIR),
|
||||
settings_path = SETTINGS_PATH,
|
||||
settings = settings,
|
||||
)
|
||||
}
|
||||
|
||||
/// The MCP configuration `claude -p --mcp-config` is pointed at.
|
||||
///
|
||||
/// Under `/root` with the hooks, never under `/mission/repo`: it carries a
|
||||
/// bearer token, and anything written into the checkout arrives in the diff the
|
||||
/// mission delivers.
|
||||
pub const MCP_CONFIG_PATH: &str = "/root/toolhooks/clawmates-mcp.json";
|
||||
|
||||
/// Where the mission container reaches this server.
|
||||
///
|
||||
/// Mission containers join `clawmates_core`, the same network the API is on, so
|
||||
/// the API is reachable by container name. The name differs between
|
||||
/// deployments (`clawmates-server-1` locally, `clawmates_server_1` on gw-04),
|
||||
/// so the default is derived from **our own** hostname — docker's embedded DNS
|
||||
/// resolves a container id on a user-defined network, which makes this
|
||||
/// self-configuring rather than a constant that is right in one place.
|
||||
/// Measured from a sibling container: both the id and the name return 200.
|
||||
pub fn api_origin() -> Option<String> {
|
||||
if let Ok(v) = std::env::var("CLAWMATES_API_ORIGIN") {
|
||||
if !v.trim().is_empty() {
|
||||
return Some(v.trim().trim_end_matches('/').to_string());
|
||||
}
|
||||
}
|
||||
let host = std::env::var("HOSTNAME").ok()?;
|
||||
let host = host.trim();
|
||||
if host.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(format!("http://{host}:8080"))
|
||||
}
|
||||
|
||||
/// The `--mcp-config` document: one HTTP server, carrying its own credential.
|
||||
///
|
||||
/// The token is a `skills:read` session and nothing else. It is written into a
|
||||
/// file the agent can read — it runs `Bash` — so the only thing keeping this
|
||||
/// safe is that the credential authenticates to exactly one route. See
|
||||
/// `cm_auth::authenticate_scoped`.
|
||||
pub fn mcp_document(origin: &str, token: &str) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"mcpServers": {
|
||||
"clawmates_skills": {
|
||||
"type": "http",
|
||||
"url": format!("{origin}/mcp/skills"),
|
||||
"headers": { "Authorization": format!("Bearer {token}") }
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// NOTE on `--allowedTools`. The provider passes it only when the config sets
|
||||
// `tools`, and the seed already does — without it `claude -p` stops mid-turn to
|
||||
// ask for write permission. Whether the MCP tools ALSO need naming there is not
|
||||
// documented anywhere we control, and the daemon exposes no config read to
|
||||
// merge into that list safely: overwriting it would take `Write` and `Bash`
|
||||
// away from every mission agent, and that failure would look like agents that
|
||||
// stopped working rather than a config that was replaced.
|
||||
//
|
||||
// So it is left alone and the question is answered by running a mission with
|
||||
// the door installed. Guessing here is how the last three defects in this file
|
||||
// were introduced.
|
||||
|
||||
/// Write the MCP configuration into a mission container.
|
||||
///
|
||||
/// Returns the path on success. `None` means the mission runs without a door —
|
||||
/// logged, never fatal, exactly like the hooks above. A phase that cannot
|
||||
/// retrieve a skill still delivers; a phase that fails to start because a
|
||||
/// config write failed delivers nothing.
|
||||
pub async fn install_door(docker: &Docker, container: &str, doc: &serde_json::Value) -> Option<String> {
|
||||
// `printf %s` with the JSON single-quoted, not a heredoc: the document is
|
||||
// one line and contains no newline to terminate on.
|
||||
let script = format!(
|
||||
"mkdir -p {HOOK_DIR} && printf '%s' {} > {MCP_CONFIG_PATH} && chmod 600 {MCP_CONFIG_PATH}",
|
||||
crate::vm_tool_tap::shell_quote(&doc.to_string()),
|
||||
);
|
||||
let argv = vec!["sh".to_string(), "-lc".to_string(), script];
|
||||
match crate::container_exec::exec_as_root(docker, container, None, &argv, INSTALL_TIMEOUT).await
|
||||
{
|
||||
Ok(out) if out.exit_code == Some(0) => Some(MCP_CONFIG_PATH.to_string()),
|
||||
other => {
|
||||
eprintln!(
|
||||
"container_tool_hooks: could not write the MCP config in {container} ({other:?}) — this mission runs without the skills door"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Event kinds under which the gate's own state lands in the mission record.
|
||||
///
|
||||
/// Recorded, not only logged, so "was this mission gated?" is answerable from
|
||||
/// the mission afterwards. Stderr is where the answer used to go, which is the
|
||||
/// same place as nowhere once the container that printed it is gone.
|
||||
pub const GATE_INSTALLED: &str = "gate.installed";
|
||||
pub const GATE_ABSENT: &str = "gate.absent";
|
||||
/// The gate ran but could not parse its input and allowed everything. See
|
||||
/// [`crate::vm_tool_gate::INERT_FILE`] — this is the reader that marker was
|
||||
/// missing in production; until now only a unit test looked for it.
|
||||
pub const GATE_INERT: &str = "gate.inert";
|
||||
/// One call the gate refused. `detail` is the hook event with `rule` set
|
||||
/// beside it — see [`crate::vm_tool_gate::denial_detail`]. Both tiers.
|
||||
pub const GATE_DENIED: &str = "gate.denied";
|
||||
/// One call a task policy would have refused while it was in shadow. Same
|
||||
/// detail shape as [`GATE_DENIED`]; the difference is that it RAN.
|
||||
pub const GATE_WOULD_DENY: &str = "gate.would_deny";
|
||||
|
||||
/// Write the install outcome into the mission record.
|
||||
pub async fn record_install(
|
||||
pool: &sqlx::PgPool,
|
||||
mission_id: uuid::Uuid,
|
||||
phase_id: Option<uuid::Uuid>,
|
||||
hooks: Option<&str>,
|
||||
) {
|
||||
let mut e = match hooks {
|
||||
Some(path) => crate::mission_events::MissionEvent::new(mission_id, GATE_INSTALLED)
|
||||
.target(path)
|
||||
.detail(serde_json::json!({ "settings": path, "tap": tap_file() })),
|
||||
None => crate::mission_events::MissionEvent::new(mission_id, GATE_ABSENT).detail(
|
||||
serde_json::json!({
|
||||
"why": "container_tool_hooks::install failed — this mission's tool \
|
||||
calls run unchecked and unrecorded"
|
||||
}),
|
||||
),
|
||||
};
|
||||
if let Some(p) = phase_id {
|
||||
e = e.phase(p);
|
||||
}
|
||||
crate::mission_events::record(pool, e).await;
|
||||
}
|
||||
|
||||
/// The inert marker's path inside the container.
|
||||
pub fn inert_file() -> String {
|
||||
format!("{HOOK_DIR}/{}", crate::vm_tool_gate::INERT_FILE)
|
||||
}
|
||||
|
||||
/// Did the gate go inert since the last drain? Reads the marker and clears
|
||||
/// it, so each occurrence is reported once.
|
||||
///
|
||||
/// `Some(text)` is the marker's contents — every line the gate appended while
|
||||
/// it could not parse. `None` is "the marker is not there", which is the
|
||||
/// normal case and also, by construction, the only case that means the gate
|
||||
/// was actually checking.
|
||||
pub async fn drain_inert(docker: &Docker, container: &str) -> Option<String> {
|
||||
let file = inert_file();
|
||||
let script = format!("cat {file} 2>/dev/null && rm -f {file} 2>/dev/null; true");
|
||||
let argv = vec!["sh".to_string(), "-lc".to_string(), script];
|
||||
match crate::container_exec::exec_as_root(docker, container, None, &argv, INSTALL_TIMEOUT).await
|
||||
{
|
||||
Ok(out) if !out.stdout.trim().is_empty() => Some(out.stdout.trim().to_string()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The gate's denial record inside the mission container.
|
||||
pub fn denied_file() -> String {
|
||||
format!("{HOOK_DIR}/{}", crate::vm_tool_gate::DENIED_FILE)
|
||||
}
|
||||
|
||||
/// Every call the gate refused since the last drain, one JSON line each
|
||||
/// (`vm_tool_gate::denial_detail` reads them). Read-then-truncate, like
|
||||
/// [`drain`], for the same reason: no cursor to keep, and the phase has
|
||||
/// finished so nothing is appending.
|
||||
pub async fn drain_denied(docker: &Docker, container: &str) -> Vec<String> {
|
||||
let file = denied_file();
|
||||
let script = format!("cat {file} 2>/dev/null || true; : > {file} 2>/dev/null || true");
|
||||
let argv = vec!["sh".to_string(), "-lc".to_string(), script];
|
||||
match crate::container_exec::exec_as_root(docker, container, None, &argv, INSTALL_TIMEOUT).await
|
||||
{
|
||||
Ok(out) => out
|
||||
.stdout
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|l| !l.is_empty())
|
||||
.map(str::to_string)
|
||||
.collect(),
|
||||
Err(_) => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The gate's shadow record: calls a policy WOULD have refused, had it been
|
||||
/// enforcing. Drained exactly like [`drain_denied`] and recorded as
|
||||
/// [`GATE_WOULD_DENY`], because a shadow mode whose output nobody reads is
|
||||
/// an off switch with extra steps.
|
||||
pub async fn drain_would_deny(docker: &Docker, container: &str) -> Vec<String> {
|
||||
let file = format!("{HOOK_DIR}/{}", crate::vm_tool_gate::WOULD_DENY_FILE);
|
||||
let script = format!("cat {file} 2>/dev/null || true; : > {file} 2>/dev/null || true");
|
||||
let argv = vec!["sh".to_string(), "-lc".to_string(), script];
|
||||
match crate::container_exec::exec_as_root(docker, container, None, &argv, INSTALL_TIMEOUT).await
|
||||
{
|
||||
Ok(out) => out
|
||||
.stdout
|
||||
.lines()
|
||||
.map(str::trim)
|
||||
.filter(|l| !l.is_empty())
|
||||
.map(str::to_string)
|
||||
.collect(),
|
||||
Err(_) => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Hosts named in fetched content, as the tap recorded them. One event per
|
||||
/// finished phase, carrying the whole list: stage 1 of argument provenance
|
||||
/// is observation only, and this is what gets inspected before any rule is
|
||||
/// built on it.
|
||||
pub const TAINT_HOSTS: &str = "taint.hosts";
|
||||
|
||||
/// Read the tap's taint file. NOT cleared, unlike every drain above: it is
|
||||
/// the state a future `untrusted-target` rule consults for the rest of the
|
||||
/// mission, so each phase's event is the set known when that phase ended.
|
||||
pub async fn drain_taint(docker: &Docker, container: &str) -> Vec<String> {
|
||||
let argv = vec![
|
||||
"sh".to_string(),
|
||||
"-lc".to_string(),
|
||||
crate::vm_tool_tap::taint_probe(TAP_DIR),
|
||||
];
|
||||
match crate::container_exec::exec_as_root(docker, container, None, &argv, INSTALL_TIMEOUT).await
|
||||
{
|
||||
Ok(out) => crate::vm_tool_tap::parse_taint(&out.stdout),
|
||||
Err(_) => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The detail of a [`TAINT_HOSTS`] event.
|
||||
pub fn taint_detail(hosts: &[String], tier: &str) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"hosts": hosts,
|
||||
"count": hosts.len(),
|
||||
"capped": hosts.len() >= crate::vm_tool_tap::MAX_TAINT_HOSTS,
|
||||
"tier": tier,
|
||||
})
|
||||
}
|
||||
|
||||
/// The tap file inside the mission container.
|
||||
pub fn tap_file() -> String {
|
||||
format!("{TAP_DIR}/tools.jsonl")
|
||||
}
|
||||
|
||||
/// Read everything the tap recorded, then clear it.
|
||||
///
|
||||
/// Read-then-truncate rather than a cursor, because this tier has no
|
||||
/// long-lived loop to hold one: the microVM path drains inside the turn it is
|
||||
/// watching, while a container turn is driven asynchronously by
|
||||
/// `topology_worker`. Truncation makes the drain idempotent — a second pass
|
||||
/// reads an empty file and records nothing — without a column to store a
|
||||
/// cursor in.
|
||||
///
|
||||
/// Called only for phases that have FINISHED, so the agent is no longer
|
||||
/// appending and the read/truncate gap cannot lose an event.
|
||||
pub async fn drain(docker: &Docker, container: &str) -> Vec<crate::vm_tool_tap::Observed> {
|
||||
let file = tap_file();
|
||||
// `cat` then truncate in one exec: two round-trips would widen the window
|
||||
// between them for no benefit.
|
||||
let script = format!("cat {file} 2>/dev/null || true; : > {file} 2>/dev/null || true");
|
||||
let argv = vec!["sh".to_string(), "-lc".to_string(), script];
|
||||
match crate::container_exec::exec_as_root(docker, container, None, &argv, INSTALL_TIMEOUT).await
|
||||
{
|
||||
Ok(out) => crate::vm_tool_tap::parse(&out.stdout),
|
||||
Err(e) => {
|
||||
// A reaped container is the normal end state, not a fault.
|
||||
eprintln!("container_tool_hooks: no tap drained from {container}: {e}");
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Every command the settings document names must be a file the installer
|
||||
/// actually writes.
|
||||
///
|
||||
/// This caught a real one: the document pointed PostToolUse at
|
||||
/// `{TAP_DIR}/tap.sh` while the installer wrote `{HOOK_DIR}/tap.sh`, so
|
||||
/// the hook referenced a file that did not exist. Claude Code does not
|
||||
/// complain about a missing hook command — it simply records nothing, and
|
||||
/// a mission ran with the tap installed, pointed at nothing, and silent.
|
||||
///
|
||||
/// Asserting that the script "mentions tap.sh" did not catch it. The paths
|
||||
/// have to be compared.
|
||||
#[test]
|
||||
fn every_hook_command_is_a_file_the_installer_writes() {
|
||||
let settings = crate::vm_tool_tap::guest_settings(None, Some(TAP_DIR), Some(HOOK_DIR));
|
||||
let script = build_install_script(None);
|
||||
|
||||
let hooks = settings["hooks"].as_object().expect("hooks");
|
||||
assert!(!hooks.is_empty(), "no hooks at all");
|
||||
for (event, entries) in hooks {
|
||||
let cmd = entries[0]["hooks"][0]["command"]
|
||||
.as_str()
|
||||
.unwrap_or_else(|| panic!("{event} has no command"));
|
||||
assert!(
|
||||
script.contains(&format!("cat > {cmd} <<")),
|
||||
"{event} points at {cmd}, which the installer never writes — \
|
||||
the hook is registered and inert"
|
||||
);
|
||||
assert!(
|
||||
script.contains(&format!("chmod +x {cmd}")),
|
||||
"{event} points at {cmd}, which is never made executable"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_script_writes_both_hooks_and_the_settings_document() {
|
||||
let s = build_install_script(None);
|
||||
assert!(s.contains("tool-gate.sh"), "the pre-execution gate is missing");
|
||||
assert!(s.contains("tap.sh"), "the tool tap is missing");
|
||||
assert!(s.contains(SETTINGS_PATH), "the settings document is missing");
|
||||
// Both hooks in ONE document — the whole reason this is composed here.
|
||||
assert!(s.contains("PreToolUse"));
|
||||
assert!(s.contains("PostToolUse"));
|
||||
}
|
||||
|
||||
/// Nothing may be written into the mission checkout.
|
||||
///
|
||||
/// A file left under `/mission/repo` shows up in the diff the mission
|
||||
/// delivers, so hook plumbing would arrive as part of the agent's work.
|
||||
#[test]
|
||||
fn nothing_is_written_into_the_checkout() {
|
||||
assert!(HOOK_DIR.starts_with("/root/"));
|
||||
assert!(SETTINGS_PATH.starts_with("/root/"));
|
||||
assert!(TAP_DIR.starts_with("/root/"));
|
||||
assert!(!build_install_script(None).contains("/mission/repo"));
|
||||
}
|
||||
|
||||
/// The two halves must stay together.
|
||||
///
|
||||
/// Writing the hooks without pointing the provider at them leaves a gate
|
||||
/// that is installed and inert — indistinguishable from a gate that found
|
||||
/// nothing, which is this codebase's signature failure. Pointing the
|
||||
/// provider at a document nobody wrote makes claude fail to start.
|
||||
#[test]
|
||||
fn the_installer_and_the_provider_prop_agree() {
|
||||
let orchestrator = include_str!("mission_orchestrator.rs");
|
||||
assert!(
|
||||
orchestrator.contains("set_claude_cli_settings")
|
||||
&& orchestrator.contains("container_tool_hooks::SETTINGS_PATH"),
|
||||
"the hooks are installed but nothing points claude at them"
|
||||
);
|
||||
let runtime = include_str!("mission_runtime.rs");
|
||||
assert!(
|
||||
runtime.contains("container_tool_hooks::install"),
|
||||
"the provider is pointed at a settings document nobody writes"
|
||||
);
|
||||
// Both container paths — created AND reused. A hook that exists only
|
||||
// on first creation disappears after a server redeploy.
|
||||
assert_eq!(
|
||||
runtime.matches("container_tool_hooks::install").count(),
|
||||
2,
|
||||
"install must run on the reuse path too"
|
||||
);
|
||||
}
|
||||
|
||||
/// The drain must clear what it read.
|
||||
///
|
||||
/// Truncation IS the idempotency here — there is no cursor column and no
|
||||
/// marker row. A drain that reads without clearing would re-record every
|
||||
/// tool call on every tick, and a phase's early files would end up weighted
|
||||
/// by how long the sweep ran.
|
||||
#[test]
|
||||
fn the_drain_reads_then_clears() {
|
||||
let file = tap_file();
|
||||
assert!(file.starts_with(TAP_DIR), "the tap must live under {TAP_DIR}");
|
||||
// The script is built inline in `drain`; assert on the shape it must
|
||||
// have, since getting this wrong duplicates every event silently.
|
||||
let script = format!("cat {file} 2>/dev/null || true; : > {file} 2>/dev/null || true");
|
||||
assert!(script.contains(&format!("cat {file}")), "must read");
|
||||
assert!(script.contains(&format!(": > {file}")), "must clear");
|
||||
}
|
||||
|
||||
/// The sweep has to exist, or the hooks write a file nobody reads.
|
||||
#[test]
|
||||
fn something_actually_collects_the_tap() {
|
||||
let runner = include_str!("phase_runner.rs");
|
||||
assert!(
|
||||
runner.contains("container_tool_hooks::drain"),
|
||||
"the tap is written and never collected — the same shape as a gate \
|
||||
that is installed and inert"
|
||||
);
|
||||
assert!(
|
||||
runner.contains("drain_finished_container_phases(pool).await?"),
|
||||
"the drain exists but the tick does not call it"
|
||||
);
|
||||
}
|
||||
|
||||
/// The taint file is never cleared, so its record needs its own
|
||||
/// once-per-phase guard. Measured without one: the first live mission
|
||||
/// recorded the same `taint.hosts` event four times, and the sweep that
|
||||
/// revisits a finished phase for 30 minutes would have kept going.
|
||||
#[test]
|
||||
fn the_taint_record_is_written_once_per_phase() {
|
||||
let runner = include_str!("phase_runner.rs");
|
||||
let body = runner
|
||||
.split("drain_taint(&docker, &container).await")
|
||||
.nth(1)
|
||||
.expect("the sweep drains the taint file");
|
||||
let guard = body.find("SELECT EXISTS").expect("no once-per-phase guard");
|
||||
let record = body.find("TAINT_HOSTS,\n").unwrap_or(usize::MAX).min(
|
||||
body.find("MissionEvent::new").expect("the record"),
|
||||
);
|
||||
assert!(guard < record, "the guard must run before the record is written");
|
||||
}
|
||||
|
||||
/// The drain must use the connector that honours DOCKER_HOST.
|
||||
///
|
||||
/// The server reaches Docker through a socket proxy, so
|
||||
/// `connect_with_local_defaults` fails there — and it failed SILENTLY,
|
||||
/// which meant the sweep did nothing while the tap filled up and every
|
||||
/// other link in the chain looked correct. Cost a full diagnostic cycle.
|
||||
#[test]
|
||||
fn the_sweep_connects_the_way_the_rest_of_the_server_does() {
|
||||
let runner = include_str!("phase_runner.rs");
|
||||
let body = runner
|
||||
.split("async fn drain_finished_container_phases(")
|
||||
.nth(1)
|
||||
.and_then(|s| s.split("\nasync fn ").next())
|
||||
.expect("sweep body");
|
||||
assert!(
|
||||
body.contains("container_exec::connect()"),
|
||||
"the sweep must use the DOCKER_HOST-aware connector"
|
||||
);
|
||||
assert!(
|
||||
// The CALL, not the word: the comment above it names the
|
||||
// connector it is warning against.
|
||||
!body.contains("connect_with_local_defaults()"),
|
||||
"the local-socket connector fails behind the socket proxy"
|
||||
);
|
||||
}
|
||||
|
||||
/// The generated installer must be valid shell — a here-doc or quoting slip
|
||||
/// makes it fail in the container, where the only symptom is a mission that
|
||||
/// silently runs unhooked.
|
||||
#[test]
|
||||
fn the_install_script_is_valid_shell() {
|
||||
if std::process::Command::new("bash").arg("-c").arg("true").status().is_err() {
|
||||
return;
|
||||
}
|
||||
let tmp = std::env::temp_dir().join(format!("cm-install-{}.sh", std::process::id()));
|
||||
std::fs::write(&tmp, build_install_script(None)).unwrap();
|
||||
let out = std::process::Command::new("bash")
|
||||
.arg("-n")
|
||||
.arg(&tmp)
|
||||
.output()
|
||||
.expect("bash -n");
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"installer will not parse: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,439 @@
|
||||
//! The harvest half of a Continuous Research mission.
|
||||
//!
|
||||
//! Finding papers is NOT agent work. `library::run_to_vault` already does arXiv
|
||||
//! search → seen-set check → PDF fetch → blob shelf → vault note, deterministically
|
||||
//! and in seconds, and it takes a `mission_id` so the run is attributed. Asking an
|
||||
//! agent to redo it would be slower, non-repeatable, and would abandon the
|
||||
//! `corpus_items` seen-set — which is the entire reason a recurring mission knows
|
||||
//! what it already covered. `corpus.rs` puts it plainly: "A recurring mission's
|
||||
//! hard problem is not running the agent — that is 23 seconds — it is knowing
|
||||
//! what it already did last time."
|
||||
//!
|
||||
//! So the harvest runs here, at launch, and the agents start from its output.
|
||||
//!
|
||||
//! The manifest path (`ContinuousResearch/<date>/harvest.jsonl`) is not invented:
|
||||
//! `templates/teams/continuous_research.toml` has told the `signal_harvester`
|
||||
//! role to write exactly that file since the template was authored. This makes
|
||||
//! the code produce what the prompt already promised, rather than leaving a role
|
||||
//! to fabricate it.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::json;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Template kind that triggers a harvest at launch.
|
||||
pub const TEMPLATE_KIND: &str = "continuous_research";
|
||||
|
||||
/// Today's manifest, relative to the vault root.
|
||||
pub fn manifest_path(date: &str) -> String {
|
||||
format!("ContinuousResearch/{date}/harvest.jsonl")
|
||||
}
|
||||
|
||||
/// UTC date stamp, the same key the vault folders use.
|
||||
pub fn today() -> String {
|
||||
let now = time::OffsetDateTime::now_utc();
|
||||
format!(
|
||||
"{:04}-{:02}-{:02}",
|
||||
now.year(),
|
||||
now.month() as u8,
|
||||
now.day()
|
||||
)
|
||||
}
|
||||
|
||||
/// The arXiv queries this mission tracks.
|
||||
///
|
||||
/// `config.topics` on the mission when the operator set them, otherwise the
|
||||
/// project-wide defaults. Read from config rather than a new column because the
|
||||
/// wizard already round-trips `config` untouched, so a topic list needs no
|
||||
/// schema change and no UI work to reach here.
|
||||
pub fn topics_for(config: &serde_json::Value) -> Vec<String> {
|
||||
config
|
||||
.get("topics")
|
||||
.and_then(|v| v.as_array())
|
||||
.map(|a| {
|
||||
a.iter()
|
||||
.filter_map(|t| t.as_str())
|
||||
.map(str::trim)
|
||||
.filter(|t| !t.is_empty())
|
||||
.map(str::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
})
|
||||
.filter(|t: &Vec<String>| !t.is_empty())
|
||||
.unwrap_or_else(crate::library::default_topics)
|
||||
}
|
||||
|
||||
/// Run the harvest for a mission and leave a manifest the agents can read.
|
||||
///
|
||||
/// Non-fatal by contract: a launch whose harvest fails still starts its phases,
|
||||
/// because a quiet day and a broken day must be distinguishable and the phase
|
||||
/// itself is what reports which happened. What is NOT acceptable is failing
|
||||
/// silently, so every outcome is logged with its counts.
|
||||
pub async fn harvest_for_mission(
|
||||
pool: &sqlx::PgPool,
|
||||
blobs: &Arc<dyn cm_files::BlobStore>,
|
||||
workspace_id: Uuid,
|
||||
mission_id: Uuid,
|
||||
topics: &[String],
|
||||
per_topic: usize,
|
||||
) -> Result<Vec<crate::papers::Paper>, String> {
|
||||
let work_root = std::env::temp_dir().join("clawmates-library");
|
||||
let run = crate::library::run_to_vault(
|
||||
pool,
|
||||
blobs,
|
||||
workspace_id,
|
||||
crate::routes::library::DEFAULT_CORPUS,
|
||||
crate::routes::library::DEFAULT_VAULT_URL,
|
||||
&work_root,
|
||||
topics,
|
||||
per_topic,
|
||||
Some(mission_id),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let shelved = run.harvest.shelved.len();
|
||||
// A quiet day is not a failure. `Harvest::healthy()` (nothing errored) is a
|
||||
// different question from `added_anything()` (something new arrived), and
|
||||
// collapsing them is the defect class this codebase keeps paying for.
|
||||
eprintln!(
|
||||
"continuous_research: mission {mission_id} harvested {} candidate(s), {} already had, \
|
||||
{} shelved, {} failed",
|
||||
run.harvest.candidates,
|
||||
run.harvest.already_had,
|
||||
shelved,
|
||||
run.harvest.failed.len()
|
||||
);
|
||||
for (source_id, why) in &run.harvest.failed {
|
||||
eprintln!("continuous_research: {source_id} not shelved: {why}");
|
||||
}
|
||||
Ok(run.harvest.papers)
|
||||
}
|
||||
|
||||
/// Write the run manifest into the MISSION's checkout.
|
||||
///
|
||||
/// Not into the vault. The manifest is per-RUN input for one mission, and the
|
||||
/// vault path is per-DATE and shared, so a second run on the same day rewrites
|
||||
/// a file that already exists — which `auto_merge` correctly refuses, because
|
||||
/// it only merges provably additive diffs:
|
||||
///
|
||||
/// "diff is not additive (1 non-add change(s), first:
|
||||
/// M ContinuousResearch/2026-08-18/harvest.jsonl); left for a human"
|
||||
///
|
||||
/// The branch was then left unmerged, `main` kept the previous run's manifest,
|
||||
/// and the next mission cloned STALE papers while every log line said the
|
||||
/// harvest succeeded. Writing into the checkout keeps the vault additive and
|
||||
/// gives each mission exactly its own papers. The agents commit it alongside
|
||||
/// their analysis through the normal delivery path.
|
||||
pub fn write_manifest(
|
||||
checkout: &std::path::Path,
|
||||
papers: &[crate::papers::Paper],
|
||||
date: &str,
|
||||
triage: &[PaperTriage],
|
||||
) -> Result<std::path::PathBuf, String> {
|
||||
let rel = manifest_path(date);
|
||||
let abs = checkout.join(&rel);
|
||||
if let Some(parent) = abs.parent() {
|
||||
std::fs::create_dir_all(parent).map_err(|e| format!("create {}: {e}", parent.display()))?;
|
||||
}
|
||||
let body = manifest_lines(papers, date, triage);
|
||||
std::fs::write(&abs, format!("{body}\n")).map_err(|e| format!("write {}: {e}", abs.display()))?;
|
||||
Ok(abs)
|
||||
}
|
||||
|
||||
/// What the decision model said about one harvested paper. `topic_tags`
|
||||
/// was written as `[]` on every manifest line from the day the manifest
|
||||
/// existed — a slot the agents were told to read and nothing filled.
|
||||
///
|
||||
/// **A score has to discriminate within the population it scores.** The
|
||||
/// first version asked "how relevant is this to an agent platform" on a
|
||||
/// four-level scale, and MEASURED on the ten papers of mission 01a0c940 it
|
||||
/// answered 2.93–3.00 — a spread of 0.07, no ranking information at all.
|
||||
/// Of course: the harvest runs the operator's own arXiv topic queries, so
|
||||
/// every paper in it is about agents by construction. "How actionable is
|
||||
/// it" saturated the same way (spread 0.20). What did discriminate on the
|
||||
/// same ten abstracts was the strength of the evidence behind the claims
|
||||
/// (1.36–3.00, spread 1.64) and what KIND of paper it is. So the manifest
|
||||
/// carries those two and no relevance number: a digest phase ranking
|
||||
/// already-relevant papers needs to know which ones measured something.
|
||||
#[derive(Debug, Clone, serde::Serialize, Default)]
|
||||
pub struct PaperTriage {
|
||||
pub topic_tags: Vec<String>,
|
||||
/// `benchmark` | `method` | `measurement` | `survey` | `position`, and
|
||||
/// how peaked that choice was — a paper the model cannot place is one
|
||||
/// the reader should look at rather than trust the label for.
|
||||
pub kind: Option<String>,
|
||||
pub kind_confidence: Option<f64>,
|
||||
/// 0 = position piece, no experiments … 3 = measured on real systems
|
||||
/// with ablations. `None` when no triage ran (no key, or a failed call).
|
||||
pub evidence: Option<f64>,
|
||||
pub evidence_confidence: Option<f64>,
|
||||
}
|
||||
|
||||
const EVIDENCE_LEVELS: [&str; 4] = [
|
||||
"Position, opinion, or framework description; no experiments.",
|
||||
"Illustrative examples, a demo, or a single small case study.",
|
||||
"Benchmarked with numbers, on a suite the authors assembled.",
|
||||
"Measured on real systems or at scale, with ablations or failure analysis.",
|
||||
];
|
||||
|
||||
const PAPER_KINDS: [(&str, &str); 5] = [
|
||||
("benchmark", "Introduces a dataset or benchmark to measure something"),
|
||||
("method", "Proposes a technique, architecture, or algorithm"),
|
||||
("measurement", "Measures the behaviour of existing systems without proposing a new one"),
|
||||
("survey", "Reviews or categorises a body of existing work"),
|
||||
("position", "Argues a viewpoint or proposes an agenda"),
|
||||
];
|
||||
|
||||
/// Triage every harvested paper in one call each. Best-effort: a missing key
|
||||
/// or a failed call leaves that paper's tags empty and relevance `None`, the
|
||||
/// state the manifest has always been in.
|
||||
pub async fn triage_papers(
|
||||
papers: &[crate::papers::Paper],
|
||||
topics: &[String],
|
||||
) -> Vec<PaperTriage> {
|
||||
use cm_decide::{Answer, Decider as _, Question};
|
||||
let Some(jev) = cm_decide::jev::Jev::from_env() else {
|
||||
return vec![PaperTriage::default(); papers.len()];
|
||||
};
|
||||
// Topics are arXiv query strings; the option NAME the model sees is the
|
||||
// readable form (`all:"agent memory" AND all:"long-term"` → `agent memory
|
||||
// long-term`), the value the query itself for precision.
|
||||
let mut criteria: std::collections::BTreeMap<String, Option<String>> = topics
|
||||
.iter()
|
||||
.map(|t| (readable_topic(t), Some(t.clone())))
|
||||
.collect();
|
||||
criteria.insert("none".into(), Some("Fits none of the listed topics".into()));
|
||||
let questions: std::collections::BTreeMap<String, Question> = [
|
||||
(
|
||||
"topic".to_string(),
|
||||
Question::Choice {
|
||||
instructions: "Which of these research topics is this paper about?".into(),
|
||||
criteria,
|
||||
},
|
||||
),
|
||||
(
|
||||
"kind".to_string(),
|
||||
Question::choice(
|
||||
"What kind of paper is this?",
|
||||
PAPER_KINDS.map(|(k, d)| (k, Some(d))),
|
||||
),
|
||||
),
|
||||
(
|
||||
"evidence".to_string(),
|
||||
Question::score(
|
||||
"How strong is the evidence behind this paper's claims?",
|
||||
EVIDENCE_LEVELS,
|
||||
),
|
||||
),
|
||||
]
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
let mut out = Vec::with_capacity(papers.len());
|
||||
for p in papers {
|
||||
let state = format!("Title: {}\n\nAbstract: {}", p.title, p.summary);
|
||||
let decided = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
jev.decide(&state, &questions),
|
||||
)
|
||||
.await;
|
||||
let mut t = PaperTriage::default();
|
||||
match decided {
|
||||
Ok(Ok(d)) => {
|
||||
if let Some(Answer::Choice { probabilities, .. }) = d.answers.get("topic") {
|
||||
let mut tags: Vec<(String, f64)> = probabilities
|
||||
.iter()
|
||||
.filter(|(k, p)| k.as_str() != "none" && **p >= 0.3)
|
||||
.map(|(k, p)| (k.clone(), *p))
|
||||
.collect();
|
||||
tags.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
|
||||
t.topic_tags = tags.into_iter().map(|(k, _)| k).collect();
|
||||
}
|
||||
if let Some(Answer::Choice { choice, confidence, .. }) = d.answers.get("kind") {
|
||||
t.kind = Some(choice.clone());
|
||||
t.kind_confidence = Some((*confidence * 100.0).round() / 100.0);
|
||||
}
|
||||
if let Some(Answer::Score { score, confidence, .. }) = d.answers.get("evidence") {
|
||||
t.evidence = Some((*score * 100.0).round() / 100.0);
|
||||
t.evidence_confidence = Some((*confidence * 100.0).round() / 100.0);
|
||||
}
|
||||
}
|
||||
Ok(Err(e)) => eprintln!("continuous_research: triage of {} failed: {e}", p.arxiv_id),
|
||||
Err(_) => eprintln!("continuous_research: triage of {} timed out", p.arxiv_id),
|
||||
}
|
||||
out.push(t);
|
||||
}
|
||||
let tagged = out.iter().filter(|t| !t.topic_tags.is_empty()).count();
|
||||
let scores: Vec<f64> = out.iter().filter_map(|t| t.evidence).collect();
|
||||
eprintln!(
|
||||
"continuous_research: triaged {} paper(s) with {}: {tagged} tagged, {} with evidence scored",
|
||||
papers.len(),
|
||||
jev.name(),
|
||||
scores.len()
|
||||
);
|
||||
// A score that came back the same for every paper ranked nothing. Said
|
||||
// out loud because the first version of this question did exactly that
|
||||
// and looked like a working feature — ten confident numbers, no
|
||||
// information. See `PaperTriage`.
|
||||
let spread = cm_decide::patterns::spread(&scores);
|
||||
if scores.len() > 2 && spread < cm_decide::patterns::SATURATED_BELOW {
|
||||
eprintln!(
|
||||
"continuous_research: WARNING — evidence scores span only {spread:.2} across \
|
||||
{} papers. The question is not separating this harvest; the ranking phase \
|
||||
gets no signal from it.",
|
||||
scores.len()
|
||||
);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// `all:"agent memory" AND all:"long-term"` → `agent memory long-term`.
|
||||
fn readable_topic(query: &str) -> String {
|
||||
let words: Vec<&str> = query
|
||||
.split(|c: char| c == '"' || c.is_whitespace() || c == '(' || c == ')')
|
||||
.filter(|w| !w.is_empty())
|
||||
.filter(|w| !matches!(*w, "AND" | "OR" | "NOT"))
|
||||
.map(|w| w.strip_prefix("all:").unwrap_or(w))
|
||||
.map(|w| w.strip_prefix("ti:").unwrap_or(w))
|
||||
.map(|w| w.strip_prefix("abs:").unwrap_or(w))
|
||||
.filter(|w| !w.is_empty())
|
||||
.collect();
|
||||
words.join(" ")
|
||||
}
|
||||
|
||||
/// The manifest lines for a set of freshly shelved papers.
|
||||
///
|
||||
/// Shape matches what `skills/research/arxiv-daily.md` documents:
|
||||
/// `{ source, url, title, snippet, first_seen, topic_tags }`, plus `kind`
|
||||
/// and `evidence` since 2026-09-22 (see [`PaperTriage`]). `triage` is
|
||||
/// positional with `papers`; shorter means the rest are untriaged.
|
||||
pub fn manifest_lines(
|
||||
papers: &[crate::papers::Paper],
|
||||
first_seen: &str,
|
||||
triage: &[PaperTriage],
|
||||
) -> String {
|
||||
papers
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, p)| {
|
||||
let t = triage.get(i).cloned().unwrap_or_default();
|
||||
json!({
|
||||
"source": p.source_id(),
|
||||
"url": format!("https://arxiv.org/abs/{}", p.arxiv_id),
|
||||
"title": p.title,
|
||||
"snippet": p.summary.chars().take(400).collect::<String>(),
|
||||
"first_seen": first_seen,
|
||||
"topic_tags": t.topic_tags,
|
||||
"kind": t.kind.as_ref().map(|k| json!({
|
||||
"is": k,
|
||||
"confidence": t.kind_confidence,
|
||||
})),
|
||||
"evidence": t.evidence.map(|e| json!({
|
||||
"score": e,
|
||||
"confidence": t.evidence_confidence,
|
||||
"scale": "0 position piece, no experiments … 3 measured on real systems with ablations",
|
||||
})),
|
||||
})
|
||||
.to_string()
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_manifest_path_matches_what_the_team_template_promises() {
|
||||
// templates/teams/continuous_research.toml tells signal_harvester to
|
||||
// write ContinuousResearch/<date>/harvest.jsonl. If this drifts, the
|
||||
// agents read a file nothing writes and silently review nothing.
|
||||
assert_eq!(
|
||||
manifest_path("2026-08-17"),
|
||||
"ContinuousResearch/2026-08-17/harvest.jsonl"
|
||||
);
|
||||
}
|
||||
|
||||
/// An operator's topic list must win over the defaults, and a blank or
|
||||
/// missing list must fall back rather than harvesting nothing.
|
||||
#[test]
|
||||
fn topics_come_from_config_and_fall_back_when_absent() {
|
||||
assert_eq!(
|
||||
topics_for(&serde_json::json!({"topics": ["world models", " robots "]})),
|
||||
vec!["world models".to_string(), "robots".to_string()],
|
||||
"operator topics win, and are trimmed"
|
||||
);
|
||||
for empty in [
|
||||
serde_json::json!({}),
|
||||
serde_json::json!({"topics": []}),
|
||||
serde_json::json!({"topics": [" "]}),
|
||||
] {
|
||||
assert_eq!(
|
||||
topics_for(&empty),
|
||||
crate::library::default_topics(),
|
||||
"an absent or blank list must fall back, not harvest nothing: {empty}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn arxiv_queries_become_readable_option_names() {
|
||||
assert_eq!(
|
||||
readable_topic(r#"all:"agent memory" AND all:"long-term""#),
|
||||
"agent memory long-term"
|
||||
);
|
||||
assert_eq!(
|
||||
readable_topic(r#"all:"agentic topology" OR all:"multi-agent topology""#),
|
||||
"agentic topology multi-agent topology"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_date_stamp_is_zero_padded() {
|
||||
let d = today();
|
||||
assert_eq!(d.len(), 10, "YYYY-MM-DD, got {d:?}");
|
||||
assert_eq!(d.matches('-').count(), 2, "{d:?}");
|
||||
}
|
||||
|
||||
/// One JSON object per line, and every key the template's prompt names —
|
||||
/// an agent instructed to read `topic_tags` must not find it absent.
|
||||
#[test]
|
||||
fn manifest_lines_carry_every_documented_key() {
|
||||
let p = crate::papers::Paper {
|
||||
arxiv_id: "2401.12345".into(),
|
||||
title: "A Paper".into(),
|
||||
authors: vec!["A. Author".into()],
|
||||
summary: "x".repeat(900),
|
||||
published: "2026-08-17".into(),
|
||||
pdf_url: "https://arxiv.org/pdf/2401.12345".into(),
|
||||
};
|
||||
let out = manifest_lines(std::slice::from_ref(&p), "2026-08-17", &[]);
|
||||
assert_eq!(out.lines().count(), 1);
|
||||
let v: serde_json::Value = serde_json::from_str(&out).expect("each line is JSON");
|
||||
for key in ["source", "url", "title", "snippet", "first_seen", "topic_tags", "kind", "evidence"] {
|
||||
assert!(v.get(key).is_some(), "missing {key} in {v}");
|
||||
}
|
||||
// Untriaged: the slots are there and empty, as they always were.
|
||||
assert_eq!(v["topic_tags"], serde_json::json!([]));
|
||||
assert!(v["kind"].is_null() && v["evidence"].is_null());
|
||||
// Triaged: the tags, the kind and the evidence score land on the line.
|
||||
let t = PaperTriage {
|
||||
topic_tags: vec!["agent memory long-term".into()],
|
||||
kind: Some("benchmark".into()),
|
||||
kind_confidence: Some(0.91),
|
||||
evidence: Some(1.36),
|
||||
evidence_confidence: Some(0.62),
|
||||
};
|
||||
let out = manifest_lines(std::slice::from_ref(&p), "2026-08-17", std::slice::from_ref(&t));
|
||||
let v: serde_json::Value = serde_json::from_str(&out).unwrap();
|
||||
assert_eq!(v["topic_tags"][0], "agent memory long-term");
|
||||
assert_eq!(v["kind"]["is"], "benchmark");
|
||||
assert_eq!(v["evidence"]["score"], 1.36);
|
||||
assert_eq!(v["source"], "arxiv:2401.12345");
|
||||
assert!(
|
||||
v["snippet"].as_str().unwrap().chars().count() <= 400,
|
||||
"snippet must be trimmed, not the whole abstract"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
//! Keep the credentials a mission can see out of what a mission delivers.
|
||||
//!
|
||||
//! Container-tier missions carry model-provider keys in their environment —
|
||||
//! Claude Code needs its own credential, and the fallback chain needs the GLM
|
||||
//! and Kimi keys (`mission_runtime::forwarded_provider_keys`). The agent runs
|
||||
//! Bash, so it can read them, and a prompt-injected page can ask it to. The
|
||||
//! cheapest place to stop the worst consequence is the one exit every mission's
|
||||
//! work passes through: delivery. Measured 2026-09-23 on prod: all three keys
|
||||
//! present in every mission container, and no gate rule mentions them.
|
||||
//!
|
||||
//! Exact, not heuristic. The server holds the real values, so this looks for
|
||||
//! THOSE strings (and their base64), not for things shaped like keys — no
|
||||
//! false positives on a README that explains what an API key looks like, and
|
||||
//! no false negatives on a key format nobody wrote a regex for.
|
||||
//!
|
||||
//! What this does not cover, stated so nobody assumes it does: a key sent
|
||||
//! straight to a host over the network (see the `untrusted-target` shadow rule
|
||||
//! and docs/TASK-PERMISSION-AND-TAINT.md), and a key transformed by anything
|
||||
//! but base64. The fix for both is keeping the keys out of the container.
|
||||
|
||||
use base64::Engine;
|
||||
|
||||
/// Every server-side secret a delivery must never carry. The provider keys a
|
||||
/// mission container receives, plus server-only keys that would be as bad to
|
||||
/// publish. Tested to be a superset of what the container is actually given.
|
||||
pub const WATCHED: &[&str] = &[
|
||||
"CLAUDE_CODE_OAUTH_TOKEN",
|
||||
"ANTHROPIC_API_KEY",
|
||||
"ZAI_API_KEY",
|
||||
"KIMI_API_KEY",
|
||||
"GROQ_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"ELEVENLABS_API_KEY",
|
||||
"TYPESAFE_API_KEY",
|
||||
// Not a credential: a random value set only on the server, watched exactly
|
||||
// like one, so the refusal can be proven end to end on a live mission
|
||||
// without ever putting a real key in an agent's output.
|
||||
"CLAWMATES_DELIVERY_CANARY",
|
||||
];
|
||||
|
||||
/// Shorter than this is not a credential, and matching it would find it in
|
||||
/// ordinary text.
|
||||
const MIN_LEN: usize = 16;
|
||||
|
||||
/// The watched secrets that are set here, as `(name, value)`.
|
||||
pub fn from_env() -> Vec<(String, String)> {
|
||||
WATCHED
|
||||
.iter()
|
||||
.filter_map(|n| {
|
||||
let v = std::env::var(n).ok()?;
|
||||
let v = v.trim().to_string();
|
||||
(v.len() >= MIN_LEN).then(|| (n.to_string(), v))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The spellings of one secret to look for: verbatim, and base64 with and
|
||||
/// without padding (the one encoding an agent reaches for to "hide" a string).
|
||||
fn spellings(value: &str) -> Vec<String> {
|
||||
let b64 = base64::engine::general_purpose::STANDARD.encode(value.as_bytes());
|
||||
let trimmed = b64.trim_end_matches('=').to_string();
|
||||
let mut v = vec![value.to_string(), b64];
|
||||
if !v.contains(&trimmed) {
|
||||
v.push(trimmed);
|
||||
}
|
||||
v
|
||||
}
|
||||
|
||||
/// Names of the secrets present in `text`, sorted and deduplicated.
|
||||
pub fn leaks_in(text: &str, secrets: &[(String, String)]) -> Vec<String> {
|
||||
let mut found: Vec<String> = secrets
|
||||
.iter()
|
||||
.filter(|(_, value)| spellings(value).iter().any(|s| text.contains(s.as_str())))
|
||||
.map(|(name, _)| name.clone())
|
||||
.collect();
|
||||
found.sort();
|
||||
found.dedup();
|
||||
found
|
||||
}
|
||||
|
||||
/// `text` with every spelling of every secret replaced by `[REDACTED:<NAME>]`.
|
||||
pub fn redact(text: &str, secrets: &[(String, String)]) -> String {
|
||||
let mut out = text.to_string();
|
||||
for (name, value) in secrets {
|
||||
// Longest first, so the unpadded base64 cannot eat part of the padded.
|
||||
let mut s = spellings(value);
|
||||
s.sort_by_key(|x| std::cmp::Reverse(x.len()));
|
||||
for spelling in s {
|
||||
out = out.replace(&spelling, &format!("[REDACTED:{name}]"));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The watched secrets, read once per process. Keys do not change under a
|
||||
/// running server; reading the environment on every recorded event would.
|
||||
pub fn cached() -> &'static [(String, String)] {
|
||||
static S: std::sync::OnceLock<Vec<(String, String)>> = std::sync::OnceLock::new();
|
||||
S.get_or_init(from_env)
|
||||
}
|
||||
|
||||
/// `text` with the process's watched secrets redacted, or unchanged (and
|
||||
/// unallocated) when none appear.
|
||||
pub fn scrub(text: &str) -> std::borrow::Cow<'_, str> {
|
||||
let secrets = cached();
|
||||
if leaks_in(text, secrets).is_empty() {
|
||||
std::borrow::Cow::Borrowed(text)
|
||||
} else {
|
||||
std::borrow::Cow::Owned(redact(text, secrets))
|
||||
}
|
||||
}
|
||||
|
||||
/// A JSON value with the watched secrets redacted from every string in it.
|
||||
///
|
||||
/// Through the serialized form: a secret has no quote or backslash in it, so
|
||||
/// replacing it with `[REDACTED:NAME]` leaves the JSON valid. If it somehow did
|
||||
/// not re-parse, the redacted TEXT is kept as a string rather than the
|
||||
/// original value — failing toward hiding the secret.
|
||||
pub fn scrub_json(v: serde_json::Value) -> serde_json::Value {
|
||||
let raw = v.to_string();
|
||||
match scrub(&raw) {
|
||||
std::borrow::Cow::Borrowed(_) => v,
|
||||
std::borrow::Cow::Owned(clean) => {
|
||||
serde_json::from_str(&clean).unwrap_or(serde_json::Value::String(clean))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The refusal recorded in place of a push.
|
||||
pub fn refusal(names: &[String]) -> String {
|
||||
format!(
|
||||
"REFUSED to push: the phase's changes contain {} (a server credential the mission \
|
||||
container can read). The work is committed on the local branch only and the stored \
|
||||
patch is redacted. Rotate the key(s) if this was not a test.",
|
||||
names.join(", ")
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn secrets() -> Vec<(String, String)> {
|
||||
vec![
|
||||
("ZAI_API_KEY".into(), "a1b2c3d4e5f6a7b8c9d0.ZyXwVuTsRqPo".into()),
|
||||
("KIMI_API_KEY".into(), "sk-kimi-0123456789abcdefghij".into()),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_verbatim_key_is_found_and_named() {
|
||||
let patch = "+export ZAI=a1b2c3d4e5f6a7b8c9d0.ZyXwVuTsRqPo\n";
|
||||
assert_eq!(leaks_in(patch, &secrets()), vec!["ZAI_API_KEY".to_string()]);
|
||||
}
|
||||
|
||||
/// The one transformation an agent reaches for to get a string past a check.
|
||||
#[test]
|
||||
fn a_base64_key_is_found_with_or_without_padding() {
|
||||
let b64 = base64::engine::general_purpose::STANDARD.encode("sk-kimi-0123456789abcdefghij");
|
||||
assert_eq!(leaks_in(&format!("+{b64}\n"), &secrets()), vec!["KIMI_API_KEY".to_string()]);
|
||||
let unpadded = b64.trim_end_matches('=');
|
||||
assert_eq!(leaks_in(&format!("+{unpadded}\n"), &secrets()), vec!["KIMI_API_KEY".to_string()]);
|
||||
}
|
||||
|
||||
/// Exact values, not shapes: text ABOUT keys is not a leak.
|
||||
#[test]
|
||||
fn text_that_merely_looks_like_a_key_is_not_a_leak() {
|
||||
let patch = "+ZAI_API_KEY=<your key here>\n+sk-kimi-XXXXXXXXXXXXXXXXXXXX\n";
|
||||
assert!(leaks_in(patch, &secrets()).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redaction_removes_every_spelling_and_names_the_key() {
|
||||
let b64 = base64::engine::general_purpose::STANDARD.encode("sk-kimi-0123456789abcdefghij");
|
||||
let patch = format!("+a1b2c3d4e5f6a7b8c9d0.ZyXwVuTsRqPo\n+{b64}\n");
|
||||
let r = redact(&patch, &secrets());
|
||||
assert!(leaks_in(&r, &secrets()).is_empty(), "{r}");
|
||||
assert!(r.contains("[REDACTED:ZAI_API_KEY]") && r.contains("[REDACTED:KIMI_API_KEY]"), "{r}");
|
||||
}
|
||||
|
||||
/// Whatever a mission container is GIVEN must be watched here, in both auth
|
||||
/// modes — or a key added to the forwarding list later leaks unwatched.
|
||||
#[test]
|
||||
fn every_forwarded_key_is_watched() {
|
||||
use crate::mission_runtime::{forwarded_provider_keys, RuntimeAuth};
|
||||
for auth in [RuntimeAuth::ApiKey, RuntimeAuth::Subscription] {
|
||||
for k in forwarded_provider_keys(auth) {
|
||||
assert!(WATCHED.contains(&k), "{k} is forwarded into mission containers but not watched");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// JSON stays JSON after redaction, including a secret inside a nested
|
||||
/// tool response — the shape a `printenv` lands in.
|
||||
#[test]
|
||||
fn json_redaction_keeps_the_document_valid() {
|
||||
let v = serde_json::json!({"response":{"stdout":"ZAI=a1b2c3d4e5f6a7b8c9d0.ZyXwVuTsRqPo\n"},"n":3});
|
||||
let raw = v.to_string();
|
||||
let clean = redact(&raw, &secrets());
|
||||
let back: serde_json::Value = serde_json::from_str(&clean).expect("still JSON");
|
||||
assert_eq!(back["n"], 3);
|
||||
assert!(back["response"]["stdout"].as_str().unwrap().contains("[REDACTED:ZAI_API_KEY]"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_values_are_never_watched() {
|
||||
// A too-short value would match ordinary text; from_env drops it.
|
||||
assert!(MIN_LEN >= 16);
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,17 @@ pub enum ApiError {
|
||||
Conflict,
|
||||
#[error("{0}")]
|
||||
Quota(String),
|
||||
/// A 400 whose REASON the caller needs.
|
||||
///
|
||||
/// Same argument as `Unavailable` below, one status code down. The
|
||||
/// proposal decide handlers each computed a precise refusal — "the mission
|
||||
/// is running, not a draft", "no node can boot that backend any more" —
|
||||
/// logged it to stderr, and returned a bare `BadRequest`. The person who
|
||||
/// needed the sentence was the one clicking Approve, and they got
|
||||
/// "bad request". `mission_plan::Refusal` exists and is written as
|
||||
/// human-readable copy; this is how it reaches them.
|
||||
#[error("{0}")]
|
||||
Refused(String),
|
||||
/// A dependency is temporarily refusing work and will accept it later —
|
||||
/// today, the Claude Code subscription's rate limit. Distinct from
|
||||
/// `Internal` because the operator's next action is different: wait and
|
||||
@@ -59,7 +70,7 @@ impl From<cm_auth::AuthError> for ApiError {
|
||||
impl IntoResponse for ApiError {
|
||||
fn into_response(self) -> Response {
|
||||
let status = match self {
|
||||
ApiError::BadRequest => StatusCode::BAD_REQUEST,
|
||||
ApiError::BadRequest | ApiError::Refused(_) => StatusCode::BAD_REQUEST,
|
||||
ApiError::Unauthorized => StatusCode::UNAUTHORIZED,
|
||||
ApiError::Forbidden => StatusCode::FORBIDDEN,
|
||||
ApiError::NotFound => StatusCode::NOT_FOUND,
|
||||
@@ -71,3 +82,31 @@ impl IntoResponse for ApiError {
|
||||
(status, Json(json!({ "error": self.to_string() }))).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::body::to_bytes;
|
||||
|
||||
/// A refusal must carry its reason into the response body.
|
||||
///
|
||||
/// The proposal decide handlers each computed a precise sentence and then
|
||||
/// returned a bare `BadRequest`, so the person clicking Approve saw
|
||||
/// "bad request" while the reason went to a server log they cannot read.
|
||||
#[tokio::test]
|
||||
async fn a_refusal_reaches_the_caller_and_a_bare_bad_request_does_not_pretend_to() {
|
||||
let refused = ApiError::Refused("this mission is running, not a draft".into());
|
||||
let response = refused.into_response();
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
|
||||
let text = String::from_utf8_lossy(&body);
|
||||
assert!(
|
||||
text.contains("running, not a draft"),
|
||||
"the reason must be in the body, not only in the server log: {text}"
|
||||
);
|
||||
|
||||
// The bare variant stays as it was — same status, no invented detail.
|
||||
let bare = ApiError::BadRequest.into_response();
|
||||
assert_eq!(bare.status(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
}
|
||||
|
||||
+661
-34
@@ -33,6 +33,22 @@ use serde_json::Value;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// The model's verdict on one pass.
|
||||
/// What one verdict cost, in provider calls and tokens.
|
||||
///
|
||||
/// Accumulated across every round of the judge's tool loop, and kept on a
|
||||
/// FAILED attempt too — that is the case that matters. `LlmEvent::Usage` was
|
||||
/// arriving on every call and being dropped on the floor (`Ok(_) => {}`), so
|
||||
/// the z.ai plan emptied twice with nothing anywhere recording a single judge
|
||||
/// token. `usage_events` had no provider or model column; the first signal
|
||||
/// was every mission failing at once.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Usage {
|
||||
/// Model requests made. One verdict is up to `MAX_TOOL_CALLS + 1` of these.
|
||||
pub requests: u32,
|
||||
pub tokens_in: u64,
|
||||
pub tokens_out: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Verdict {
|
||||
pub met: bool,
|
||||
@@ -66,6 +82,14 @@ pub struct Verdict {
|
||||
/// read back as "not independent", which is what they were.
|
||||
#[serde(default)]
|
||||
pub independent: bool,
|
||||
/// What this attempt cost. Recorded to `usage_events` by [`record`].
|
||||
#[serde(default)]
|
||||
pub usage: Usage,
|
||||
/// The judge's verification plan, committed from the condition alone
|
||||
/// BEFORE it read the evidence. See [`commit_expectation`]. `None` when
|
||||
/// the judge had no commit round (fallback paths, or the round failed).
|
||||
#[serde(default)]
|
||||
pub expectation: Option<String>,
|
||||
}
|
||||
|
||||
impl Verdict {
|
||||
@@ -92,6 +116,8 @@ impl Verdict {
|
||||
independent: false,
|
||||
error,
|
||||
checks: Vec::new(),
|
||||
usage: Usage::default(),
|
||||
expectation: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -212,7 +238,117 @@ require content the condition does not ask for.
|
||||
Judge the condition AS WRITTEN. Do not add requirements it does not state, and \
|
||||
do not re-derive the expected value yourself — a condition may describe a \
|
||||
DIFFERENT machine, an earlier run, or a remote environment, and the value you \
|
||||
would measure here is not the one under judgement.";
|
||||
would measure here is not the one under judgement.
|
||||
You have a limited number of commands. A file you `cat` comes back COMPLETE \
|
||||
unless the output says bytes were omitted; do not read it again with head, \
|
||||
tail, sed or grep — check the claim against the text you already have. Decide \
|
||||
what you need to verify first, then read each file once. Outputs from earlier \
|
||||
rounds are shortened to their opening lines to save space, so read a file in \
|
||||
the round you intend to check it.";
|
||||
|
||||
/// Prompt for the commit round: the judge writes its verification plan from
|
||||
/// the condition alone, before it has seen a word of what the agents claim.
|
||||
///
|
||||
/// Self-Play Reward Hacking of Reference-Free Judges (arXiv 2607.05904)
|
||||
/// measured a judge's pass rate climbing 0.72 → 0.94 across rounds while the
|
||||
/// answers stayed 0.20 correct: a judge that reads the candidate first is
|
||||
/// argued into the candidate's framing. Cross-family judges and three-judge
|
||||
/// ensembles did not help. The one mitigation that did was making the judge
|
||||
/// commit to its own answer first (false-positive rate 0.719 → 0.012).
|
||||
///
|
||||
/// The commitment here is a plan, not an answer — a judge that "commits" to
|
||||
/// an expected VALUE re-derives a measurement the condition may describe for
|
||||
/// a different machine, which `EVAL_SYSTEM_VERIFYING` already forbids. What
|
||||
/// it commits to is which files, strings and tests would show MET, and which
|
||||
/// commands would show it. The verifying prompt then holds it to that.
|
||||
const EVAL_SYSTEM_COMMIT: &str = "\
|
||||
You are about to judge whether a phase of automated work is complete. You \
|
||||
have NOT yet seen what the agents produced, and you must not guess at it.
|
||||
|
||||
From the COMPLETION CONDITION alone, write down what MET would look like:
|
||||
|
||||
- each requirement the condition ACTUALLY STATES, one per line — do not add \
|
||||
requirements it does not state, and do not re-derive expected values;
|
||||
- for each, the concrete evidence that would show it: which file, which \
|
||||
string or symbol, which test name, which command output;
|
||||
- the commands you intend to run to check it, fewest first — a `git diff` \
|
||||
or `rg` that settles several requirements at once beats one command each.
|
||||
|
||||
Plain text, at most 20 lines. No verdict yet.";
|
||||
|
||||
/// The prompt the verifying judge reads, with its own commitment placed
|
||||
/// between the condition and the evidence so it meets the agents' claims
|
||||
/// already knowing what it is looking for.
|
||||
fn judge_user(condition: &str, evidence: &str, expectation: Option<&str>) -> String {
|
||||
match expectation {
|
||||
Some(plan) => format!(
|
||||
"COMPLETION CONDITION:\n{condition}\n\n\
|
||||
YOUR VERIFICATION PLAN (you wrote this before seeing the evidence — \
|
||||
check what it names, and if the evidence pulls you toward a different \
|
||||
reading of the condition, say so in `reason` rather than silently \
|
||||
adopting it):\n{plan}\n\n\
|
||||
EVIDENCE (agent claims — verify them):\n{evidence}"
|
||||
),
|
||||
None => format!(
|
||||
"COMPLETION CONDITION:\n{condition}\n\nEVIDENCE (agent claims — verify them):\n{evidence}"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// One tool-free request on the condition alone. Counted in `usage` like any
|
||||
/// other; a failure here is logged and the verdict proceeds without a plan,
|
||||
/// because the round exists to make the judge harder to argue with, and a
|
||||
/// judge that cannot be reached at all fails on the next request anyway.
|
||||
async fn commit_expectation(
|
||||
provider: &dyn cm_llm::LlmProvider,
|
||||
condition: &str,
|
||||
model: &str,
|
||||
usage: &mut Usage,
|
||||
) -> Option<String> {
|
||||
use cm_llm::{ChatMessage, ChatRequest, ChatRole, ContentPart, LlmEvent};
|
||||
use futures::StreamExt as _;
|
||||
let request = ChatRequest {
|
||||
system: EVAL_SYSTEM_COMMIT.to_string(),
|
||||
model: model.to_string(),
|
||||
messages: vec![ChatMessage {
|
||||
role: ChatRole::User,
|
||||
parts: vec![ContentPart::text(format!("COMPLETION CONDITION:\n{condition}"))],
|
||||
}],
|
||||
tools: vec![],
|
||||
// A reasoning model thinks before the 20 lines; see the verdict
|
||||
// request for why the budget is generous and only emitted tokens bill.
|
||||
max_tokens: 8192,
|
||||
web_search: false,
|
||||
};
|
||||
usage.requests += 1;
|
||||
let mut stream = match provider.stream(request).await {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
eprintln!("evaluator: commit round failed for {model}, judging without a plan: {e}");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let mut text = String::new();
|
||||
while let Some(event) = stream.next().await {
|
||||
match event {
|
||||
Ok(LlmEvent::TextDelta(t)) => text.push_str(&t),
|
||||
Ok(LlmEvent::Usage { input_tokens, output_tokens }) => {
|
||||
usage.tokens_in += u64::from(input_tokens);
|
||||
usage.tokens_out += u64::from(output_tokens);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
eprintln!("evaluator: commit round failed for {model}, judging without a plan: {e}");
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
let text = text.trim();
|
||||
if text.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(head(text, 4000))
|
||||
}
|
||||
|
||||
/// Which provider family a model spec belongs to.
|
||||
///
|
||||
@@ -271,13 +407,27 @@ fn names_a_provider(spec: &str) -> bool {
|
||||
spec.contains(':')
|
||||
}
|
||||
|
||||
/// The provider family the mission's agent ran on.
|
||||
/// The provider family the mission's agent ran on, from `missions.backend`.
|
||||
///
|
||||
/// Today every mission backend is Claude Code (`agent-claude`), including the
|
||||
/// microVM path. When `agent-glm` / `agent-kimi` images exist this should read
|
||||
/// `missions.backend`; until then, hardcoding the truth is better than plumbing a
|
||||
/// parameter that only ever has one value.
|
||||
const IMPLEMENTER_FAMILY: &str = "anthropic";
|
||||
/// Mirrors `mission_runtime::microvm_credential_for`: the backend decides which
|
||||
/// credential the guest gets and which host its egress proxy allows, so it is
|
||||
/// the one honest source for "who answered the agent's turns". This was a
|
||||
/// hardcoded `"anthropic"` while every backend was Claude Code on Anthropic;
|
||||
/// once `glm` and `kimi` rootfs existed that constant made a glm-backend
|
||||
/// mission judged by `glm:glm-5.3` read as `independent = true`, which is the
|
||||
/// one claim this path exists to make honestly.
|
||||
///
|
||||
/// `unknown` for anything unrecognised, for the same reason `provider_family`
|
||||
/// says it: a guess in either direction misstates independence.
|
||||
pub fn implementer_family(backend: Option<&str>) -> &'static str {
|
||||
match backend.map(str::trim) {
|
||||
None | Some("") | Some("default") | Some("claude") | Some("canary-claude") => "anthropic",
|
||||
Some("glm") => "glm",
|
||||
Some("kimi") => "kimi",
|
||||
Some("local-ornith") => "local",
|
||||
Some(_) => "unknown",
|
||||
}
|
||||
}
|
||||
|
||||
/// Which validator spec applies, given the mission's own setting and the
|
||||
/// deployment default.
|
||||
@@ -313,9 +463,23 @@ fn resolve_validator_spec(mission: Option<&str>, deployment: Option<&str>) -> Op
|
||||
/// back Claude while the caller believed it had asked for GLM. The fallback is
|
||||
/// detectable because the returned model still carries the `name:` prefix, and it
|
||||
/// is checked here rather than trusted.
|
||||
/// The mission's implementer family, read from its row. `anthropic` when the
|
||||
/// row cannot be read — the pre-2026-09-18 behaviour, and the family every
|
||||
/// backend actually had until then.
|
||||
async fn mission_implementer_family(runtime: &cm_runtime::Runtime, mission_id: Uuid) -> &'static str {
|
||||
let backend: Option<String> = sqlx::query_scalar("SELECT backend FROM missions WHERE id = $1")
|
||||
.bind(mission_id)
|
||||
.fetch_optional(runtime.pool())
|
||||
.await
|
||||
.unwrap_or(None)
|
||||
.flatten();
|
||||
implementer_family(backend.as_deref())
|
||||
}
|
||||
|
||||
async fn cross_provider_judge(
|
||||
runtime: &cm_runtime::Runtime,
|
||||
mission_id: Uuid,
|
||||
implementer: &str,
|
||||
) -> Option<(std::sync::Arc<dyn cm_llm::LlmProvider>, String)> {
|
||||
// Read per mission rather than widening `Mission` for one caller. One extra
|
||||
// query per evaluation, against a path that is about to make a model call.
|
||||
@@ -330,12 +494,60 @@ async fn cross_provider_judge(
|
||||
per_mission.as_deref(),
|
||||
std::env::var("CLAWMATES_VALIDATOR_MODEL").ok().as_deref(),
|
||||
)?;
|
||||
let spec = spec.as_str();
|
||||
independent_judge(runtime, &spec, implementer)
|
||||
}
|
||||
|
||||
/// The second independent judge, used only when the first could not answer.
|
||||
///
|
||||
/// `CLAWMATES_VALIDATOR_FALLBACK_MODEL`, e.g. `kimi:kimi-for-coding`. Added
|
||||
/// 2026-09-23 after GLM's plan limit ran out for the second time in a month:
|
||||
/// with one judge, every conditioned phase on every mission fails on the judge
|
||||
/// until the quota resets (days). Held to every check the primary is, plus one
|
||||
/// more: a different family from the primary as well, or it is the same outage
|
||||
/// twice.
|
||||
///
|
||||
/// Measured before it was wired: `scripts/judge-eval.sh`, 15 cases x 3 draws,
|
||||
/// kimi-for-coding 44/45 against glm-5.3's 43/45. `goodhart`, the false
|
||||
/// positive that once ruled Kimi out, was 3/3. The miss was one draw of
|
||||
/// `should-panic-hack`, the shape GLM also misses.
|
||||
async fn fallback_judge(
|
||||
runtime: &cm_runtime::Runtime,
|
||||
implementer: &str,
|
||||
primary_model_family: &str,
|
||||
) -> Option<(std::sync::Arc<dyn cm_llm::LlmProvider>, String)> {
|
||||
let spec = fallback_spec(
|
||||
std::env::var("CLAWMATES_VALIDATOR_FALLBACK_MODEL").ok().as_deref(),
|
||||
primary_model_family,
|
||||
)?;
|
||||
independent_judge(runtime, &spec, implementer)
|
||||
}
|
||||
|
||||
/// The fallback spec, if one is set and it is not the primary's own family.
|
||||
fn fallback_spec(env: Option<&str>, primary_family: &str) -> Option<String> {
|
||||
let spec = env.map(str::trim).filter(|s| !s.is_empty())?;
|
||||
if provider_family(spec) == primary_family {
|
||||
eprintln!(
|
||||
"evaluator: CLAWMATES_VALIDATOR_FALLBACK_MODEL={spec} is the primary judge's own \
|
||||
family ({primary_family}) — a quota or outage takes both down; ignoring it"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
Some(spec.to_string())
|
||||
}
|
||||
|
||||
/// Resolve `spec` to a judge that is genuinely independent of `implementer`,
|
||||
/// or `None` with the reason logged. Shared by the primary and the fallback so
|
||||
/// neither can be held to a weaker standard than the other.
|
||||
fn independent_judge(
|
||||
runtime: &cm_runtime::Runtime,
|
||||
spec: &str,
|
||||
implementer: &str,
|
||||
) -> Option<(std::sync::Arc<dyn cm_llm::LlmProvider>, String)> {
|
||||
let family = provider_family(spec);
|
||||
if family == IMPLEMENTER_FAMILY {
|
||||
if family == implementer {
|
||||
eprintln!(
|
||||
"evaluator: CLAWMATES_VALIDATOR_MODEL={spec} is the same provider family as the \
|
||||
agent ({IMPLEMENTER_FAMILY}) — that is not an independent check, ignoring it"
|
||||
agent ({implementer}) — that is not an independent check, ignoring it"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
@@ -387,11 +599,22 @@ pub fn evaluator_model() -> String {
|
||||
std::env::var("CLAWMATES_EVALUATOR_MODEL").unwrap_or_else(|_| cm_runtime::judge_model())
|
||||
}
|
||||
|
||||
/// The model the direct subscription path judges with. Small and fast by
|
||||
/// default — a verdict is a classification, not a composition.
|
||||
/// The model the direct subscription path judges with.
|
||||
///
|
||||
/// This defaulted to haiku on the reasoning that "a verdict is a
|
||||
/// classification, not a composition". The shape of the ANSWER is a boolean;
|
||||
/// the WORK is not. Reaching a verdict means reading a phase's evidence and
|
||||
/// checking it against the repository — on mission 01a00bbb the judge had to
|
||||
/// notice that the agents claimed six INT items while git history contained
|
||||
/// three, and then write guidance for the next pass.
|
||||
///
|
||||
/// It is also the single component whose failure mode is passing work that was
|
||||
/// never done, which is the recurring defect in this codebase. Under the
|
||||
/// operator's model policy (haiku only for genuine yes/no lookups, thinking on
|
||||
/// opus) this is a thinking job.
|
||||
fn subscription_model() -> String {
|
||||
std::env::var("CLAWMATES_EVALUATOR_SUBSCRIPTION_MODEL")
|
||||
.unwrap_or_else(|_| "claude-haiku-4-5-20251001".to_string())
|
||||
.unwrap_or_else(|_| "claude-opus-5".to_string())
|
||||
}
|
||||
|
||||
/// A judge that talks to the Messages API directly on the subscription token,
|
||||
@@ -427,10 +650,22 @@ pub async fn evaluate(
|
||||
condition: &str,
|
||||
evidence: &str,
|
||||
) -> Verdict {
|
||||
let user = format!(
|
||||
"COMPLETION CONDITION:\n{condition}\n\nEVIDENCE (agent claims — verify them):\n{evidence}"
|
||||
);
|
||||
let sandbox = crate::evaluator_tools::Sandbox::for_mission(mission_id);
|
||||
// A JavaScript project arrives without node_modules (excluded from the
|
||||
// copy on purpose) in a container with no registry. Install offline from
|
||||
// the lockfile first, and tell the judge how that went.
|
||||
let deps_note = match &sandbox {
|
||||
Some(sb) => sb.prepare_dependencies(mission_id).await,
|
||||
None => None,
|
||||
};
|
||||
let evidence_with_deps;
|
||||
let evidence = match deps_note {
|
||||
Some(note) => {
|
||||
evidence_with_deps = format!("{evidence}\n\n{note}");
|
||||
evidence_with_deps.as_str()
|
||||
}
|
||||
None => evidence,
|
||||
};
|
||||
// Purged explicitly at every exit below: `Drop` runs as uid 65532 and cannot
|
||||
// delete the root-owned `target/` the judge's own `cargo test` leaves behind.
|
||||
// Wrapped so the purge below runs on EVERY exit: this function returns
|
||||
@@ -443,7 +678,32 @@ pub async fn evaluate(
|
||||
// failure — the model that talked itself into a shortcut is the one disposed
|
||||
// to accept it — and the tool loop is what makes the check evidence rather
|
||||
// than opinion, so an independent judge must have it too.
|
||||
if let Some((provider, model)) = cross_provider_judge(runtime, mission_id).await {
|
||||
let implementer = mission_implementer_family(runtime, mission_id).await;
|
||||
// Skip a judge whose plan is about to run out, BEFORE spending a call
|
||||
// that would fail with a 429. Only on a real reading
|
||||
// (`judge_quota::near_limit` is None without one), and only when a
|
||||
// fallback that passes the same independence checks exists.
|
||||
let chosen = match cross_provider_judge(runtime, mission_id, implementer).await {
|
||||
Some((p, m)) => {
|
||||
let family = provider_family(&m);
|
||||
match crate::judge_quota::near_limit(&family) {
|
||||
Some(w) => match fallback_judge(runtime, implementer, &family).await {
|
||||
Some((fp, fm)) => {
|
||||
eprintln!(
|
||||
"evaluator: {m}'s plan is at {:.0}% of its {} window — judging \
|
||||
with {fm} instead of spending a call that would fail",
|
||||
w.used_pct, w.name
|
||||
);
|
||||
Some((fp, fm))
|
||||
}
|
||||
None => Some((p, m)),
|
||||
},
|
||||
None => Some((p, m)),
|
||||
}
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
if let Some((provider, model)) = chosen {
|
||||
let system = match &sandbox {
|
||||
Some(_) => format!("{EVAL_SYSTEM_VERIFYING}\n\n{VERDICT_CONTRACT}"),
|
||||
None => format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}"),
|
||||
@@ -453,12 +713,27 @@ pub async fn evaluate(
|
||||
model,
|
||||
provider_family(&model)
|
||||
);
|
||||
match judge_with_tools(provider.as_ref(), &system, &user, &model, sandbox.as_ref()).await {
|
||||
let mut usage = Usage::default();
|
||||
let expectation =
|
||||
commit_expectation(provider.as_ref(), condition, &model, &mut usage).await;
|
||||
let user = judge_user(condition, evidence, expectation.as_deref());
|
||||
match judge_with_tools(
|
||||
provider.as_ref(),
|
||||
&system,
|
||||
&user,
|
||||
&model,
|
||||
sandbox.as_ref(),
|
||||
&mut usage,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok((text, checks)) => {
|
||||
let mut v = parse_verdict(&model, &text);
|
||||
v.guidance = sanitize_guidance(condition, evidence, &v.guidance);
|
||||
v.checks = checks;
|
||||
v.independent = true;
|
||||
v.usage = usage;
|
||||
v.expectation = expectation;
|
||||
return v;
|
||||
}
|
||||
// Deliberately NOT a silent fall-through to the house judge. An
|
||||
@@ -467,14 +742,76 @@ pub async fn evaluate(
|
||||
// not have. The phase stays unmet this pass and says why; the next
|
||||
// sweep retries.
|
||||
Err(e) => {
|
||||
// A SECOND independent family, when one is configured. Still
|
||||
// never the agent's own: `fallback_judge` applies the same
|
||||
// checks as the primary.
|
||||
let primary_family = provider_family(&model);
|
||||
let primary_family = if primary_family == "unknown" {
|
||||
std::env::var("CLAWMATES_VALIDATOR_MODEL")
|
||||
.map(|s| provider_family(&s))
|
||||
.unwrap_or(primary_family)
|
||||
} else {
|
||||
primary_family
|
||||
};
|
||||
if let Some((fb, fb_model)) =
|
||||
fallback_judge(runtime, implementer, &primary_family).await
|
||||
{
|
||||
eprintln!(
|
||||
"evaluator: the independent judge ({model}) failed — NOT falling back to the agent's own provider: {e}"
|
||||
"evaluator: the independent judge ({model}) failed ({}) — falling \
|
||||
back to {fb_model}, also independent of the agent",
|
||||
e.chars().take(160).collect::<String>()
|
||||
);
|
||||
return Verdict::not_met(
|
||||
let mut fb_usage = Usage::default();
|
||||
let fb_expectation =
|
||||
commit_expectation(fb.as_ref(), condition, &fb_model, &mut fb_usage).await;
|
||||
let fb_user = judge_user(condition, evidence, fb_expectation.as_deref());
|
||||
match judge_with_tools(
|
||||
fb.as_ref(),
|
||||
&system,
|
||||
&fb_user,
|
||||
&fb_model,
|
||||
sandbox.as_ref(),
|
||||
&mut fb_usage,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok((text, checks)) => {
|
||||
let mut v = parse_verdict(&fb_model, &text);
|
||||
v.guidance = sanitize_guidance(condition, evidence, &v.guidance);
|
||||
v.checks = checks;
|
||||
v.independent = true;
|
||||
v.usage = fb_usage;
|
||||
v.expectation = fb_expectation;
|
||||
return v;
|
||||
}
|
||||
// Both down. The PRIMARY's error leads: the phase
|
||||
// runner reads it to tell a plan limit (do not
|
||||
// spend the pass) from a transient failure.
|
||||
Err(e2) => {
|
||||
eprintln!("evaluator: the fallback judge ({fb_model}) failed too: {e2}");
|
||||
let mut v = Verdict::not_met(
|
||||
&model,
|
||||
"neither independent validator could be reached this pass",
|
||||
Some(format!("{e} | fallback {fb_model}: {e2}")),
|
||||
);
|
||||
v.usage = usage;
|
||||
v.expectation = expectation;
|
||||
return v;
|
||||
}
|
||||
}
|
||||
}
|
||||
eprintln!(
|
||||
"evaluator: the independent judge ({model}) failed — NOT falling back to \
|
||||
the agent's own provider: {e}"
|
||||
);
|
||||
let mut v = Verdict::not_met(
|
||||
&model,
|
||||
"the independent validator could not be reached this pass",
|
||||
Some(e),
|
||||
);
|
||||
v.usage = usage;
|
||||
v.expectation = expectation;
|
||||
return v;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -487,9 +824,16 @@ pub async fn evaluate(
|
||||
Some(_) => format!("{EVAL_SYSTEM_VERIFYING}\n\n{VERDICT_CONTRACT}"),
|
||||
None => format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}"),
|
||||
};
|
||||
let outcome = judge_with_tools(&provider, &system, &user, &model, sandbox.as_ref()).await;
|
||||
// Same family as the agent; `independent` stays false below.
|
||||
return match outcome {
|
||||
let mut usage = Usage::default();
|
||||
let expectation = commit_expectation(&provider, condition, &model, &mut usage).await;
|
||||
let user = judge_user(condition, evidence, expectation.as_deref());
|
||||
let outcome =
|
||||
judge_with_tools(&provider, &system, &user, &model, sandbox.as_ref(), &mut usage)
|
||||
.await;
|
||||
// Independent exactly when the agent did NOT run on Anthropic: a
|
||||
// glm- or kimi-backend mission judged by the Anthropic subscription
|
||||
// is a cross-provider check, and a claude-backend one is not.
|
||||
let mut v = match outcome {
|
||||
Err(e) => Verdict::not_met(
|
||||
&model,
|
||||
"could not evaluate the completion condition this pass",
|
||||
@@ -502,11 +846,15 @@ pub async fn evaluate(
|
||||
v
|
||||
}
|
||||
};
|
||||
v.usage = usage;
|
||||
v.independent = implementer != "anthropic";
|
||||
v.expectation = expectation;
|
||||
return v;
|
||||
}
|
||||
|
||||
// Fallback paths have no tool loop, so they judge claims only and must say so.
|
||||
let system = format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}");
|
||||
let (eval_system, user) = (system.as_str(), user);
|
||||
let (eval_system, user) = (system.as_str(), judge_user(condition, evidence, None));
|
||||
|
||||
let model = evaluator_model();
|
||||
// Same routing as the door governor (mcp_door.rs): `runtime:<alias>` goes
|
||||
@@ -593,6 +941,7 @@ async fn judge_with_tools(
|
||||
user: &str,
|
||||
model: &str,
|
||||
sandbox: Option<&crate::evaluator_tools::Sandbox>,
|
||||
usage: &mut Usage,
|
||||
) -> Result<(String, Vec<crate::evaluator_tools::CheckOutcome>), String> {
|
||||
use cm_llm::{ChatMessage, ChatRequest, ChatRole, ContentPart, LlmEvent};
|
||||
use futures::StreamExt as _;
|
||||
@@ -614,9 +963,28 @@ async fn judge_with_tools(
|
||||
model: model.to_string(),
|
||||
messages: messages.clone(),
|
||||
tools: tools.clone(),
|
||||
max_tokens: 1024,
|
||||
// Room to actually ANALYSE. glm-5.3 is a reasoning model: it
|
||||
// spends most of this budget on a `thinking` block and only then
|
||||
// writes the verdict JSON. On a realistic phase prompt it used
|
||||
// 819 tokens; a phase with 25 items and 120 KB of evidence has far
|
||||
// more to work through, and running out mid-thought truncates the
|
||||
// verdict. A truncated verdict parses as empty and FAILS CLOSED,
|
||||
// burning one of the phase's passes on a judge that never answered
|
||||
// — how mission 01a00bbb lost one.
|
||||
//
|
||||
// Measured ceiling: z.ai accepts max_tokens up to 131072 on
|
||||
// glm-5.1 and glm-5.3 (131073 -> 400, "限制数值范围[1,131072]"),
|
||||
// so this is nowhere near a limit. It is chosen for cost and
|
||||
// latency, not capability, and only what the model actually emits
|
||||
// is billed — `stop_reason: end_turn` well under the cap is the
|
||||
// normal case.
|
||||
max_tokens: 16384,
|
||||
web_search: false,
|
||||
};
|
||||
// Counted BEFORE the stream is opened: a request the provider refused
|
||||
// with a 429 is still a request we made, and the storm of those is the
|
||||
// thing this accounting exists to make visible.
|
||||
usage.requests += 1;
|
||||
let mut stream = provider.stream(request).await.map_err(|e| e.to_string())?;
|
||||
let mut text = String::new();
|
||||
let mut calls: Vec<(String, String, Value)> = Vec::new();
|
||||
@@ -624,6 +992,10 @@ async fn judge_with_tools(
|
||||
match event {
|
||||
Ok(LlmEvent::TextDelta(t)) => text.push_str(&t),
|
||||
Ok(LlmEvent::ToolUse { id, name, input }) => calls.push((id, name, input)),
|
||||
Ok(LlmEvent::Usage { input_tokens, output_tokens }) => {
|
||||
usage.tokens_in += u64::from(input_tokens);
|
||||
usage.tokens_out += u64::from(output_tokens);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(e) => return Err(e.to_string()),
|
||||
}
|
||||
@@ -694,6 +1066,9 @@ async fn judge_with_tools(
|
||||
content: Value::String(evidence),
|
||||
});
|
||||
}
|
||||
// Everything the judge has already read shrinks to a reminder before
|
||||
// this round's results go in full. See `compact_earlier_results`.
|
||||
compact_earlier_results(&mut messages);
|
||||
messages.push(ChatMessage {
|
||||
role: ChatRole::User,
|
||||
parts: results,
|
||||
@@ -702,6 +1077,56 @@ async fn judge_with_tools(
|
||||
Err("evaluator exceeded its verification budget without reaching a verdict".into())
|
||||
}
|
||||
|
||||
/// How much of an earlier check's output stays in the history.
|
||||
///
|
||||
/// Enough to recognise the command and its outcome — a test summary line, a
|
||||
/// grep hit, an error — not enough to re-read the whole thing, which the judge
|
||||
/// already did in the round it arrived.
|
||||
const KEPT_OF_EARLIER_RESULT: usize = 800;
|
||||
|
||||
/// Shrink every tool result from EARLIER rounds to a short head.
|
||||
///
|
||||
/// The judge's history is resent whole on every round, and each check's
|
||||
/// output is bounded at `evaluator_tools::MAX_OUTPUT_BYTES` (12 KB). Measured
|
||||
/// on prod, 7 of 9 verdicts ran to the 12-check cap, so by the last round the
|
||||
/// history carried ~144 KB of outputs the judge had already read, on top of
|
||||
/// up to 120 KB of evidence — and every round paid for all of it again. That
|
||||
/// is the quadratic term in a verdict's cost, and it is why one blocked phase
|
||||
/// could empty a weekly plan.
|
||||
///
|
||||
/// The round that just ran keeps its results in full; only what came before
|
||||
/// is compacted, and it is compacted once — a result already carrying the
|
||||
/// marker is left alone. The judge's budget of checks is unchanged: this
|
||||
/// makes each check cheaper to remember, not fewer to run.
|
||||
fn compact_earlier_results(messages: &mut [cm_llm::ChatMessage]) {
|
||||
use cm_llm::ContentPart;
|
||||
const MARKER: &str = "\n[… output elided here — it was shown in full when this check ran]";
|
||||
// The most recent round's results stay whole, so a read lives through
|
||||
// TWO model calls — the one that receives it and the one after. With
|
||||
// everything compacted at once, mission 01a0b803 read REPORT.md in full
|
||||
// (18,698 B, the 64 KB window working) and then, the round after, ran
|
||||
// wc/head/tail/sed/grep against it anyway: the file was already down to
|
||||
// 800 bytes by the time the judge went to check a claim against it.
|
||||
let keep = messages
|
||||
.iter()
|
||||
.rposition(|m| m.parts.iter().any(|p| matches!(p, ContentPart::ToolResult { .. })));
|
||||
for (i, m) in messages.iter_mut().enumerate() {
|
||||
if Some(i) == keep {
|
||||
continue;
|
||||
}
|
||||
for part in m.parts.iter_mut() {
|
||||
if let ContentPart::ToolResult { content, .. } = part {
|
||||
if let Some(text) = content.as_str() {
|
||||
if text.len() > KEPT_OF_EARLIER_RESULT && !text.ends_with(MARKER) {
|
||||
let kept = head(text, KEPT_OF_EARLIER_RESULT);
|
||||
*content = Value::String(format!("{kept}{MARKER}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse the model's reply into a verdict, failing closed.
|
||||
fn parse_verdict(model: &str, text: &str) -> Verdict {
|
||||
let trimmed = text.trim();
|
||||
@@ -752,6 +1177,8 @@ fn parse_verdict(model: &str, text: &str) -> Verdict {
|
||||
independent: false,
|
||||
error: None,
|
||||
checks: Vec::new(),
|
||||
usage: Usage::default(),
|
||||
expectation: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -775,13 +1202,14 @@ pub async fn record(
|
||||
sqlx::query(
|
||||
"INSERT INTO mission_phase_evaluations
|
||||
(id, mission_id, phase_id, iteration, met, reason, guidance, model, error,
|
||||
checks, independent)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||
checks, independent, expectation)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
||||
ON CONFLICT (phase_id, iteration) DO UPDATE
|
||||
SET met = EXCLUDED.met, reason = EXCLUDED.reason,
|
||||
guidance = EXCLUDED.guidance, model = EXCLUDED.model,
|
||||
error = EXCLUDED.error, checks = EXCLUDED.checks,
|
||||
independent = EXCLUDED.independent",
|
||||
independent = EXCLUDED.independent,
|
||||
expectation = EXCLUDED.expectation",
|
||||
)
|
||||
.bind(Uuid::now_v7())
|
||||
.bind(mission_id)
|
||||
@@ -794,9 +1222,32 @@ pub async fn record(
|
||||
.bind(v.error.as_deref())
|
||||
.bind(serde_json::json!(v.checks))
|
||||
.bind(v.independent)
|
||||
.bind(v.expectation.as_deref())
|
||||
.execute(pool)
|
||||
.await
|
||||
.map(|_| ())
|
||||
.await?;
|
||||
|
||||
// The judge's spend, beside the verdict it bought. `kind = 'judge'` keeps
|
||||
// it apart from the agents' `llm_tokens`, and `provider` is what makes a
|
||||
// plan-limit question answerable before the plan answers it for you.
|
||||
// Recorded for a failed attempt too: `requests` on those is the number
|
||||
// that emptied the plan.
|
||||
if v.usage.requests > 0 {
|
||||
sqlx::query(
|
||||
"INSERT INTO usage_events
|
||||
(workspace_id, kind, tokens_in, tokens_out, provider, model, mission_id, requests)
|
||||
SELECT workspace_id, 'judge', $2, $3, $4, $5, id, $6
|
||||
FROM missions WHERE id = $1",
|
||||
)
|
||||
.bind(mission_id)
|
||||
.bind(v.usage.tokens_in as i64)
|
||||
.bind(v.usage.tokens_out as i64)
|
||||
.bind(provider_family(&v.model))
|
||||
.bind(&v.model)
|
||||
.bind(v.usage.requests as i32)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The most recent verdict for a phase, used to carry guidance into the next
|
||||
@@ -829,6 +1280,87 @@ pub async fn latest(
|
||||
|
||||
#[cfg(test)]
|
||||
mod cross_provider_tests {
|
||||
/// The quadratic term: earlier outputs resent whole every round.
|
||||
#[test]
|
||||
fn earlier_tool_results_shrink_and_the_latest_stays_whole() {
|
||||
use cm_llm::{ChatMessage, ChatRole, ContentPart};
|
||||
let big = "line of output\n".repeat(900); // ~13 KB
|
||||
let mut messages = vec![
|
||||
ChatMessage {
|
||||
role: ChatRole::User,
|
||||
parts: vec![ContentPart::text("judge this")],
|
||||
},
|
||||
ChatMessage {
|
||||
role: ChatRole::User,
|
||||
parts: vec![
|
||||
ContentPart::ToolResult { tool_use_id: "a".into(), content: Value::String(big.clone()) },
|
||||
ContentPart::ToolResult { tool_use_id: "b".into(), content: Value::String(big.clone()) },
|
||||
],
|
||||
},
|
||||
];
|
||||
// A second, later round of results: the compaction must leave THIS one
|
||||
// whole and shrink only the round before it.
|
||||
messages.push(ChatMessage {
|
||||
role: ChatRole::User,
|
||||
parts: vec![ContentPart::ToolResult { tool_use_id: "c".into(), content: Value::String(big.clone()) }],
|
||||
});
|
||||
compact_earlier_results(&mut messages);
|
||||
for part in &messages[1].parts {
|
||||
let ContentPart::ToolResult { content, .. } = part else { panic!() };
|
||||
let s = content.as_str().unwrap();
|
||||
assert!(s.len() < KEPT_OF_EARLIER_RESULT + 120, "not compacted: {} bytes", s.len());
|
||||
assert!(s.starts_with("line of output"), "the head survives");
|
||||
assert!(s.contains("elided"), "and says so");
|
||||
}
|
||||
let ContentPart::ToolResult { content, .. } = &messages[2].parts[0] else { panic!() };
|
||||
assert_eq!(content.as_str().unwrap().len(), big.len(), "the latest round stays whole");
|
||||
// Idempotent: a second pass must not shrink the reminder further.
|
||||
let once: Vec<String> = messages[1].parts.iter().map(|p| serde_json::to_string(p).unwrap()).collect();
|
||||
compact_earlier_results(&mut messages);
|
||||
let twice: Vec<String> = messages[1].parts.iter().map(|p| serde_json::to_string(p).unwrap()).collect();
|
||||
assert_eq!(once, twice);
|
||||
// And the latest round is still whole after the second pass.
|
||||
let ContentPart::ToolResult { content, .. } = &messages[2].parts[0] else { panic!() };
|
||||
assert_eq!(content.as_str().unwrap().len(), big.len());
|
||||
// Plain text parts are untouched.
|
||||
assert!(matches!(&messages[0].parts[0], ContentPart::Text { text } if text == "judge this"));
|
||||
}
|
||||
|
||||
/// `LlmEvent::Usage` arrives on every provider call. It was matched by
|
||||
/// `Ok(_) => {}` and dropped, which is how two plan exhaustions happened
|
||||
/// with no row anywhere saying a judge token was spent.
|
||||
#[tokio::test]
|
||||
async fn a_verdict_records_what_it_cost() {
|
||||
let provider = cm_llm::ScriptedProvider::from_toml("").expect("empty scenario file");
|
||||
let mut usage = Usage::default();
|
||||
let out = judge_with_tools(&provider, "system", "judge this", "scripted:echo", None, &mut usage)
|
||||
.await
|
||||
.expect("the echo provider answers");
|
||||
assert!(!out.0.is_empty());
|
||||
assert_eq!(usage.requests, 1, "one round, no tool calls, one request");
|
||||
assert!(usage.tokens_in > 0 && usage.tokens_out > 0, "{usage:?}");
|
||||
}
|
||||
|
||||
/// The count is what the plan limit sees, so it must include the request
|
||||
/// that failed — the retry storm was made of those.
|
||||
#[tokio::test]
|
||||
async fn a_refused_request_still_counts() {
|
||||
struct Refuses;
|
||||
#[async_trait::async_trait]
|
||||
impl cm_llm::LlmProvider for Refuses {
|
||||
async fn stream(&self, _: cm_llm::ChatRequest) -> Result<cm_llm::EventStream, cm_llm::LlmError> {
|
||||
Err(cm_llm::LlmError::Scenario("429 Too Many Requests".into()))
|
||||
}
|
||||
}
|
||||
let mut usage = Usage::default();
|
||||
let err = judge_with_tools(&Refuses, "s", "u", "glm:glm-5.3", None, &mut usage)
|
||||
.await
|
||||
.expect_err("refused");
|
||||
assert!(err.contains("429"), "{err}");
|
||||
assert_eq!(usage.requests, 1);
|
||||
assert_eq!((usage.tokens_in, usage.tokens_out), (0, 0));
|
||||
}
|
||||
|
||||
use super::*;
|
||||
|
||||
/// A bare model name must never be accepted as a validator spec.
|
||||
@@ -881,7 +1413,23 @@ mod cross_provider_tests {
|
||||
#[test]
|
||||
fn an_unrecognised_model_is_not_assumed_to_be_ours() {
|
||||
assert_eq!(provider_family("some-new-model-v9"), "unknown");
|
||||
assert_ne!(provider_family("some-new-model-v9"), IMPLEMENTER_FAMILY);
|
||||
assert_ne!(provider_family("some-new-model-v9"), implementer_family(None));
|
||||
}
|
||||
|
||||
/// The implementer family comes from the mission's backend, and the two
|
||||
/// readers have to agree on the spelling of a family or a glm mission
|
||||
/// judged by glm reads as independent — which it did, while this was a
|
||||
/// constant.
|
||||
#[test]
|
||||
fn the_implementer_family_follows_the_backend() {
|
||||
assert_eq!(implementer_family(None), "anthropic");
|
||||
for b in ["", "default", "claude", "canary-claude"] {
|
||||
assert_eq!(implementer_family(Some(b)), "anthropic", "{b}");
|
||||
}
|
||||
assert_eq!(implementer_family(Some("glm")), provider_family("glm:glm-5.3"));
|
||||
assert_eq!(implementer_family(Some("kimi")), provider_family("kimi:kimi-k2"));
|
||||
assert_ne!(implementer_family(Some("claude")), provider_family("glm:glm-5.3"));
|
||||
assert_eq!(implementer_family(Some("something-else")), "unknown");
|
||||
}
|
||||
|
||||
/// The whole point: a judge in the implementer's own family is not
|
||||
@@ -891,13 +1439,15 @@ mod cross_provider_tests {
|
||||
for spec in ["claude-opus-4-8", "runtime:claw_x", "sonnet"] {
|
||||
assert_eq!(
|
||||
provider_family(spec),
|
||||
IMPLEMENTER_FAMILY,
|
||||
"{spec} would have to be rejected as a validator"
|
||||
implementer_family(Some("claude")),
|
||||
"{spec} would have to be rejected as a validator of a claude mission"
|
||||
);
|
||||
}
|
||||
for spec in ["glm:glm-4.7", "kimi:kimi-k2"] {
|
||||
assert_ne!(provider_family(spec), IMPLEMENTER_FAMILY, "{spec}");
|
||||
assert_ne!(provider_family(spec), implementer_family(Some("claude")), "{spec}");
|
||||
}
|
||||
// And the other way round: glm judging a glm mission is the same trap.
|
||||
assert_eq!(provider_family("glm:glm-5.3"), implementer_family(Some("glm")));
|
||||
}
|
||||
|
||||
/// A mission's own choice wins over the deployment default.
|
||||
@@ -1005,6 +1555,35 @@ mod cross_provider_tests {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Commit-first: the plan sits between the condition and the evidence,
|
||||
/// so the judge meets the claims already knowing what it is looking for.
|
||||
/// Without a plan the prompt is byte-for-byte what it was before the
|
||||
/// round existed — older recorded prompts stay comparable.
|
||||
#[test]
|
||||
fn plan_sits_between_condition_and_evidence() {
|
||||
let with = judge_user("COND", "EVID", Some("PLAN"));
|
||||
let c = with.find("COND").unwrap();
|
||||
let p = with.find("PLAN").unwrap();
|
||||
let e = with.find("EVID").unwrap();
|
||||
assert!(c < p && p < e, "{with}");
|
||||
assert!(with.contains("before seeing the evidence"));
|
||||
|
||||
let without = judge_user("COND", "EVID", None);
|
||||
assert_eq!(
|
||||
without,
|
||||
"COMPLETION CONDITION:\nCOND\n\nEVIDENCE (agent claims — verify them):\nEVID"
|
||||
);
|
||||
}
|
||||
|
||||
/// The commit prompt must not invite the judge to add requirements or
|
||||
/// re-derive values — the two failure modes `done-when-wording` measured.
|
||||
#[test]
|
||||
fn commit_prompt_forbids_invented_requirements() {
|
||||
assert!(EVAL_SYSTEM_COMMIT.contains("do not add"));
|
||||
assert!(EVAL_SYSTEM_COMMIT.contains("do not re-derive"));
|
||||
assert!(EVAL_SYSTEM_COMMIT.contains("No verdict yet"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_a_well_formed_verdict() {
|
||||
let v = parse_verdict("m", r#"{"met": true, "reason": "tests pass"}"#);
|
||||
@@ -1204,3 +1783,51 @@ mod tests {
|
||||
assert!(head("abc", 200).ends_with('c'));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod fallback_judge_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_fallback_from_another_family_is_taken() {
|
||||
assert_eq!(
|
||||
fallback_spec(Some("kimi:kimi-for-coding"), "glm").as_deref(),
|
||||
Some("kimi:kimi-for-coding")
|
||||
);
|
||||
}
|
||||
|
||||
/// The same family is the same outage: GLM's plan limit covers every GLM
|
||||
/// model, so `glm:glm-4.7` behind `glm:glm-5.3` would fail in the same breath.
|
||||
#[test]
|
||||
fn a_fallback_from_the_primarys_family_is_refused() {
|
||||
assert_eq!(fallback_spec(Some("glm:glm-4.7"), "glm"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_fallback_configured_means_none() {
|
||||
assert_eq!(fallback_spec(None, "glm"), None);
|
||||
assert_eq!(fallback_spec(Some(" "), "glm"), None);
|
||||
}
|
||||
|
||||
/// The fallback goes through the SAME independence checks as the primary —
|
||||
/// one function, so a Kimi fallback on a Kimi-implemented mission is refused
|
||||
/// exactly as a Kimi primary would be.
|
||||
#[test]
|
||||
fn both_judges_share_one_independence_check() {
|
||||
let src = include_str!("evaluator.rs");
|
||||
let body = src.split("async fn fallback_judge(").nth(1).unwrap();
|
||||
let body = &body[..body.find("\n}\n").unwrap()];
|
||||
assert!(body.contains("independent_judge(runtime, &spec, implementer)"), "{body}");
|
||||
let primary = src.split("async fn cross_provider_judge(").nth(1).unwrap();
|
||||
let primary = &primary[..primary.find("\n}\n").unwrap()];
|
||||
assert!(primary.contains("independent_judge(runtime, &spec, implementer)"));
|
||||
}
|
||||
|
||||
/// When both fail, the primary's error leads — the phase runner reads it
|
||||
/// for z.ai's plan-limit code to avoid spending the pass.
|
||||
#[test]
|
||||
fn when_both_fail_the_primary_error_leads() {
|
||||
let src = include_str!("evaluator.rs");
|
||||
assert!(src.contains(r#"Some(format!("{e} | fallback {fb_model}: {e2}"))"#));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,7 +50,16 @@ const COMMAND_TIMEOUT: Duration = Duration::from_secs(180);
|
||||
/// Cap on what one command may return to the model. Test suites are chatty and
|
||||
/// the judge pays for every byte; the tail is where failures live, so when
|
||||
/// output overflows we keep both ends and drop the middle.
|
||||
const MAX_OUTPUT_BYTES: usize = 12_000;
|
||||
///
|
||||
/// 64 KB, up from 12 KB on 2026-09-18. The smaller cap was sized for test
|
||||
/// output and applied to deliverables: a research REPORT.md of ~18 KB came
|
||||
/// back truncated from `cat`, and the judge — correctly — reassembled it with
|
||||
/// `head -119`, `tail -120`, `sed -n 80,200p` and three greps, five extra
|
||||
/// rounds each resending the whole conversation. 7 of 9 verdicts ran to the
|
||||
/// 12-check cap that way. Since `compact_earlier_results` shrinks a result to
|
||||
/// 800 bytes once its round is over, one 64 KB read costs one round; the
|
||||
/// slicing it replaces cost five.
|
||||
const MAX_OUTPUT_BYTES: usize = 64_000;
|
||||
|
||||
/// Programs the judge may run. Every one either reports state or runs a
|
||||
/// project's own checks — none of them edit the tree.
|
||||
@@ -357,7 +366,10 @@ impl Sandbox {
|
||||
ran: true,
|
||||
refused: false,
|
||||
exit_code: out.exit_code,
|
||||
evidence: clamp_output(&body),
|
||||
// Scrubbed BEFORE the judge sees it: the judge is another
|
||||
// company's model, and `cat` on an agent's file would
|
||||
// otherwise send a leaked credential to it.
|
||||
evidence: crate::delivery_secrets::scrub(&clamp_output(&body)).into_owned(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -394,7 +406,117 @@ fn git_ownership_env(workdir: &str) -> Vec<String> {
|
||||
]
|
||||
}
|
||||
|
||||
/// Where a mission agent's npm cache lands on the host.
|
||||
///
|
||||
/// The mission container's `HOME` is `/zeroclaw-data`, bind-mounted from
|
||||
/// `<mission>/runtime-data`, so an agent's `npm install` has already filled
|
||||
/// `<mission>/runtime-data/.npm` on the host — in a path the judge's container
|
||||
/// can see. No copy out of the mission container is needed.
|
||||
pub fn mission_npm_cache(mission_id: Uuid) -> PathBuf {
|
||||
crate::mission_workspace::missions_root()
|
||||
.join(mission_id.to_string())
|
||||
.join("runtime-data")
|
||||
.join(".npm")
|
||||
}
|
||||
|
||||
/// How long an offline install may take. Longer than a check: a cold `npm ci`
|
||||
/// of a Vite app unpacks a few hundred packages.
|
||||
const INSTALL_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
|
||||
/// The script that gives the judge a `node_modules` it built itself.
|
||||
///
|
||||
/// The verify copy excludes `node_modules` on purpose (the transport packer's
|
||||
/// list: the judge must not run agent-built binaries), and the judge's
|
||||
/// container has no route to a registry (`clawmates_core` has no gateway), so
|
||||
/// `npm test` in a copy used to fail with `vitest: not found` however good the
|
||||
/// work was. Measured on the frontend team's first run: correct component,
|
||||
/// 10/10 tests when re-run by hand, failed twice by a judge that could not
|
||||
/// install.
|
||||
///
|
||||
/// Not piped into `tail`: a pipe exits with its LAST command's status, so
|
||||
/// `npm ci | tail` reports success over a failed install. The log is written,
|
||||
/// its tail printed, and npm's own status returned.
|
||||
///
|
||||
/// `npm ci --offline` rebuilds `node_modules` from the lockfile using only the
|
||||
/// cache, and checks every tarball against the lockfile's integrity hash. The
|
||||
/// cache is COPIED into the verify root first: `npm ci` writes to its cache,
|
||||
/// the judge runs as root, and pointing it at the mission's own cache would
|
||||
/// leave root-owned files in a tree uid 65532 owns — the single-writer breach
|
||||
/// the verify copy exists to prevent. The copy goes with the verify root when
|
||||
/// the sandbox is purged.
|
||||
pub fn npm_offline_script(cache: &Path, verify_root: &Path) -> String {
|
||||
let local = verify_root.join("npm-cache");
|
||||
format!(
|
||||
"cp -a {cache} {local} || exit 3\n\
|
||||
npm ci --offline --no-audit --no-fund --cache {local} > {log} 2>&1\n\
|
||||
rc=$?\n\
|
||||
tail -25 {log}\n\
|
||||
exit $rc\n",
|
||||
cache = crate::vm_tool_tap::shell_quote(&cache.display().to_string()),
|
||||
local = crate::vm_tool_tap::shell_quote(&local.display().to_string()),
|
||||
log = crate::vm_tool_tap::shell_quote(&verify_root.join("npm-ci.log").display().to_string()),
|
||||
)
|
||||
}
|
||||
|
||||
impl Sandbox {
|
||||
/// Install a JavaScript project's dependencies for the judge, offline,
|
||||
/// when the copy has a `package-lock.json`. Returns a note for the judge's
|
||||
/// evidence, or `None` when there is nothing to install.
|
||||
///
|
||||
/// Server-driven, not a judge tool call: `npm ci` is not on the judge's
|
||||
/// allow-list and should not be — installing is the harness's job, and the
|
||||
/// judge only needs to know whether it worked, so that "could not install"
|
||||
/// never reads as "the tests fail".
|
||||
pub async fn prepare_dependencies(&self, mission_id: Uuid) -> Option<String> {
|
||||
if !self.workdir.join("package-lock.json").is_file() {
|
||||
return None;
|
||||
}
|
||||
let cache = mission_npm_cache(mission_id);
|
||||
if !cache.join("_cacache").is_dir() {
|
||||
return Some(format!(
|
||||
"DEPENDENCIES: not installed. This is an npm project, but the mission \
|
||||
left no npm cache at {} to install from offline. A test command that \
|
||||
cannot find its runner is a missing install, not a failing suite.",
|
||||
cache.display()
|
||||
));
|
||||
}
|
||||
let root = self.workdir.parent()?.to_path_buf();
|
||||
let docker = crate::container_exec::connect().ok()?;
|
||||
let argv = vec![
|
||||
"sh".to_string(),
|
||||
"-lc".to_string(),
|
||||
npm_offline_script(&cache, &root),
|
||||
];
|
||||
let workdir = self.workdir.display().to_string();
|
||||
let out = crate::container_exec::exec_with_env(
|
||||
&docker,
|
||||
&self.container,
|
||||
Some(&workdir),
|
||||
&argv,
|
||||
&git_ownership_env(&workdir),
|
||||
INSTALL_TIMEOUT,
|
||||
)
|
||||
.await;
|
||||
let note = match out {
|
||||
Ok(o) if o.exit_code == Some(0) && self.workdir.join("node_modules/.bin").is_dir() => {
|
||||
"DEPENDENCIES: installed by the harness with `npm ci --offline` from \
|
||||
package-lock.json, every package checked against the lockfile's \
|
||||
integrity hashes. node_modules is present; run the project's test \
|
||||
command directly."
|
||||
.to_string()
|
||||
}
|
||||
Ok(o) => format!(
|
||||
"DEPENDENCIES: offline install FAILED (exit {:?}). A test command that \
|
||||
cannot find its runner is a missing install, not a failing suite.\n{}",
|
||||
o.exit_code,
|
||||
clamp_output(&format!("{}{}", o.stdout, o.stderr)),
|
||||
),
|
||||
Err(e) => format!("DEPENDENCIES: offline install could not run: {e}"),
|
||||
};
|
||||
eprintln!("evaluator_tools: mission {mission_id} — {}", note.lines().next().unwrap_or(""));
|
||||
Some(note)
|
||||
}
|
||||
|
||||
/// Remove the copy, from inside the container that wrote it.
|
||||
///
|
||||
/// `Drop` cannot do this. The judge runs `cargo test` in a container as
|
||||
@@ -431,11 +553,17 @@ impl Drop for Sandbox {
|
||||
return;
|
||||
}
|
||||
if let Some(root) = self.workdir.parent() {
|
||||
if let Err(e) = std::fs::remove_dir_all(root) {
|
||||
eprintln!(
|
||||
match std::fs::remove_dir_all(root) {
|
||||
Ok(()) => {}
|
||||
// Already gone, because `purge` ran first and worked. That is
|
||||
// the SUCCESS path, and reporting it as a failure is how a
|
||||
// real cleanup error gets read as noise — the exact habit that
|
||||
// let two root-owned copies sit stranded for hours.
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(e) => eprintln!(
|
||||
"evaluator_tools: could not remove the verification copy at {} ({e})",
|
||||
root.display()
|
||||
);
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -762,3 +890,79 @@ mod tests {
|
||||
let _ = clamp_output(&long);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod npm_offline_tests {
|
||||
use super::*;
|
||||
|
||||
/// Run the generated script with a fake `npm` that records its arguments
|
||||
/// and exits `npm_rc`. Returns (exit code, recorded npm argv).
|
||||
fn run_with_fake_npm(npm_rc: i32, with_cache: bool) -> (Option<i32>, String) {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let bin = tmp.path().join("bin");
|
||||
std::fs::create_dir_all(&bin).unwrap();
|
||||
let argv_log = tmp.path().join("npm-argv");
|
||||
let fake = bin.join("npm");
|
||||
std::fs::write(
|
||||
&fake,
|
||||
format!("#!/bin/sh\necho \"$@\" > {}\necho npm said hello\nexit {npm_rc}\n", argv_log.display()),
|
||||
)
|
||||
.unwrap();
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
std::fs::set_permissions(&fake, std::fs::Permissions::from_mode(0o755)).unwrap();
|
||||
|
||||
let cache = tmp.path().join("mission-cache");
|
||||
if with_cache {
|
||||
std::fs::create_dir_all(cache.join("_cacache")).unwrap();
|
||||
}
|
||||
let root = tmp.path().join("verify");
|
||||
std::fs::create_dir_all(&root).unwrap();
|
||||
|
||||
let out = std::process::Command::new("sh")
|
||||
.arg("-c")
|
||||
.arg(npm_offline_script(&cache, &root))
|
||||
.env("PATH", format!("{}:{}", bin.display(), std::env::var("PATH").unwrap_or_default()))
|
||||
.output()
|
||||
.unwrap();
|
||||
let recorded = std::fs::read_to_string(&argv_log).unwrap_or_default();
|
||||
if with_cache {
|
||||
assert!(root.join("npm-cache/_cacache").is_dir(), "the cache must be copied into the verify root");
|
||||
}
|
||||
(out.status.code(), recorded)
|
||||
}
|
||||
|
||||
/// npm's own failure must survive: `npm ci | tail` exits 0 over a failed
|
||||
/// install, and the judge would then be told dependencies were ready.
|
||||
#[test]
|
||||
fn a_failed_install_is_reported_as_failed() {
|
||||
let (rc, _) = run_with_fake_npm(7, true);
|
||||
assert_eq!(rc, Some(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_clean_install_exits_zero_offline_against_the_copy() {
|
||||
let (rc, argv) = run_with_fake_npm(0, true);
|
||||
assert_eq!(rc, Some(0));
|
||||
assert!(argv.starts_with("ci --offline"), "{argv}");
|
||||
// Pointed at the COPY in the verify root, never the mission's cache:
|
||||
// npm writes to its cache and the judge runs as root.
|
||||
assert!(argv.contains("/verify/npm-cache"), "{argv}");
|
||||
assert!(!argv.contains("mission-cache"), "{argv}");
|
||||
}
|
||||
|
||||
/// No cache to copy stops before npm runs, with its own status.
|
||||
#[test]
|
||||
fn a_missing_cache_never_reaches_npm() {
|
||||
let (rc, argv) = run_with_fake_npm(0, false);
|
||||
assert_eq!(rc, Some(3));
|
||||
assert!(argv.is_empty(), "npm ran without a cache: {argv}");
|
||||
}
|
||||
|
||||
/// The cache path is where the mission container's HOME is bound.
|
||||
#[test]
|
||||
fn the_npm_cache_is_under_the_bound_home() {
|
||||
let id = Uuid::nil();
|
||||
let p = mission_npm_cache(id);
|
||||
assert!(p.ends_with(format!("{id}/runtime-data/.npm")), "{}", p.display());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
//! Is the mission gateway configured, and is anything listening?
|
||||
//!
|
||||
//! The third sibling of [`crate::runtime_preflight`] and
|
||||
//! [`crate::validator_preflight`], for the same class of failure: the
|
||||
//! configuration is absent or wrong, and nothing says so until a mission pays
|
||||
//! for it.
|
||||
//!
|
||||
//! `ZEROCLAW_GATEWAY_URL` and `ZEROCLAW_TOKEN` have no defaults and are read at
|
||||
//! FIRST USE, inside `ZeroClawDriveExecutor::from_env`. So a deployment missing
|
||||
//! them boots clean, serves every page, lists every mission — and fails the
|
||||
//! first time someone presses run, with an error that surfaces on a phase
|
||||
//! rather than at startup. The information exists the whole time; nobody is
|
||||
//! told until it is expensive.
|
||||
//!
|
||||
//! A report, not a gate, matching its siblings. A server with no gateway should
|
||||
//! still boot: the frontend, the catalogue and every read path work without it,
|
||||
//! and refusing to start would turn a degraded deployment into a dead one.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
const PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// What the preflight found.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Verdict {
|
||||
/// No gateway configured. Missions on the container tier cannot run.
|
||||
NotConfigured { missing: Vec<String> },
|
||||
/// Configured but nothing answered at that address.
|
||||
Unreachable { url: String, error: String },
|
||||
/// Configured and something answered.
|
||||
Reachable { url: String },
|
||||
}
|
||||
|
||||
impl Verdict {
|
||||
/// The line to print at boot.
|
||||
///
|
||||
/// Each names the CONSEQUENCE, not just the state. "ZEROCLAW_TOKEN not set"
|
||||
/// tells an operator what is missing; it does not tell them that every
|
||||
/// container-tier mission they launch will fail on its first phase.
|
||||
pub fn message(&self) -> String {
|
||||
match self {
|
||||
Verdict::NotConfigured { missing } => format!(
|
||||
"gateway_preflight: NOT CONFIGURED ({}) — container-tier missions \
|
||||
cannot run. They will launch, provision a runtime, and fail on \
|
||||
the first turn; the server is otherwise healthy",
|
||||
missing.join(", ")
|
||||
),
|
||||
Verdict::Unreachable { url, error } => format!(
|
||||
"gateway_preflight: {url} is configured but did not answer ({error}) \
|
||||
— container-tier missions will fail on their first turn. The \
|
||||
config is right and the machine is not"
|
||||
),
|
||||
Verdict::Reachable { url } => {
|
||||
format!("gateway_preflight: {url} answered")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Which required variables are absent.
|
||||
///
|
||||
/// Split from the network probe so the rule is testable without a gateway:
|
||||
/// this is the half that is pure, and it is the half that is wrong most often.
|
||||
pub fn missing_config(url: Option<&str>, token: Option<&str>, pairing: Option<&str>) -> Vec<String> {
|
||||
let mut missing = Vec::new();
|
||||
if url.map(str::trim).unwrap_or("").is_empty() {
|
||||
missing.push("ZEROCLAW_GATEWAY_URL".to_string());
|
||||
}
|
||||
// Either credential works: a durable token, or a one-time pairing code the
|
||||
// executor exchanges on first use.
|
||||
let has_token = !token.map(str::trim).unwrap_or("").is_empty();
|
||||
let has_pairing = !pairing.map(str::trim).unwrap_or("").is_empty();
|
||||
if !has_token && !has_pairing {
|
||||
missing.push("ZEROCLAW_TOKEN or ZEROCLAW_PAIRING_CODE".to_string());
|
||||
}
|
||||
missing
|
||||
}
|
||||
|
||||
fn env_opt(key: &str) -> Option<String> {
|
||||
std::env::var(key).ok().filter(|v| !v.trim().is_empty())
|
||||
}
|
||||
|
||||
/// Probe the configured gateway.
|
||||
pub async fn check() -> Verdict {
|
||||
let url = env_opt("ZEROCLAW_GATEWAY_URL");
|
||||
let missing = missing_config(
|
||||
url.as_deref(),
|
||||
env_opt("ZEROCLAW_TOKEN").as_deref(),
|
||||
env_opt("ZEROCLAW_PAIRING_CODE").as_deref(),
|
||||
);
|
||||
if !missing.is_empty() {
|
||||
return Verdict::NotConfigured { missing };
|
||||
}
|
||||
let url = url.expect("checked above");
|
||||
|
||||
// Any HTTP answer proves something is listening and routable, which is the
|
||||
// question this preflight exists to answer. Authenticating here would need
|
||||
// a pairing exchange that BURNS a one-time code — a preflight that costs
|
||||
// the deployment its credential is worse than no preflight.
|
||||
let client = match reqwest::Client::builder().timeout(PROBE_TIMEOUT).build() {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
return Verdict::Unreachable {
|
||||
url,
|
||||
error: e.to_string(),
|
||||
}
|
||||
}
|
||||
};
|
||||
match client.get(&url).send().await {
|
||||
Ok(_) => Verdict::Reachable { url },
|
||||
Err(e) => Verdict::Unreachable {
|
||||
url,
|
||||
error: e.to_string(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the probe and print the verdict. Never panics, never blocks boot.
|
||||
pub fn report_at_boot() {
|
||||
tokio::spawn(async {
|
||||
eprintln!("{}", check().await.message());
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_fully_configured_deployment_is_missing_nothing() {
|
||||
assert!(missing_config(Some("http://gw:42617"), Some("tok"), None).is_empty());
|
||||
// A pairing code alone is enough — the executor exchanges it on first use.
|
||||
assert!(missing_config(Some("http://gw:42617"), None, Some("123456")).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_string_counts_as_absent() {
|
||||
// The failure this whole module exists for: `unwrap_or_default` and an
|
||||
// empty env var turn "unconfigured" into "configured with nothing",
|
||||
// which fails later as a 401 rather than now as a missing setting.
|
||||
let missing = missing_config(Some(" "), Some(""), Some(" "));
|
||||
assert_eq!(missing.len(), 2, "both must be reported: {missing:?}");
|
||||
assert!(missing[0].contains("GATEWAY_URL"));
|
||||
assert!(missing[1].contains("ZEROCLAW_TOKEN"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_message_names_the_consequence_not_just_the_state() {
|
||||
let v = Verdict::NotConfigured {
|
||||
missing: vec!["ZEROCLAW_GATEWAY_URL".into()],
|
||||
};
|
||||
let m = v.message();
|
||||
assert!(m.contains("ZEROCLAW_GATEWAY_URL"));
|
||||
assert!(
|
||||
m.contains("cannot run"),
|
||||
"an operator needs to know what stops working, not only what is \
|
||||
unset: {m}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,14 @@ pub struct Harvest {
|
||||
pub failed: Vec<(String, String)>,
|
||||
/// Vault-relative paths of the notes written.
|
||||
pub notes_written: Vec<String>,
|
||||
/// The papers actually shelved this run, in shelve order.
|
||||
///
|
||||
/// `shelved` carries only source ids, which is all the seen-set needs. The
|
||||
/// run manifest a Continuous Research mission hands its agents needs the
|
||||
/// title and abstract too, and re-reading them back out of the notes we
|
||||
/// just wrote would be a parse of our own output — one more place for the
|
||||
/// two to drift.
|
||||
pub papers: Vec<crate::papers::Paper>,
|
||||
}
|
||||
|
||||
impl Harvest {
|
||||
@@ -167,6 +175,7 @@ pub async fn shelve(
|
||||
.await?;
|
||||
|
||||
out.notes_written.push(paper.note_path());
|
||||
out.papers.push(paper.clone());
|
||||
out.shelved.push(sid);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
//! How much of each judge provider's plan is left, read from the providers'
|
||||
//! own usage APIs.
|
||||
//!
|
||||
//! Built 2026-09-23 after GLM's weekly limit ran out for the second time in a
|
||||
//! month. Measured then: the judge spent about 0.1% of what the shared z.ai key
|
||||
//! used that week (275 requests of 6,959; ~0.4M of ~470M tokens). Other
|
||||
//! consumers of the same key starve it, and nothing in ClawMates could see it
|
||||
//! coming: the first sign was every conditioned phase failing on a 1310.
|
||||
//!
|
||||
//! Two jobs:
|
||||
//!
|
||||
//! - **Warn** once per window when it crosses [`WARN_AT`] percent, so the
|
||||
//! operator hears about it days early, not from a failed mission.
|
||||
//! - **Switch** the judge before the wall: once the primary judge's provider
|
||||
//! crosses [`SWITCH_AT`] in any window, the evaluator goes straight to the
|
||||
//! fallback judge (`evaluator::fallback_judge`) instead of spending a call
|
||||
//! that is about to fail with a 429.
|
||||
//!
|
||||
//! Best-effort throughout: a usage API that is down or changes shape leaves
|
||||
//! the readings empty, and an empty reading NEVER switches anything. The 429
|
||||
//! fallback in the evaluator is still there behind this.
|
||||
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Mutex, OnceLock};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Percent of a window at which the operator is warned.
|
||||
pub const WARN_AT: f64 = 80.0;
|
||||
/// Percent of a window at which the primary judge is skipped for the fallback.
|
||||
pub const SWITCH_AT: f64 = 95.0;
|
||||
|
||||
/// One quota window as a provider reported it.
|
||||
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
|
||||
pub struct Window {
|
||||
/// `5h`, `7d`, … — the window's length, as the provider describes it.
|
||||
pub name: String,
|
||||
/// 0–100.
|
||||
pub used_pct: f64,
|
||||
/// RFC 3339, when the provider said.
|
||||
pub resets_at: Option<String>,
|
||||
}
|
||||
|
||||
/// Latest readings per provider family (`glm`, `kimi`), with when they were
|
||||
/// taken.
|
||||
#[derive(Debug, Clone, Default, serde::Serialize)]
|
||||
pub struct Snapshot {
|
||||
pub providers: HashMap<String, Reading>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct Reading {
|
||||
pub windows: Vec<Window>,
|
||||
pub read_at: String,
|
||||
}
|
||||
|
||||
fn state() -> &'static Mutex<Snapshot> {
|
||||
static S: OnceLock<Mutex<Snapshot>> = OnceLock::new();
|
||||
S.get_or_init(|| Mutex::new(Snapshot::default()))
|
||||
}
|
||||
|
||||
/// Windows already warned about, keyed `family/window/resets_at`, so a window
|
||||
/// warns once per reset cycle, not on every poll.
|
||||
fn warned() -> &'static Mutex<std::collections::HashSet<String>> {
|
||||
static W: OnceLock<Mutex<std::collections::HashSet<String>>> = OnceLock::new();
|
||||
W.get_or_init(|| Mutex::new(Default::default()))
|
||||
}
|
||||
|
||||
/// The current readings.
|
||||
pub fn snapshot() -> Snapshot {
|
||||
state().lock().map(|s| s.clone()).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Should the evaluator skip a judge of this family for the fallback? True
|
||||
/// only on a REAL reading at or past [`SWITCH_AT`]; no reading means no.
|
||||
pub fn near_limit(family: &str) -> Option<Window> {
|
||||
let snap = snapshot();
|
||||
snap.providers
|
||||
.get(family)?
|
||||
.windows
|
||||
.iter()
|
||||
.find(|w| w.used_pct >= SWITCH_AT)
|
||||
.cloned()
|
||||
}
|
||||
|
||||
/// Parse z.ai's `GET /api/monitor/usage/quota/limit`.
|
||||
///
|
||||
/// Its `TOKENS_LIMIT` entries carry `unit` + `number` for the window and a
|
||||
/// `percentage`. Measured on the Pro plan 2026-09-23: `unit 3, number 5` is the
|
||||
/// 5-hour window and `unit 6, number 1` the weekly one (its reset matched the
|
||||
/// 1310 error's own "will reset at"). `nextResetTime` is epoch milliseconds.
|
||||
pub fn parse_zai(v: &Value) -> Vec<Window> {
|
||||
let Some(limits) = v.pointer("/data/limits").and_then(Value::as_array) else {
|
||||
return Vec::new();
|
||||
};
|
||||
limits
|
||||
.iter()
|
||||
.filter(|l| l.get("type").and_then(Value::as_str) == Some("TOKENS_LIMIT"))
|
||||
.filter_map(|l| {
|
||||
let pct = l.get("percentage").and_then(Value::as_f64)?;
|
||||
let number = l.get("number").and_then(Value::as_i64).unwrap_or(1);
|
||||
let name = match l.get("unit").and_then(Value::as_i64) {
|
||||
Some(3) => format!("{number}h"),
|
||||
Some(6) => format!("{}d", number * 7),
|
||||
Some(u) => format!("unit{u}x{number}"),
|
||||
None => "unknown".to_string(),
|
||||
};
|
||||
let resets_at = l
|
||||
.get("nextResetTime")
|
||||
.and_then(Value::as_i64)
|
||||
.and_then(|ms| {
|
||||
time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(ms) * 1_000_000).ok()
|
||||
})
|
||||
.and_then(|t| t.format(&time::format_description::well_known::Rfc3339).ok());
|
||||
Some(Window { name, used_pct: pct, resets_at })
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Parse Kimi's `GET https://api.kimi.com/coding/v1/usages`.
|
||||
///
|
||||
/// `usages.limit_5h` / `usages.limit_7d` carry `used_ratio` (0–1) and
|
||||
/// `reset_time`. A ratio above 1 is taken as already a percentage, so a unit
|
||||
/// change on their side reads as "very used", which fails toward warning.
|
||||
pub fn parse_kimi(v: &Value) -> Vec<Window> {
|
||||
let Some(usages) = v.get("usages").and_then(Value::as_object) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut out: Vec<Window> = usages
|
||||
.iter()
|
||||
.filter_map(|(k, u)| {
|
||||
let ratio = u.get("used_ratio").and_then(Value::as_f64)?;
|
||||
let pct = if ratio <= 1.0 { ratio * 100.0 } else { ratio };
|
||||
Some(Window {
|
||||
name: k.trim_start_matches("limit_").to_string(),
|
||||
used_pct: pct,
|
||||
resets_at: u.get("reset_time").and_then(Value::as_str).map(str::to_string),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
out.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
out
|
||||
}
|
||||
|
||||
async fn fetch(client: &reqwest::Client, url: &str, auth: &str) -> Option<Value> {
|
||||
let resp = client
|
||||
.get(url)
|
||||
.header("Authorization", auth)
|
||||
.header("Accept-Language", "en-US,en")
|
||||
.timeout(Duration::from_secs(20))
|
||||
.send()
|
||||
.await
|
||||
.ok()?;
|
||||
if !resp.status().is_success() {
|
||||
eprintln!("judge_quota: {url} answered {}", resp.status());
|
||||
return None;
|
||||
}
|
||||
resp.json().await.ok()
|
||||
}
|
||||
|
||||
/// One poll of both providers. Keys come from the same env vars the provider
|
||||
/// registry uses; a provider whose key is unset is simply not read.
|
||||
pub async fn poll_once(client: &reqwest::Client) {
|
||||
let mut readings: Vec<(&str, Vec<Window>)> = Vec::new();
|
||||
if let Some(key) = std::env::var("ZAI_API_KEY").ok().filter(|k| !k.is_empty()) {
|
||||
// z.ai takes the bare key, no `Bearer` (measured).
|
||||
if let Some(v) = fetch(client, "https://api.z.ai/api/monitor/usage/quota/limit", &key).await {
|
||||
readings.push(("glm", parse_zai(&v)));
|
||||
}
|
||||
}
|
||||
if let Some(key) = std::env::var("KIMI_API_KEY").ok().filter(|k| !k.is_empty()) {
|
||||
if let Some(v) =
|
||||
fetch(client, "https://api.kimi.com/coding/v1/usages", &format!("Bearer {key}")).await
|
||||
{
|
||||
readings.push(("kimi", parse_kimi(&v)));
|
||||
}
|
||||
}
|
||||
let now = time::OffsetDateTime::now_utc()
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.unwrap_or_default();
|
||||
for (family, windows) in readings {
|
||||
if windows.is_empty() {
|
||||
eprintln!("judge_quota: {family} usage API answered but no window parsed — shape changed?");
|
||||
continue;
|
||||
}
|
||||
for w in &windows {
|
||||
if w.used_pct >= WARN_AT {
|
||||
let key = format!("{family}/{}/{}", w.name, w.resets_at.as_deref().unwrap_or(""));
|
||||
let first = warned().lock().map(|mut s| s.insert(key)).unwrap_or(false);
|
||||
if first {
|
||||
eprintln!(
|
||||
"judge_quota: WARNING {family} {} window at {:.0}% (resets {}){}",
|
||||
w.name,
|
||||
w.used_pct,
|
||||
w.resets_at.as_deref().unwrap_or("?"),
|
||||
if w.used_pct >= SWITCH_AT {
|
||||
" — the evaluator now skips this judge for the fallback"
|
||||
} else {
|
||||
""
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Ok(mut s) = state().lock() {
|
||||
s.providers
|
||||
.insert(family.to_string(), Reading { windows, read_at: now.clone() });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Poll forever.
|
||||
pub fn spawn_poller(interval: Duration) {
|
||||
tokio::spawn(async move {
|
||||
let client = reqwest::Client::new();
|
||||
let mut tick = tokio::time::interval(interval);
|
||||
loop {
|
||||
tick.tick().await;
|
||||
poll_once(&client).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
/// The shape z.ai actually returned on 2026-09-23, on the day the weekly
|
||||
/// window was exhausted.
|
||||
#[test]
|
||||
fn zai_reading_names_both_windows() {
|
||||
let v = json!({"code":200,"data":{"limits":[
|
||||
{"type":"TIME_LIMIT","unit":5,"number":1,"usage":1000,"percentage":0},
|
||||
{"type":"TOKENS_LIMIT","unit":3,"number":5,"percentage":23,"nextResetTime":1790167210082i64},
|
||||
{"type":"TOKENS_LIMIT","unit":6,"number":1,"percentage":100,"nextResetTime":1790301693982i64}
|
||||
],"level":"pro"}});
|
||||
let w = parse_zai(&v);
|
||||
assert_eq!(w.len(), 2, "TIME_LIMIT (tool calls) is not a token window: {w:?}");
|
||||
assert_eq!(w[0].name, "5h");
|
||||
assert_eq!(w[0].used_pct, 23.0);
|
||||
assert_eq!(w[1].name, "7d");
|
||||
assert_eq!(w[1].used_pct, 100.0);
|
||||
assert!(w[1].resets_at.as_deref().unwrap().starts_with("2026-09-25T02:01"), "{:?}", w[1]);
|
||||
}
|
||||
|
||||
/// Kimi's measured shape; ratios become percentages.
|
||||
#[test]
|
||||
fn kimi_reading_converts_ratios() {
|
||||
let v = json!({"usages":{
|
||||
"limit_5h":{"used_ratio":0.01,"reset_time":"2026-09-23T15:49:20Z"},
|
||||
"limit_7d":{"used_ratio":0.97,"reset_time":"2026-09-28T19:49:20Z"}}});
|
||||
let w = parse_kimi(&v);
|
||||
assert_eq!(w.iter().map(|w| w.name.as_str()).collect::<Vec<_>>(), ["5h", "7d"]);
|
||||
assert!((w[0].used_pct - 1.0).abs() < 1e-9);
|
||||
assert!((w[1].used_pct - 97.0).abs() < 1e-9);
|
||||
}
|
||||
|
||||
/// A changed or empty shape yields no windows — and no windows never
|
||||
/// switches the judge.
|
||||
#[test]
|
||||
fn an_unreadable_answer_switches_nothing() {
|
||||
assert!(parse_zai(&json!({"data":{}})).is_empty());
|
||||
assert!(parse_kimi(&json!({"error":"x"})).is_empty());
|
||||
assert!(near_limit("some-family-never-read").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn near_limit_fires_only_at_the_switch_threshold() {
|
||||
{
|
||||
let mut s = state().lock().unwrap();
|
||||
s.providers.insert(
|
||||
"test-fam-a".into(),
|
||||
Reading {
|
||||
windows: vec![Window { name: "7d".into(), used_pct: SWITCH_AT - 0.5, resets_at: None }],
|
||||
read_at: String::new(),
|
||||
},
|
||||
);
|
||||
s.providers.insert(
|
||||
"test-fam-b".into(),
|
||||
Reading {
|
||||
windows: vec![
|
||||
Window { name: "5h".into(), used_pct: 10.0, resets_at: None },
|
||||
Window { name: "7d".into(), used_pct: SWITCH_AT, resets_at: None },
|
||||
],
|
||||
read_at: String::new(),
|
||||
},
|
||||
);
|
||||
}
|
||||
assert!(near_limit("test-fam-a").is_none());
|
||||
assert_eq!(near_limit("test-fam-b").unwrap().name, "7d");
|
||||
}
|
||||
}
|
||||
@@ -225,6 +225,128 @@ pub async fn apply(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Is autonomous skill authoring on?
|
||||
///
|
||||
/// Default OFF since 2026-09-20, by operator decision. It shipped default ON,
|
||||
/// and in the months since no agent-authored skill was ever delivered to a
|
||||
/// mission or scored by the Skill-Use scorer — prod's `level_up_proposals`
|
||||
/// held zero rows on the day of the flip. An auto-apply loop whose output has
|
||||
/// never been measured is a supply chain of our own making (the shape Cisco
|
||||
/// found in OpenClaw's third-party skills), so it waits for a human until
|
||||
/// `promoted_from_brain` skills go through the `files` delivery arm and get
|
||||
/// a Trigger/Compliance score like the hand-authored ones. Stated at boot
|
||||
/// either way: a safety gate that changes state silently is how nobody
|
||||
/// notices it changed.
|
||||
pub fn self_authoring_enabled() -> bool {
|
||||
matches!(
|
||||
std::env::var("CLAWMATES_SKILL_SELF_AUTHORING")
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_ascii_lowercase()
|
||||
.as_str(),
|
||||
"1" | "on" | "true"
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod self_authoring_flag_tests {
|
||||
/// Serialised through one env var; each case restores the prior state.
|
||||
fn with(value: Option<&str>, f: impl FnOnce()) {
|
||||
let key = "CLAWMATES_SKILL_SELF_AUTHORING";
|
||||
let prior = std::env::var(key).ok();
|
||||
match value {
|
||||
Some(v) => std::env::set_var(key, v),
|
||||
None => std::env::remove_var(key),
|
||||
}
|
||||
f();
|
||||
match prior {
|
||||
Some(v) => std::env::set_var(key, v),
|
||||
None => std::env::remove_var(key),
|
||||
}
|
||||
}
|
||||
|
||||
/// Off unless switched on. The previous default was the reverse.
|
||||
#[test]
|
||||
fn off_by_default_on_by_explicit_opt_in() {
|
||||
with(None, || assert!(!super::self_authoring_enabled()));
|
||||
with(Some(""), || assert!(!super::self_authoring_enabled()));
|
||||
with(Some("0"), || assert!(!super::self_authoring_enabled()));
|
||||
with(Some("yes"), || assert!(!super::self_authoring_enabled()));
|
||||
with(Some("1"), || assert!(super::self_authoring_enabled()));
|
||||
with(Some("on"), || assert!(super::self_authoring_enabled()));
|
||||
with(Some("TRUE"), || assert!(super::self_authoring_enabled()));
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply a pending proposal's `skill_candidate` items with no human decision.
|
||||
///
|
||||
/// ONLY `skill_candidate`. The other item kinds are deliberately left to the
|
||||
/// human gate: `identity_refinement` rewrites an agent's system prompt and
|
||||
/// `brain_consolidation` edits its memory, and both change what the agent IS
|
||||
/// rather than adding a procedure it can consult. Self-authoring a skill is
|
||||
/// recoverable — the row is workspace-scoped, versioned and revertible, and
|
||||
/// cannot take a hand-authored name. Rewriting an identity autonomously is not
|
||||
/// the same bet, and it is not the one that was asked for.
|
||||
///
|
||||
/// The remaining items stay pending, so a human still sees them.
|
||||
pub async fn apply_autonomous(
|
||||
pool: &PgPool,
|
||||
workspace_id: cm_domain::WorkspaceId,
|
||||
proposal_id: Uuid,
|
||||
) -> Result<Vec<String>, String> {
|
||||
let proposal = cm_db::repo::level_up::get(pool, proposal_id, workspace_id.as_uuid())
|
||||
.await
|
||||
.map_err(|e| format!("load proposal: {e}"))?
|
||||
.ok_or_else(|| "proposal not found".to_string())?;
|
||||
if proposal.status != "pending" {
|
||||
return Err(format!("proposal already {}", proposal.status));
|
||||
}
|
||||
|
||||
let items = proposal
|
||||
.payload
|
||||
.get("suggested_items")
|
||||
.and_then(|v| v.as_array())
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut applied: Vec<String> = Vec::new();
|
||||
let mut candidates = 0usize;
|
||||
for item in items {
|
||||
let Some(item_id) = item.get("id").and_then(|v| v.as_str()) else {
|
||||
continue;
|
||||
};
|
||||
if item.get("kind").and_then(|v| v.as_str()) != Some("skill_candidate") {
|
||||
continue;
|
||||
}
|
||||
candidates += 1;
|
||||
match apply_skill_candidate(pool, &proposal, &item).await {
|
||||
Ok(()) => applied.push(item_id.to_string()),
|
||||
// A refused draft is a normal outcome (a name collision with a
|
||||
// hand-authored skill is the common one), not a failure of the
|
||||
// sweep. Said out loud so a refusal is never mistaken for the
|
||||
// agent simply not having proposed anything.
|
||||
Err(e) => eprintln!(
|
||||
"level_up: autonomous apply refused {item_id} for workspace {}: {e}",
|
||||
workspace_id.as_uuid()
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
if candidates == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
cm_db::repo::level_up::mark_applied_autonomously(
|
||||
pool,
|
||||
proposal_id,
|
||||
workspace_id.as_uuid(),
|
||||
&applied,
|
||||
applied.len() != candidates,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("mark applied: {e}"))?;
|
||||
Ok(applied)
|
||||
}
|
||||
|
||||
// ── Appliers ───────────────────────────────────────────────────
|
||||
|
||||
async fn apply_identity(
|
||||
@@ -304,20 +426,61 @@ async fn apply_skill_candidate(
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
// A draft may never take the name of a hand-authored skill.
|
||||
//
|
||||
// The row itself is safe — ids are workspace-scoped, so this cannot
|
||||
// overwrite a builtin, and bindings resolve by skill_id rather than name,
|
||||
// so it cannot shadow one either. What it CAN do is put two different
|
||||
// procedures under one name in the same agent's bundle, and then nobody
|
||||
// reading a transcript can tell which one the agent followed. That
|
||||
// ambiguity is the whole problem in a system where the skill is the
|
||||
// standard the behaviour is graded against.
|
||||
let collides: Option<Uuid> = sqlx::query_scalar(
|
||||
"SELECT id FROM skills WHERE name = $1 AND workspace_id IS NULL",
|
||||
)
|
||||
.bind(name)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(|e| format!("check builtin collision: {e}"))?;
|
||||
if collides.is_some() {
|
||||
return Err(format!(
|
||||
"skill name {name:?} is hand-authored — an agent-authored draft \
|
||||
cannot take the name of a skill it is graded against"
|
||||
));
|
||||
}
|
||||
|
||||
// Workspace-scoped custom skill. Deterministic id per
|
||||
// (workspace, name) so re-approving the same draft updates in
|
||||
// place rather than duplicating.
|
||||
let id = workspace_skill_id(proposal.workspace_id, name);
|
||||
|
||||
// Versioned, for the same reason builtins are: a self-authored skill that
|
||||
// silently replaces its own body has no undo, and the version a run was
|
||||
// judged under is the only way to read that run back honestly later.
|
||||
let mut tx = pool.begin().await.map_err(|e| format!("begin: {e}"))?;
|
||||
let existing: Option<(i32, String)> =
|
||||
sqlx::query_as("SELECT current_version, body FROM skills WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| format!("read current skill: {e}"))?;
|
||||
let (next_version, bump) = match &existing {
|
||||
Some((v, prev)) if prev == body => (*v, false),
|
||||
Some((v, _)) => (v + 1, true),
|
||||
None => (1, true),
|
||||
};
|
||||
|
||||
sqlx::query(
|
||||
"INSERT INTO skills
|
||||
(id, name, title, author, description, when_to_use, tags,
|
||||
source_kind, workspace_id, current_version, body)
|
||||
VALUES ($1,$2,$2,'level_up',$3,$4,$5,'promoted_from_brain',$6,1,$7)
|
||||
VALUES ($1,$2,$2,'level_up',$3,$4,$5,'promoted_from_brain',$6,$8,$7)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
description = EXCLUDED.description,
|
||||
when_to_use = EXCLUDED.when_to_use,
|
||||
tags = EXCLUDED.tags,
|
||||
body = EXCLUDED.body,
|
||||
current_version = EXCLUDED.current_version,
|
||||
updated_at = now()",
|
||||
)
|
||||
.bind(id)
|
||||
@@ -327,9 +490,28 @@ async fn apply_skill_candidate(
|
||||
.bind(&tags)
|
||||
.bind(proposal.workspace_id)
|
||||
.bind(body)
|
||||
.execute(pool)
|
||||
.bind(next_version)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| format!("upsert skill draft: {e}"))?;
|
||||
|
||||
if bump {
|
||||
sqlx::query(
|
||||
"INSERT INTO skill_versions
|
||||
(skill_id, version, body_md, description, when_to_use)
|
||||
VALUES ($1,$2,$3,$4,$5)
|
||||
ON CONFLICT DO NOTHING",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(next_version)
|
||||
.bind(body)
|
||||
.bind(description)
|
||||
.bind(when_to_use)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| format!("record skill version: {e}"))?;
|
||||
}
|
||||
tx.commit().await.map_err(|e| format!("commit: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
+62
-24
@@ -1,65 +1,81 @@
|
||||
//! REST API for Clawmates (spec §13). One route resource per module.
|
||||
|
||||
pub mod agent_lifecycle;
|
||||
pub mod agent_names;
|
||||
pub mod auto_merge;
|
||||
pub mod benchmark_runner;
|
||||
pub mod beszel;
|
||||
pub mod brain_seed;
|
||||
pub mod cleanup_sweeper;
|
||||
pub mod agent_names;
|
||||
pub mod mission_gc;
|
||||
pub mod container_exec;
|
||||
pub mod corpus;
|
||||
mod error;
|
||||
pub mod evaluator;
|
||||
pub mod evaluator_tools;
|
||||
pub mod judge_quota;
|
||||
mod extract;
|
||||
pub mod fleet;
|
||||
pub mod fleet_herdr;
|
||||
pub mod harvest;
|
||||
pub mod level_up;
|
||||
pub mod library;
|
||||
pub mod live_bus;
|
||||
mod mcp_door;
|
||||
mod mcp_skills;
|
||||
pub mod mission_orchestrator;
|
||||
pub mod mission_refiner;
|
||||
pub mod auto_merge;
|
||||
pub mod corpus;
|
||||
pub mod harvest;
|
||||
pub mod library;
|
||||
pub mod mission_delivery;
|
||||
pub mod mission_events;
|
||||
pub mod microvm_client;
|
||||
pub mod microvm_executor;
|
||||
pub mod microvm_turn_executor;
|
||||
pub mod subscription;
|
||||
pub mod vm_placement;
|
||||
pub mod vm_stop_gate;
|
||||
pub mod vm_tool_tap;
|
||||
pub mod continuous_research;
|
||||
pub mod delivery_secrets;
|
||||
pub mod llm_proxy;
|
||||
pub mod mission_delivery;
|
||||
pub mod podcast;
|
||||
pub mod mission_events;
|
||||
pub mod mission_fs;
|
||||
pub mod mission_memory;
|
||||
pub mod mission_gc;
|
||||
pub mod mission_orchestrator;
|
||||
pub mod mission_schedule;
|
||||
pub mod mission_outputs;
|
||||
pub mod papers;
|
||||
pub mod phase_config;
|
||||
pub mod session_executor;
|
||||
pub mod repo_digest;
|
||||
pub mod runtime_preflight;
|
||||
pub mod validator_preflight;
|
||||
pub mod mission_plan;
|
||||
pub mod mission_refiner;
|
||||
pub mod mission_roster;
|
||||
pub mod mission_runtime;
|
||||
pub mod mission_workspace;
|
||||
pub mod node_rules;
|
||||
pub mod papers;
|
||||
pub mod phase_config;
|
||||
pub mod phase_runner;
|
||||
pub mod root_copy;
|
||||
pub mod phase_summarizer;
|
||||
pub mod quota;
|
||||
mod recursive_exec;
|
||||
pub mod repo_digest;
|
||||
pub mod root_copy;
|
||||
mod routes;
|
||||
mod runtime_provision;
|
||||
pub mod runtime_preflight;
|
||||
pub mod runtime_provision;
|
||||
pub mod security_scan;
|
||||
pub mod session_executor;
|
||||
pub mod container_tool_hooks;
|
||||
pub mod gateway_preflight;
|
||||
pub mod skill_delivery;
|
||||
pub mod skill_self_authoring;
|
||||
pub mod skill_triage;
|
||||
pub mod skill_use;
|
||||
pub mod skills_loader;
|
||||
pub mod subscription;
|
||||
pub mod swarm;
|
||||
pub mod task_card_parser;
|
||||
pub mod task_card_worker;
|
||||
pub mod team_template_loader;
|
||||
pub mod tool_versions;
|
||||
mod topology_exec;
|
||||
pub mod topology_exec;
|
||||
pub mod topology_worker;
|
||||
pub mod validator_preflight;
|
||||
pub mod vm_placement;
|
||||
pub mod vm_stop_gate;
|
||||
pub mod vm_tool_gate;
|
||||
pub mod vm_tool_tap;
|
||||
pub mod workflow_registry;
|
||||
|
||||
use axum::routing::{delete, get, patch, post};
|
||||
@@ -177,6 +193,7 @@ pub fn router(state: AppState) -> Router {
|
||||
.route("/api/nodes", get(routes::nodes::list))
|
||||
.route("/api/fleet/capacity", get(routes::nodes::capacity))
|
||||
.route("/api/fleet/backends", get(routes::nodes::backends))
|
||||
.route("/api/judge/quota", get(routes::nodes::judge_quota))
|
||||
.route("/api/nodes/pair", post(routes::nodes::pair))
|
||||
.route("/api/nodes/live", get(routes::nodes::live))
|
||||
.route("/api/nodes/agent", get(routes::nodes::agent_ws))
|
||||
@@ -242,6 +259,11 @@ pub fn router(state: AppState) -> Router {
|
||||
.route("/api/user/me", get(routes::identity::me))
|
||||
.route("/api/claws", post(routes::claws::create))
|
||||
.route("/api/claws/batch-delete", post(routes::claws::batch_delete))
|
||||
.route("/api/claws/lifecycle", get(routes::claws::lifecycle_census))
|
||||
.route(
|
||||
"/api/claws/lifecycle/sweep",
|
||||
post(routes::claws::lifecycle_sweep),
|
||||
)
|
||||
.route("/api/claws/{id}", patch(routes::claws::patch))
|
||||
.route("/api/claws/{id}", delete(routes::claws::delete))
|
||||
.route("/api/claws/{id}/model", patch(routes::claws::set_model))
|
||||
@@ -490,6 +512,15 @@ pub fn router(state: AppState) -> Router {
|
||||
// The workflow recipe catalog (templates/workflows/*.toml). Serving it
|
||||
// lets the client stop mirroring the phase composition table inline.
|
||||
.route("/api/workflows", get(routes::missions::list_workflows))
|
||||
// The private podcast feed. Token in the query string, not a header:
|
||||
// no podcast app can set headers. See `routes::podcast`.
|
||||
.route("/api/podcast/feed.xml", get(routes::podcast::feed))
|
||||
.route("/api/podcast/episodes", get(routes::podcast::list_episodes))
|
||||
.route("/api/podcast/subscription", get(routes::podcast::subscription))
|
||||
.route(
|
||||
"/api/podcast/episodes/{file}",
|
||||
get(routes::podcast::episode_audio),
|
||||
)
|
||||
.route(
|
||||
"/api/missions/{id}",
|
||||
get(routes::missions::get)
|
||||
@@ -509,7 +540,10 @@ pub fn router(state: AppState) -> Router {
|
||||
"/api/missions/refine-draft",
|
||||
post(routes::missions::refine_draft),
|
||||
)
|
||||
.route("/api/missions/{id}/merge", post(routes::missions::merge_branch))
|
||||
.route(
|
||||
"/api/missions/{id}/merge",
|
||||
post(routes::missions::merge_branch),
|
||||
)
|
||||
.route(
|
||||
"/api/missions/{id}/artifacts/{artifact_id}/content",
|
||||
get(routes::missions::artifact_content),
|
||||
@@ -567,6 +601,10 @@ pub fn router(state: AppState) -> Router {
|
||||
"/api/missions/{id}/phases/{phase_id}/evaluations",
|
||||
get(routes::missions::list_phase_evaluations),
|
||||
)
|
||||
.route(
|
||||
"/api/missions/{id}/skill-use",
|
||||
get(routes::missions::skill_use),
|
||||
)
|
||||
.route(
|
||||
"/api/missions/{id}/teams",
|
||||
get(routes::missions::list_teams),
|
||||
|
||||
@@ -153,6 +153,7 @@ pub async fn run_to_vault(
|
||||
total.shelved.extend(h.shelved);
|
||||
total.failed.extend(h.failed);
|
||||
total.notes_written.extend(h.notes_written);
|
||||
total.papers.extend(h.papers);
|
||||
}
|
||||
|
||||
// The TAIL of the uuid, not the head. UUIDv7 leads with a 48-bit
|
||||
@@ -175,6 +176,7 @@ pub async fn run_to_vault(
|
||||
}
|
||||
|
||||
git(&vault, &["checkout", "-B", &branch]).await?;
|
||||
|
||||
git(&vault, &["add", "--", "60 Papers"]).await?;
|
||||
let message = format!(
|
||||
"library: {} new paper(s)\n\n{}\n\nShelved in the blob store; this commit is the catalogue.",
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
//! A process-wide push bus for live taxonomy events.
|
||||
//!
|
||||
//! `/api/world/live` is a 2-second database poll. That is the right shape for
|
||||
//! state you can query — statuses, phases, telemetry — and the wrong shape for a
|
||||
//! token stream: an agent's reasoning only becomes visible after the step
|
||||
//! finishes and its text is persisted, so the REASONING STREAM card showed
|
||||
//! completed paragraphs rather than an agent thinking.
|
||||
//!
|
||||
//! This carries the frames that cannot wait for a round trip through Postgres.
|
||||
//! `topology_exec` publishes as the runtime's WebSocket delivers them; the SSE
|
||||
//! handler subscribes and forwards, so a chunk reaches the browser in one hop.
|
||||
//!
|
||||
//! **Why a global rather than a field on `AppState`.** The publisher is
|
||||
//! `topology_exec`, reached through `phase_runner` → `topology_worker` →
|
||||
//! `MissionTap`, none of which hold `AppState`. Threading a handle through all
|
||||
//! of them would put a UI concern into four layers that have no other reason to
|
||||
//! know about one. There is exactly one bus per process and it holds no
|
||||
//! per-request state, so a `OnceLock` is the honest representation.
|
||||
//!
|
||||
//! **Lossy on purpose.** A slow reader lags and skips rather than applying
|
||||
//! backpressure to the agent that is producing. Dropping frames degrades a live
|
||||
//! view; blocking would slow the mission to the speed of the slowest open tab.
|
||||
//! The durable record is `mission_events` — this bus is the fast path, never the
|
||||
//! source of truth.
|
||||
|
||||
use std::sync::{Arc, OnceLock};
|
||||
|
||||
use serde_json::Value;
|
||||
use tokio::sync::broadcast;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Bounded so a stalled subscriber costs memory once, not unboundedly. At
|
||||
/// token granularity a busy mission produces a few hundred frames a second;
|
||||
/// this is roughly a couple of seconds of slack before a slow reader starts
|
||||
/// skipping.
|
||||
const CAPACITY: usize = 2048;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LiveEvent {
|
||||
/// Every subscriber is workspace-scoped; the bus is not.
|
||||
pub workspace_id: Uuid,
|
||||
/// A taxonomy type, e.g. `agent.reasoning.delta`.
|
||||
pub kind: String,
|
||||
pub data: Value,
|
||||
}
|
||||
|
||||
pub struct LiveBus {
|
||||
tx: broadcast::Sender<LiveEvent>,
|
||||
}
|
||||
|
||||
impl LiveBus {
|
||||
fn new() -> LiveBus {
|
||||
let (tx, _rx) = broadcast::channel(CAPACITY);
|
||||
LiveBus { tx }
|
||||
}
|
||||
|
||||
/// Publish. Returns immediately, and succeeds even with no subscribers —
|
||||
/// nobody watching is the normal case, not an error.
|
||||
pub fn publish(&self, workspace_id: Uuid, kind: &str, data: Value) {
|
||||
let _ = self.tx.send(LiveEvent {
|
||||
workspace_id,
|
||||
kind: kind.to_string(),
|
||||
data,
|
||||
});
|
||||
}
|
||||
|
||||
pub fn subscribe(&self) -> broadcast::Receiver<LiveEvent> {
|
||||
self.tx.subscribe()
|
||||
}
|
||||
}
|
||||
|
||||
static BUS: OnceLock<Arc<LiveBus>> = OnceLock::new();
|
||||
|
||||
pub fn global() -> &'static Arc<LiveBus> {
|
||||
BUS.get_or_init(|| Arc::new(LiveBus::new()))
|
||||
}
|
||||
|
||||
/// The claw alias the runtime dispatches on (`claw_<uuid>`) → the agent id the
|
||||
/// UI keys on. Returns `None` for any other alias — the governor, the door and
|
||||
/// the evaluator all drive turns under names that are not claws, and attributing
|
||||
/// their output to an agent would put words in someone's mouth.
|
||||
pub fn agent_id_from_alias(alias: &str) -> Option<Uuid> {
|
||||
Uuid::parse_str(alias.strip_prefix("claw_")?).ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn only_claw_aliases_resolve_to_an_agent() {
|
||||
let id = Uuid::now_v7();
|
||||
assert_eq!(
|
||||
agent_id_from_alias(&format!("claw_{id}")),
|
||||
Some(id),
|
||||
"the runtime's own alias form must resolve"
|
||||
);
|
||||
// These drive real turns and must NOT be attributed to an agent.
|
||||
for other in ["scout", "coordinator", "door", "evaluator", "claw_nonsense"] {
|
||||
assert_eq!(agent_id_from_alias(other), None, "{other}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_subscriber_receives_what_is_published() {
|
||||
let bus = LiveBus::new();
|
||||
let mut rx = bus.subscribe();
|
||||
let ws = Uuid::now_v7();
|
||||
bus.publish(
|
||||
ws,
|
||||
"agent.reasoning.delta",
|
||||
serde_json::json!({"text": "hi"}),
|
||||
);
|
||||
let ev = rx.recv().await.expect("delivered");
|
||||
assert_eq!(ev.workspace_id, ws);
|
||||
assert_eq!(ev.kind, "agent.reasoning.delta");
|
||||
}
|
||||
|
||||
/// Publishing with nobody listening must not error — that is the common
|
||||
/// case (no browser open) and it must never disturb the mission.
|
||||
#[test]
|
||||
fn publishing_into_the_void_is_fine() {
|
||||
let bus = LiveBus::new();
|
||||
bus.publish(Uuid::now_v7(), "agent.tool.call", serde_json::json!({}));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,446 @@
|
||||
//! Model calls from mission containers, with the real credential added here.
|
||||
//!
|
||||
//! Container-tier missions ran Claude Code with the platform's provider keys in
|
||||
//! their environment (`mission_runtime::forwarded_provider_env`), readable by
|
||||
//! an agent that has Bash and public egress. `delivery_secrets` stops those keys
|
||||
//! leaving through a delivery; nothing stopped a `curl` carrying one.
|
||||
//!
|
||||
//! With this on (`CLAWMATES_LLM_PROXY=1`), a mission container holds a
|
||||
//! per-mission TOKEN where each key used to be, and `ANTHROPIC_BASE_URL` (plus
|
||||
//! the GLM/Kimi hops' base URLs in the mission's config copy) point here. This
|
||||
//! swaps the token for the real credential and forwards the request unchanged,
|
||||
//! streaming the answer back. The agent never holds a real key.
|
||||
//!
|
||||
//! Measured before building (2026-09-23 spike on gw-04): Claude Code logged in
|
||||
//! with a subscription OAuth token, given only a placeholder and a base URL,
|
||||
//! sent nothing but `POST /v1/messages` to it and answered correctly once the
|
||||
//! placeholder was swapped. Nothing bypassed the base URL.
|
||||
//!
|
||||
//! # Who can use it
|
||||
//!
|
||||
//! Its own listener, [`PORT`], which is neither published nor routed by
|
||||
//! Traefik: reachable only from containers on the server's Docker networks. A
|
||||
//! token is `HMAC(secret, mission_id)` — stateless, so it survives a server
|
||||
//! redeploy under a running mission — and is honoured only while that mission
|
||||
//! is `running`. A token that escapes is worth one mission's model calls, from
|
||||
//! inside the network, until the mission ends.
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::extract::{Path, State};
|
||||
use axum::http::{HeaderMap, Method, StatusCode, Uri};
|
||||
use axum::response::Response;
|
||||
use hmac::{Hmac, Mac};
|
||||
use sha2::Sha256;
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// The proxy's own port inside the server container.
|
||||
pub const PORT: u16 = 8089;
|
||||
|
||||
const TOKEN_PREFIX: &str = "cmlp";
|
||||
|
||||
/// On only when asked for AND a secret exists to sign tokens with. A flag
|
||||
/// without a secret would mint tokens nobody can verify, and every mission
|
||||
/// would fail to reach its model.
|
||||
pub fn enabled() -> bool {
|
||||
std::env::var("CLAWMATES_LLM_PROXY").is_ok_and(|v| v.trim() == "1") && secret().is_some()
|
||||
}
|
||||
|
||||
fn secret() -> Option<Vec<u8>> {
|
||||
std::env::var("CLAWMATES_LLM_PROXY_SECRET")
|
||||
.ok()
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| s.len() >= 32)
|
||||
.map(String::into_bytes)
|
||||
}
|
||||
|
||||
fn sign(secret: &[u8], mission_id: Uuid) -> String {
|
||||
let mut mac = Hmac::<Sha256>::new_from_slice(secret).expect("hmac takes any key length");
|
||||
mac.update(mission_id.as_bytes());
|
||||
hex::encode(&mac.finalize().into_bytes()[..20])
|
||||
}
|
||||
|
||||
/// The token a mission's container gets in place of every provider key.
|
||||
pub fn token_for_with(secret: &[u8], mission_id: Uuid) -> String {
|
||||
format!("{TOKEN_PREFIX}.{}.{}", mission_id.simple(), sign(secret, mission_id))
|
||||
}
|
||||
|
||||
pub fn token_for(mission_id: Uuid) -> Option<String> {
|
||||
secret().map(|s| token_for_with(&s, mission_id))
|
||||
}
|
||||
|
||||
/// The mission a token was minted for, if the signature holds.
|
||||
pub fn verify_with(secret: &[u8], token: &str) -> Option<Uuid> {
|
||||
let mut parts = token.trim().splitn(3, '.');
|
||||
if parts.next()? != TOKEN_PREFIX {
|
||||
return None;
|
||||
}
|
||||
let mission = Uuid::parse_str(parts.next()?).ok()?;
|
||||
let sig = parts.next()?;
|
||||
let want = sign(secret, mission);
|
||||
// Constant-time compare: the signature is the whole credential.
|
||||
let ok = sig.len() == want.len()
|
||||
&& sig.bytes().zip(want.bytes()).fold(0u8, |a, (x, y)| a | (x ^ y)) == 0;
|
||||
ok.then_some(mission)
|
||||
}
|
||||
|
||||
/// Where a mission container reaches the proxy: the server's own hostname on
|
||||
/// the Docker network (the same self-configuring rule as the skills door's
|
||||
/// `api_origin`), overridable for other deployments.
|
||||
pub fn base_url() -> Option<String> {
|
||||
if let Ok(v) = std::env::var("CLAWMATES_LLM_PROXY_URL") {
|
||||
if !v.trim().is_empty() {
|
||||
return Some(v.trim().trim_end_matches('/').to_string());
|
||||
}
|
||||
}
|
||||
let host = std::env::var("HOSTNAME").ok()?;
|
||||
let host = host.trim();
|
||||
(!host.is_empty()).then(|| format!("http://{host}:{PORT}"))
|
||||
}
|
||||
|
||||
/// Where fleet nodes reach the proxy: the server's TAILNET address and the
|
||||
/// proxy port, e.g. `100.102.112.85:8089` (published on that address only,
|
||||
/// never on a public interface). Set = microVM missions may be relayed.
|
||||
pub fn node_relay_addr() -> Option<String> {
|
||||
std::env::var("CLAWMATES_LLM_PROXY_NODE_ADDR")
|
||||
.ok()
|
||||
.map(|v| v.trim().to_string())
|
||||
.filter(|v| !v.is_empty())
|
||||
}
|
||||
|
||||
/// The proxy route a microVM backend's CLI speaks to, or `None` for a backend
|
||||
/// that reaches no hosted provider (`local-ornith`) or that nobody taught this
|
||||
/// function about — the same fail-closed rule as the node's `provider_hosts`.
|
||||
pub fn microvm_route(backend: Option<&str>) -> Option<&'static str> {
|
||||
match backend {
|
||||
None | Some("") | Some("default") | Some("claude") | Some("canary-claude") => Some("anthropic"),
|
||||
Some("glm") => Some("glm"),
|
||||
Some("kimi") => Some("kimi"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Should this microVM phase reach its model through the proxy? Only when the
|
||||
/// proxy is on, a node address is configured, the backend has a route, and the
|
||||
/// NODE says it can relay (daemon 0.5.0+ reports `model_relay`). An older node
|
||||
/// keeps the old path — the key in the guest — rather than a guest whose model
|
||||
/// calls go nowhere.
|
||||
pub async fn microvm_relay(pool: &PgPool, node: Uuid, backend: Option<&str>) -> Option<String> {
|
||||
if !enabled() || microvm_route(backend).is_none() {
|
||||
return None;
|
||||
}
|
||||
let addr = node_relay_addr()?;
|
||||
let relays: bool = sqlx::query_scalar(
|
||||
"SELECT coalesce((capabilities->>'model_relay')::boolean, false) FROM nodes WHERE id = $1",
|
||||
)
|
||||
.bind(node)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.unwrap_or(false);
|
||||
if !relays {
|
||||
eprintln!("llm_proxy: node {node} cannot relay model calls (daemon < 0.5.0) — its guest gets the provider key");
|
||||
return None;
|
||||
}
|
||||
Some(addr)
|
||||
}
|
||||
|
||||
/// One upstream: where it lives and the real credential it takes.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub struct Upstream {
|
||||
pub base: &'static str,
|
||||
/// `(header, value)`.
|
||||
pub auth: (&'static str, String),
|
||||
}
|
||||
|
||||
/// The upstream for a route, with the credential read from the server's env.
|
||||
/// `None` for an unknown route or a provider whose key is not set here.
|
||||
pub fn upstream(provider: &str) -> Option<Upstream> {
|
||||
let env = |k: &str| std::env::var(k).ok().map(|v| v.trim().to_string()).filter(|v| !v.is_empty());
|
||||
match provider {
|
||||
"anthropic" => match crate::mission_runtime::runtime_auth_mode() {
|
||||
crate::mission_runtime::RuntimeAuth::Subscription => Some(Upstream {
|
||||
base: "https://api.anthropic.com",
|
||||
auth: ("authorization", format!("Bearer {}", env("CLAUDE_CODE_OAUTH_TOKEN")?)),
|
||||
}),
|
||||
crate::mission_runtime::RuntimeAuth::ApiKey => Some(Upstream {
|
||||
base: "https://api.anthropic.com",
|
||||
auth: ("x-api-key", env("ANTHROPIC_API_KEY")?),
|
||||
}),
|
||||
},
|
||||
"glm" => Some(Upstream {
|
||||
base: "https://api.z.ai/api/anthropic",
|
||||
auth: ("authorization", format!("Bearer {}", env("ZAI_API_KEY")?)),
|
||||
}),
|
||||
"kimi" => Some(Upstream {
|
||||
base: "https://api.kimi.com/coding",
|
||||
auth: ("authorization", format!("Bearer {}", env("KIMI_API_KEY")?)),
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The token a request presents, as Claude Code sends it: `Authorization:
|
||||
/// Bearer` for an OAuth or auth-token credential, `x-api-key` for an API key.
|
||||
fn presented_token(headers: &HeaderMap) -> Option<String> {
|
||||
if let Some(v) = headers.get("authorization").and_then(|v| v.to_str().ok()) {
|
||||
return Some(v.trim().trim_start_matches("Bearer ").trim().to_string());
|
||||
}
|
||||
headers
|
||||
.get("x-api-key")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|v| v.trim().to_string())
|
||||
}
|
||||
|
||||
/// Request headers that must not be forwarded: the placeholder credential,
|
||||
/// and the hop-by-hop / length headers the client rebuilds.
|
||||
const DROP_REQUEST: &[&str] = &[
|
||||
"authorization",
|
||||
"x-api-key",
|
||||
"host",
|
||||
"content-length",
|
||||
"connection",
|
||||
"accept-encoding",
|
||||
"transfer-encoding",
|
||||
];
|
||||
const DROP_RESPONSE: &[&str] = &["content-length", "transfer-encoding", "connection", "content-encoding"];
|
||||
|
||||
fn deny(status: StatusCode, why: &str) -> Response {
|
||||
Response::builder()
|
||||
.status(status)
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(
|
||||
serde_json::json!({"type":"error","error":{"type":"clawmates_llm_proxy","message":why}})
|
||||
.to_string(),
|
||||
))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct ProxyState {
|
||||
pool: PgPool,
|
||||
client: reqwest::Client,
|
||||
}
|
||||
|
||||
async fn handle(
|
||||
State(st): State<ProxyState>,
|
||||
Path((provider, rest)): Path<(String, String)>,
|
||||
method: Method,
|
||||
uri: Uri,
|
||||
headers: HeaderMap,
|
||||
body: axum::body::Bytes,
|
||||
) -> Response {
|
||||
let Some(secret) = secret() else {
|
||||
return deny(StatusCode::SERVICE_UNAVAILABLE, "proxy has no signing secret");
|
||||
};
|
||||
let Some(mission) = presented_token(&headers).and_then(|t| verify_with(&secret, &t)) else {
|
||||
return deny(StatusCode::UNAUTHORIZED, "not a valid mission token");
|
||||
};
|
||||
let running: bool = sqlx::query_scalar("SELECT status = 'running' FROM missions WHERE id = $1")
|
||||
.bind(mission)
|
||||
.fetch_optional(&st.pool)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.unwrap_or(false);
|
||||
if !running {
|
||||
return deny(StatusCode::FORBIDDEN, "mission is not running");
|
||||
}
|
||||
let Some(up) = upstream(&provider) else {
|
||||
return deny(StatusCode::NOT_FOUND, "unknown or unconfigured provider");
|
||||
};
|
||||
let query = uri.query().map(|q| format!("?{q}")).unwrap_or_default();
|
||||
let url = format!("{}/{rest}{query}", up.base);
|
||||
let mut req = st.client.request(method, &url);
|
||||
for (k, v) in headers.iter() {
|
||||
if !DROP_REQUEST.contains(&k.as_str()) {
|
||||
req = req.header(k, v);
|
||||
}
|
||||
}
|
||||
req = req.header(up.auth.0, up.auth.1).body(body);
|
||||
let resp = match req.send().await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
eprintln!("llm_proxy: mission {mission} → {provider}: {e}");
|
||||
return deny(StatusCode::BAD_GATEWAY, "upstream unreachable");
|
||||
}
|
||||
};
|
||||
let mut out = Response::builder().status(resp.status().as_u16());
|
||||
for (k, v) in resp.headers().iter() {
|
||||
if !DROP_RESPONSE.contains(&k.as_str()) {
|
||||
out = out.header(k.as_str(), v.as_bytes());
|
||||
}
|
||||
}
|
||||
out.body(Body::from_stream(resp.bytes_stream()))
|
||||
.unwrap_or_else(|_| deny(StatusCode::BAD_GATEWAY, "could not relay the response"))
|
||||
}
|
||||
|
||||
/// Serve the proxy on [`PORT`]. A no-op unless [`enabled`].
|
||||
pub fn spawn(pool: PgPool) {
|
||||
if !enabled() {
|
||||
eprintln!("llm_proxy: off (CLAWMATES_LLM_PROXY != 1 or no CLAWMATES_LLM_PROXY_SECRET) — mission containers hold provider keys");
|
||||
return;
|
||||
}
|
||||
tokio::spawn(async move {
|
||||
let client = reqwest::Client::builder()
|
||||
// A long agent turn streams for many minutes; the CLI's own
|
||||
// API_TIMEOUT_MS is 50 minutes.
|
||||
.timeout(std::time::Duration::from_secs(3000))
|
||||
.build()
|
||||
.expect("reqwest client");
|
||||
let app = axum::Router::new()
|
||||
.route("/{provider}/{*rest}", axum::routing::any(handle))
|
||||
.with_state(ProxyState { pool, client });
|
||||
match tokio::net::TcpListener::bind(("0.0.0.0", PORT)).await {
|
||||
Ok(l) => {
|
||||
eprintln!("llm_proxy: listening on :{PORT} — mission containers get tokens, not keys");
|
||||
if let Err(e) = axum::serve(l, app).await {
|
||||
eprintln!("llm_proxy: stopped: {e}");
|
||||
}
|
||||
}
|
||||
Err(e) => eprintln!("llm_proxy: could not bind :{PORT}: {e}"),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Point a mission config's GLM and Kimi hops at the proxy. Their base URLs are
|
||||
/// literal in the seed config (the default hop takes `ANTHROPIC_BASE_URL` from
|
||||
/// the container env), so they are rewritten in the mission's own copy.
|
||||
/// Returns the edited document and how many hops were redirected.
|
||||
pub fn route_config_through(raw: &str, proxy: &str) -> Result<(String, usize), String> {
|
||||
let mut doc = raw
|
||||
.parse::<toml_edit::DocumentMut>()
|
||||
.map_err(|e| format!("parse runtime config.toml: {e}"))?;
|
||||
let mut n = 0;
|
||||
for hop in ["glm", "kimi"] {
|
||||
let Some(env) = doc
|
||||
.get_mut("providers")
|
||||
.and_then(|p| p.get_mut("models"))
|
||||
.and_then(|m| m.get_mut("claude_cli"))
|
||||
.and_then(|c| c.get_mut(hop))
|
||||
.and_then(|h| h.get_mut("env"))
|
||||
.and_then(|e| e.as_table_like_mut())
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if env.get("ANTHROPIC_BASE_URL").is_some() {
|
||||
env.insert("ANTHROPIC_BASE_URL", toml_edit::value(format!("{proxy}/{hop}")));
|
||||
n += 1;
|
||||
}
|
||||
}
|
||||
Ok((doc.to_string(), n))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const S: &[u8] = b"0123456789abcdef0123456789abcdef-test";
|
||||
|
||||
#[test]
|
||||
fn a_token_verifies_to_its_own_mission() {
|
||||
let m = Uuid::now_v7();
|
||||
assert_eq!(verify_with(S, &token_for_with(S, m)), Some(m));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tampered_or_foreign_token_is_refused() {
|
||||
let m = Uuid::now_v7();
|
||||
let t = token_for_with(S, m);
|
||||
// Another mission's id with this signature.
|
||||
let other = Uuid::now_v7();
|
||||
let forged = t.replace(&m.simple().to_string(), &other.simple().to_string());
|
||||
assert_eq!(verify_with(S, &forged), None);
|
||||
// Signed with a different secret.
|
||||
assert_eq!(verify_with(b"another-secret-another-secret-xx", &t), None);
|
||||
// Garbage and a real provider key are not tokens.
|
||||
assert_eq!(verify_with(S, "sk-ant-oat01-whatever"), None);
|
||||
assert_eq!(verify_with(S, ""), None);
|
||||
}
|
||||
|
||||
/// The token must never look like, or contain, a real key — it is what the
|
||||
/// agent can read now.
|
||||
#[test]
|
||||
fn a_token_names_its_mission_and_nothing_else() {
|
||||
let m = Uuid::now_v7();
|
||||
let t = token_for_with(S, m);
|
||||
assert!(t.starts_with("cmlp."), "{t}");
|
||||
assert!(t.contains(&m.simple().to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_placeholder_is_what_gets_checked_either_way_claude_sends_it() {
|
||||
let mut h = HeaderMap::new();
|
||||
h.insert("authorization", "Bearer cmlp.x.y".parse().unwrap());
|
||||
assert_eq!(presented_token(&h).as_deref(), Some("cmlp.x.y"));
|
||||
let mut h = HeaderMap::new();
|
||||
h.insert("x-api-key", "cmlp.a.b".parse().unwrap());
|
||||
assert_eq!(presented_token(&h).as_deref(), Some("cmlp.a.b"));
|
||||
}
|
||||
|
||||
/// The placeholder never travels upstream: both credential headers are
|
||||
/// dropped before the real one is added.
|
||||
#[test]
|
||||
fn the_placeholder_is_never_forwarded() {
|
||||
assert!(DROP_REQUEST.contains(&"authorization") && DROP_REQUEST.contains(&"x-api-key"));
|
||||
}
|
||||
|
||||
/// A relayed guest holds the token under the name its CLI reads and points
|
||||
/// at its own loopback model port — never a provider host, never a key.
|
||||
#[test]
|
||||
fn a_relayed_guest_gets_the_token_and_a_loopback_base_url() {
|
||||
for (backend, cred, route) in [
|
||||
(Some("claude"), "CLAUDE_CODE_OAUTH_TOKEN", "anthropic"),
|
||||
(None, "CLAUDE_CODE_OAUTH_TOKEN", "anthropic"),
|
||||
(Some("glm"), "ANTHROPIC_AUTH_TOKEN", "glm"),
|
||||
(Some("kimi"), "ANTHROPIC_AUTH_TOKEN", "kimi"),
|
||||
] {
|
||||
let env = crate::mission_runtime::microvm_proxied_env(backend, "cmlp.m.s").unwrap();
|
||||
let get = |k: &str| env.iter().find(|(n, _)| n == k).map(|(_, v)| v.as_str());
|
||||
assert_eq!(get(cred), Some("cmlp.m.s"), "{backend:?}");
|
||||
assert_eq!(get("ANTHROPIC_BASE_URL"), Some(format!("http://127.0.0.1:11434/{route}").as_str()));
|
||||
assert_eq!(env.len(), 2, "nothing else — in particular no other key: {env:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A local model and an unknown backend have no route, so they are never
|
||||
/// relayed (the local one keeps its own pipe to the node's model).
|
||||
#[test]
|
||||
fn local_and_unknown_backends_are_not_relayed() {
|
||||
assert_eq!(microvm_route(Some("local-ornith")), None);
|
||||
assert_eq!(microvm_route(Some("something-new")), None);
|
||||
assert!(crate::mission_runtime::microvm_proxied_env(Some("local-ornith"), "t").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_known_providers_route() {
|
||||
assert!(upstream("evil.example").is_none());
|
||||
assert!(upstream("").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_glm_and_kimi_hops_are_redirected_and_nothing_else_moves() {
|
||||
let raw = r#"
|
||||
[providers.models.claude_cli.default]
|
||||
model = "claude-sonnet-4-6"
|
||||
env = { CLAUDE_CODE_OAUTH_TOKEN = "$CLAUDE_CODE_OAUTH_TOKEN" }
|
||||
|
||||
[providers.models.claude_cli.glm]
|
||||
model = "glm-4.7"
|
||||
env = { HOME = "/zeroclaw-data/glm-home", ANTHROPIC_BASE_URL = "https://api.z.ai/api/anthropic", ANTHROPIC_AUTH_TOKEN = "$ZAI_API_KEY" }
|
||||
|
||||
[providers.models.claude_cli.kimi]
|
||||
model = "kimi-for-coding"
|
||||
env = { HOME = "/zeroclaw-data/kimi-home", ANTHROPIC_BASE_URL = "https://api.kimi.com/coding", ANTHROPIC_AUTH_TOKEN = "$KIMI_API_KEY" }
|
||||
"#;
|
||||
let (out, n) = route_config_through(raw, "http://srv:8089").unwrap();
|
||||
assert_eq!(n, 2);
|
||||
assert!(out.contains(r#"ANTHROPIC_BASE_URL = "http://srv:8089/glm""#), "{out}");
|
||||
assert!(out.contains(r#"ANTHROPIC_BASE_URL = "http://srv:8089/kimi""#), "{out}");
|
||||
assert!(!out.contains("api.z.ai") && !out.contains("api.kimi.com"), "{out}");
|
||||
// The credential REFERENCES are untouched; the env behind them changes.
|
||||
assert!(out.contains(r#"ANTHROPIC_AUTH_TOKEN = "$ZAI_API_KEY""#));
|
||||
assert!(out.contains(r#"HOME = "/zeroclaw-data/glm-home""#));
|
||||
}
|
||||
}
|
||||
+266
-54
@@ -7,9 +7,19 @@
|
||||
//! (broker-executed tools reveal secrets only inside the broker), every action
|
||||
//! is journaled to the append-only audit log, and a central policy decides each
|
||||
//! call — but the **human approver is replaced by an automated policy/governor**
|
||||
//! ("agents control their destiny"). The default policy is allow-all, so agents
|
||||
//! are autonomous out of the gate; recipient allowlists, spend caps, taint
|
||||
//! blocks, or a governor agent plug into [`policy_decide`].
|
||||
//! ("agents control their destiny"). Recipient allowlists, spend caps, taint
|
||||
//! blocks, or a governor plug into [`policy_decide`]. Since 2026-09-20 the
|
||||
//! door is **closed by default**: a call is approved only by a governor that
|
||||
//! answered ALLOW, or by an explicit `CLAWMATES_DOOR_POLICY=allow`.
|
||||
//!
|
||||
//! Since 2026-09-21 the governor has THREE outcomes, not two. With
|
||||
//! `TYPESAFE_API_KEY` set the decision is a calibrated one
|
||||
//! (`cm_decide::door`: three Nouls, the max is the deny probability):
|
||||
//! above `DENY_AT` refused, below `ALLOW_BELOW` executed, and in between
|
||||
//! **held** — a pending approval a person decides, executed on approve.
|
||||
//! Measured on 24 labelled actions: AUROC 1.0, no false denies, no misses,
|
||||
//! 4 held. The chat-model governor (`CLAWMATES_DOOR_GOVERNOR`) remains the
|
||||
//! fallback when no key is set; it has no middle band.
|
||||
//!
|
||||
//! v1 exposes `email_send` (runtime-executed → `outbox`, observable, no external
|
||||
//! creds). Broker-backed tools (e.g. `slack_post`) are the next increment — they
|
||||
@@ -77,6 +87,9 @@ fn tool_result(id: Option<Value>, is_error: bool, text: String) -> Json<Value> {
|
||||
enum PolicyOutcome {
|
||||
Approve,
|
||||
Deny(String),
|
||||
/// Not refused, not executed: a person decides. Carries the reason a
|
||||
/// reviewer reads.
|
||||
Hold(String),
|
||||
}
|
||||
|
||||
/// Decides each door call in place of a human. The human is removed; autonomy
|
||||
@@ -85,10 +98,14 @@ enum PolicyOutcome {
|
||||
/// 2. a per-workspace hourly rate cap (`CLAWMATES_DOOR_RATE_LIMIT`, counts
|
||||
/// executed door actions in the audit log);
|
||||
/// 3. an email recipient-domain allowlist (`CLAWMATES_DOOR_EMAIL_ALLOW`);
|
||||
/// 4. a governor hook (extension point) — a deterministic rule set or a
|
||||
/// governor agent can veto here.
|
||||
/// 4. a governor agent (`CLAWMATES_DOOR_GOVERNOR`) — must answer ALLOW;
|
||||
/// unreachable, silent, or off-contract means DENY;
|
||||
/// 5. with no governor, an explicit `CLAWMATES_DOOR_POLICY=allow`.
|
||||
///
|
||||
/// Default (no env set) = allow-all → agents fully autonomous.
|
||||
/// Default (no env set) = **deny**. This was allow-all until 2026-09-20, and
|
||||
/// the governor failed open on top of that, so with the judge plan emptied
|
||||
/// every outbound action was approved by nobody. See [`ungoverned_default`]
|
||||
/// and `cm_runtime::Runtime::judge` for the measurement behind the flip.
|
||||
async fn policy_decide(
|
||||
state: &AppState,
|
||||
workspace: cm_domain::WorkspaceId,
|
||||
@@ -139,10 +156,18 @@ async fn policy_decide(
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Governor agent: when CLAWMATES_DOOR_GOVERNOR is set, an LLM judges the
|
||||
// action and can veto — the "self-governing topology" path. Fail-open
|
||||
// (a governor outage doesn't halt agents); deterministic rules above are
|
||||
// the hard floor.
|
||||
// 4. Calibrated governor: three questions, one call, a probability with
|
||||
// a middle band. Preferred over the chat governor whenever a key is
|
||||
// set. Fail-CLOSED: an unreachable or malformed answer denies.
|
||||
if let Some(jev) = cm_decide::jev::Jev::from_env() {
|
||||
return calibrated_decision(&jev, mcp_tool, args).await;
|
||||
}
|
||||
|
||||
// 4b. Governor agent: when CLAWMATES_DOOR_GOVERNOR is set, an LLM judges the
|
||||
// action — the "self-governing topology" path. Fail-CLOSED: a governor
|
||||
// that cannot be reached, or that does not say ALLOW, denies. The
|
||||
// deterministic rules above are the hard floor; this is the only
|
||||
// approver.
|
||||
if std::env::var("CLAWMATES_DOOR_GOVERNOR").is_ok() {
|
||||
let system = "You are a security governor for an autonomous agent's outbound actions. \
|
||||
Reply with exactly ALLOW or DENY on the first line, then one short reason. \
|
||||
@@ -166,24 +191,98 @@ async fn policy_decide(
|
||||
} else {
|
||||
state.runtime.judge(system, &request).await
|
||||
};
|
||||
// Fail-open is deliberate, but a governor that is failing open on EVERY
|
||||
// request is a security control that has quietly stopped existing —
|
||||
// and the caller drops `reason` whenever it allows, so nothing said so.
|
||||
// `judge()` returns this exact prefix when the provider never answered,
|
||||
// which a rate-limited or uncredited judge model does on every call.
|
||||
if allow && reason.starts_with("governor unreachable") {
|
||||
// Still logged loudly: a door that denies everything because its
|
||||
// governor is down is safe, and is also a platform with no outbound
|
||||
// actions until someone reads this line.
|
||||
if reason.starts_with("governor unreachable") {
|
||||
eprintln!(
|
||||
"mcp_door: WARNING — the door governor is FAILING OPEN for {mcp_tool} \
|
||||
({reason}). Every outbound action is being approved unjudged. Point \
|
||||
CLAWMATES_JUDGE_MODEL at a reachable model."
|
||||
"mcp_door: WARNING — the door governor is unreachable, DENYING {mcp_tool} \
|
||||
({reason}). Point CLAWMATES_JUDGE_MODEL at a reachable model."
|
||||
);
|
||||
}
|
||||
if !allow {
|
||||
return PolicyOutcome::Deny(format!("governor agent vetoed — {reason}"));
|
||||
}
|
||||
return PolicyOutcome::Approve;
|
||||
}
|
||||
|
||||
PolicyOutcome::Approve
|
||||
// 5. No governor. The door is closed unless the operator opened it.
|
||||
ungoverned_default(std::env::var("CLAWMATES_DOOR_POLICY").ok().as_deref())
|
||||
}
|
||||
|
||||
/// How long the calibrated governor may take. It measures ~170 ms; a door
|
||||
/// that waits ten seconds on it is a door whose provider is down.
|
||||
const DECISION_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
||||
/// Ask the three door questions and read the band. `DENY_AT` and
|
||||
/// `ALLOW_BELOW` are `cm_decide::door`'s, overridable per deployment by
|
||||
/// `CLAWMATES_DOOR_DENY_AT` / `CLAWMATES_DOOR_ALLOW_BELOW`.
|
||||
async fn calibrated_decision(
|
||||
jev: &cm_decide::jev::Jev,
|
||||
mcp_tool: &str,
|
||||
args: &Value,
|
||||
) -> PolicyOutcome {
|
||||
use cm_decide::Decider as _;
|
||||
let deny_at = env_f64("CLAWMATES_DOOR_DENY_AT", cm_decide::door::DENY_AT);
|
||||
let allow_below = env_f64("CLAWMATES_DOOR_ALLOW_BELOW", cm_decide::door::ALLOW_BELOW);
|
||||
let state = cm_decide::door::state(mcp_tool, args);
|
||||
let decision = match tokio::time::timeout(
|
||||
DECISION_TIMEOUT,
|
||||
jev.decide(&state, &cm_decide::door::questions()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(d)) => d,
|
||||
Ok(Err(e)) => {
|
||||
eprintln!("mcp_door: the calibrated governor failed, DENYING {mcp_tool}: {e}");
|
||||
return PolicyOutcome::Deny(format!("the door's governor could not decide ({e})"));
|
||||
}
|
||||
Err(_) => {
|
||||
eprintln!("mcp_door: the calibrated governor timed out, DENYING {mcp_tool}");
|
||||
return PolicyOutcome::Deny("the door's governor did not answer in time".into());
|
||||
}
|
||||
};
|
||||
let Some(risk) = cm_decide::door::Risk::from_answers(&decision.answers) else {
|
||||
return PolicyOutcome::Deny("the door's governor answered in an unexpected shape".into());
|
||||
};
|
||||
let why = format!(
|
||||
"{} judged this {:.0}% likely to be {} (exfil {:.2}, secret {:.2}, spam {:.2})",
|
||||
decision.model,
|
||||
risk.deny * 100.0,
|
||||
risk.dominant(),
|
||||
risk.exfil,
|
||||
risk.secret,
|
||||
risk.spam
|
||||
);
|
||||
match cm_decide::patterns::gate_noul(risk.deny, allow_below, deny_at) {
|
||||
cm_decide::patterns::Gate::Act => PolicyOutcome::Deny(why),
|
||||
cm_decide::patterns::Gate::Dismiss => PolicyOutcome::Approve,
|
||||
cm_decide::patterns::Gate::Review => PolicyOutcome::Hold(why),
|
||||
}
|
||||
}
|
||||
|
||||
fn env_f64(key: &str, default: f64) -> f64 {
|
||||
std::env::var(key)
|
||||
.ok()
|
||||
.and_then(|v| v.trim().parse::<f64>().ok())
|
||||
.filter(|v| (0.0..=1.0).contains(v))
|
||||
.unwrap_or(default)
|
||||
}
|
||||
|
||||
/// The posture with no governor configured. Only the literal `allow` opens
|
||||
/// the door; unset, empty, or anything else keeps it shut and says how to
|
||||
/// open it. `deny` is handled earlier as the kill switch and lands here too.
|
||||
fn ungoverned_default(policy: Option<&str>) -> PolicyOutcome {
|
||||
match policy.map(str::trim) {
|
||||
Some("allow") => PolicyOutcome::Approve,
|
||||
_ => PolicyOutcome::Deny(
|
||||
"the door has no governor and no allow policy — set CLAWMATES_DOOR_GOVERNOR=1 \
|
||||
or, to run ungoverned, CLAWMATES_DOOR_POLICY=allow"
|
||||
.into(),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Mint an auto-approved approval + single-use execution grant for a
|
||||
@@ -198,31 +297,7 @@ async fn mint_grant(
|
||||
category: Option<cm_domain::GatedCategory>,
|
||||
args: &Value,
|
||||
) -> Result<uuid::Uuid, String> {
|
||||
// approvals.run_id / requested_by_agent are strict FKs → agent → session → run.
|
||||
let session =
|
||||
cm_db::repo::sessions::create(&state.pool, agent_id, user.workspace_id, "mcp-door")
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let run_id = cm_db::repo::runs::create(&state.pool, session.id)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let approval = cm_safety::approvals::create(
|
||||
&state.pool,
|
||||
cm_safety::NewApproval {
|
||||
workspace_id: user.workspace_id,
|
||||
run_id,
|
||||
session_key: session.id.as_uuid().to_string(),
|
||||
action_type: internal.to_string(),
|
||||
category: category.unwrap_or(cm_domain::GatedCategory::OutboundMessage),
|
||||
payload: args.clone(),
|
||||
preview: state.runtime.tool_preview(internal, args),
|
||||
requested_by_agent: agent_id,
|
||||
taint_sources: Vec::new(),
|
||||
expires_at: Some(time::OffsetDateTime::now_utc() + time::Duration::hours(1)),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let approval = door_approval(state, user, agent_id, internal, category, args, "mcp-door").await?;
|
||||
// Auto-decide (policy already approved above): mints the single-use grant
|
||||
// and writes the decision to the audit log.
|
||||
cm_safety::approvals::decide(
|
||||
@@ -236,13 +311,103 @@ async fn mint_grant(
|
||||
Ok(approval.id)
|
||||
}
|
||||
|
||||
/// Marks a held door action's approval so the approve route knows to
|
||||
/// execute the tool rather than resume a chat run. It is the `session_key`
|
||||
/// prefix; a chat approval's key is a `SessionKey`, which never starts so.
|
||||
pub const HELD_SESSION_KEY_PREFIX: &str = "door:";
|
||||
|
||||
/// The approval row a door action gets. Pending; the caller decides it —
|
||||
/// immediately for an approved action ([`mint_grant`]), or a person later
|
||||
/// for a held one. `title` names the session so the row is recognisable.
|
||||
async fn door_approval(
|
||||
state: &AppState,
|
||||
user: &cm_auth::AuthedUser,
|
||||
agent_id: cm_domain::AgentId,
|
||||
internal: &str,
|
||||
category: Option<cm_domain::GatedCategory>,
|
||||
args: &Value,
|
||||
title: &str,
|
||||
) -> Result<cm_safety::Approval, String> {
|
||||
// approvals.run_id / requested_by_agent are strict FKs → agent → session → run.
|
||||
let session = cm_db::repo::sessions::create(&state.pool, agent_id, user.workspace_id, title)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let run_id = cm_db::repo::runs::create(&state.pool, session.id)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let session_key = if title == "mcp-door-held" {
|
||||
format!("{HELD_SESSION_KEY_PREFIX}{}", session.id.as_uuid())
|
||||
} else {
|
||||
session.id.as_uuid().to_string()
|
||||
};
|
||||
cm_safety::approvals::create(
|
||||
&state.pool,
|
||||
cm_safety::NewApproval {
|
||||
workspace_id: user.workspace_id,
|
||||
run_id,
|
||||
session_key,
|
||||
action_type: internal.to_string(),
|
||||
category: category.unwrap_or(cm_domain::GatedCategory::OutboundMessage),
|
||||
payload: args.clone(),
|
||||
preview: state.runtime.tool_preview(internal, args),
|
||||
requested_by_agent: agent_id,
|
||||
taint_sources: Vec::new(),
|
||||
expires_at: Some(time::OffsetDateTime::now_utc() + time::Duration::hours(24)),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// Execute a held door action that a person has just approved. Called by
|
||||
/// the approvals route; the grant was minted by the decide it just made.
|
||||
pub async fn execute_held(state: &AppState, approval: &cm_safety::Approval) -> Result<Value, String> {
|
||||
let out = state
|
||||
.runtime
|
||||
.execute_door_tool(
|
||||
approval.workspace_id,
|
||||
approval.requested_by_agent,
|
||||
&approval.action_type,
|
||||
approval.payload.clone(),
|
||||
Some(approval.id),
|
||||
)
|
||||
.await;
|
||||
let (event, detail) = match &out {
|
||||
Ok(output) => (
|
||||
"door.executed",
|
||||
json!({ "auto_decided": false, "approval_id": approval.id, "args": approval.payload, "output": output }),
|
||||
),
|
||||
Err(e) => ("door.error", json!({ "approval_id": approval.id, "error": e, "args": approval.payload })),
|
||||
};
|
||||
let _ = cm_db::repo::audit::append(
|
||||
&state.pool,
|
||||
approval.workspace_id,
|
||||
cm_db::repo::audit::Actor::Agent(approval.requested_by_agent),
|
||||
event,
|
||||
"tool",
|
||||
&approval.action_type,
|
||||
detail,
|
||||
)
|
||||
.await;
|
||||
out
|
||||
}
|
||||
|
||||
/// Authenticate the bearer header → workspace/user. `None` if missing/invalid.
|
||||
///
|
||||
/// Accepts [`cm_auth::SCOPE_AGENT_DOOR`] as well as a person's session. This
|
||||
/// route is the one that can `delegate`, and the thing that will eventually
|
||||
/// hold a token for it is an agent runtime — so the narrow credential has to
|
||||
/// exist before something reaches for the only one that does.
|
||||
async fn authed(state: &AppState, headers: &HeaderMap) -> Option<cm_auth::AuthedUser> {
|
||||
let token = headers
|
||||
.get(AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.strip_prefix("Bearer "))?;
|
||||
state.auth.authenticate(token).await.ok()
|
||||
state
|
||||
.auth
|
||||
.authenticate_scoped(token, cm_auth::SCOPE_AGENT_DOOR)
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// Resolve the specific claw making the call. Our ZeroClaw fork stamps the
|
||||
@@ -501,10 +666,18 @@ pub async fn mcp(
|
||||
|
||||
let category = state.runtime.tool_gate_category(internal);
|
||||
|
||||
// The gate — human replaced by automated policy.
|
||||
if let PolicyOutcome::Deny(reason) =
|
||||
policy_decide(&state, user.workspace_id, mcp_name, category, &args).await
|
||||
{
|
||||
// Attribute the action to the specific calling claw (X-ZeroClaw-Agent
|
||||
// header), or the workspace's first agent as a legacy fallback.
|
||||
let agent_id = match caller_agent(&state, &user, &headers).await {
|
||||
Ok(id) => id,
|
||||
Err(msg) => return tool_result(req.id, true, msg),
|
||||
};
|
||||
|
||||
// The gate — human replaced by automated policy, with a way back
|
||||
// to the human for the actions the policy will not decide alone.
|
||||
match policy_decide(&state, user.workspace_id, mcp_name, category, &args).await {
|
||||
PolicyOutcome::Approve => {}
|
||||
PolicyOutcome::Deny(reason) => {
|
||||
let _ = cm_db::repo::audit::append(
|
||||
&state.pool,
|
||||
user.workspace_id,
|
||||
@@ -517,13 +690,40 @@ pub async fn mcp(
|
||||
.await;
|
||||
return tool_result(req.id, true, format!("denied by policy: {reason}"));
|
||||
}
|
||||
|
||||
// Attribute the action to the specific calling claw (X-ZeroClaw-Agent
|
||||
// header), or the workspace's first agent as a legacy fallback.
|
||||
let agent_id = match caller_agent(&state, &user, &headers).await {
|
||||
Ok(id) => id,
|
||||
Err(msg) => return tool_result(req.id, true, msg),
|
||||
PolicyOutcome::Hold(reason) => {
|
||||
let internal_name = internal;
|
||||
let approval = match door_approval(
|
||||
&state, &user, agent_id, internal_name, category, &args, "mcp-door-held",
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(a) => a,
|
||||
Err(e) => {
|
||||
return tool_result(req.id, true, format!("held, but could not queue it for review: {e}"))
|
||||
}
|
||||
};
|
||||
let _ = cm_db::repo::audit::append(
|
||||
&state.pool,
|
||||
user.workspace_id,
|
||||
cm_db::repo::audit::Actor::System,
|
||||
"door.held",
|
||||
"tool",
|
||||
mcp_name,
|
||||
json!({ "category": category.map(|c| c.as_str()), "reason": reason, "approval_id": approval.id, "args": args }),
|
||||
)
|
||||
.await;
|
||||
return tool_result(
|
||||
req.id,
|
||||
true,
|
||||
format!(
|
||||
"held for human review — NOT executed. {reason}. It is in the approvals \
|
||||
queue as {}; if a person approves it, it will be executed then. Do not \
|
||||
retry it with different wording.",
|
||||
approval.id
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Gated delegation bridge: `delegate` causes a sibling claw to run a
|
||||
// full turn and returns its result, gated + audited here rather than
|
||||
@@ -600,6 +800,18 @@ pub async fn mcp(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The default posture is closed. Before 2026-09-20 an unset policy
|
||||
/// meant allow-all.
|
||||
#[test]
|
||||
fn the_door_is_closed_unless_opened() {
|
||||
assert!(matches!(ungoverned_default(None), PolicyOutcome::Deny(_)));
|
||||
assert!(matches!(ungoverned_default(Some("")), PolicyOutcome::Deny(_)));
|
||||
assert!(matches!(ungoverned_default(Some("deny")), PolicyOutcome::Deny(_)));
|
||||
assert!(matches!(ungoverned_default(Some("yes")), PolicyOutcome::Deny(_)));
|
||||
assert!(matches!(ungoverned_default(Some("allow")), PolicyOutcome::Approve));
|
||||
assert!(matches!(ungoverned_default(Some(" allow ")), PolicyOutcome::Approve));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exposed_tool_name_maps_to_registry_name() {
|
||||
assert_eq!(internal_name("email_send"), Some("email.send"));
|
||||
|
||||
@@ -60,12 +60,26 @@ fn err(id: Option<Value>, code: i64, message: &str) -> Json<Value> {
|
||||
|
||||
// ── Auth ─────────────────────────────────────────────────────────
|
||||
|
||||
/// This endpoint accepts a **narrow** credential as well as a person's session.
|
||||
///
|
||||
/// It is the one route a mission container is given a token for, and that token
|
||||
/// sits in a file the agent can `cat`. Mission agents run arbitrary `Bash` with
|
||||
/// egress and no read gate, so a full session here would be an owner-privileged
|
||||
/// API key handed to something explicitly untrusted — which is why
|
||||
/// `SCOPE_SKILLS_READ` exists and why this is the only call site that names it.
|
||||
///
|
||||
/// `authenticate_scoped` still accepts `full`, so the UI and any human caller
|
||||
/// are unaffected.
|
||||
async fn authed(state: &AppState, headers: &HeaderMap) -> Option<cm_auth::AuthedUser> {
|
||||
let token = headers
|
||||
.get(AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.strip_prefix("Bearer "))?;
|
||||
state.auth.authenticate(token).await.ok()
|
||||
state
|
||||
.auth
|
||||
.authenticate_scoped(token, cm_auth::SCOPE_SKILLS_READ)
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// Resolve the calling agent via `X-ZeroClaw-Agent` header
|
||||
@@ -92,7 +106,7 @@ async fn caller_agent(
|
||||
|
||||
// ── URI helpers ──────────────────────────────────────────────────
|
||||
|
||||
fn skill_uri(workspace_id: Option<Uuid>, name: &str) -> String {
|
||||
pub(crate) fn skill_uri(workspace_id: Option<Uuid>, name: &str) -> String {
|
||||
match workspace_id {
|
||||
Some(ws) => format!("{URI_PREFIX_WORKSPACE}{ws}/{name}"),
|
||||
None => format!("{URI_PREFIX_GLOBAL}{name}"),
|
||||
@@ -100,7 +114,7 @@ fn skill_uri(workspace_id: Option<Uuid>, name: &str) -> String {
|
||||
}
|
||||
|
||||
/// Parse `skill:global/<name>` or `skill:workspace/<ws>/<name>`.
|
||||
fn parse_uri(uri: &str) -> Option<(Option<Uuid>, String)> {
|
||||
pub(crate) fn parse_uri(uri: &str) -> Option<(Option<Uuid>, String)> {
|
||||
if let Some(name) = uri.strip_prefix(URI_PREFIX_GLOBAL) {
|
||||
return Some((None, name.to_string()));
|
||||
}
|
||||
|
||||
@@ -98,14 +98,18 @@ impl<'a> MicroVm<'a> {
|
||||
vcpus: u32,
|
||||
mem_mib: u32,
|
||||
backend: Option<&str>,
|
||||
model_relay: Option<&str>,
|
||||
) -> Result<Value, String> {
|
||||
// 60s, not the hub default: a create that has to copy a rootfs and boot
|
||||
// is measured near 1s, but a node under load has no reason to be fast.
|
||||
self.call(
|
||||
"vm_create",
|
||||
json!({ "vcpus": vcpus, "mem_mib": mem_mib, "backend": backend }),
|
||||
60,
|
||||
)
|
||||
//
|
||||
// `model_relay` is omitted, not sent as null, when absent: an older node
|
||||
// ignores unknown fields either way, but the absence is the old path.
|
||||
let mut req = json!({ "vcpus": vcpus, "mem_mib": mem_mib, "backend": backend });
|
||||
if let Some(r) = model_relay {
|
||||
req["model_relay"] = json!(r);
|
||||
}
|
||||
self.call("vm_create", req, 60)
|
||||
.await
|
||||
}
|
||||
|
||||
|
||||
@@ -232,7 +232,13 @@ fn agent_definitions() -> serde_json::Value {
|
||||
// so `acceptEdits` reaches it either way. It is also why the image is
|
||||
// pinned to 2.1.223: 2.1.222 fixed background subagents being able to
|
||||
// bypass tool restrictions, and this is the tool restriction in question.
|
||||
"tools": "Read, Grep, Glob, Bash",
|
||||
// A JSON ARRAY. The comma-separated string is frontmatter syntax,
|
||||
// not `--agents` syntax, and every CLI before 2.1.243 dropped an
|
||||
// invalid definition SILENTLY — so from the day this shipped until
|
||||
// the 2.1.276 canary on 2026-09-18 refused it ("verifier.tools:
|
||||
// Invalid input"), this restriction may never have been applied.
|
||||
// The guarantee below is only as real as this line's shape.
|
||||
"tools": ["Read", "Grep", "Glob", "Bash"],
|
||||
// Foreground, against the default since 2.1.198. A background verifier
|
||||
// lets the lead carry on and write its report before the check has
|
||||
// finished — the finding would arrive after the conclusion. The whole
|
||||
@@ -256,7 +262,7 @@ fn agent_definitions() -> serde_json::Value {
|
||||
"description": "Reads and searches the codebase to answer a specific \
|
||||
question. Use when finding something out would otherwise \
|
||||
fill the main context with files and search output.",
|
||||
"tools": "Read, Grep, Glob",
|
||||
"tools": ["Read", "Grep", "Glob"],
|
||||
"prompt": format!(
|
||||
"You answer one question about this codebase by reading it. Return the \
|
||||
answer and the paths that support it — not a transcript of your \
|
||||
@@ -306,6 +312,55 @@ const TEAMMATE_PROBE: &str = "cat /root/.claude/teams/*/config.json 2>/dev/null
|
||||
/// says so, which is the difference between losing a check and losing the work.
|
||||
const SETTINGS_PROBE: &str = "claude --help 2>&1 | grep -q -- '--settings' && echo SETTINGS-OK";
|
||||
|
||||
/// What the guest's `claude` reports itself as. Recorded beside the rootfs the
|
||||
/// node said it booted, so "which CLI did this mission run on" is a query
|
||||
/// against `topology_runs`, not an archaeology of image mtimes. Found necessary
|
||||
/// on 2026-09-18: every rootfs on the fleet had been on 2.1.223–2.1.226 for a
|
||||
/// month while the container tier moved to 2.1.276, and nothing recorded either.
|
||||
const CLI_VERSION_PROBE: &str = "claude --version 2>/dev/null | head -c 80";
|
||||
|
||||
/// Read the tool gate's denials, its shadow record and the inert marker out
|
||||
/// of the guest, in one exec. The marker is a line count of "gave up"
|
||||
/// events; the other two are the gate's own JSONL. A missing file is an
|
||||
/// empty section, not an error.
|
||||
///
|
||||
/// The two JSONL sections are separated by [`WOULD_MARKER`] rather than by
|
||||
/// shape: both are JSON objects with the same keys, and telling them apart
|
||||
/// by content would mean a refusal and a call that RAN could be confused —
|
||||
/// which is the one distinction shadow mode exists to make.
|
||||
fn tool_gate_probe(dir: &str) -> String {
|
||||
format!(
|
||||
"echo INERT=$(wc -l < {dir}/{inert} 2>/dev/null || echo 0); cat {dir}/{denied} 2>/dev/null; echo {marker}; cat {dir}/{would} 2>/dev/null",
|
||||
inert = crate::vm_tool_gate::INERT_FILE,
|
||||
denied = crate::vm_tool_gate::DENIED_FILE,
|
||||
would = crate::vm_tool_gate::WOULD_DENY_FILE,
|
||||
marker = WOULD_MARKER,
|
||||
)
|
||||
}
|
||||
|
||||
const WOULD_MARKER: &str = "--CM-WOULD-DENY--";
|
||||
|
||||
/// Parse [`tool_gate_probe`]'s output.
|
||||
fn parse_tool_gate_probe(out: &str) -> ToolGateOutcome {
|
||||
let mut o = ToolGateOutcome::default();
|
||||
let mut shadow = false;
|
||||
for line in out.lines() {
|
||||
let line = line.trim();
|
||||
if let Some(n) = line.strip_prefix("INERT=") {
|
||||
o.inert = n.trim().parse().unwrap_or(0);
|
||||
} else if line == WOULD_MARKER {
|
||||
shadow = true;
|
||||
} else if !line.is_empty() {
|
||||
if shadow {
|
||||
o.would_deny.push(line.to_string());
|
||||
} else {
|
||||
o.denied.push(line.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
o
|
||||
}
|
||||
|
||||
/// How many times the stop gate refused to let the agent finish.
|
||||
const BLOCKS_PROBE: &str = "cat /root/gate/blocks 2>/dev/null || echo 0";
|
||||
|
||||
@@ -350,6 +405,23 @@ fn agent_command(prompt: &str, settings: Option<&str>) -> String {
|
||||
)
|
||||
}
|
||||
|
||||
/// Re-assert the base URL inside the command, after the login shell's profile.
|
||||
///
|
||||
/// fcagent passes the turn's env into the process, but the command runs in a
|
||||
/// login shell that then sources `/etc/profile.d/00-image-env.sh` — written at
|
||||
/// rootfs build time from the image's `ENV` — and the glm and kimi images bake
|
||||
/// `ANTHROPIC_BASE_URL` there. MEASURED on the first relayed kimi mission
|
||||
/// (01a0cfc3): the node relay was bound, yet the guest dialled `api.kimi.com`
|
||||
/// through egress with the proxy token, because the profile overwrote the relay
|
||||
/// URL. The claude image bakes none, which is why that backend worked. The URL
|
||||
/// is not a secret, so it goes in the command; credentials stay env-only.
|
||||
fn with_base_url_override(cmd: String, base_url: Option<&str>) -> String {
|
||||
match base_url {
|
||||
Some(url) => format!("export ANTHROPIC_BASE_URL={} && {cmd}", shell_quote(url)),
|
||||
None => cmd,
|
||||
}
|
||||
}
|
||||
|
||||
/// How often the tap is drained while a turn runs.
|
||||
///
|
||||
/// A turn can last an hour; the World is meant to show what is happening now.
|
||||
@@ -400,6 +472,35 @@ pub struct VmOutcome {
|
||||
/// field cannot tell them apart — the unmatched-frame log and the install
|
||||
/// error are what separate them.
|
||||
pub tools: Vec<crate::vm_tool_tap::Observed>,
|
||||
/// The rootfs the node reported booting (`vm_create` reply), e.g.
|
||||
/// `/opt/clawmates-fc/rootfs-glm.ext4`. `None` if the reply carried none.
|
||||
pub rootfs: Option<String>,
|
||||
/// The guest's own `claude --version`, e.g. `2.1.276 (Claude Code)`.
|
||||
/// `None` if the probe failed — which is a fact worth seeing, not a zero.
|
||||
pub cli_version: Option<String>,
|
||||
/// What the PreToolUse gate refused (`denied.jsonl` lines) and whether it
|
||||
/// ever went inert. `None` when no tool gate was installed. The guest
|
||||
/// wrote these files from the first day the gate existed; nothing read
|
||||
/// them out of a VM until 2026-09-18, so a denial — or a gate that had
|
||||
/// silently given up parsing — left no trace in the mission.
|
||||
pub tool_gate: Option<ToolGateOutcome>,
|
||||
/// Hosts named in content the agent fetched — the tap's taint file
|
||||
/// ([`crate::vm_tool_tap::TAINT_FILE`]). Empty when no tap was installed or
|
||||
/// nothing was fetched. Observed only: no rule reads it yet.
|
||||
pub taint_hosts: Vec<String>,
|
||||
}
|
||||
|
||||
/// The tool gate's own record of a phase.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
|
||||
pub struct ToolGateOutcome {
|
||||
/// Raw `denied.jsonl` lines — each one a call the gate refused.
|
||||
pub denied: Vec<String>,
|
||||
/// Calls a task policy WOULD have refused, had it been enforcing. These
|
||||
/// ran. Kept apart from `denied` because the difference between "was
|
||||
/// refused" and "would have been refused" is the whole of shadow mode.
|
||||
pub would_deny: Vec<String>,
|
||||
/// Times the gate could not parse its input and allowed the call anyway.
|
||||
pub inert: u32,
|
||||
}
|
||||
|
||||
/// Boot a VM, run the phase in it, collect the result, and destroy it.
|
||||
@@ -409,10 +510,18 @@ pub struct VmOutcome {
|
||||
pub async fn run_phase_in_vm(hub: &NodeHub, p: VmPhase<'_>) -> Result<VmOutcome, String> {
|
||||
// Resolved BEFORE the VM boots: a missing subscription token must fail the
|
||||
// phase, not boot a VM whose agent will sit there unauthenticated.
|
||||
let env = crate::mission_runtime::microvm_provider_env(p.backend)?;
|
||||
//
|
||||
// With a relay, the guest holds the mission's proxy token instead of the
|
||||
// provider key, and its model calls go loopback → vsock → node → server
|
||||
// proxy, which adds the credential. See `llm_proxy`.
|
||||
let env = match (&p.model_relay, crate::llm_proxy::token_for(p.mission_id)) {
|
||||
(Some(_), Some(token)) => crate::mission_runtime::microvm_proxied_env(p.backend, &token)?,
|
||||
_ => crate::mission_runtime::microvm_provider_env(p.backend)?,
|
||||
};
|
||||
let relay = p.model_relay.as_deref().filter(|_| crate::llm_proxy::token_for(p.mission_id).is_some());
|
||||
|
||||
let vm = MicroVm::new(hub, p.node_id, vm_id_for(p.phase_id, p.iteration, p.step));
|
||||
let created = vm.create(VCPUS, MEM_MIB, p.backend).await?;
|
||||
let created = vm.create(VCPUS, MEM_MIB, p.backend, relay).await?;
|
||||
|
||||
// From here on every early return must still destroy the VM, so the work is
|
||||
// one call whose result is held while teardown runs unconditionally.
|
||||
@@ -427,6 +536,7 @@ pub async fn run_phase_in_vm(hub: &NodeHub, p: VmPhase<'_>) -> Result<VmOutcome,
|
||||
p.gate,
|
||||
p.run_id,
|
||||
p.tap_sink.as_ref(),
|
||||
p.task_policy,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -457,6 +567,10 @@ pub struct VmPhase<'a> {
|
||||
pub task: &'a str,
|
||||
/// `missions.backend` — which rootfs image. `None` boots the node's default.
|
||||
pub backend: Option<&'a str>,
|
||||
/// The server's LLM proxy as the node should reach it, when this phase's
|
||||
/// model calls are relayed (`llm_proxy::microvm_relay`). `None` keeps the
|
||||
/// provider key in the guest.
|
||||
pub model_relay: Option<String>,
|
||||
/// The host checkout, injected as a tar and collected back over the same
|
||||
/// path so `mission_delivery` needs no change.
|
||||
pub repo: &'a std::path::Path,
|
||||
@@ -477,6 +591,10 @@ pub struct VmPhase<'a> {
|
||||
/// Claude Code `Stop` hook inside the guest. `None` leaves the turn exactly
|
||||
/// as it was.
|
||||
pub gate: Option<&'a crate::vm_stop_gate::StopGate>,
|
||||
/// The tools this phase's agents may use at all, installed with the
|
||||
/// gate. `None` keeps the gate exactly as it was — no task policy, the
|
||||
/// floor and the role policies only.
|
||||
pub task_policy: Option<&'a crate::vm_tool_gate::TaskPolicy>,
|
||||
/// Which node of a composed graph this VM is running, if any. `None` is the
|
||||
/// solo path, where the phase is one VM and the id needs no further
|
||||
/// qualification. Part of the vm id, so the nodes of one phase-iteration
|
||||
@@ -545,6 +663,8 @@ async fn run_inside(
|
||||
// `VmPhase::tap_sink` — `Some` means the sink owns recording and the
|
||||
// returned `tools` is empty.
|
||||
tap_sink: Option<&tokio::sync::mpsc::UnboundedSender<Vec<crate::vm_tool_tap::Observed>>>,
|
||||
// The tools this phase may use at all, baked into the gate at install.
|
||||
task_policy: Option<&crate::vm_tool_gate::TaskPolicy>,
|
||||
) -> Result<VmOutcome, String> {
|
||||
// An agent CLI cannot reach its API without the tunnel, and a turn without
|
||||
// egress does not fail — it hangs, or reports a network error the operator
|
||||
@@ -618,6 +738,16 @@ async fn run_inside(
|
||||
.await
|
||||
.map(|p| p.stdout.contains("SETTINGS-OK"))
|
||||
.unwrap_or(false);
|
||||
let cli_version = vm
|
||||
.exec(CLI_VERSION_PROBE, None, 60, &[])
|
||||
.await
|
||||
.ok()
|
||||
.map(|p| p.stdout.trim().to_string())
|
||||
.filter(|v| !v.is_empty());
|
||||
let rootfs = created
|
||||
.get("rootfs")
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::to_string);
|
||||
let gate_dir = match gate {
|
||||
None => None,
|
||||
Some(g) => {
|
||||
@@ -691,10 +821,40 @@ async fn run_inside(
|
||||
// ONE settings document, written once, carrying whichever hooks installed.
|
||||
// Two writers here is the silent clobber `guest_settings` exists to stop:
|
||||
// whichever ran second would erase the other's hook with no error at all.
|
||||
let settings = match (gate_dir, tap_dir) {
|
||||
(None, None) => None,
|
||||
(g, t) => {
|
||||
let doc = crate::vm_tool_tap::guest_settings(g, t);
|
||||
// The PRE-execution gate. Installed on the same terms as the tap: a
|
||||
// failure here degrades to no gate rather than failing the phase, because
|
||||
// a mission that runs ungated is what we have today and a mission that
|
||||
// refuses to run is a regression.
|
||||
let tool_gate_dir = match settings_supported {
|
||||
false => None,
|
||||
true => match vm
|
||||
.exec(
|
||||
&crate::vm_tool_gate::install_command_with(
|
||||
crate::vm_tool_gate::GUEST_DIR,
|
||||
task_policy,
|
||||
),
|
||||
None,
|
||||
60,
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(o) if o.rc == 0 => Some(crate::vm_tool_gate::GUEST_DIR),
|
||||
other => {
|
||||
eprintln!(
|
||||
"microvm_executor: could not install the tool gate on {} ({other:?}) \
|
||||
— this phase's tool calls will run unchecked",
|
||||
vm.vm_id()
|
||||
);
|
||||
None
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
let settings = match (gate_dir, tap_dir, tool_gate_dir) {
|
||||
(None, None, None) => None,
|
||||
(g, t, tg) => {
|
||||
let doc = crate::vm_tool_tap::guest_settings(g, t, tg);
|
||||
let cmd = crate::vm_tool_tap::settings_command(
|
||||
crate::vm_tool_tap::SETTINGS_PATH,
|
||||
&doc,
|
||||
@@ -719,7 +879,10 @@ async fn run_inside(
|
||||
// Concurrency here is safe because `fcagent` is thread-per-connection: the
|
||||
// live log tail already relies on exactly that, on a second connection, for
|
||||
// the whole length of a turn. So this needs no fleet-node change.
|
||||
let turn_cmd = agent_command(&prompt, settings.as_deref());
|
||||
let turn_cmd = with_base_url_override(
|
||||
agent_command(&prompt, settings.as_deref()),
|
||||
env.iter().find(|(k, _)| k == "ANTHROPIC_BASE_URL").map(|(_, v)| v.as_str()),
|
||||
);
|
||||
let turn = vm.exec_attributed(
|
||||
&turn_cmd,
|
||||
None,
|
||||
@@ -819,6 +982,18 @@ async fn run_inside(
|
||||
}
|
||||
None => tools,
|
||||
};
|
||||
// The taint file, from the same directory and for the same reason: /root
|
||||
// goes with the VM.
|
||||
let taint_hosts = match tap_dir {
|
||||
None => Vec::new(),
|
||||
Some(dir) => match vm.exec(&crate::vm_tool_tap::taint_probe(dir), None, 60, &[]).await {
|
||||
Ok(o) => crate::vm_tool_tap::parse_taint(&o.stdout),
|
||||
Err(e) => {
|
||||
eprintln!("microvm_executor: taint probe failed on {}: {e}", vm.vm_id());
|
||||
Vec::new()
|
||||
}
|
||||
},
|
||||
};
|
||||
if tap_dir.is_some() && tools.is_empty() {
|
||||
// A tap that installed and drained nothing is the silent case: the
|
||||
// phase looks the same as it did before the tap existed. Say so, or the
|
||||
@@ -841,6 +1016,27 @@ async fn run_inside(
|
||||
None
|
||||
}
|
||||
};
|
||||
// The tool gate's confession, while /root is still there to read.
|
||||
let tool_gate = match tool_gate_dir {
|
||||
None => None,
|
||||
Some(dir) => match vm.exec(&tool_gate_probe(dir), None, 60, &[]).await {
|
||||
Ok(p) => Some(parse_tool_gate_probe(&p.stdout)),
|
||||
Err(e) => {
|
||||
eprintln!("microvm_executor: tool gate probe failed on {}: {e}", vm.vm_id());
|
||||
None
|
||||
}
|
||||
},
|
||||
};
|
||||
if let Some(g) = &tool_gate {
|
||||
if g.inert > 0 {
|
||||
eprintln!(
|
||||
"microvm_executor: the tool gate in {} went INERT {} time(s) — those calls were \
|
||||
allowed unchecked",
|
||||
vm.vm_id(),
|
||||
g.inert
|
||||
);
|
||||
}
|
||||
}
|
||||
// Teammates, when a team was asked for. A team mission that formed no team is
|
||||
// silently solo otherwise — it would still deliver, still look fine, and the
|
||||
// only difference from a solo run would be the tokens it did not spend.
|
||||
@@ -927,6 +1123,10 @@ async fn run_inside(
|
||||
stop_blocks,
|
||||
released_at_cap,
|
||||
tools,
|
||||
rootfs,
|
||||
cli_version,
|
||||
tool_gate,
|
||||
taint_hosts,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -940,6 +1140,17 @@ mod tests {
|
||||
/// live, and the command's own stdout is what becomes `VmOutcome::summary`.
|
||||
/// A redirect would give a live view and an empty summary — which is the
|
||||
/// same "green and empty" shape this codebase keeps finding.
|
||||
/// The relay URL must win over the image's profile, which the login shell
|
||||
/// sources after fcagent sets the env; and a non-relayed turn is untouched.
|
||||
#[test]
|
||||
fn a_relayed_turn_re_exports_the_base_url_after_the_profile() {
|
||||
let base = agent_command("t", None);
|
||||
let cmd = with_base_url_override(base.clone(), Some("http://127.0.0.1:11434/kimi"));
|
||||
assert!(cmd.starts_with("export ANTHROPIC_BASE_URL='http://127.0.0.1:11434/kimi' && "), "{cmd}");
|
||||
assert!(cmd.ends_with(&base));
|
||||
assert_eq!(with_base_url_override(base.clone(), None), base);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_turn_is_teed_so_it_streams_and_still_reports() {
|
||||
let cmd = agent_command("do the thing", None);
|
||||
@@ -1063,12 +1274,71 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// The gate's files come out of the guest as one exec: an INERT= count line
|
||||
/// and the raw denied.jsonl. A missing file is an empty section.
|
||||
#[test]
|
||||
fn the_tool_gate_probe_parses_denials_and_the_inert_count() {
|
||||
let out = "INERT=2\n{\"tool\":\"Bash\",\"reason\":\"exfil\"}\n\n{\"tool\":\"Bash\",\"reason\":\"rm -rf\"}\n";
|
||||
let g = parse_tool_gate_probe(out);
|
||||
assert_eq!(g.inert, 2);
|
||||
assert_eq!(g.denied.len(), 2);
|
||||
assert!(g.would_deny.is_empty(), "no marker, no shadow section");
|
||||
assert!(g.denied[1].contains("rm -rf"));
|
||||
let none = parse_tool_gate_probe("INERT=0\n");
|
||||
assert_eq!(none, ToolGateOutcome::default());
|
||||
// The probe reads both files from the gate's own directory.
|
||||
let cmd = tool_gate_probe(crate::vm_tool_gate::GUEST_DIR);
|
||||
assert!(cmd.contains("/root/toolgate/inert") && cmd.contains("/root/toolgate/denied.jsonl"), "{cmd}");
|
||||
assert!(cmd.contains("/root/toolgate/would-deny.jsonl"), "{cmd}");
|
||||
}
|
||||
|
||||
/// A refusal and a call that merely WOULD have been refused must never
|
||||
/// be read as each other: both are JSON objects with the same keys, so
|
||||
/// the marker is what separates them.
|
||||
#[test]
|
||||
fn the_probe_keeps_refusals_apart_from_the_shadow_record() {
|
||||
let out = format!(
|
||||
"INERT=1\n{denied}\n{WOULD_MARKER}\n{would}\n{would}",
|
||||
denied = r#"{"rule":"force-push"}"#,
|
||||
would = r#"{"rule":"task-permission"}"#,
|
||||
);
|
||||
let g = parse_tool_gate_probe(&out);
|
||||
assert_eq!(g.inert, 1);
|
||||
assert_eq!(g.denied.len(), 1, "{g:?}");
|
||||
assert!(g.denied[0].contains("force-push"));
|
||||
assert_eq!(g.would_deny.len(), 2, "{g:?}");
|
||||
assert!(g.would_deny.iter().all(|l| l.contains("task-permission")));
|
||||
}
|
||||
|
||||
/// The CLI's `--agents` schema takes `tools` as an array. 2.1.276 refused
|
||||
/// the string form with "verifier.tools: Invalid input" and exited before
|
||||
/// a single API call; every earlier CLI ignored the definition silently.
|
||||
#[test]
|
||||
fn every_tools_field_is_an_array_the_cli_accepts() {
|
||||
for (name, d) in agent_definitions().as_object().expect("object") {
|
||||
let tools = d.get("tools").unwrap_or_else(|| panic!("{name} has no tools"));
|
||||
assert!(tools.is_array(), "{name}.tools must be a JSON array, got {tools}");
|
||||
assert!(
|
||||
tools.as_array().unwrap().iter().all(|t| t.is_string()),
|
||||
"{name}.tools entries must be strings"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// A verifier that can edit will fix what it was asked to check and then
|
||||
/// report success, and the report is about a tree nobody reviewed.
|
||||
#[test]
|
||||
fn the_verifier_cannot_modify_what_it_checks() {
|
||||
let defs = agent_definitions();
|
||||
let tools = defs["verifier"]["tools"].as_str().expect("a tools allowlist");
|
||||
// An ARRAY, as `--agents` requires. `as_str` here would have kept
|
||||
// passing on the string form the CLI was silently discarding.
|
||||
let tools: Vec<&str> = defs["verifier"]["tools"]
|
||||
.as_array()
|
||||
.expect("a tools allowlist, as a JSON array")
|
||||
.iter()
|
||||
.map(|t| t.as_str().expect("tool names are strings"))
|
||||
.collect();
|
||||
let tools = tools.join(", ");
|
||||
for forbidden in ["Edit", "Write"] {
|
||||
assert!(
|
||||
!tools.contains(forbidden),
|
||||
|
||||
@@ -191,6 +191,13 @@ impl<V: PhaseVm> TurnExecutor for MicroVmTurnExecutor<V> {
|
||||
let outcome = self
|
||||
.vms
|
||||
.run(VmPhase {
|
||||
model_relay: crate::llm_proxy::microvm_relay(
|
||||
&self.pool,
|
||||
fleet_node.as_uuid(),
|
||||
backend.as_deref(),
|
||||
)
|
||||
.await,
|
||||
task_policy: None,
|
||||
// Every node of a composed graph streams to the same outer run,
|
||||
// which is the one the operator is watching.
|
||||
run_id: Some(self.run_id),
|
||||
@@ -233,6 +240,12 @@ impl<V: PhaseVm> TurnExecutor for MicroVmTurnExecutor<V> {
|
||||
self.phase_id,
|
||||
self.run_id,
|
||||
&outcome.tools,
|
||||
// No turn agents supplied, so nothing is attributed — the same
|
||||
// `agent_id: None` this path has always written. Resolving the
|
||||
// graph node to an agent uuid is the fix, and it cannot be tested
|
||||
// while the fleet is offline; guessing at it here would put one
|
||||
// node's actions on another node's record.
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -289,6 +302,7 @@ impl<V: PhaseVm> TurnExecutor for MicroVmTurnExecutor<V> {
|
||||
// the honest value for "not measured on this path".
|
||||
tokens: 0,
|
||||
gated: Vec::new(),
|
||||
spend: Default::default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -422,6 +436,10 @@ mod tests {
|
||||
stop_blocks: None,
|
||||
released_at_cap: None,
|
||||
tools: Vec::new(),
|
||||
rootfs: None,
|
||||
cli_version: None,
|
||||
tool_gate: None,
|
||||
taint_hosts: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -617,6 +635,10 @@ mod tests {
|
||||
stop_blocks: None,
|
||||
released_at_cap: None,
|
||||
tools: Vec::new(),
|
||||
rootfs: None,
|
||||
cli_version: None,
|
||||
tool_gate: None,
|
||||
taint_hosts: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -649,6 +671,10 @@ mod tests {
|
||||
stop_blocks: Some(crate::vm_stop_gate::MAX_BLOCKS),
|
||||
released_at_cap: Some(true),
|
||||
tools: Vec::new(),
|
||||
rootfs: None,
|
||||
cli_version: None,
|
||||
tool_gate: None,
|
||||
taint_hosts: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -676,6 +702,10 @@ mod tests {
|
||||
stop_blocks: Some(crate::vm_stop_gate::MAX_BLOCKS),
|
||||
released_at_cap: Some(false),
|
||||
tools: Vec::new(),
|
||||
rootfs: None,
|
||||
cli_version: None,
|
||||
tool_gate: None,
|
||||
taint_hosts: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -315,6 +315,20 @@ pub async fn capture_phase_diff_at(
|
||||
// `--name-status` line means (renames are three fields; the NEW path is the
|
||||
// one that changed).
|
||||
let all_paths = crate::auto_merge::changed_paths(&name_status);
|
||||
// Server credentials in the outgoing work: named here, redacted from the
|
||||
// stored patch (it is served to the UI), and the push refused below.
|
||||
let secrets = crate::delivery_secrets::from_env();
|
||||
let mut leaked = crate::delivery_secrets::leaks_in(&patch, &secrets);
|
||||
let patch = if leaked.is_empty() {
|
||||
patch
|
||||
} else {
|
||||
eprintln!(
|
||||
"mission_delivery: mission {mission_id} phase {phase_id} — the diff contains {} \
|
||||
— redacting the stored patch and refusing to push",
|
||||
leaked.join(", ")
|
||||
);
|
||||
crate::delivery_secrets::redact(&patch, &secrets)
|
||||
};
|
||||
let files_truncated = all_paths.len() > MAX_CAPTURED_PATHS;
|
||||
let files: Vec<(char, String)> = all_paths.into_iter().take(MAX_CAPTURED_PATHS).collect();
|
||||
let empty = patch.trim().is_empty();
|
||||
@@ -380,6 +394,9 @@ pub async fn capture_phase_diff_at(
|
||||
// [`untrusted_empty_reason`].
|
||||
let mut outcome: Option<TestOutcome> = None;
|
||||
let mut published: Option<Publish> = None;
|
||||
// Set only for a recipe whose output accrues into its repository; `None`
|
||||
// means "not that kind of mission", which is different from "refused".
|
||||
let mut merged: Option<crate::auto_merge::MergeOutcome> = None;
|
||||
let mut publish_error: Option<String> = untrusted_empty_reason(empty, diff_error.as_deref());
|
||||
if let Some(c) = committed.as_ref() {
|
||||
if !empty {
|
||||
@@ -420,11 +437,38 @@ pub async fn capture_phase_diff_at(
|
||||
}
|
||||
outcome = Some(o);
|
||||
}
|
||||
match push_url_for(pool, mission_id).await {
|
||||
// Commit messages leave with the branch too.
|
||||
if let Ok(log) = git(&repo, &["log", "--format=%B", &format!("{base_sha}..HEAD")]).await {
|
||||
leaked.extend(crate::delivery_secrets::leaks_in(&log, &secrets));
|
||||
leaked.sort();
|
||||
leaked.dedup();
|
||||
}
|
||||
let url_or_refusal = if leaked.is_empty() {
|
||||
push_url_for(pool, mission_id).await
|
||||
} else {
|
||||
crate::mission_events::record(
|
||||
pool,
|
||||
crate::mission_events::MissionEvent::new(mission_id, "delivery.secret_blocked")
|
||||
.phase(phase_id)
|
||||
.detail(serde_json::json!({ "keys": leaked, "branch": c.branch })),
|
||||
)
|
||||
.await;
|
||||
publish_error = Some(crate::delivery_secrets::refusal(&leaked));
|
||||
Ok(None)
|
||||
};
|
||||
match url_or_refusal {
|
||||
Ok(Some(url)) => {
|
||||
let verified = outcome.as_ref().and_then(TestOutcome::verified);
|
||||
match publish_phase_branch(&repo, &url, &c.branch, gate, verified).await {
|
||||
Ok(p) => published = Some(p),
|
||||
Ok(p) => {
|
||||
if p.pushed {
|
||||
merged = try_accrue_to_default_branch(
|
||||
pool, mission_id, phase_id, &repo, &url, &p.branch,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
published = Some(p);
|
||||
}
|
||||
// `publish_phase_branch` only returns Err for a local
|
||||
// git failure; a rejected push is Ok with an error
|
||||
// inside. Both must reach the artifact.
|
||||
@@ -445,6 +489,8 @@ pub async fn capture_phase_diff_at(
|
||||
}
|
||||
}
|
||||
}
|
||||
// Refused above for a leaked credential; that reason stands.
|
||||
Ok(_) if !leaked.is_empty() => {}
|
||||
Ok(None) => {
|
||||
// Legitimate: a mission with no repo bound has nowhere to
|
||||
// push. Still recorded, because "not pushed" with no reason
|
||||
@@ -484,6 +530,11 @@ pub async fn capture_phase_diff_at(
|
||||
"tests_status": outcome.as_ref().map(TestOutcome::status),
|
||||
"tests_detail": outcome.as_ref().and_then(TestOutcome::detail),
|
||||
"pushed": published.as_ref().map(|p| p.pushed),
|
||||
// Whether the branch was accrued into the repo's default branch, and
|
||||
// why not when it was not. Null for every recipe that is reviewed by
|
||||
// a human, which is all of them but continuous_research.
|
||||
"merged": merged.as_ref().map(|m| m.merged),
|
||||
"merge_reason": merged.as_ref().map(|m| m.reason.clone()),
|
||||
"push_error": published
|
||||
.as_ref()
|
||||
.and_then(|p| p.error.clone())
|
||||
@@ -698,7 +749,26 @@ pub async fn commit_phase_work(
|
||||
.map(|p| format!(":(exclude){p}"))
|
||||
.collect();
|
||||
add.extend(excludes.iter().map(String::as_str));
|
||||
git(repo, &add).await?;
|
||||
// `git add` EXITS 1 whenever the pathspec walked over a gitignored path,
|
||||
// even though it staged everything else correctly and even though we
|
||||
// excluded that path ourselves. Measured against git 2.x: `-c
|
||||
// advice.addIgnoredFile=false`, `--ignore-errors`, `-A` and `:/` all still
|
||||
// exit 1, and all still stage the right files. There is no flag that makes
|
||||
// this command's exit code mean "nothing was staged".
|
||||
//
|
||||
// Propagating it with `?` therefore aborted the commit AFTER a successful
|
||||
// staging, so the branch was never made, nothing was committed and nothing
|
||||
// was pushed — for any repo with a populated `target/`, which is every Rust
|
||||
// repo an agent has built in. `capture_phase_diff_at` above already treats
|
||||
// the same command as advisory (`let _ = ...`); this is the same command
|
||||
// and gets the same policy. The index is the source of truth, and the
|
||||
// `diff --cached` immediately below is what actually reads it.
|
||||
if let Err(e) = git(repo, &add).await {
|
||||
eprintln!(
|
||||
"mission_delivery: `git add` reported {e} — continuing, the staged \
|
||||
index below is what decides whether there is anything to commit"
|
||||
);
|
||||
}
|
||||
|
||||
// `--cached` compares the index against HEAD: empty means the agents left
|
||||
// nothing unstaged for us, which is the normal case when they committed
|
||||
@@ -848,6 +918,113 @@ pub fn discover_test_command(repo: &Path) -> Option<Vec<String>> {
|
||||
/// after clone because agents run as root in a container that mounts the
|
||||
/// checkout. Building it here also means a rotated token takes effect
|
||||
/// immediately instead of at the next clone.
|
||||
/// Merge a delivered branch into the repository's default branch, when the
|
||||
/// mission is one whose output is meant to accrue rather than be reviewed.
|
||||
///
|
||||
/// **Why this exists.** Paper notes reached the vault automatically from the
|
||||
/// day the library shipped (`library.rs` calls `auto_merge::try_merge`); the
|
||||
/// DIGEST that analyses them did not, because nothing on the mission path
|
||||
/// ever called it. A mission branch waited for an operator merge
|
||||
/// (`routes::missions::merge_branch`) that, measured on 2026-09-22, had not
|
||||
/// happened since 2026-08-18: every `continuous_research` run in that month
|
||||
/// produced `analysis.md`, `script.md` and `episode.json` onto a branch
|
||||
/// nobody merged. The pipeline was half-continuous — papers flowed, the
|
||||
/// thinking about them did not.
|
||||
///
|
||||
/// **Why it is safe to do automatically here.** Three independent limits,
|
||||
/// none of which trusts the mission type on its own:
|
||||
/// * `MergePolicy::AdditiveOnly` — `try_merge` re-reads the diff against
|
||||
/// the REMOTE base and refuses on any modify, delete or rename. A digest
|
||||
/// writes a new `ContinuousResearch/<date>/` folder, so it is all adds;
|
||||
/// the day that stops being true the merge stops, loudly.
|
||||
/// * `verified` — the phase's own judge said `met`. Read from
|
||||
/// `mission_phase_evaluations` rather than inferred from the phase's
|
||||
/// status, the same way `library.rs` measures `healthy() && shelved`
|
||||
/// instead of assuming a clean run.
|
||||
/// * the recipe — only `continuous_research`, whose whole point is an
|
||||
/// unattended loop into the operator's own vault. Every other recipe's
|
||||
/// branch is left exactly as it was, for a human.
|
||||
///
|
||||
/// Returns `None` when this mission is not one of those; `Some` otherwise,
|
||||
/// including when the merge was refused, because a refusal is the interesting
|
||||
/// half and belongs in the artifact beside the branch.
|
||||
/// Which recipes deliver into a repository that ACCRUES rather than one a
|
||||
/// human reviews. Pure, so the list is readable and testable without a
|
||||
/// database — and so adding one is a deliberate edit here rather than a
|
||||
/// condition buried in a query.
|
||||
///
|
||||
/// Only `continuous_research`: its output is a dated folder in the
|
||||
/// operator's own vault, produced on a schedule, and a human merge gate in
|
||||
/// front of it means the digest is never read (measured: a month of them).
|
||||
/// Every other recipe writes into a code repository where a review gate is
|
||||
/// the point.
|
||||
fn accrues_automatically(template_kind: &str) -> bool {
|
||||
template_kind == crate::continuous_research::TEMPLATE_KIND
|
||||
}
|
||||
|
||||
async fn try_accrue_to_default_branch(
|
||||
pool: &sqlx::PgPool,
|
||||
mission_id: Uuid,
|
||||
phase_id: Uuid,
|
||||
repo: &std::path::Path,
|
||||
push_url: &str,
|
||||
branch: &str,
|
||||
) -> Option<crate::auto_merge::MergeOutcome> {
|
||||
let row: (String, Option<String>) = sqlx::query_as::<_, (String, Option<String>)>(
|
||||
"SELECT m.template_kind, r.default_branch
|
||||
FROM missions m LEFT JOIN repos r ON r.id = m.repo_id
|
||||
WHERE m.id = $1",
|
||||
)
|
||||
.bind(mission_id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(|e| eprintln!("mission_delivery: accrue lookup for {mission_id} failed: {e}"))
|
||||
.ok()
|
||||
.flatten()?;
|
||||
let (template_kind, default_branch) = row;
|
||||
if !accrues_automatically(&template_kind) {
|
||||
return None;
|
||||
}
|
||||
|
||||
// The judge's own verdict for THIS phase, not the phase status. A phase
|
||||
// with no completion condition completes without ever being judged, and
|
||||
// merging that into a knowledge base on the strength of "it finished"
|
||||
// is the kind of inference this codebase keeps paying for.
|
||||
let met: Option<bool> = sqlx::query_scalar(
|
||||
"SELECT met FROM mission_phase_evaluations
|
||||
WHERE phase_id = $1 ORDER BY iteration DESC LIMIT 1",
|
||||
)
|
||||
.bind(phase_id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.unwrap_or(None);
|
||||
let verified = met == Some(true);
|
||||
|
||||
let base = default_branch.unwrap_or_else(|| "main".to_string());
|
||||
let outcome = crate::auto_merge::try_merge(
|
||||
repo,
|
||||
push_url,
|
||||
branch,
|
||||
&base,
|
||||
crate::auto_merge::MergePolicy::AdditiveOnly,
|
||||
verified,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|e| crate::auto_merge::MergeOutcome {
|
||||
merged: false,
|
||||
reason: format!("merge attempt failed: {e}"),
|
||||
});
|
||||
eprintln!(
|
||||
"mission_delivery: {branch} -> {base} — {} ({})",
|
||||
outcome.reason,
|
||||
match verified {
|
||||
true => "judge met",
|
||||
false => "not judged met",
|
||||
}
|
||||
);
|
||||
Some(outcome)
|
||||
}
|
||||
|
||||
async fn push_url_for(pool: &sqlx::PgPool, mission_id: Uuid) -> Result<Option<String>, String> {
|
||||
let url: Option<String> = sqlx::query_scalar(
|
||||
"SELECT r.clone_url FROM missions m JOIN repos r ON r.id = m.repo_id WHERE m.id = $1",
|
||||
@@ -1010,6 +1187,39 @@ pub struct Publish {
|
||||
/// reason: this codebase has repeatedly found things reporting success while
|
||||
/// doing nothing, and a test suite that never ran must not license a push to a
|
||||
/// mission branch.
|
||||
/// Did the toolchain fail to BUILD the project, as opposed to building it and
|
||||
/// finding failing tests?
|
||||
///
|
||||
/// Deliberately narrow. These three phrases are emitted by cargo/rustc only
|
||||
/// when compilation or linking did not complete; a failing `assert!` produces
|
||||
/// none of them. Anything not matched here stays a red suite, because guessing
|
||||
/// "probably an environment problem" over a genuine test failure is the far
|
||||
/// more expensive mistake — it would let broken code through the gate.
|
||||
fn build_failed(output: &str) -> bool {
|
||||
let o = output.to_ascii_lowercase();
|
||||
o.contains("error: could not compile")
|
||||
|| o.contains("error: linking with")
|
||||
|| o.contains("error: failed to run custom build command")
|
||||
}
|
||||
|
||||
/// The first line that explains a build failure, for the artifact.
|
||||
fn build_failure_excerpt(output: &str) -> String {
|
||||
output
|
||||
.lines()
|
||||
.find(|l| {
|
||||
let l = l.to_ascii_lowercase();
|
||||
l.contains("error: could not compile")
|
||||
|| l.contains("error: linking with")
|
||||
|| l.contains("error: failed to run custom build command")
|
||||
|| l.contains("cannot find -l")
|
||||
|| l.contains("not installed")
|
||||
})
|
||||
.unwrap_or("")
|
||||
.chars()
|
||||
.take(300)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub async fn verify_tests(repo: &Path, container: &str) -> TestOutcome {
|
||||
let Some(argv) = discover_test_command(repo) else {
|
||||
return TestOutcome::NoSuite;
|
||||
@@ -1027,14 +1237,32 @@ pub async fn verify_tests(repo: &Path, container: &str) -> TestOutcome {
|
||||
argv.join(" "),
|
||||
out.exit_code
|
||||
);
|
||||
let text = out.combined();
|
||||
match out.exit_code {
|
||||
Some(0) => TestOutcome::Passed,
|
||||
// A suite that never COMPILED is not a red suite. Both are
|
||||
// non-zero (cargo exits 101 either way), and calling the
|
||||
// difference is what stops a missing toolchain being reported
|
||||
// as the user's code being broken.
|
||||
//
|
||||
// Measured twice on clawhdf5 in one sitting: no `cmake` gave
|
||||
// "is `cmake` not installed?", and no `python3-dev` gave
|
||||
// "cannot find -lpython3.11" — both exit 101, both would have
|
||||
// been recorded as `tests_status: "failed"` on a repo whose
|
||||
// tests were never run. The branch suffix is `-wip` either way,
|
||||
// so nothing ships differently; what changes is that the
|
||||
// artifact now says which of the two happened.
|
||||
Some(_) if build_failed(&text) => TestOutcome::CouldNotRun(format!(
|
||||
"`{}` could not build the project: {}",
|
||||
argv.join(" "),
|
||||
build_failure_excerpt(&text)
|
||||
)),
|
||||
// An unreadable status is not a pass, and it is not a red
|
||||
// suite either — the command may never have started.
|
||||
None => TestOutcome::CouldNotRun(format!(
|
||||
"`{}` produced no exit status: {}",
|
||||
argv.join(" "),
|
||||
out.combined().chars().take(300).collect::<String>()
|
||||
text.chars().take(300).collect::<String>()
|
||||
)),
|
||||
Some(code) => TestOutcome::Failed(code),
|
||||
}
|
||||
@@ -1244,6 +1472,24 @@ mod changed_path_capture_tests {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
/// Exactly one recipe accrues without a human. The others deliver into
|
||||
/// code repositories where the review gate is the point, and a recipe
|
||||
/// added to that list should be an edit somebody reviewed.
|
||||
#[test]
|
||||
fn only_continuous_research_accrues_automatically() {
|
||||
assert!(accrues_automatically(crate::continuous_research::TEMPLATE_KIND));
|
||||
for kind in [
|
||||
"research_and_code",
|
||||
"research_only",
|
||||
"security_hardening",
|
||||
"benchmark",
|
||||
"refactor",
|
||||
"",
|
||||
] {
|
||||
assert!(!accrues_automatically(kind), "{kind} must not auto-merge");
|
||||
}
|
||||
}
|
||||
use super::*;
|
||||
|
||||
/// Git says "your history diverged" several ways, and the one production
|
||||
@@ -1324,6 +1570,103 @@ mod tests {
|
||||
assert_eq!(Gate::OnGreenTests.branch_suffix(None), "-wip");
|
||||
}
|
||||
|
||||
/// The regression that destroyed a mission's work.
|
||||
///
|
||||
/// `git add -- . :(exclude)target` EXITS NON-ZERO when the tree contains a
|
||||
/// gitignored `target/`, while correctly staging everything else. The
|
||||
/// commit path used to propagate that exit with `?`, so a successful
|
||||
/// staging still aborted the commit — no branch, no commit, no push — and
|
||||
/// the agents' work was reaped with the container.
|
||||
///
|
||||
/// This test asserts the git behaviour itself, because the fix is only
|
||||
/// correct for as long as the behaviour holds: if a future git makes this
|
||||
/// command exit 0, this test fails and tells the next reader the workaround
|
||||
/// can go. What must never regress is the second assertion — that the files
|
||||
/// ARE staged regardless of the exit code, which is why the index and not
|
||||
/// the exit code is the source of truth.
|
||||
#[tokio::test]
|
||||
async fn git_add_exits_nonzero_over_an_ignored_path_yet_still_stages() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let repo = dir.path();
|
||||
let run = |args: Vec<&str>| {
|
||||
std::process::Command::new("git")
|
||||
.args(&args)
|
||||
.current_dir(repo)
|
||||
.output()
|
||||
.expect("git runs")
|
||||
};
|
||||
run(vec!["init", "-q", "."]);
|
||||
run(vec!["config", "user.email", "[email protected]"]);
|
||||
run(vec!["config", "user.name", "T"]);
|
||||
std::fs::write(repo.join(".gitignore"), "target\n").unwrap();
|
||||
run(vec!["add", ".gitignore"]);
|
||||
run(vec!["commit", "-qm", "base"]);
|
||||
|
||||
// The shape every Rust repo an agent has built in ends up with.
|
||||
std::fs::create_dir_all(repo.join("target")).unwrap();
|
||||
std::fs::write(repo.join("target/build.bin"), "junk").unwrap();
|
||||
std::fs::create_dir_all(repo.join("research")).unwrap();
|
||||
std::fs::write(repo.join("research/summary.md"), "findings").unwrap();
|
||||
|
||||
let mut add: Vec<&str> = vec!["add", "--", "."];
|
||||
let excludes: Vec<String> = EXCLUDED_PATHS
|
||||
.iter()
|
||||
.map(|p| format!(":(exclude){p}"))
|
||||
.collect();
|
||||
add.extend(excludes.iter().map(String::as_str));
|
||||
let out = run(add);
|
||||
|
||||
assert!(
|
||||
!out.status.success(),
|
||||
"if this now succeeds, git changed and the advisory handling in \
|
||||
commit_and_branch can be simplified"
|
||||
);
|
||||
|
||||
let staged = run(vec!["diff", "--cached", "--name-only"]);
|
||||
let staged = String::from_utf8_lossy(&staged.stdout);
|
||||
assert!(
|
||||
staged.contains("research/summary.md"),
|
||||
"the work MUST be staged despite the non-zero exit; this is the \
|
||||
assertion that stops a phase's output being silently discarded \
|
||||
again. staged: {staged:?}"
|
||||
);
|
||||
assert!(
|
||||
!staged.contains("target/"),
|
||||
"the exclude pathspec must still keep build output out: {staged:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// A missing toolchain must not be reported as the user's tests failing.
|
||||
/// Both are cargo exit 101; only the output distinguishes them, and this
|
||||
/// session produced both real samples on clawhdf5.
|
||||
#[test]
|
||||
fn a_build_failure_is_not_a_red_suite() {
|
||||
let no_cmake = "error: failed to run custom build command for `libz-ng-sys v1.1.29`\n\
|
||||
is `cmake` not installed?";
|
||||
let no_python = "= note: /usr/bin/ld: cannot find -lpython3.11: No such file or directory\n\
|
||||
error: could not compile `clawhdf5-py` (lib) due to 1 previous error";
|
||||
for sample in [no_cmake, no_python] {
|
||||
assert!(build_failed(sample), "must read as a build failure: {sample}");
|
||||
assert!(
|
||||
!build_failure_excerpt(sample).is_empty(),
|
||||
"the artifact needs a reason, not an empty string"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The half that protects the gate: a genuinely failing test must STAY a
|
||||
/// red suite. Mistaking one for an environment problem would let broken
|
||||
/// code past `on_green_tests`, which is the expensive direction to be
|
||||
/// wrong in.
|
||||
#[test]
|
||||
fn a_failing_test_is_still_a_red_suite() {
|
||||
let red = "running 3 tests\n\
|
||||
test math::adds ... FAILED\n\
|
||||
failures:\n math::adds\n\
|
||||
test result: FAILED. 2 passed; 1 failed; 0 ignored";
|
||||
assert!(!build_failed(red), "a failing assertion is not a build failure");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_command_is_discovered_from_the_tree() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -23,6 +23,36 @@ pub const PHASE_COMPLETED: &str = "phase.completed";
|
||||
pub const TOOL_CALL: &str = "tool.call";
|
||||
/// A tool touched a path. `target` is the path, repo-relative where known.
|
||||
pub const FILE_TOUCH: &str = "file.touch";
|
||||
/// The exact prompt text an agent was given. `detail.text` is the full string,
|
||||
/// `target` is the role or tier that composed it.
|
||||
///
|
||||
/// The durable answer to "what did this agent actually receive". Skills, the
|
||||
/// task, the evaluator's feedback and the tool preamble are assembled from four
|
||||
/// places across three tiers, so re-deriving the prompt after the fact means
|
||||
/// re-running that assembly against data that has since changed. Recording it
|
||||
/// is the only way the question stays answerable.
|
||||
pub const PROMPT_COMPOSED: &str = "prompt.composed";
|
||||
/// The agent's own narrative for a turn. `detail.text`.
|
||||
///
|
||||
/// Written by `topology_worker` and pushed live once by `live_bus`. Until the
|
||||
/// reader below existed, the stored row was never read again by anything: both
|
||||
/// database readers in `routes/world.rs` filter to `tool.call`/`file.touch`,
|
||||
/// and the only other statement touching the table is the GC that deletes it.
|
||||
pub const REASONING: &str = "reasoning";
|
||||
|
||||
/// Kinds the per-phase cap applies to.
|
||||
///
|
||||
/// The cap exists to bound the two unbounded kinds: a coding phase can call
|
||||
/// thousands of tools and touch thousands of paths. The others are bounded by
|
||||
/// the phase's own structure — one start, one completion, one prompt per turn —
|
||||
/// and counting them against the same budget meant a busy phase could push out
|
||||
/// its OWN terminal event, leaving a phase that looks like it never finished.
|
||||
const CAPPED_KINDS: &[&str] = &[TOOL_CALL, FILE_TOUCH];
|
||||
|
||||
/// Does this kind count against, and get dropped by, `PER_PHASE_CAP`?
|
||||
pub fn is_capped(kind: &str) -> bool {
|
||||
CAPPED_KINDS.contains(&kind)
|
||||
}
|
||||
|
||||
/// Most events one phase may record.
|
||||
///
|
||||
@@ -84,26 +114,40 @@ impl MissionEvent {
|
||||
/// writers there are. `INSERT … SELECT … WHERE (subquery) < cap` makes the
|
||||
/// decision inside the statement.
|
||||
pub async fn record(pool: &PgPool, e: MissionEvent) {
|
||||
// Never store a server credential. Every event a mission produces passes
|
||||
// here — tool output (a `printenv`), judge verdicts, prompts — and all of
|
||||
// it is served to the UI. See `delivery_secrets`.
|
||||
let detail = if e.detail.is_null() {
|
||||
Value::Object(Default::default())
|
||||
} else {
|
||||
e.detail
|
||||
crate::delivery_secrets::scrub_json(e.detail)
|
||||
};
|
||||
let target = e
|
||||
.target
|
||||
.as_deref()
|
||||
.map(|t| crate::delivery_secrets::scrub(t).into_owned());
|
||||
// The cap is still decided INSIDE the insert (see the test below), and now
|
||||
// only counts the kinds it is meant to bound.
|
||||
let capped = is_capped(&e.kind);
|
||||
let res = sqlx::query(
|
||||
"INSERT INTO mission_events
|
||||
(mission_id, phase_id, run_id, agent_id, kind, target, detail)
|
||||
SELECT $1, $2, $3, $4, $5, $6, $7
|
||||
WHERE $2::uuid IS NULL
|
||||
OR (SELECT count(*) FROM mission_events WHERE phase_id = $2) < $8",
|
||||
OR NOT $9
|
||||
OR (SELECT count(*) FROM mission_events
|
||||
WHERE phase_id = $2 AND kind = ANY($10)) < $8",
|
||||
)
|
||||
.bind(e.mission_id)
|
||||
.bind(e.phase_id)
|
||||
.bind(e.run_id)
|
||||
.bind(e.agent_id)
|
||||
.bind(&e.kind)
|
||||
.bind(&e.target)
|
||||
.bind(&target)
|
||||
.bind(&detail)
|
||||
.bind(PER_PHASE_CAP)
|
||||
.bind(capped)
|
||||
.bind(CAPPED_KINDS)
|
||||
.execute(pool)
|
||||
.await;
|
||||
if let Err(err) = res {
|
||||
@@ -112,6 +156,109 @@ pub async fn record(pool: &PgPool, e: MissionEvent) {
|
||||
}
|
||||
|
||||
/// Record several events under one round trip's worth of intent.
|
||||
/// Every recorded prompt and narrative for a mission, oldest first.
|
||||
///
|
||||
/// The read side of `PROMPT_COMPOSED` / `REASONING`. Both kinds were write-only
|
||||
/// before this: the prompt was never stored at all, and the narrative was
|
||||
/// stored and then read by nothing. Together they answer "what did this agent
|
||||
/// receive, and what did it say it did", which is the question
|
||||
/// `docs/PROVENANCE-ASSESSMENT.md` records as unanswerable.
|
||||
pub async fn narrative_for_mission(
|
||||
pool: &PgPool,
|
||||
mission_id: Uuid,
|
||||
) -> Result<Vec<(String, Option<Uuid>, Option<String>, String)>, sqlx::Error> {
|
||||
let rows: Vec<(String, Option<Uuid>, Option<String>, Value)> = sqlx::query_as(
|
||||
"SELECT kind, agent_id, target, detail
|
||||
FROM mission_events
|
||||
WHERE mission_id = $1 AND kind = ANY($2)
|
||||
ORDER BY id",
|
||||
)
|
||||
.bind(mission_id)
|
||||
.bind(&[PROMPT_COMPOSED, REASONING][..])
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|(kind, agent, target, detail)| {
|
||||
let text = detail
|
||||
.get("text")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
(kind, agent, target, text)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// One action an agent took, as a reader gets it back.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ToolEvidence {
|
||||
/// The tool's name, e.g. `Bash`, `Write`.
|
||||
pub tool: String,
|
||||
/// The absolute path inside the sandbox, when the tool named one.
|
||||
///
|
||||
/// Absolute, unlike the sibling `file.touch` row's `target`. See the note
|
||||
/// in `phase_runner::record_vm_tools`: normalising is what destroys the
|
||||
/// only question a path can settle.
|
||||
pub path: Option<String>,
|
||||
/// The tool's arguments, bounded by `vm_tool_tap::bounded_input`.
|
||||
pub input: Value,
|
||||
/// What a command produced, bounded by `vm_tool_tap::bounded_response`.
|
||||
///
|
||||
/// Null for every tool that is not a command. This is where a failing test
|
||||
/// run is visible, and it is the only place it is — the recorded stream has
|
||||
/// no exit codes.
|
||||
pub response: Value,
|
||||
}
|
||||
|
||||
impl ToolEvidence {
|
||||
/// The shell command, for the tools that run one.
|
||||
pub fn command(&self) -> Option<&str> {
|
||||
self.input.get("command").and_then(Value::as_str)
|
||||
}
|
||||
}
|
||||
|
||||
/// Every tool call recorded for a mission, in order.
|
||||
///
|
||||
/// The counterpart to [`narrative_for_mission`], and the reason it exists: the
|
||||
/// narrative is what an agent *said* it did. These rows are what it did. A
|
||||
/// measurement built on the narrative alone scores prose, and prose is written
|
||||
/// by the thing being measured.
|
||||
///
|
||||
/// **Bounded by [`PER_PHASE_CAP`].** A phase that ran more tools than the cap
|
||||
/// returns the first `PER_PHASE_CAP` and no marker saying so, so a check that
|
||||
/// concludes "this never happened" from an empty result is only sound for
|
||||
/// phases under the cap. Every check in `skill_use` is one-sided in the safe
|
||||
/// direction for that reason: it reports a violation it can see, never
|
||||
/// compliance it inferred from silence.
|
||||
pub async fn tool_evidence_for_mission(
|
||||
pool: &PgPool,
|
||||
mission_id: Uuid,
|
||||
) -> Result<Vec<ToolEvidence>, sqlx::Error> {
|
||||
let rows: Vec<(Option<String>, Value)> = sqlx::query_as(
|
||||
"SELECT target, detail
|
||||
FROM mission_events
|
||||
WHERE mission_id = $1 AND kind = $2
|
||||
ORDER BY id",
|
||||
)
|
||||
.bind(mission_id)
|
||||
.bind(TOOL_CALL)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|(target, detail)| ToolEvidence {
|
||||
tool: target.unwrap_or_default(),
|
||||
path: detail
|
||||
.get("path")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string),
|
||||
input: detail.get("input").cloned().unwrap_or(Value::Null),
|
||||
response: detail.get("response").cloned().unwrap_or(Value::Null),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn record_all(pool: &PgPool, events: Vec<MissionEvent>) {
|
||||
for e in events {
|
||||
record(pool, e).await;
|
||||
|
||||
@@ -128,8 +128,52 @@ pub fn unpack_into(archive: &[u8], dest: &Path) -> Result<(), String> {
|
||||
// Ownership in the archive is the container's root; re-applying it on the
|
||||
// host would recreate the very uid split this module exists to remove.
|
||||
ar.set_preserve_permissions(false);
|
||||
ar.unpack(dest)
|
||||
.map_err(|e| format!("unpack into {}: {e}", dest.display()))
|
||||
|
||||
// Filter on the way OUT as well as on the way in.
|
||||
//
|
||||
// `pack_dir` (host -> container) skips `transport_excludes`, but `copy_out`
|
||||
// (container -> host) is the raw Docker archive API, which carries the whole
|
||||
// tree — `target/` included. The asymmetry was invisible for as long as the
|
||||
// runtime image had no `cmake`, because nothing could compile and no
|
||||
// `target/` existed. The moment missions could build, every collection
|
||||
// failed on a build artifact:
|
||||
//
|
||||
// failed to unpack `…/repo/target/debug/build/ahash-…/build_script_build-…`
|
||||
//
|
||||
// and `phase_runner` correctly refused to capture a stale tree — so a
|
||||
// coding phase that HAD done the work delivered nothing, retrying forever.
|
||||
//
|
||||
// Entries are skipped by NAME at any depth, the same rule `is_excluded`
|
||||
// uses, because a workspace has a `target/` per crate.
|
||||
let mut skipped = 0usize;
|
||||
for entry in ar
|
||||
.entries()
|
||||
.map_err(|e| format!("read archive for {}: {e}", dest.display()))?
|
||||
{
|
||||
let mut entry = entry.map_err(|e| format!("read entry for {}: {e}", dest.display()))?;
|
||||
let path = entry
|
||||
.path()
|
||||
.map_err(|e| format!("entry path for {}: {e}", dest.display()))?
|
||||
.into_owned();
|
||||
if path
|
||||
.components()
|
||||
.any(|c| is_excluded(&c.as_os_str().to_string_lossy()))
|
||||
{
|
||||
skipped += 1;
|
||||
continue;
|
||||
}
|
||||
entry
|
||||
.unpack_in(dest)
|
||||
.map_err(|e| format!("unpack into {}: {e}", dest.display()))?;
|
||||
}
|
||||
if skipped > 0 {
|
||||
eprintln!(
|
||||
"mission_fs: unpack into {} skipped {skipped} excluded entr{} (build output)",
|
||||
dest.display(),
|
||||
if skipped == 1 { "y" } else { "ies" }
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Copy a host directory into a running container at [`CONTAINER_MISSION_DIR`].
|
||||
@@ -203,6 +247,51 @@ pub async fn put_file(
|
||||
.map_err(|e| format!("upload {path} to {container}: {e}"))
|
||||
}
|
||||
|
||||
/// Build a flat tar of several files. [`single_file_archive`] for many.
|
||||
fn files_archive(files: &[(String, Vec<u8>)]) -> Result<Vec<u8>, String> {
|
||||
let mut builder = tar::Builder::new(Vec::new());
|
||||
for (name, contents) in files {
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header
|
||||
.set_path(name)
|
||||
.map_err(|e| format!("tar path {name}: {e}"))?;
|
||||
header.set_size(contents.len() as u64);
|
||||
// World-readable, unlike `single_file_archive`'s 0600: that one carries
|
||||
// a credential, this one carries procedures the agent is meant to read.
|
||||
header.set_mode(0o644);
|
||||
header.set_entry_type(tar::EntryType::Regular);
|
||||
header.set_cksum();
|
||||
builder
|
||||
.append(&header, contents.as_slice())
|
||||
.map_err(|e| format!("tar {name}: {e}"))?;
|
||||
}
|
||||
builder
|
||||
.into_inner()
|
||||
.map_err(|e| format!("finish archive of {} files: {e}", files.len()))
|
||||
}
|
||||
|
||||
/// Write several files into one directory of a container, in one upload.
|
||||
///
|
||||
/// `dir` must already exist — `upload_to_container` will not create it, the
|
||||
/// same constraint [`sync_in`] works around. Size-independent for the reason
|
||||
/// [`put_file`] gives; fifty skill bodies would be well past `ARG_MAX` as a
|
||||
/// printf.
|
||||
pub async fn put_files(
|
||||
docker: &Docker,
|
||||
container: &str,
|
||||
dir: &str,
|
||||
files: &[(String, Vec<u8>)],
|
||||
) -> Result<(), String> {
|
||||
let archive = files_archive(files)?;
|
||||
let opts = bollard::query_parameters::UploadToContainerOptionsBuilder::default()
|
||||
.path(dir)
|
||||
.build();
|
||||
docker
|
||||
.upload_to_container(container, Some(opts), bollard::body_full(archive.into()))
|
||||
.await
|
||||
.map_err(|e| format!("upload {} files to {container}:{dir}: {e}", files.len()))
|
||||
}
|
||||
|
||||
/// Copy a directory back out of a container onto the host.
|
||||
pub async fn copy_out(
|
||||
docker: &Docker,
|
||||
@@ -249,14 +338,52 @@ fn host_repo(mission_id: uuid::Uuid) -> std::path::PathBuf {
|
||||
|
||||
/// Push the host checkout into the container before a phase runs.
|
||||
///
|
||||
/// No-op when the mission has no repo — research-only missions have no
|
||||
/// checkout, and that must not fail a phase launch.
|
||||
/// A repo-less mission has no checkout to push, but it still needs
|
||||
/// `/mission/repo` to EXIST inside the container: the phase prompt tells the
|
||||
/// agent that is its working directory, `mission_orchestrator` pins every
|
||||
/// claw's `workspace.path` to it, and `mission_outputs` copies it back out to
|
||||
/// register artifacts. This used to return early instead, so none of those three
|
||||
/// were true — the pin resolved to nothing, ZeroClaw fell back to each agent's
|
||||
/// own sandbox, and the agents (correctly) reported they had no such directory
|
||||
/// and refused to work. Creating it empty is what the microVM tier already does,
|
||||
/// for the same reason: see `microvm_executor::inject` ("the guest needs the
|
||||
/// workspace to exist before the agent writes into it").
|
||||
///
|
||||
/// Creating it host-side rather than `mkdir`-ing in the container keeps the copy
|
||||
/// cycle symmetric — `sync_out` unpacks over this same path, so work written by
|
||||
/// one phase survives into the next instead of being wiped by the next
|
||||
/// `sync_in`.
|
||||
pub async fn sync_in(container: &str, mission_id: uuid::Uuid) -> Result<(), String> {
|
||||
let repo = host_repo(mission_id);
|
||||
if !repo.is_dir() {
|
||||
return Ok(());
|
||||
tokio::fs::create_dir_all(&repo)
|
||||
.await
|
||||
.map_err(|e| format!("create empty workspace {}: {e}", repo.display()))?;
|
||||
}
|
||||
let docker = crate::container_exec::connect()?;
|
||||
// `upload_to_container` requires the DESTINATION to exist: uploading into
|
||||
// `/mission` when the container has no `/mission` fails with
|
||||
// "404 Could not find the file /mission in container", which reads like a
|
||||
// missing source file rather than a missing target directory. Nothing else
|
||||
// creates it — not the image, not the container spec (in copy mode there is
|
||||
// no `/mission` bind) — so create it here, immediately before the copy that
|
||||
// depends on it.
|
||||
let mkdir = [
|
||||
"mkdir".to_string(),
|
||||
"-p".to_string(),
|
||||
CONTAINER_MISSION_DIR.to_string(),
|
||||
];
|
||||
if let Err(e) = crate::container_exec::exec_as_root(
|
||||
&docker,
|
||||
container,
|
||||
None,
|
||||
&mkdir,
|
||||
std::time::Duration::from_secs(20),
|
||||
)
|
||||
.await
|
||||
{
|
||||
return Err(format!("create {CONTAINER_MISSION_DIR} in {container}: {e}"));
|
||||
}
|
||||
copy_in(&docker, container, &repo, "repo").await
|
||||
}
|
||||
|
||||
@@ -289,6 +416,44 @@ mod tests {
|
||||
std::fs::write(root.join(".git/HEAD"), "ref: refs/heads/main\n").unwrap();
|
||||
}
|
||||
|
||||
/// Build output must be dropped on the way BACK, not only on the way out.
|
||||
///
|
||||
/// `copy_out` uses the raw Docker archive API, which carries `target/`
|
||||
/// whatever `pack_dir` did. Unpacking it failed on a build-script binary
|
||||
/// and took the whole collection down with it, so a coding phase that had
|
||||
/// really done the work delivered nothing.
|
||||
#[test]
|
||||
fn unpacking_drops_build_output_but_keeps_the_source() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let src = tmp.path().join("repo");
|
||||
std::fs::create_dir_all(src.join("src")).unwrap();
|
||||
std::fs::create_dir_all(src.join("target/debug/build")).unwrap();
|
||||
std::fs::create_dir_all(src.join("crates/inner/target")).unwrap();
|
||||
std::fs::write(src.join("src/lib.rs"), "pub fn x() {}\n").unwrap();
|
||||
std::fs::write(src.join("target/debug/build/script"), "ELF").unwrap();
|
||||
std::fs::write(src.join("crates/inner/target/blob"), "ELF").unwrap();
|
||||
|
||||
// Built WITHOUT the filter, the way the Docker API hands it to us.
|
||||
let mut buf = Vec::new();
|
||||
{
|
||||
let mut b = tar::Builder::new(&mut buf);
|
||||
b.append_dir_all("repo", &src).unwrap();
|
||||
b.finish().unwrap();
|
||||
}
|
||||
|
||||
let dest = tmp.path().join("out");
|
||||
unpack_into(&buf, &dest).expect("must not fail on build output");
|
||||
assert!(dest.join("repo/src/lib.rs").is_file(), "source must survive");
|
||||
assert!(
|
||||
!dest.join("repo/target").exists(),
|
||||
"root target/ must be dropped"
|
||||
);
|
||||
assert!(
|
||||
!dest.join("repo/crates/inner/target").exists(),
|
||||
"a per-crate target/ must be dropped too — matched by NAME at any depth"
|
||||
);
|
||||
}
|
||||
|
||||
/// A checkout must survive the round trip intact — including `.git`,
|
||||
/// without which the whole delivery path (diff, commit, push) is dead.
|
||||
#[test]
|
||||
|
||||
@@ -137,12 +137,21 @@ const EVENT_RETENTION_DAYS: i32 = 7;
|
||||
/// deployment that has been accumulating for months would otherwise take a long
|
||||
/// lock on its first sweep after this ships. The sweep runs on a timer, so a
|
||||
/// large backlog simply drains over several passes.
|
||||
async fn reap_mission_events(pool: &PgPool, out: &mut Reclaimed) {
|
||||
pub async fn reap_mission_events(pool: &PgPool, out: &mut Reclaimed) {
|
||||
let res = sqlx::query(
|
||||
"DELETE FROM mission_events
|
||||
WHERE id IN (
|
||||
SELECT id FROM mission_events
|
||||
WHERE created_at < now() - make_interval(days => $1)
|
||||
SELECT e.id FROM mission_events e
|
||||
JOIN missions m ON m.id = e.mission_id
|
||||
WHERE e.created_at < now() - make_interval(days => $1)
|
||||
-- A mission under measurement or investigation keeps its
|
||||
-- events. Without this the evidence a Skill-Use baseline or a
|
||||
-- provenance question depends on expires while the question
|
||||
-- is still open, and the answer degrades silently into
|
||||
-- \"there are no events\" — which reads identically to
|
||||
-- \"nothing happened\".
|
||||
AND (m.retain_events_until IS NULL
|
||||
OR m.retain_events_until < now())
|
||||
LIMIT 10000
|
||||
)",
|
||||
)
|
||||
|
||||
@@ -0,0 +1,452 @@
|
||||
//! Project memory: what past missions on a repository learned.
|
||||
//!
|
||||
//! Until 2026-09-20 missions wrote no memory at all. The chat path records
|
||||
//! every turn into the claw's `.brain`, but a mission's crew is minted per
|
||||
//! mission (`per-mission-crews`: reuse is OFF by operator decision), so a
|
||||
//! brain keyed by agent would be written once and never read. What persists
|
||||
//! across missions is the repository. So the memory is keyed by `repo_id`:
|
||||
//! one `.brain` per repo, holding the judge's verdicts, recalled by the next
|
||||
//! mission's task text and placed in its brief.
|
||||
//!
|
||||
//! What is remembered is the verdict, not the work: for a met phase the
|
||||
//! judge's `reason` (what it found), for an unmet one its `guidance` — the
|
||||
//! agent-facing half, already stripped of acceptance literals by
|
||||
//! `evaluator::sanitize_guidance`, because a verdict quoted verbatim into
|
||||
//! the next brief is how the 2026-08-01 Goodhart incident happened.
|
||||
//!
|
||||
//! Recall is BM25 over the keyword index (`cm_brain::ClawBrain::recall`);
|
||||
//! there is no embedder. Measured before anything richer is built: the test
|
||||
//! is a second mission on the same repo recalling the first's verdict.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use cm_brain::ClawBrain;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// How many past verdicts a brief carries. Three is enough to say "this was
|
||||
/// tried" without becoming the prompt.
|
||||
pub const RECALL_K: usize = 3;
|
||||
|
||||
/// How many BM25 candidates the reranker sees. Wider than `RECALL_K` so a
|
||||
/// relevant verdict that keyword overlap ranked fourth can still make the
|
||||
/// brief; narrow enough that one call stays one call.
|
||||
pub const CANDIDATES: usize = 8;
|
||||
|
||||
/// Below this a candidate is dropped even if fewer than `RECALL_K` remain:
|
||||
/// a brief that carries an irrelevant verdict is worse than a shorter one.
|
||||
pub const RELEVANT_AT: f64 = 0.3;
|
||||
|
||||
/// The heading the recalled lines go under. Named here because the scorer and
|
||||
/// the prompt-order tests read it back.
|
||||
pub const SECTION_HEADING: &str = "# What past missions on this repository learned";
|
||||
|
||||
fn brain_path(dir: &Path, repo_id: Uuid) -> PathBuf {
|
||||
dir.join(format!("repo_{repo_id}.h5"))
|
||||
}
|
||||
|
||||
/// One line of memory from a verdict. Pure, so the shape is testable without
|
||||
/// a brain file.
|
||||
pub fn verdict_line(
|
||||
mission_id: Uuid,
|
||||
phase_kind: &str,
|
||||
brief: &str,
|
||||
condition: &str,
|
||||
verdict: &crate::evaluator::Verdict,
|
||||
) -> Option<String> {
|
||||
// A judge that could not be reached has not judged; there is no lesson.
|
||||
if verdict.error.is_some() {
|
||||
return None;
|
||||
}
|
||||
let outcome = if verdict.met { "MET" } else { "UNMET" };
|
||||
let finding = if verdict.met {
|
||||
verdict.reason.trim()
|
||||
} else {
|
||||
verdict.guidance.trim()
|
||||
};
|
||||
if finding.is_empty() {
|
||||
return None;
|
||||
}
|
||||
// The TAIL. A UUIDv7 leads with its timestamp, so two missions launched
|
||||
// seconds apart share their first eight characters — measured: two planted
|
||||
// missions 34 s apart both rendered as `01a0cb38`, and the self-audit read
|
||||
// them as one mission failing twice.
|
||||
let short = mission_id.simple().to_string();
|
||||
let short = &short[short.len() - 8..];
|
||||
// The brief is what the agent was TOLD; the condition is what it was
|
||||
// judged against, and working agents are not shown it. Without the brief a
|
||||
// reader of the record cannot tell "the agent skipped a requirement" from
|
||||
// "nobody asked for it" — the first self-audit on a planted brief/condition
|
||||
// mismatch diagnosed the former and proposed a fix that would not have
|
||||
// helped.
|
||||
let brief = brief.trim();
|
||||
let told = if brief.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!(" — brief: {}", head(&brief.split_whitespace().collect::<Vec<_>>().join(" "), 200))
|
||||
};
|
||||
Some(format!(
|
||||
"{outcome} — {phase_kind} phase of mission {short}{told} — condition: {} — judge: {}",
|
||||
head(condition, 200),
|
||||
head(finding, 400),
|
||||
))
|
||||
}
|
||||
|
||||
/// Record a verdict in the repo's brain. Best-effort and loud on failure:
|
||||
/// memory must never fail a phase, and a brain that silently stopped
|
||||
/// recording is the kind of thing that stays broken for a month.
|
||||
pub fn remember_verdict(
|
||||
repo_id: Uuid,
|
||||
mission_id: Uuid,
|
||||
phase_kind: &str,
|
||||
brief: &str,
|
||||
condition: &str,
|
||||
verdict: &crate::evaluator::Verdict,
|
||||
) {
|
||||
remember_in(
|
||||
&cm_runtime::brain::brain_dir(),
|
||||
repo_id,
|
||||
mission_id,
|
||||
phase_kind,
|
||||
brief,
|
||||
condition,
|
||||
verdict,
|
||||
)
|
||||
}
|
||||
|
||||
fn remember_in(
|
||||
dir: &Path,
|
||||
repo_id: Uuid,
|
||||
mission_id: Uuid,
|
||||
phase_kind: &str,
|
||||
brief: &str,
|
||||
condition: &str,
|
||||
verdict: &crate::evaluator::Verdict,
|
||||
) {
|
||||
let Some(line) = verdict_line(mission_id, phase_kind, brief, condition, verdict) else {
|
||||
return;
|
||||
};
|
||||
let path = brain_path(dir, repo_id);
|
||||
if let Some(dir) = path.parent() {
|
||||
let _ = std::fs::create_dir_all(dir);
|
||||
}
|
||||
match ClawBrain::open_or_create(&path, &format!("repo_{repo_id}")) {
|
||||
Ok(mut brain) => {
|
||||
if let Err(e) = brain.remember("judge", &line, &mission_id.to_string()) {
|
||||
eprintln!("mission_memory: could not record verdict for repo {repo_id}: {e}");
|
||||
}
|
||||
}
|
||||
Err(e) => eprintln!("mission_memory: could not open brain for repo {repo_id}: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The past verdicts most relevant to `query` (the phase's task text).
|
||||
/// Empty when the repo has no brain yet, which is every repo's first mission.
|
||||
///
|
||||
/// Two stages when a decision model is configured: BM25 proposes
|
||||
/// `CANDIDATES`, one Noul per candidate — "is this past verdict relevant
|
||||
/// to the task?" — reorders them and drops the ones below `RELEVANT_AT`.
|
||||
/// Keyword overlap is what BM25 measures, and a verdict about MICROVM.md
|
||||
/// shares words with every task that mentions a file; the rerank is the
|
||||
/// vendor's own pattern and costs one ~200 ms call. Without a key the
|
||||
/// BM25 order stands, as before.
|
||||
pub async fn recall(repo_id: Uuid, query: &str) -> Vec<String> {
|
||||
let candidates = recall_in(&cm_runtime::brain::brain_dir(), repo_id, query, CANDIDATES);
|
||||
match cm_decide::jev::Jev::from_env() {
|
||||
Some(jev) if candidates.len() > 1 => rerank(&jev, query, candidates).await,
|
||||
_ => candidates.into_iter().take(RECALL_K).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn rerank(jev: &cm_decide::jev::Jev, query: &str, candidates: Vec<String>) -> Vec<String> {
|
||||
use cm_decide::{Answer, Decider as _, Question};
|
||||
let questions: std::collections::BTreeMap<String, Question> = candidates
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, c)| {
|
||||
(
|
||||
format!("c{i}"),
|
||||
Question::noul(format!(
|
||||
"This earlier judge verdict is relevant to the task and would help an \
|
||||
agent doing it: {c}"
|
||||
)),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let decided = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
jev.decide(query, &questions),
|
||||
)
|
||||
.await;
|
||||
let decision = match decided {
|
||||
Ok(Ok(d)) => d,
|
||||
Ok(Err(e)) => {
|
||||
eprintln!("mission_memory: rerank failed ({e}); keeping the BM25 order");
|
||||
return candidates.into_iter().take(RECALL_K).collect();
|
||||
}
|
||||
Err(_) => {
|
||||
eprintln!("mission_memory: rerank timed out; keeping the BM25 order");
|
||||
return candidates.into_iter().take(RECALL_K).collect();
|
||||
}
|
||||
};
|
||||
let scored: Vec<(String, f64)> = candidates
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(i, c)| {
|
||||
let p = match decision.answers.get(&format!("c{i}")) {
|
||||
Some(Answer::Noul { noul }) => *noul,
|
||||
_ => 0.0,
|
||||
};
|
||||
(c, p)
|
||||
})
|
||||
.collect();
|
||||
let kept: Vec<String> = cm_decide::patterns::rerank(scored)
|
||||
.into_iter()
|
||||
.filter(|(_, p)| *p >= RELEVANT_AT)
|
||||
.take(RECALL_K)
|
||||
.map(|(c, _)| c)
|
||||
.collect();
|
||||
eprintln!(
|
||||
"mission_memory: reranked {} candidate(s) with {}, kept {} ({} ms)",
|
||||
questions.len(),
|
||||
decision.model,
|
||||
kept.len(),
|
||||
decision.latency.as_millis()
|
||||
);
|
||||
kept
|
||||
}
|
||||
|
||||
fn recall_in(dir: &Path, repo_id: Uuid, query: &str, k: usize) -> Vec<String> {
|
||||
let path = brain_path(dir, repo_id);
|
||||
if !path.exists() {
|
||||
return Vec::new();
|
||||
}
|
||||
match ClawBrain::open_or_create(&path, &format!("repo_{repo_id}")) {
|
||||
Ok(brain) => brain
|
||||
.recall(query, k)
|
||||
.into_iter()
|
||||
// `remember` stores "role: text"; the role is ours and not a lesson.
|
||||
.map(|m| m.strip_prefix("judge: ").map(str::to_string).unwrap_or(m))
|
||||
.collect(),
|
||||
Err(e) => {
|
||||
eprintln!("mission_memory: could not open brain for repo {repo_id}: {e}");
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a mission finds the whole of its repository's memory, readable.
|
||||
///
|
||||
/// Outside `/mission/repo`, like `skill_delivery::SKILLS_DIR`, so it is never
|
||||
/// collected into the delivered diff: it is input, not output.
|
||||
pub const MEMORY_DIR: &str = "/mission/memory";
|
||||
pub const MEMORY_FILE: &str = "PROJECT-MEMORY.md";
|
||||
|
||||
/// Most entries an export carries. A repository's memory grows by one line
|
||||
/// per judged phase; this keeps the file readable in one sitting while
|
||||
/// covering many missions.
|
||||
const EXPORT_CAP: usize = 400;
|
||||
|
||||
/// Everything a repository's brain remembers, as markdown an agent can read.
|
||||
///
|
||||
/// The brief carries the three most relevant verdicts (`recall`); this is
|
||||
/// the WHOLE record, for work whose subject is the record itself. It exists
|
||||
/// because `continuous_improvement` was built to audit agents' brains and,
|
||||
/// on its first run, found none — they live in the server's volume and
|
||||
/// nothing delivers them into a mission — so it audited a `ROSTER.md` in a
|
||||
/// scratch repo instead. Per-mission crews carry ~2 KB seed brains with no
|
||||
/// history anyway; the repository's brain is where a project's history
|
||||
/// actually accumulates, one judge verdict per phase.
|
||||
///
|
||||
/// Rendered, not shipped raw: the `.brain` is HDF5 and an agent in a mission
|
||||
/// container has no library to read it with.
|
||||
///
|
||||
/// `None` when the repository has no brain yet or it holds nothing.
|
||||
pub fn export(repo_id: Uuid) -> Option<String> {
|
||||
export_in(&cm_runtime::brain::brain_dir(), repo_id)
|
||||
}
|
||||
|
||||
fn export_in(dir: &Path, repo_id: Uuid) -> Option<String> {
|
||||
let path = brain_path(dir, repo_id);
|
||||
if !path.exists() {
|
||||
return None;
|
||||
}
|
||||
let brain = ClawBrain::open_or_create(&path, &format!("repo_{repo_id}")).ok()?;
|
||||
let entries = brain.recent_memory(EXPORT_CAP);
|
||||
if entries.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let total = brain.memory_count();
|
||||
let mut out = format!(
|
||||
"# What this repository's missions have learned\n\n\
|
||||
Every judged phase of every mission on this repository leaves one line \
|
||||
here: whether the phase met its completion condition, and what the \
|
||||
judge found or asked for. Newest first. {} of {} entr{} shown.\n\n\
|
||||
This is the record, not instructions. `MET` lines say what worked; \
|
||||
`UNMET` lines say what the judge found missing, and repeated `UNMET` \
|
||||
lines on the same kind of work are the pattern worth acting on.\n\n",
|
||||
entries.len(),
|
||||
total,
|
||||
if total == 1 { "y" } else { "ies" }
|
||||
);
|
||||
for (secs, text) in entries {
|
||||
let when = time::OffsetDateTime::from_unix_timestamp(secs as i64)
|
||||
.ok()
|
||||
.and_then(|t| t.format(&time::format_description::well_known::Rfc3339).ok())
|
||||
.unwrap_or_else(|| "unknown time".to_string());
|
||||
let line = text.strip_prefix("judge: ").unwrap_or(&text);
|
||||
out.push_str(&format!("- `{when}` {line}\n"));
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// The section a brief carries, or nothing when there is nothing to say —
|
||||
/// an empty heading tells the agent there is history and then shows none.
|
||||
pub fn section(recalled: &[String]) -> Option<String> {
|
||||
if recalled.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut out = String::from(SECTION_HEADING);
|
||||
out.push_str(
|
||||
"\n\nJudge verdicts from earlier missions here, most relevant first. \
|
||||
They say what was checked and what was found; they are not the task.\n",
|
||||
);
|
||||
for line in recalled {
|
||||
out.push_str("- ");
|
||||
out.push_str(line);
|
||||
out.push('\n');
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
fn head(s: &str, n: usize) -> String {
|
||||
match s.char_indices().nth(n) {
|
||||
Some((i, _)) => format!("{}…", &s[..i]),
|
||||
None => s.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::evaluator::{Usage, Verdict};
|
||||
|
||||
fn verdict(met: bool, reason: &str, guidance: &str, error: Option<&str>) -> Verdict {
|
||||
Verdict {
|
||||
met,
|
||||
reason: reason.into(),
|
||||
guidance: guidance.into(),
|
||||
model: "m".into(),
|
||||
error: error.map(str::to_string),
|
||||
checks: Vec::new(),
|
||||
independent: true,
|
||||
usage: Usage::default(),
|
||||
expectation: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Unmet carries the sanitized guidance, never the operator reason —
|
||||
/// the reason may quote the acceptance text the next mission must earn.
|
||||
#[test]
|
||||
fn unmet_remembers_guidance_not_reason() {
|
||||
let v = verdict(false, "token ZZQX-9 is absent", "the required marker is absent", None);
|
||||
let line = verdict_line(Uuid::nil(), "coding", "", "cond", &v).unwrap();
|
||||
assert!(line.starts_with("UNMET — coding phase"));
|
||||
assert!(line.contains("the required marker is absent"));
|
||||
assert!(!line.contains("ZZQX-9"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn met_remembers_what_the_judge_found() {
|
||||
let v = verdict(true, "MICROVM.md holds both lines", "", None);
|
||||
let line = verdict_line(Uuid::nil(), "coding", "", "cond", &v).unwrap();
|
||||
assert!(line.starts_with("MET — "));
|
||||
assert!(line.contains("MICROVM.md holds both lines"));
|
||||
}
|
||||
|
||||
/// What the agent was told sits beside what it was judged against, and two
|
||||
/// missions launched back to back stay two missions. Both were missing when
|
||||
/// the first self-audit read a planted brief/condition mismatch as "the
|
||||
/// agent skipped the section" and two missions as one.
|
||||
#[test]
|
||||
fn a_line_carries_the_brief_and_a_distinguishing_mission_id() {
|
||||
let v = verdict(false, "r", "add a Limitations section", None);
|
||||
let a = Uuid::now_v7();
|
||||
let b = Uuid::now_v7();
|
||||
let brief = "Write NOTES.md:\n five bullet points";
|
||||
let la = verdict_line(a, "research", brief, "ends with Limitations", &v).unwrap();
|
||||
let lb = verdict_line(b, "research", brief, "ends with Limitations", &v).unwrap();
|
||||
assert!(la.contains(" — brief: Write NOTES.md: five bullet points — condition: "), "{la}");
|
||||
assert_ne!(la, lb, "same-second UUIDv7s must not render as one mission");
|
||||
let tail = a.simple().to_string();
|
||||
assert!(la.contains(&format!("mission {}", &tail[tail.len() - 8..])), "{la}");
|
||||
let none = verdict_line(a, "research", " ", "c", &v).unwrap();
|
||||
assert!(!none.contains("brief:"), "an empty brief adds no segment: {none}");
|
||||
}
|
||||
|
||||
/// No judgement, no lesson.
|
||||
#[test]
|
||||
fn an_unreachable_judge_leaves_no_memory() {
|
||||
let v = verdict(false, "could not evaluate", "could not evaluate", Some("429"));
|
||||
assert!(verdict_line(Uuid::nil(), "coding", "", "cond", &v).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn section_is_absent_when_nothing_was_recalled() {
|
||||
assert!(section(&[]).is_none());
|
||||
let s = section(&["MET — x".into()]).unwrap();
|
||||
assert!(s.starts_with(SECTION_HEADING));
|
||||
assert!(s.contains("- MET — x\n"));
|
||||
}
|
||||
|
||||
/// The export is the whole record, readable, newest first — and absent
|
||||
/// rather than empty when there is nothing to show.
|
||||
#[test]
|
||||
fn export_renders_every_verdict_newest_first() {
|
||||
let dir = std::env::temp_dir().join(format!("cm-mission-export-{}", Uuid::now_v7()));
|
||||
let repo = Uuid::now_v7();
|
||||
assert!(export_in(&dir, repo).is_none(), "no brain, no export");
|
||||
|
||||
remember_in(&dir, repo, Uuid::now_v7(), "coding", "", "first",
|
||||
&verdict(false, "r", "the tests do not cover the empty case", None));
|
||||
std::thread::sleep(std::time::Duration::from_millis(5));
|
||||
remember_in(&dir, repo, Uuid::now_v7(), "coding", "", "second",
|
||||
&verdict(true, "all three tests pass", "", None));
|
||||
|
||||
let md = export_in(&dir, repo).expect("two verdicts, so an export");
|
||||
assert!(md.starts_with("# What this repository's missions have learned"));
|
||||
assert!(md.contains("2 of 2 entries shown"), "{md}");
|
||||
let met = md.find("MET — coding").unwrap();
|
||||
let unmet = md.find("UNMET — coding").unwrap();
|
||||
assert!(met < unmet, "newest (MET) must come first:\n{md}");
|
||||
assert!(md.contains("the tests do not cover the empty case"));
|
||||
// `remember` stores "judge: <line>"; that ROLE prefix must not follow
|
||||
// the timestamp. (The line itself legitimately says "— judge: …".)
|
||||
assert!(!md.contains("` judge: "), "the storage prefix leaked:\n{md}");
|
||||
assert!(md.contains("` MET — coding"), "{md}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// Round trip through a real brain file: what one mission's verdict
|
||||
/// wrote, a query shaped like the next mission's task recalls.
|
||||
#[test]
|
||||
fn a_second_mission_recalls_the_first_verdict() {
|
||||
let dir = std::env::temp_dir().join(format!("cm-mission-memory-{}", Uuid::now_v7()));
|
||||
let repo = Uuid::now_v7();
|
||||
let v = verdict(
|
||||
true,
|
||||
"BASELINE.md records 0.689 ns/iter from benches/add_bench.rs",
|
||||
"",
|
||||
None,
|
||||
);
|
||||
remember_in(&dir, repo, Uuid::now_v7(), "benchmark", "", "a baseline is recorded", &v);
|
||||
let got = recall_in(&dir, repo, "record a performance baseline for the hot path", RECALL_K);
|
||||
assert_eq!(got.len(), 1, "{got:?}");
|
||||
assert!(got[0].starts_with("MET — benchmark phase"), "{}", got[0]);
|
||||
assert!(!got[0].starts_with("judge: "));
|
||||
// A repo with no history recalls nothing and creates no file.
|
||||
let other = Uuid::now_v7();
|
||||
assert!(recall_in(&dir, other, "anything", RECALL_K).is_empty());
|
||||
assert!(!brain_path(&dir, other).exists());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,7 @@ pub async fn on_launch(
|
||||
user_id: cm_domain::UserId,
|
||||
mission_id: Uuid,
|
||||
node_hub: Option<std::sync::Arc<crate::fleet::NodeHub>>,
|
||||
blobs: Option<std::sync::Arc<dyn cm_files::BlobStore>>,
|
||||
) -> Result<Option<Uuid>, String> {
|
||||
eprintln!("mission_orchestrator::on_launch fired mission_id={mission_id}");
|
||||
let Some(mission) = cm_db::repo::missions::get(pool, mission_id, workspace_id.as_uuid())
|
||||
@@ -53,16 +54,102 @@ pub async fn on_launch(
|
||||
return Err("mission not found".into());
|
||||
};
|
||||
|
||||
// A Continuous Research mission harvests BEFORE its checkout is taken.
|
||||
//
|
||||
// The ORDER here is load-bearing and was wrong: the harvest ran after
|
||||
// `ensure_checkout`, so the mission cloned the vault before the manifest
|
||||
// was pushed to it. The reader agent found no harvest.jsonl, and — being
|
||||
// resourceful — queried arXiv itself and wrote its own. That is precisely
|
||||
// what `skills/research/arxiv-daily.md` forbids: the papers it found are
|
||||
// not checked off in `corpus_items`, so the next run re-offers them, and
|
||||
// the 13 the real harvest DID shelve went unread. Harvest first, then
|
||||
// clone, so the checkout contains the manifest.
|
||||
//
|
||||
// Finding papers is not agent work: `library::run_to_vault` searches arXiv,
|
||||
// checks the `corpus_items` seen-set, fetches and verifies each PDF, shelves
|
||||
// it and writes the catalogue note — deterministically, in seconds. The
|
||||
// seen-set is the entire reason a recurring mission knows what it already
|
||||
// covered, and an agent re-searching arXiv would leave it wrong.
|
||||
//
|
||||
// Deliberately NON-FATAL. A harvest that fails still lets the phases run,
|
||||
// because the phase is what reports whether today was quiet or broken, and
|
||||
// those must stay distinguishable. What is never acceptable is silence, so
|
||||
// both outcomes are logged with their counts.
|
||||
let mut harvested: Vec<crate::papers::Paper> = Vec::new();
|
||||
if mission.template_kind == crate::continuous_research::TEMPLATE_KIND {
|
||||
match blobs.as_ref() {
|
||||
Some(b) => {
|
||||
let topics = crate::continuous_research::topics_for(&mission.config);
|
||||
match crate::continuous_research::harvest_for_mission(
|
||||
pool,
|
||||
b,
|
||||
workspace_id.as_uuid(),
|
||||
mission_id,
|
||||
&topics,
|
||||
5,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(papers) => {
|
||||
eprintln!(
|
||||
"mission_orchestrator: continuous research harvest shelved {} paper(s) for mission {mission_id}",
|
||||
papers.len()
|
||||
);
|
||||
harvested = papers;
|
||||
}
|
||||
Err(e) => eprintln!(
|
||||
"mission_orchestrator: continuous research harvest FAILED for {mission_id} (phases still start, and will report an empty day): {e}"
|
||||
),
|
||||
}
|
||||
}
|
||||
// Not a warning to bury: without blob storage there is nowhere to
|
||||
// shelve a PDF, so the mission will find an empty manifest and
|
||||
// correctly report that nothing arrived.
|
||||
None => eprintln!(
|
||||
"mission_orchestrator: mission {mission_id} is continuous_research but blob storage is not configured — no harvest, so today's manifest will be empty"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ensure_checkout is idempotent (fetch+reset on existing clones,
|
||||
// clone on missing dirs) so we run it BEFORE the team_id short-
|
||||
// circuit: a re-launched or retried mission still needs a fresh
|
||||
// repo checkout even though its team was minted on the first
|
||||
// launch. Non-fatal — logs and continues on failure.
|
||||
match crate::mission_workspace::ensure_checkout(pool, workspace_id, mission_id).await {
|
||||
Ok(Some(path)) => eprintln!(
|
||||
Ok(Some(path)) => {
|
||||
eprintln!(
|
||||
"mission_orchestrator: repo checked out at {} for mission {mission_id}",
|
||||
path.display()
|
||||
);
|
||||
// The manifest goes in the CHECKOUT, not the vault: it is this run's
|
||||
// input, and the vault path is per-date and shared, so a second run
|
||||
// the same day rewrites a file that already exists and auto_merge
|
||||
// rightly refuses the branch. See `write_manifest`.
|
||||
if mission.template_kind == crate::continuous_research::TEMPLATE_KIND {
|
||||
let date = crate::continuous_research::today();
|
||||
// Tag and score each paper before the agents see the list;
|
||||
// an untriaged manifest (no key) is the old, empty-tags one.
|
||||
let topics = crate::continuous_research::topics_for(&mission.config);
|
||||
let triage =
|
||||
crate::continuous_research::triage_papers(&harvested, &topics).await;
|
||||
match crate::continuous_research::write_manifest(&path, &harvested, &date, &triage) {
|
||||
Ok(at) => eprintln!(
|
||||
"mission_orchestrator: wrote {} paper(s) to {}",
|
||||
harvested.len(),
|
||||
at.display()
|
||||
),
|
||||
// Loud: the reader phase would find no manifest and, being
|
||||
// resourceful, go and search arXiv itself — which corrupts
|
||||
// the seen-set. Better to see why here.
|
||||
Err(e) => eprintln!(
|
||||
"mission_orchestrator: could NOT write the harvest manifest for \
|
||||
{mission_id} — the reader phase will see no papers: {e}"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => eprintln!(
|
||||
"mission_orchestrator: mission {mission_id} has no repo bound, skipping checkout"
|
||||
),
|
||||
@@ -88,6 +175,13 @@ pub async fn on_launch(
|
||||
match prov.ensure_container(mission_id).await {
|
||||
Ok(ec) => {
|
||||
mission_gateway = Some(ec.endpoint.clone());
|
||||
crate::container_tool_hooks::record_install(
|
||||
pool,
|
||||
mission_id,
|
||||
None,
|
||||
ec.hooks.as_deref(),
|
||||
)
|
||||
.await;
|
||||
let container_name = crate::mission_runtime::container_name(mission_id);
|
||||
if let Err(e) = cm_db::repo::missions::set_runtime_binding(
|
||||
pool,
|
||||
@@ -249,6 +343,59 @@ pub async fn on_launch(
|
||||
Some(url) => RuntimeProvisioner::for_gateway(url),
|
||||
None => RuntimeProvisioner::from_env(),
|
||||
};
|
||||
|
||||
// Tell the daemon where the hooks are. `container_tool_hooks::install`
|
||||
// wrote them; this is what makes claude read them. Doing one without the
|
||||
// other leaves a gate that is installed and inert, which looks exactly
|
||||
// like a gate that found nothing.
|
||||
if let Some(p) = provisioner.as_ref() {
|
||||
if let Err(e) = p
|
||||
.set_claude_cli_settings(crate::container_tool_hooks::SETTINGS_PATH)
|
||||
.await
|
||||
{
|
||||
eprintln!(
|
||||
"mission_orchestrator: could not point claude_cli at the hook \
|
||||
settings ({e}) — this mission's tool calls run unchecked"
|
||||
);
|
||||
}
|
||||
// Only when this mission got its OWN container — the shared runtime is
|
||||
// not ours to reconfigure, and `mission_gateway` being Some is exactly
|
||||
// the signal that `ensure_container` ran.
|
||||
// What a retrieval arm retrieves FROM is installed here, per arm: the
|
||||
// MCP door for `index`, the skill files for `files`. `inline` installs
|
||||
// nothing and `installed` is irrelevant to it.
|
||||
let requested = crate::skill_delivery::requested_for(&mission.config);
|
||||
let container = crate::mission_runtime::container_name(mission_id);
|
||||
let installed = match requested {
|
||||
_ if mission_gateway.is_none() => false,
|
||||
crate::skill_delivery::Mode::Index => {
|
||||
install_skills_door(pool, user_id, mission_id, &container, p).await
|
||||
}
|
||||
crate::skill_delivery::Mode::Files => {
|
||||
install_skill_files(pool, workspace_id, mission_id, &container).await
|
||||
}
|
||||
crate::skill_delivery::Mode::Inline => false,
|
||||
};
|
||||
// Decided here and recorded, not re-derived per turn: this is the only
|
||||
// point that knows whether the door actually installed, and an arm that
|
||||
// could change mid-mission would make the run unattributable.
|
||||
record_skill_delivery(
|
||||
pool,
|
||||
mission_id,
|
||||
crate::skill_delivery::resolve(requested, installed),
|
||||
)
|
||||
.await;
|
||||
|
||||
// The repository's whole memory, readable, beside the skills. The
|
||||
// brief already carries the three most relevant verdicts; this is
|
||||
// the full record, for work whose subject IS the record — see
|
||||
// `mission_memory::export` for why it was needed.
|
||||
if mission_gateway.is_some() {
|
||||
if let Some(repo) = mission.repo_id {
|
||||
install_project_memory(repo, mission_id, &container).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut first_team_id: Option<Uuid> = None;
|
||||
let mut provisioned_claws: Vec<cm_domain::AgentId> = Vec::new();
|
||||
for (purpose, template_id) in &picks {
|
||||
@@ -597,7 +744,12 @@ async fn mint_team_from_template(
|
||||
// out-of-band via MissionRuntimeProvisioner::pin_agent_workspaces.
|
||||
if let Some(p) = provisioner {
|
||||
match p
|
||||
.provision_claw(claw_id, role_model, &template.template.risk_profile)
|
||||
.provision_claw(
|
||||
claw_id,
|
||||
role_model,
|
||||
&template.template.risk_profile,
|
||||
&template.template.mcp_bundles,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => provisioned_claws.push(agent_id),
|
||||
@@ -780,3 +932,254 @@ fn default_accent_for(slot: &str) -> &'static str {
|
||||
_ => "#8a8a92",
|
||||
}
|
||||
}
|
||||
|
||||
/// Give this mission's agents a reachable, narrow door to the skills catalogue.
|
||||
///
|
||||
/// Two halves that must both happen: the document goes into the container, and
|
||||
/// the daemon is told to pass it to `claude -p --mcp-config`. Doing one without
|
||||
/// the other leaves a door that is installed and unreachable, which looks
|
||||
/// exactly like a door nobody walked through — the same shape as the hooks that
|
||||
/// were installed and inert.
|
||||
///
|
||||
/// # The credential
|
||||
///
|
||||
/// A `skills:read` session, not a user's. It is written into a file the agent
|
||||
/// can `cat` — it runs `Bash` with egress — so the only thing keeping this safe
|
||||
/// is that the token authenticates to exactly one route and nowhere else. See
|
||||
/// `cm_auth::AuthService::authenticate_scoped`. A full session here would be an
|
||||
/// owner-privileged API key handed to something explicitly untrusted, which is
|
||||
/// why the door went undeployed rather than being deployed the easy way.
|
||||
///
|
||||
/// Every failure degrades to "no door", never to a failed launch. A mission
|
||||
/// that cannot retrieve a skill still delivers.
|
||||
/// Returns whether the door is installed AND reachable. The caller needs the
|
||||
/// answer, not just the log line: the `index` delivery arm hands agents a list
|
||||
/// of uris to fetch, and without a door every one of them is a dead end that
|
||||
/// reads as an agent ignoring its skills.
|
||||
async fn install_skills_door(
|
||||
pool: &PgPool,
|
||||
user_id: cm_domain::UserId,
|
||||
mission_id: Uuid,
|
||||
container: &str,
|
||||
prov: &RuntimeProvisioner,
|
||||
) -> bool {
|
||||
let Some(origin) = crate::container_tool_hooks::api_origin() else {
|
||||
eprintln!(
|
||||
"mission_orchestrator: no API origin for the skills door (set \
|
||||
CLAWMATES_API_ORIGIN) — mission {mission_id} runs without it"
|
||||
);
|
||||
return false;
|
||||
};
|
||||
// Bound to the mission: revoked by `revoke_mission_credentials` the
|
||||
// moment it reaches a terminal status. The 24 h TTL is the backstop for a
|
||||
// mission nothing ever closes, not the credential's lifetime — until
|
||||
// 2026-09-20 it was, and a twenty-minute mission left a live token in
|
||||
// its container for the other twenty-three hours.
|
||||
let auth = cm_auth::AuthService::new(pool.clone());
|
||||
let token = match auth
|
||||
.mint_scoped_for_mission(
|
||||
user_id,
|
||||
cm_auth::SCOPE_SKILLS_READ,
|
||||
time::Duration::hours(24),
|
||||
mission_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"mission_orchestrator: could not mint a skills token ({e}) — \
|
||||
mission {mission_id} runs without the door"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let docker = match crate::container_exec::connect() {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
eprintln!("mission_orchestrator: cannot reach docker for the skills door: {e}");
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let doc = crate::container_tool_hooks::mcp_document(&origin, &token);
|
||||
let Some(path) = crate::container_tool_hooks::install_door(&docker, container, &doc).await
|
||||
else {
|
||||
// `install_door` already said why.
|
||||
return false;
|
||||
};
|
||||
if let Err(e) = prov.set_claude_cli_mcp_config(&path).await {
|
||||
eprintln!(
|
||||
"mission_orchestrator: wrote the MCP config but could not point \
|
||||
claude_cli at it ({e}) — the door is installed and unreachable"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
eprintln!(
|
||||
"mission_orchestrator: skills door installed for mission {mission_id} \
|
||||
({origin}/mcp/skills)"
|
||||
);
|
||||
true
|
||||
}
|
||||
|
||||
/// Write every skill the workspace can see into the mission container as a
|
||||
/// file, for the `files` arm.
|
||||
///
|
||||
/// Every visible skill and not only the bound ones, because bindings are
|
||||
/// resolved per AGENT at turn time (`effective_for_agent`) and this runs once
|
||||
/// per mission before any turn — the same reason the MCP door serves the whole
|
||||
/// catalogue rather than a per-mission subset. A few KB each; the whole
|
||||
/// catalogue is smaller than one phase's evidence.
|
||||
///
|
||||
/// Returns whether the files are in place. `false` means the mission falls
|
||||
/// back to `inline` (see `skill_delivery::resolve`) — an entry that points at
|
||||
/// a file which is not there reads exactly like an agent ignoring its skills,
|
||||
/// which is the failure this arm exists to stop misdiagnosing.
|
||||
async fn install_skill_files(
|
||||
pool: &PgPool,
|
||||
workspace_id: WorkspaceId,
|
||||
mission_id: Uuid,
|
||||
container: &str,
|
||||
) -> bool {
|
||||
let skills = match cm_db::repo::skills_catalog::list_visible(pool, workspace_id.as_uuid()).await
|
||||
{
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"mission_orchestrator: could not list skills for the files arm ({e}) — \
|
||||
mission {mission_id} delivers skills inline"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let docker = match crate::container_exec::connect() {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
eprintln!("mission_orchestrator: cannot reach docker for the skill files: {e}");
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let dir = crate::skill_delivery::SKILLS_DIR;
|
||||
// `upload_to_container` will not create the directory.
|
||||
let argv = vec!["sh".to_string(), "-lc".to_string(), format!("mkdir -p {dir}")];
|
||||
match crate::container_exec::exec_as_root(
|
||||
&docker,
|
||||
container,
|
||||
None,
|
||||
&argv,
|
||||
crate::container_tool_hooks::INSTALL_TIMEOUT,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(out) if out.exit_code == Some(0) => {}
|
||||
other => {
|
||||
eprintln!(
|
||||
"mission_orchestrator: could not create {dir} in {container} ({other:?}) — \
|
||||
mission {mission_id} delivers skills inline"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
let files: Vec<(String, Vec<u8>)> = skills
|
||||
.iter()
|
||||
.map(|sk| (format!("{}.md", sk.name), sk.body.clone().into_bytes()))
|
||||
.collect();
|
||||
let n = files.len();
|
||||
if let Err(e) = crate::mission_fs::put_files(&docker, container, dir, &files).await {
|
||||
eprintln!(
|
||||
"mission_orchestrator: could not write the skill files ({e}) — mission \
|
||||
{mission_id} delivers skills inline"
|
||||
);
|
||||
return false;
|
||||
}
|
||||
eprintln!("mission_orchestrator: {n} skill file(s) installed for mission {mission_id} under {dir}");
|
||||
true
|
||||
}
|
||||
|
||||
/// Write the repository's memory export into the mission container.
|
||||
///
|
||||
/// Best-effort and loud: a mission with no memory to read is an ordinary
|
||||
/// mission, and a first mission on a repository has none. Outside the
|
||||
/// checkout (`mission_memory::MEMORY_DIR`) so it never lands in the diff.
|
||||
async fn install_project_memory(repo_id: Uuid, mission_id: Uuid, container: &str) {
|
||||
let Some(md) = crate::mission_memory::export(repo_id) else {
|
||||
return;
|
||||
};
|
||||
let docker = match crate::container_exec::connect() {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
eprintln!("mission_orchestrator: cannot reach docker for project memory: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let dir = crate::mission_memory::MEMORY_DIR;
|
||||
let argv = vec!["sh".to_string(), "-lc".to_string(), format!("mkdir -p {dir}")];
|
||||
if !matches!(
|
||||
crate::container_exec::exec_as_root(
|
||||
&docker,
|
||||
container,
|
||||
None,
|
||||
&argv,
|
||||
crate::container_tool_hooks::INSTALL_TIMEOUT,
|
||||
)
|
||||
.await,
|
||||
Ok(out) if out.exit_code == Some(0)
|
||||
) {
|
||||
eprintln!("mission_orchestrator: could not create {dir} for mission {mission_id}");
|
||||
return;
|
||||
}
|
||||
let bytes = md.len();
|
||||
let files = vec![(crate::mission_memory::MEMORY_FILE.to_string(), md.into_bytes())];
|
||||
match crate::mission_fs::put_files(&docker, container, dir, &files).await {
|
||||
Ok(()) => eprintln!(
|
||||
"mission_orchestrator: project memory ({bytes} bytes) installed for mission \
|
||||
{mission_id} at {dir}/{}",
|
||||
crate::mission_memory::MEMORY_FILE
|
||||
),
|
||||
Err(e) => eprintln!(
|
||||
"mission_orchestrator: could not write project memory for {mission_id}: {e}"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Record which arm this mission runs, so every turn composes the same one and
|
||||
/// the score can be attributed to it afterwards.
|
||||
///
|
||||
/// A write failure is not fatal: `skill_delivery_mode` reads NULL as `inline`,
|
||||
/// which is the arm that needs nothing installed. A mission that quietly ran
|
||||
/// the control arm is a lost data point; a mission that failed to launch over
|
||||
/// a telemetry column is a lost mission.
|
||||
async fn record_skill_delivery(pool: &PgPool, mission_id: Uuid, mode: crate::skill_delivery::Mode) {
|
||||
if let Err(e) = sqlx::query("UPDATE missions SET skill_delivery = $2 WHERE id = $1")
|
||||
.bind(mission_id)
|
||||
.bind(mode.as_str())
|
||||
.execute(pool)
|
||||
.await
|
||||
{
|
||||
eprintln!(
|
||||
"mission_orchestrator: could not record skill_delivery={} for mission \
|
||||
{mission_id} ({e}) — its turns will compose skills inline",
|
||||
mode.as_str()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Revoke every credential minted for a mission. Called on every path that
|
||||
/// takes a mission to a terminal status — the runner's close and the
|
||||
/// operator's stop — so the authority a mission was given ends with it.
|
||||
/// Best-effort and loud: a revocation that failed is logged with the count
|
||||
/// it could not clear, which is the number an operator needs.
|
||||
pub async fn revoke_mission_credentials(pool: &PgPool, mission_id: Uuid) {
|
||||
match cm_auth::AuthService::new(pool.clone())
|
||||
.revoke_mission_sessions(mission_id)
|
||||
.await
|
||||
{
|
||||
Ok(0) => {}
|
||||
Ok(n) => eprintln!(
|
||||
"mission_orchestrator: revoked {n} credential(s) for mission {mission_id} at close"
|
||||
),
|
||||
Err(e) => eprintln!(
|
||||
"mission_orchestrator: could NOT revoke credentials for mission {mission_id}: {e} \
|
||||
— they expire on their own within 24 h"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,6 +35,7 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use sqlx::{PgPool, Row};
|
||||
use time::{Duration, OffsetDateTime};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Directories never worth capturing, whatever an agent leaves behind.
|
||||
@@ -56,6 +57,14 @@ const SKIP_DIRS: &[&str] = &[
|
||||
/// How many phases to capture per tick, matching `CAPTURE_BATCH`.
|
||||
const BATCH: i64 = 5;
|
||||
|
||||
/// How long a phase's outputs may stay uncollectable before the sweep stops
|
||||
/// retrying and calls it empty.
|
||||
///
|
||||
/// Generous on purpose: the container is torn down asynchronously after a
|
||||
/// phase, so an early tick can legitimately fail. What must NOT happen is
|
||||
/// retrying forever — that is the state this constant exists to end.
|
||||
const COLLECT_GRACE: Duration = Duration::minutes(10);
|
||||
|
||||
/// The artifact kind this path registers. Also the idempotency key: a phase with
|
||||
/// one of these has already been captured.
|
||||
pub const OUTPUT_KIND: &str = "document";
|
||||
@@ -66,7 +75,7 @@ const EMPTY_MARKER: &str = "NO-OUTPUT.md";
|
||||
/// Capture the outputs of finished phases on missions that have no repo.
|
||||
pub async fn capture_repo_less_phases(pool: &PgPool) -> Result<(), String> {
|
||||
let rows = sqlx::query(
|
||||
"SELECT mp.id, mp.mission_id, mp.kind, mp.config, m.runtime_kind
|
||||
"SELECT mp.id, mp.mission_id, mp.kind, mp.config, mp.completed_at, m.runtime_kind
|
||||
FROM mission_phases mp
|
||||
JOIN missions m ON m.id = mp.mission_id
|
||||
WHERE mp.status IN ('completed', 'failed')
|
||||
@@ -97,22 +106,42 @@ pub async fn capture_repo_less_phases(pool: &PgPool) -> Result<(), String> {
|
||||
let mission_id: Uuid = row.get("mission_id");
|
||||
let kind: String = row.get("kind");
|
||||
let config: serde_json::Value = row.get("config");
|
||||
let completed_at: Option<OffsetDateTime> = row.get("completed_at");
|
||||
let runtime_kind: String = row.get("runtime_kind");
|
||||
|
||||
let dest = outputs_dir(mission_id, phase_id);
|
||||
let captured = match collect_into(mission_id, &dest, &runtime_kind).await {
|
||||
Ok(files) => files,
|
||||
Err(e) => {
|
||||
// Loud and retryable, never silently "captured nothing": the
|
||||
// whole defect this module exists for is work disappearing
|
||||
// without a word. The next tick tries again; if the container is
|
||||
// already gone the phase is failed below on the next pass.
|
||||
// Retryable, but BOUNDED. A bare `continue` here is how a phase
|
||||
// whose collect can never succeed stayed `completed` with zero
|
||||
// artifacts forever: the fail-empty rule and the NO-OUTPUT
|
||||
// marker both live below this point, so neither was ever
|
||||
// reached, and the phase was re-attempted on every tick for the
|
||||
// life of the deployment.
|
||||
//
|
||||
// The grace window exists because the container may legitimately
|
||||
// not be ready on the first tick after a phase finishes. Past
|
||||
// that, "cannot collect" and "collected nothing" are the same
|
||||
// fact for the operator, so we fall through and let the rules
|
||||
// below fail the phase and leave a marker explaining why.
|
||||
let settled = completed_at
|
||||
.map(|t| OffsetDateTime::now_utc() - t > COLLECT_GRACE)
|
||||
.unwrap_or(true);
|
||||
if !settled {
|
||||
eprintln!(
|
||||
"mission_outputs: could NOT collect outputs for phase {phase_id} \
|
||||
of mission {mission_id}: {e}"
|
||||
of mission {mission_id} (will retry): {e}"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
eprintln!(
|
||||
"mission_outputs: giving up collecting phase {phase_id} of mission \
|
||||
{mission_id} after {}s: {e} — treating it as having produced nothing",
|
||||
COLLECT_GRACE.whole_seconds()
|
||||
);
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
|
||||
for file in &captured {
|
||||
@@ -251,7 +280,11 @@ async fn register_empty_marker(
|
||||
/// the end of the turn (`microvm_executor` writes them over
|
||||
/// `mission_workspace::checkout_path`). Asking docker for a VM mission's files
|
||||
/// would query a container that never existed.
|
||||
async fn collect_into(mission_id: Uuid, dest: &Path, runtime_kind: &str) -> Result<Vec<PathBuf>, String> {
|
||||
async fn collect_into(
|
||||
mission_id: Uuid,
|
||||
dest: &Path,
|
||||
runtime_kind: &str,
|
||||
) -> Result<Vec<PathBuf>, String> {
|
||||
// A stale copy from an earlier attempt would be registered as this pass's
|
||||
// output — the same "captured a tree nobody wrote" shape capture avoids.
|
||||
let _ = std::fs::remove_dir_all(dest);
|
||||
@@ -279,8 +312,7 @@ async fn collect_into(mission_id: Uuid, dest: &Path, runtime_kind: &str) -> Resu
|
||||
/// shell-out, and this runs as the server's own uid against its own directory.
|
||||
fn copy_tree(src: &Path, dest: &Path) -> Result<(), String> {
|
||||
std::fs::create_dir_all(dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
|
||||
let entries =
|
||||
std::fs::read_dir(src).map_err(|e| format!("read {}: {e}", src.display()))?;
|
||||
let entries = std::fs::read_dir(src).map_err(|e| format!("read {}: {e}", src.display()))?;
|
||||
for entry in entries.flatten() {
|
||||
let from = entry.path();
|
||||
let to = dest.join(entry.file_name());
|
||||
@@ -437,9 +469,7 @@ mod tests {
|
||||
|
||||
// ...and the traversal shape this guards against is not, once resolved.
|
||||
let escaped = root.join("..").join("..").join("etc/passwd");
|
||||
let normalised: PathBuf = escaped
|
||||
.components()
|
||||
.fold(PathBuf::new(), |mut acc, c| {
|
||||
let normalised: PathBuf = escaped.components().fold(PathBuf::new(), |mut acc, c| {
|
||||
match c {
|
||||
std::path::Component::ParentDir => {
|
||||
acc.pop();
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEFAULT_MODEL: &str = "claude-opus-4-8";
|
||||
const DEFAULT_MODEL: &str = "claude-opus-5";
|
||||
|
||||
fn model_name() -> String {
|
||||
std::env::var("CLAWMATES_REFINER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
||||
|
||||
@@ -282,6 +282,24 @@ fn microvm_provider_env_from(
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
/// The guest env for a microVM whose model calls are RELAYED to the server's
|
||||
/// LLM proxy: the credential variable the backend's CLI reads carries the
|
||||
/// mission's proxy token, and `ANTHROPIC_BASE_URL` points at the guest's own
|
||||
/// loopback model port, which fcagent pipes to the node and the node relays.
|
||||
/// The per-turn env overrides the base URL the image bakes in.
|
||||
pub fn microvm_proxied_env(
|
||||
backend: Option<&str>,
|
||||
token: &str,
|
||||
) -> Result<Vec<(String, String)>, String> {
|
||||
let want = microvm_credential_for(backend)?;
|
||||
let route = crate::llm_proxy::microvm_route(backend)
|
||||
.ok_or_else(|| format!("backend {backend:?} has no LLM proxy route"))?;
|
||||
Ok(vec![
|
||||
(want.target.to_string(), token.to_string()),
|
||||
("ANTHROPIC_BASE_URL".to_string(), format!("http://127.0.0.1:11434/{route}")),
|
||||
])
|
||||
}
|
||||
|
||||
/// The testable half of [`forwarded_provider_env`]. The lookup is a parameter
|
||||
/// because a test cannot set process environment variables here — the workspace
|
||||
/// denies `unsafe`, and `set_var` is racy across test threads regardless.
|
||||
@@ -323,11 +341,24 @@ pub fn runtime_auth_mode() -> RuntimeAuth {
|
||||
}
|
||||
}
|
||||
|
||||
/// Docker networks the runtime container must be attached to.
|
||||
/// - `clawmates_core`: talks to the server + database
|
||||
/// - `clawmates_edge`: has egress for outbound provider calls
|
||||
/// Docker networks the mission runtime container must be attached to.
|
||||
/// - `clawmates_core`: talks to the server (skills door, API) + database
|
||||
/// - `clawmates_missions`: has egress for outbound provider calls and fetches
|
||||
///
|
||||
/// Missions used to egress from `clawmates_edge`, the same network as the
|
||||
/// SERVER. That made the host's egress policy impossible to scope: a mission
|
||||
/// agent runs model-generated shell over content it fetched from the open web
|
||||
/// and must not reach the tailnet, but the server on the same subnet must —
|
||||
/// Beszel, Ollama, the node daemons. Measured 2026-09-18: the tailnet rule
|
||||
/// written for missions cut the server off from architect:8090 within the
|
||||
/// minute. A subnet of their own is what lets `clawmates-egress.sh` on gw-04
|
||||
/// drop tailnet/private/link-local/ssh for missions and nothing else.
|
||||
///
|
||||
/// The network is declared by compose (prod: /opt/clawmates/docker-compose.yml;
|
||||
/// local: deploy/compose/docker-compose.yml) with the same shape `edge` has —
|
||||
/// not internal, so it routes off the host.
|
||||
const CORE_NETWORK: &str = "clawmates_core";
|
||||
const EDGE_NETWORK: &str = "clawmates_edge";
|
||||
const EDGE_NETWORK: &str = "clawmates_missions";
|
||||
|
||||
/// Host path (as seen by the docker engine, NOT the server container)
|
||||
/// where the mission's checkouts live. Matches the mount source used
|
||||
@@ -361,8 +392,6 @@ use crate::container_exec::MISSION_UID;
|
||||
|
||||
const DEFAULT_SEED_DIR: &str = "/root/clawmates-runtime/data";
|
||||
|
||||
|
||||
|
||||
/// What a mission gets its own copy of.
|
||||
///
|
||||
/// An allow-list, not the whole directory. The seed dir is **1.7 GB** on
|
||||
@@ -560,6 +589,14 @@ pub struct MissionRuntimeProvisioner {
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EnsuredContainer {
|
||||
pub endpoint: String,
|
||||
/// Where the tool hooks were written, or `None` if installing them failed.
|
||||
///
|
||||
/// Carried out to the caller rather than only logged, because the caller
|
||||
/// has the pool and this struct's producer does not. Before this field
|
||||
/// the outcome went to stderr and nowhere else, so a mission whose gate
|
||||
/// never installed left a record indistinguishable from one whose gate
|
||||
/// stood there all night and matched nothing.
|
||||
pub hooks: Option<String>,
|
||||
/// One-time pairing code minted by the daemon at boot; may be
|
||||
/// None on the reuse-existing path when we couldn't scrape it
|
||||
/// back (log rotation). Callers keep the previously-persisted
|
||||
@@ -593,6 +630,20 @@ impl MissionRuntimeProvisioner {
|
||||
Some(MissionRuntimeProvisioner { docker, image })
|
||||
}
|
||||
|
||||
/// Is this container actually attached to the egress network?
|
||||
///
|
||||
/// Asked only when the attach reported an error, to tell "already connected"
|
||||
/// apart from "not connected". Inspect failing is treated as NOT attached:
|
||||
/// the whole point is to stop guessing that egress is present.
|
||||
async fn is_on_edge_network(&self, name: &str) -> bool {
|
||||
self.docker
|
||||
.inspect_container(name, None::<bollard::query_parameters::InspectContainerOptions>)
|
||||
.await
|
||||
.ok()
|
||||
.and_then(|c| c.network_settings?.networks)
|
||||
.is_some_and(|nets| nets.contains_key(EDGE_NETWORK))
|
||||
}
|
||||
|
||||
/// Idempotent: returns the endpoint URL, creating the container
|
||||
/// on first call. If the container exists but is stopped, starts
|
||||
/// it. If it exists and is running, returns its endpoint.
|
||||
@@ -613,10 +664,26 @@ impl MissionRuntimeProvisioner {
|
||||
// Reuse; try to re-scrape the pairing code from logs,
|
||||
// but it may have rotated out — caller falls back to
|
||||
// the previously-persisted value in that case.
|
||||
// Re-install on reuse: the container outlives the server
|
||||
// process, and a hook that exists only on first creation is a
|
||||
// hook that quietly disappears after a redeploy.
|
||||
let hooks = crate::container_tool_hooks::install_with(
|
||||
&self.docker,
|
||||
&name,
|
||||
// A container serves every phase of the mission, so the
|
||||
// policy installed here is the mission-wide default. A
|
||||
// phase's own `agent_tools` is honoured on the microVM
|
||||
// tier, where the VM is per-phase; narrowing per phase
|
||||
// here would need a re-install between phases and is not
|
||||
// done.
|
||||
Some(&crate::vm_tool_gate::TaskPolicy::default_shadow()),
|
||||
)
|
||||
.await;
|
||||
let pairing_code = self.mint_pairing_code(&name).await;
|
||||
return Ok(EnsuredContainer {
|
||||
endpoint: endpoint_url(&name),
|
||||
pairing_code,
|
||||
hooks,
|
||||
});
|
||||
}
|
||||
// Exists but not running — remove + recreate below rather
|
||||
@@ -730,8 +797,30 @@ impl MissionRuntimeProvisioner {
|
||||
// The other three are unrelated providers (Gemini/Groq/OpenAI) with no
|
||||
// subscription equivalent, so they forward in both modes.
|
||||
let auth_mode = runtime_auth_mode();
|
||||
// With the LLM proxy on, the container gets a per-mission token where
|
||||
// each key would be, and Claude Code's base URL points at the proxy,
|
||||
// which adds the real credential. See `llm_proxy`. Without a reachable
|
||||
// proxy address the keys forward as before, loudly.
|
||||
let proxied = match (crate::llm_proxy::enabled(), crate::llm_proxy::base_url(), crate::llm_proxy::token_for(mission_id)) {
|
||||
(true, Some(base), Some(token)) => Some((base, token)),
|
||||
(true, _, _) => {
|
||||
eprintln!(
|
||||
"mission_runtime: CLAWMATES_LLM_PROXY is on but the proxy address or token \
|
||||
could not be derived — mission {mission_id} gets the real provider keys"
|
||||
);
|
||||
None
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
for (key, v) in forwarded_provider_env(auth_mode) {
|
||||
env.push(format!("{key}={v}"));
|
||||
match &proxied {
|
||||
Some((_, token)) => env.push(format!("{key}={token}")),
|
||||
None => env.push(format!("{key}={v}")),
|
||||
}
|
||||
}
|
||||
if let Some((base, _)) = &proxied {
|
||||
env.push(format!("ANTHROPIC_BASE_URL={base}/anthropic"));
|
||||
eprintln!("mission_runtime: mission {mission_id} reaches its models through {base} — no provider key in the container");
|
||||
}
|
||||
eprintln!(
|
||||
"mission_runtime: mission {mission_id} container auth mode = {} \
|
||||
@@ -773,7 +862,19 @@ impl MissionRuntimeProvisioner {
|
||||
.map_err(|e| format!("create mission runtime container: {e}"))?;
|
||||
|
||||
// Attach to the edge network for outbound provider egress.
|
||||
let _ = self
|
||||
//
|
||||
// `clawmates_core` is `internal: true` and has NO default route —
|
||||
// verified from a container on it, where every external address is
|
||||
// unreachable. So this attach is not an optimisation: without it the
|
||||
// mission cannot reach a provider, cannot fetch anything, and cannot do
|
||||
// its work. The result used to be discarded, which made a failure here
|
||||
// indistinguishable from success and produced the green-with-nothing
|
||||
// shape this codebase keeps meeting.
|
||||
//
|
||||
// Not fatal on the error alone: re-attaching an already-connected
|
||||
// container is an error too, and a benign one on any relaunch path. The
|
||||
// container's own network list is the fact that settles it.
|
||||
if let Err(e) = self
|
||||
.docker
|
||||
.connect_network(
|
||||
EDGE_NETWORK,
|
||||
@@ -782,7 +883,24 @@ impl MissionRuntimeProvisioner {
|
||||
endpoint_config: Some(EndpointSettings::default()),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
{
|
||||
if self.is_on_edge_network(&name).await {
|
||||
eprintln!(
|
||||
"mission_runtime: {name} was already on {EDGE_NETWORK} ({e}) — \
|
||||
egress is present, continuing"
|
||||
);
|
||||
} else {
|
||||
return Err(format!(
|
||||
"attach mission runtime container to {EDGE_NETWORK}: {e} — \
|
||||
{CORE_NETWORK} is internal and has no route off the host, so \
|
||||
this mission would run with no egress at all: every provider \
|
||||
call and every fetch would fail while the phase still \
|
||||
reported completion. Is the `missions` network declared in \
|
||||
the compose file and created (`docker network ls`)?"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
self.docker
|
||||
.start_container(&name, None::<StartContainerOptions>)
|
||||
@@ -794,9 +912,27 @@ impl MissionRuntimeProvisioner {
|
||||
// slow boot doesn't hang the launch — the topology_worker
|
||||
// will retry pairing later if we came up empty.
|
||||
let pairing_code = self.wait_for_pairing_code(&name).await;
|
||||
|
||||
// Gate and observe the tools claude runs inside its own subprocess.
|
||||
// Best-effort by design: a phase that runs unhooked still delivers, and
|
||||
// failing the launch to protect telemetry would be the wrong trade.
|
||||
let hooks = crate::container_tool_hooks::install_with(
|
||||
&self.docker,
|
||||
&name,
|
||||
// A container serves every phase of the mission, so the
|
||||
// policy installed here is the mission-wide default. A
|
||||
// phase's own `agent_tools` is honoured on the microVM
|
||||
// tier, where the VM is per-phase; narrowing per phase
|
||||
// here would need a re-install between phases and is not
|
||||
// done.
|
||||
Some(&crate::vm_tool_gate::TaskPolicy::default_shadow()),
|
||||
)
|
||||
.await;
|
||||
|
||||
Ok(EnsuredContainer {
|
||||
endpoint: endpoint_url(&name),
|
||||
pairing_code,
|
||||
hooks,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -897,8 +1033,20 @@ impl MissionRuntimeProvisioner {
|
||||
.map(|c| crate::runtime_provision::claw_alias(c.as_uuid()))
|
||||
.collect();
|
||||
let (edited, pinned) = stamp_workspace_paths(&raw, &aliases, workspace_path)?;
|
||||
// Pinning NOTHING is a failure, not a no-op. Returning Ok here meant the
|
||||
// caller's deliberately-fatal guard could not fire, so a mission whose
|
||||
// aliases were missing from the config launched anyway with every agent
|
||||
// writing into its own sandbox and delivering nothing — the outcome that
|
||||
// guard's error message already describes. Name the aliases: the only
|
||||
// way this happens is a config/alias mismatch, and the aliases are the
|
||||
// evidence needed to find it.
|
||||
if pinned == 0 {
|
||||
return Ok(());
|
||||
return Err(format!(
|
||||
"pinned 0 of {} agent workspace(s) to {workspace_path} — none of these \
|
||||
aliases exist in {CONFIG_PATH}: {}",
|
||||
aliases.len(),
|
||||
aliases.join(", ")
|
||||
));
|
||||
}
|
||||
|
||||
// Upload rather than exec. This used to base64 the whole config into a
|
||||
@@ -908,6 +1056,22 @@ impl MissionRuntimeProvisioner {
|
||||
// pin never applied, its agents never saw `/mission/repo`, and phase 0
|
||||
// completed having written nothing. The tar API has no argv limit, so
|
||||
// the failure mode is gone rather than merely further away.
|
||||
// The GLM and Kimi hops carry literal base URLs in the config; point
|
||||
// them at the proxy too, or a fallback would send the placeholder token
|
||||
// straight to the provider and fail.
|
||||
let edited = match (crate::llm_proxy::enabled(), crate::llm_proxy::base_url()) {
|
||||
(true, Some(base)) => {
|
||||
let (routed, n) = crate::llm_proxy::route_config_through(&edited, &base)?;
|
||||
if n < 2 {
|
||||
eprintln!(
|
||||
"mission_runtime: routed only {n} of 2 fallback hops through the LLM proxy \
|
||||
for mission {mission_id} — an unrouted hop will fail rather than leak"
|
||||
);
|
||||
}
|
||||
routed
|
||||
}
|
||||
_ => edited,
|
||||
};
|
||||
crate::mission_fs::put_file(&self.docker, &name, CONFIG_PATH, edited.as_bytes())
|
||||
.await
|
||||
.map_err(|e| format!("write runtime config.toml: {e}"))?;
|
||||
@@ -1050,6 +1214,103 @@ impl MissionRuntimeProvisioner {
|
||||
/// this to decide whether to KEEP a binding for a retry, and answering
|
||||
/// "still there" when docker cannot be reached would pin the binding open
|
||||
/// on an unreachable daemon rather than on a real container.
|
||||
/// Every `cm-runtime-mission-*` container on this engine, running or not.
|
||||
///
|
||||
/// The piece the row-driven sweep never had. Without it "which containers
|
||||
/// exist" is a question the platform cannot ask, and a container the
|
||||
/// database has forgotten is not merely unreaped — it is unseeable.
|
||||
pub async fn list_mission_containers(&self) -> Result<Vec<(String, Option<i64>)>, String> {
|
||||
let mut filters = std::collections::HashMap::new();
|
||||
filters.insert("name".to_string(), vec!["cm-runtime-mission-".to_string()]);
|
||||
let opts = bollard::query_parameters::ListContainersOptionsBuilder::default()
|
||||
.all(true)
|
||||
.filters(&filters)
|
||||
.build();
|
||||
let list = self
|
||||
.docker
|
||||
.list_containers(Some(opts))
|
||||
.await
|
||||
.map_err(|e| format!("list mission containers: {e}"))?;
|
||||
Ok(list
|
||||
.into_iter()
|
||||
.filter_map(|c| {
|
||||
let name = c
|
||||
.names
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
// Docker returns names with a leading slash.
|
||||
.map(|n| n.trim_start_matches('/').to_string())
|
||||
.find(|n| n.starts_with("cm-runtime-mission-"))?;
|
||||
Some((name, c.created))
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// How long ago docker says this container was created.
|
||||
///
|
||||
/// Taken from the listing rather than a second `inspect`: `created` is
|
||||
/// already a unix timestamp there, so this needs neither a date parser nor
|
||||
/// another round-trip. `None` when docker reported none, and the caller
|
||||
/// treats that as "do not reap" — a container we cannot date is exactly the
|
||||
/// one worth leaving.
|
||||
pub fn container_age(created_epoch: Option<i64>) -> Option<std::time::Duration> {
|
||||
let created = created_epoch?;
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.ok()?
|
||||
.as_secs() as i64;
|
||||
u64::try_from(now - created).ok().map(std::time::Duration::from_secs)
|
||||
}
|
||||
|
||||
/// Does this container's checkout hold commits no remote has?
|
||||
///
|
||||
/// Answered by `git` inside the container, because only it knows which
|
||||
/// refs the remote had. `--not --remotes` lists every commit reachable
|
||||
/// from any local ref and from no remote-tracking ref — which is exactly
|
||||
/// "work that exists only here".
|
||||
///
|
||||
/// Every failure path returns `SomeOrUnknown`. A container we cannot
|
||||
/// question is not a container we may delete.
|
||||
pub async fn unpushed_commits(&self, name: &str) -> UnpushedWork {
|
||||
let script = "cd /mission/repo 2>/dev/null || exit 91; \
|
||||
git rev-list --all --not --remotes 2>/dev/null | wc -l";
|
||||
let argv = vec!["sh".to_string(), "-lc".to_string(), script.to_string()];
|
||||
let out = match crate::container_exec::exec_as_root(
|
||||
&self.docker,
|
||||
name,
|
||||
None,
|
||||
&argv,
|
||||
std::time::Duration::from_secs(30),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(o) => o,
|
||||
Err(e) => {
|
||||
return UnpushedWork::SomeOrUnknown(format!("could not ask git ({e})"));
|
||||
}
|
||||
};
|
||||
if out.exit_code == Some(91) {
|
||||
// No checkout at all — nothing to lose.
|
||||
return UnpushedWork::None;
|
||||
}
|
||||
if out.exit_code != Some(0) {
|
||||
return UnpushedWork::SomeOrUnknown(format!(
|
||||
"git probe exited {:?}",
|
||||
out.exit_code
|
||||
));
|
||||
}
|
||||
match out.stdout.trim().parse::<u64>() {
|
||||
Ok(0) => UnpushedWork::None,
|
||||
Ok(n) => UnpushedWork::SomeOrUnknown(format!(
|
||||
"{n} commit(s) in its checkout are on no remote"
|
||||
)),
|
||||
Err(_) => UnpushedWork::SomeOrUnknown(format!(
|
||||
"unreadable git output {:?}",
|
||||
out.stdout.trim()
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn container_exists(&self, mission_id: Uuid) -> bool {
|
||||
self.docker
|
||||
.inspect_container(&container_name(mission_id), None::<InspectContainerOptions>)
|
||||
@@ -1132,10 +1393,114 @@ pub fn spawn_sweeper(pool: sqlx::PgPool, grace: std::time::Duration) {
|
||||
if let Err(e) = sweep_once(&pool, grace).await {
|
||||
eprintln!("mission_runtime::sweeper: sweep failed: {e}");
|
||||
}
|
||||
if let Err(e) = sweep_orphans(&pool, ORPHAN_GRACE).await {
|
||||
eprintln!("mission_runtime::sweeper: orphan sweep failed: {e}");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// How long a container with no mission row may sit before it is reaped.
|
||||
///
|
||||
/// Long, deliberately. The row-driven sweep above handles every container the
|
||||
/// platform still knows about, so anything reaching this path is already
|
||||
/// unexpected — and the one real orphan we have seen held ten unpushed commits.
|
||||
/// A day of disk is cheaper than being wrong about that.
|
||||
const ORPHAN_GRACE: std::time::Duration = std::time::Duration::from_secs(24 * 3600);
|
||||
|
||||
/// Reap `cm-runtime-mission-*` containers that no `missions` row points at.
|
||||
///
|
||||
/// [`sweep_once`] selects `FROM missions`, and `teardown_container` is only
|
||||
/// ever called with an id that came from that query. So a container whose row
|
||||
/// is gone is invisible to every reaper: nothing enumerates docker, nothing
|
||||
/// errors, and the only symptom is disk.
|
||||
///
|
||||
/// Found on gw-04 2026-08-21 — a container `Up` for nine days holding 2.5G,
|
||||
/// against a `missions` table with **zero rows**.
|
||||
///
|
||||
/// # It refuses to reap work that exists nowhere else
|
||||
///
|
||||
/// That container's checkout held **ten commits on a branch that had never
|
||||
/// been pushed** (+3451/-30 across 30 files). A reaper that deleted on sight
|
||||
/// would have destroyed all of it, silently, as its designed behaviour. So
|
||||
/// before removing anything this asks the checkout whether it holds commits
|
||||
/// that no remote has, and leaves the container alone — loudly, every tick —
|
||||
/// when it does.
|
||||
///
|
||||
/// The check is deliberately one-sided in the safe direction: an inspection
|
||||
/// that fails for any reason counts as "might hold work", never as "safe to
|
||||
/// delete". Losing a day of disk to an unreadable container is recoverable;
|
||||
/// the other way round is not.
|
||||
pub async fn sweep_orphans(pool: &sqlx::PgPool, grace: std::time::Duration) -> Result<(), String> {
|
||||
let Some(prov) = MissionRuntimeProvisioner::from_env() else {
|
||||
return Ok(());
|
||||
};
|
||||
let names = prov.list_mission_containers().await?;
|
||||
if names.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
for (name, created) in names {
|
||||
let Some(id) = mission_id_from_container(&name) else {
|
||||
continue;
|
||||
};
|
||||
// `WHERE id = $1` across every workspace on purpose: the question is
|
||||
// whether ANY row still points at this container, not whether one the
|
||||
// caller can see does.
|
||||
let known: Option<(Uuid,)> = sqlx::query_as("SELECT id FROM missions WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(|e| format!("look up mission {id}: {e}"))?;
|
||||
if known.is_some() {
|
||||
continue;
|
||||
}
|
||||
match MissionRuntimeProvisioner::container_age(created) {
|
||||
Some(age) if age < grace => continue,
|
||||
None => continue,
|
||||
Some(_) => {}
|
||||
}
|
||||
match prov.unpushed_commits(&name).await {
|
||||
// The safe answer, and the one an error also produces.
|
||||
UnpushedWork::SomeOrUnknown(why) => {
|
||||
eprintln!(
|
||||
"mission_runtime::orphans: {name} has no mission row and is older than \
|
||||
the grace period, but it is NOT safe to reap: {why}. Recover the work \
|
||||
(`git bundle create … origin/main..HEAD`, or push the branch) and then \
|
||||
remove it by hand."
|
||||
);
|
||||
}
|
||||
UnpushedWork::None => {
|
||||
eprintln!(
|
||||
"mission_runtime::orphans: reaping {name} — no mission row, older than \
|
||||
the grace period, and its checkout holds nothing a remote does not"
|
||||
);
|
||||
if let Err(e) = prov.teardown_container(id).await {
|
||||
eprintln!("mission_runtime::orphans: reap {name}: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether an orphan's checkout holds commits no remote has.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum UnpushedWork {
|
||||
/// Every commit is reachable from a remote ref — nothing is lost.
|
||||
None,
|
||||
/// There IS unpushed work, or the question could not be answered. One
|
||||
/// variant for both, because the reaper must treat them identically.
|
||||
SomeOrUnknown(String),
|
||||
}
|
||||
|
||||
/// The mission id encoded in a runtime container's name, if it is one.
|
||||
///
|
||||
/// The inverse of [`container_name`], which formats the uuid `simple` (no
|
||||
/// dashes). Anything that does not parse is not ours and is left alone.
|
||||
pub fn mission_id_from_container(name: &str) -> Option<Uuid> {
|
||||
Uuid::parse_str(name.strip_prefix("cm-runtime-mission-")?).ok()
|
||||
}
|
||||
|
||||
async fn sweep_once(pool: &sqlx::PgPool, grace: std::time::Duration) -> Result<(), String> {
|
||||
use sqlx::Row;
|
||||
let grace_secs = grace.as_secs() as f64;
|
||||
@@ -1454,7 +1819,10 @@ mod tests {
|
||||
"GROQ_API_KEY" => Some("real".into()),
|
||||
_ => None,
|
||||
});
|
||||
assert_eq!(pairs, vec![("GROQ_API_KEY".to_string(), "real".to_string())]);
|
||||
assert_eq!(
|
||||
pairs,
|
||||
vec![("GROQ_API_KEY".to_string(), "real".to_string())]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1493,8 +1861,8 @@ mod tests {
|
||||
fn the_two_anthropic_credentials_are_mutually_exclusive() {
|
||||
for mode in [RuntimeAuth::ApiKey, RuntimeAuth::Subscription] {
|
||||
let keys = forwarded_provider_keys(mode);
|
||||
let both = keys.contains(&"ANTHROPIC_API_KEY")
|
||||
&& keys.contains(&"CLAUDE_CODE_OAUTH_TOKEN");
|
||||
let both =
|
||||
keys.contains(&"ANTHROPIC_API_KEY") && keys.contains(&"CLAUDE_CODE_OAUTH_TOKEN");
|
||||
assert!(!both, "{mode:?} forwards both credentials: {keys:?}");
|
||||
}
|
||||
}
|
||||
@@ -1546,6 +1914,48 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// The name→id round trip the orphan sweep depends on.
|
||||
///
|
||||
/// If this is wrong the sweep either skips every orphan (harmless) or
|
||||
/// resolves a container to the WRONG mission id and asks the database
|
||||
/// about a mission that does exist — reading "still live, leave it" for a
|
||||
/// container that is not. Cheap to get right, expensive to get wrong.
|
||||
#[test]
|
||||
fn a_container_name_round_trips_to_its_mission() {
|
||||
let id = Uuid::now_v7();
|
||||
assert_eq!(mission_id_from_container(&container_name(id)), Some(id));
|
||||
// Not ours, and not a panic.
|
||||
assert_eq!(mission_id_from_container("clawmates_server_1"), None);
|
||||
assert_eq!(mission_id_from_container("cm-runtime-mission-nonsense"), None);
|
||||
assert_eq!(mission_id_from_container("cm-sandbox-abc"), None);
|
||||
}
|
||||
|
||||
/// A container docker will not date must not be reaped.
|
||||
#[test]
|
||||
fn an_undatable_container_has_no_age() {
|
||||
assert_eq!(MissionRuntimeProvisioner::container_age(None), None);
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs() as i64;
|
||||
let age = MissionRuntimeProvisioner::container_age(Some(now - 3600)).expect("age");
|
||||
assert!(age.as_secs() >= 3500 && age.as_secs() <= 3700, "{age:?}");
|
||||
// A clock skew that puts creation in the future must not underflow into
|
||||
// a colossal age that reads as "long past the grace period".
|
||||
assert_eq!(MissionRuntimeProvisioner::container_age(Some(now + 600)), None);
|
||||
}
|
||||
|
||||
/// The grace period is long, and that is the point.
|
||||
#[test]
|
||||
fn the_orphan_grace_is_generous() {
|
||||
assert!(
|
||||
ORPHAN_GRACE >= std::time::Duration::from_secs(12 * 3600),
|
||||
"the row-driven sweep already handles everything the platform knows \
|
||||
about, so anything reaching the orphan path is unexpected — and the \
|
||||
one real orphan held ten unpushed commits"
|
||||
);
|
||||
}
|
||||
|
||||
const SAMPLE_CONFIG: &str = r#"# top comment
|
||||
[agents.claw_a]
|
||||
model_provider = "anthropic.default"
|
||||
@@ -1749,7 +2159,10 @@ allowed_tools = ["file_read", "file_edit"]
|
||||
let script = copy_script();
|
||||
// A missing entry is normal on a fresh deployment (no .kimi-code
|
||||
// until Kimi is first used) and must not fail the copy.
|
||||
assert!(script.contains("if [ -e "), "absent paths must be tolerated");
|
||||
assert!(
|
||||
script.contains("if [ -e "),
|
||||
"absent paths must be tolerated"
|
||||
);
|
||||
assert!(script.contains("/seed/.zeroclaw"));
|
||||
assert!(!script.contains("/seed/.rustup"));
|
||||
for p in SEEDED_PATHS {
|
||||
|
||||
@@ -0,0 +1,321 @@
|
||||
//! Launching missions that are due.
|
||||
//!
|
||||
//! `missions.schedule` has carried a cron since `0047_missions.sql` — the
|
||||
//! wizard collects it, the API persists it — and until this module nothing ever
|
||||
//! read it back. The only due-work enumerator in the codebase was
|
||||
//! `routines::claim_due`, so **every scheduled mission ever created sat in
|
||||
//! `draft` forever** while the UI reported it was on a schedule. Measured
|
||||
//! before this was written: a mission with `* * * * *` did not move for four
|
||||
//! minutes and started no runs.
|
||||
//!
|
||||
//! The shape here is deliberately `cm-scheduler`'s, not a second invention:
|
||||
//!
|
||||
//! - **Claim atomically** (`FOR UPDATE SKIP LOCKED`) so replicas fire once.
|
||||
//! - **Advance the clock before dispatching**, so a failing launch cannot stall
|
||||
//! the schedule.
|
||||
//! - **Record the claim first** in `mission_fires`, keyed by the occurrence's
|
||||
//! own timestamp, so a crash between those two is retried rather than
|
||||
//! silently dropped — and a slot already launched is never launched twice.
|
||||
//! - **Cap the fan-out**, because a backlog would otherwise start one container
|
||||
//! per missed occurrence.
|
||||
//!
|
||||
//! The one thing it does NOT share with routines is the launch itself: a due
|
||||
//! mission goes through `mission_orchestrator::on_launch` and
|
||||
//! `missions::set_status`, exactly as the draft→running transition in
|
||||
//! `routes::missions::set_status` does, so there is one path that mints a crew.
|
||||
|
||||
use cm_db::repo::missions as missions_repo;
|
||||
use sqlx::{PgPool, Row};
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Most missions one tick will launch.
|
||||
///
|
||||
/// Lower than the scheduler's 25: a mission firing is a container, a repo
|
||||
/// checkout and real model spend, where a routine firing may be a single turn.
|
||||
/// The remainder stays due and is taken by the next tick.
|
||||
const MAX_LAUNCHES_PER_TICK: usize = 5;
|
||||
|
||||
/// A mission whose occurrence has come due and been claimed.
|
||||
#[derive(Debug)]
|
||||
pub struct DueMission {
|
||||
pub id: Uuid,
|
||||
pub workspace_id: Uuid,
|
||||
pub title: String,
|
||||
pub cron: Option<String>,
|
||||
/// The occurrence that came due — the value `next_run_at` held. Identifies
|
||||
/// the slot in `mission_fires`, so it must not be re-read from the clock.
|
||||
pub slot: OffsetDateTime,
|
||||
}
|
||||
|
||||
/// Claim every mission due at `now`, atomically.
|
||||
///
|
||||
/// `next_run_at` is cleared by the claim. The caller recomputes it from the
|
||||
/// cron and writes it back; a mission whose cron no longer yields an occurrence
|
||||
/// simply stays cleared and stops firing, which is the correct end state for
|
||||
/// a one-shot or an exhausted schedule.
|
||||
pub async fn claim_due(pool: &PgPool, now: OffsetDateTime) -> Result<Vec<DueMission>, String> {
|
||||
let rows = sqlx::query(
|
||||
"UPDATE missions SET next_run_at = NULL
|
||||
WHERE id IN (
|
||||
SELECT id FROM missions
|
||||
WHERE next_run_at IS NOT NULL
|
||||
AND next_run_at <= $1
|
||||
-- Never relaunch a mission that is mid-flight. A daily cron on
|
||||
-- a mission that takes longer than a day must skip the
|
||||
-- occurrence, not stack a second crew on the same workspace.
|
||||
AND status <> 'running'
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
RETURNING id, workspace_id, title, schedule ->> 'cron' AS cron, $1::timestamptz AS slot",
|
||||
)
|
||||
.bind(now)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| format!("claim due missions: {e}"))?;
|
||||
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|r| DueMission {
|
||||
id: r.get("id"),
|
||||
workspace_id: r.get("workspace_id"),
|
||||
title: r.get("title"),
|
||||
cron: r.get("cron"),
|
||||
slot: r.get("slot"),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Record that this occurrence was taken. `false` means another replica (or an
|
||||
/// earlier attempt) already has it and this one must not launch.
|
||||
async fn claim_slot(pool: &PgPool, mission_id: Uuid, slot: OffsetDateTime) -> Result<bool, String> {
|
||||
let inserted = sqlx::query(
|
||||
"INSERT INTO mission_fires (mission_id, scheduled_at, status)
|
||||
VALUES ($1, $2, 'claimed')
|
||||
ON CONFLICT (mission_id, scheduled_at) DO NOTHING",
|
||||
)
|
||||
.bind(mission_id)
|
||||
.bind(slot)
|
||||
.execute(pool)
|
||||
.await
|
||||
.map_err(|e| format!("claim mission fire: {e}"))?;
|
||||
Ok(inserted.rows_affected() == 1)
|
||||
}
|
||||
|
||||
async fn settle_slot(
|
||||
pool: &PgPool,
|
||||
mission_id: Uuid,
|
||||
slot: OffsetDateTime,
|
||||
status: &str,
|
||||
detail: Option<&str>,
|
||||
) {
|
||||
if let Err(e) = sqlx::query(
|
||||
"UPDATE mission_fires SET status = $3, detail = $4, completed_at = now()
|
||||
WHERE mission_id = $1 AND scheduled_at = $2",
|
||||
)
|
||||
.bind(mission_id)
|
||||
.bind(slot)
|
||||
.bind(status)
|
||||
.bind(detail)
|
||||
.execute(pool)
|
||||
.await
|
||||
{
|
||||
eprintln!("mission_schedule: settling {mission_id} @ {slot} as {status}: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute and persist the next occurrence.
|
||||
///
|
||||
/// A cron that will not parse is reported and the mission left un-scheduled
|
||||
/// rather than skipped in silence — the whole point of this module is that a
|
||||
/// schedule which does nothing must never look like a schedule that works.
|
||||
async fn reschedule(pool: &PgPool, m: &DueMission, after: OffsetDateTime) {
|
||||
let Some(cron) = m.cron.as_deref().map(str::trim).filter(|c| !c.is_empty()) else {
|
||||
return;
|
||||
};
|
||||
match cm_runtime::scheduling::next_occurrence(cron, after) {
|
||||
Ok(next) => {
|
||||
if let Err(e) = sqlx::query("UPDATE missions SET next_run_at = $2 WHERE id = $1")
|
||||
.bind(m.id)
|
||||
.bind(next)
|
||||
.execute(pool)
|
||||
.await
|
||||
{
|
||||
eprintln!("mission_schedule: could not set next_run_at for {}: {e}", m.id);
|
||||
}
|
||||
}
|
||||
Err(e) => eprintln!(
|
||||
"mission_schedule: mission {} ({}) has an unusable cron {cron:?} — it will NOT run \
|
||||
again until the schedule is corrected: {e}",
|
||||
m.id, m.title
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// One pass. Returns how many missions were launched.
|
||||
pub async fn tick(
|
||||
pool: &PgPool,
|
||||
node_hub: Option<std::sync::Arc<crate::fleet::NodeHub>>,
|
||||
blobs: Option<std::sync::Arc<dyn cm_files::BlobStore>>,
|
||||
now: OffsetDateTime,
|
||||
) -> Result<usize, String> {
|
||||
let due = claim_due(pool, now).await?;
|
||||
let mut launched = 0usize;
|
||||
|
||||
for m in due.iter().take(MAX_LAUNCHES_PER_TICK) {
|
||||
// Clock first: a launch that fails must not stall the schedule.
|
||||
reschedule(pool, m, now).await;
|
||||
|
||||
if !claim_slot(pool, m.id, m.slot).await? {
|
||||
continue;
|
||||
}
|
||||
|
||||
// An unattended launch still needs an actor. Missions carry no creator
|
||||
// column, so the workspace owner stands in — the same identity the
|
||||
// audit trail already attributes workspace-level action to.
|
||||
let workspace = cm_domain::WorkspaceId::from(m.workspace_id);
|
||||
let owner = match cm_db::repo::users::owner_of_workspace(pool, workspace).await {
|
||||
Ok(u) => u,
|
||||
Err(e) => {
|
||||
// `fetch_one`, so "no owner" arrives as RowNotFound rather than
|
||||
// None. Either way the occurrence is settled `failed` with the
|
||||
// reason, never dropped quietly.
|
||||
let why = format!("no owner to launch as: {e}");
|
||||
eprintln!("mission_schedule: cannot launch {} — {why}", m.id);
|
||||
settle_slot(pool, m.id, m.slot, "failed", Some(&why)).await;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
match crate::mission_orchestrator::on_launch(
|
||||
pool,
|
||||
workspace,
|
||||
owner,
|
||||
m.id,
|
||||
node_hub.clone(),
|
||||
blobs.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
if let Err(e) =
|
||||
missions_repo::set_status(pool, m.id, m.workspace_id, "running").await
|
||||
{
|
||||
let why = format!("launched but could not mark running: {e}");
|
||||
eprintln!("mission_schedule: {} — {why}", m.id);
|
||||
settle_slot(pool, m.id, m.slot, "failed", Some(&why)).await;
|
||||
continue;
|
||||
}
|
||||
settle_slot(pool, m.id, m.slot, "fired", None).await;
|
||||
launched += 1;
|
||||
eprintln!(
|
||||
"mission_schedule: launched {} ({}) for occurrence {}",
|
||||
m.id, m.title, m.slot
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("mission_schedule: on_launch failed for {}: {e}", m.id);
|
||||
settle_slot(pool, m.id, m.slot, "failed", Some(&e)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if due.len() > MAX_LAUNCHES_PER_TICK {
|
||||
eprintln!(
|
||||
"mission_schedule: {} due, launched {} this tick (cap {}); the rest stay due",
|
||||
due.len(),
|
||||
launched,
|
||||
MAX_LAUNCHES_PER_TICK
|
||||
);
|
||||
}
|
||||
Ok(launched)
|
||||
}
|
||||
|
||||
/// Spawn the sweep.
|
||||
pub fn spawn(
|
||||
pool: PgPool,
|
||||
node_hub: Option<std::sync::Arc<crate::fleet::NodeHub>>,
|
||||
blobs: Option<std::sync::Arc<dyn cm_files::BlobStore>>,
|
||||
interval: std::time::Duration,
|
||||
) {
|
||||
tokio::spawn(async move {
|
||||
let mut ticker = tokio::time::interval(interval);
|
||||
// Skip the immediate first tick so a restart loop cannot become a
|
||||
// launch loop.
|
||||
ticker.tick().await;
|
||||
loop {
|
||||
ticker.tick().await;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
match tick(&pool, node_hub.clone(), blobs.clone(), now).await {
|
||||
Ok(n) if n > 0 => eprintln!("mission_schedule: launched {n} due mission(s)"),
|
||||
Ok(_) => {}
|
||||
Err(e) => eprintln!("mission_schedule: sweep failed: {e}"),
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The cap is what stops a backlog turning into a container stampede. A
|
||||
/// clock jump or a cron that resolves to "every minute" can leave hundreds
|
||||
/// of occurrences owed; each mission launch is a container, a checkout and
|
||||
/// real model spend, so this must stay well below the routine scheduler's
|
||||
/// 25.
|
||||
#[test]
|
||||
fn the_launch_cap_is_conservative() {
|
||||
assert!(
|
||||
MAX_LAUNCHES_PER_TICK <= 5,
|
||||
"a mission firing costs far more than a routine firing"
|
||||
);
|
||||
assert!(MAX_LAUNCHES_PER_TICK >= 1, "a cap of zero never launches");
|
||||
}
|
||||
|
||||
/// The claim must never pick up a mission that is already running.
|
||||
///
|
||||
/// A daily cron on a mission that takes longer than a day would otherwise
|
||||
/// stack a second crew on the same workspace — two containers, two vault
|
||||
/// branches, and a seen-set race. Asserted against the SQL text because the
|
||||
/// predicate is the whole safety property and it lives only in the query.
|
||||
#[test]
|
||||
fn the_claim_skips_missions_that_are_still_running() {
|
||||
// Re-read the source of the query this module issues.
|
||||
let src = include_str!("mission_schedule.rs");
|
||||
let claim = src
|
||||
.split("pub async fn claim_due")
|
||||
.nth(1)
|
||||
.expect("claim_due exists");
|
||||
let body = &claim[..claim.find("fetch_all").unwrap_or(claim.len())];
|
||||
assert!(
|
||||
body.contains("status <> 'running'"),
|
||||
"claim_due must not relaunch a mission that is mid-flight"
|
||||
);
|
||||
assert!(
|
||||
body.contains("FOR UPDATE SKIP LOCKED"),
|
||||
"the claim must be atomic or replicas double-launch"
|
||||
);
|
||||
assert!(
|
||||
body.contains("next_run_at <= $1"),
|
||||
"only occurrences that have come due may be claimed"
|
||||
);
|
||||
}
|
||||
|
||||
/// The clock advances BEFORE the launch, and the slot is claimed before the
|
||||
/// launch too. Both orderings matter: reschedule-first means a failing
|
||||
/// launch cannot stall the schedule; claim-first means a crash mid-launch
|
||||
/// is retried rather than dropped.
|
||||
#[test]
|
||||
fn the_clock_advances_before_the_launch_is_attempted() {
|
||||
let src = include_str!("mission_schedule.rs");
|
||||
let tick = src.split("pub async fn tick").nth(1).expect("tick exists");
|
||||
let resched = tick.find("reschedule(pool, m, now)").expect("reschedules");
|
||||
let claim = tick.find("claim_slot(pool, m.id, m.slot)").expect("claims");
|
||||
let launch = tick.find("on_launch(").expect("launches");
|
||||
assert!(
|
||||
resched < claim && claim < launch,
|
||||
"order must be reschedule -> claim -> launch (got {resched}, {claim}, {launch})"
|
||||
);
|
||||
}
|
||||
}
|
||||
+140
-1
@@ -143,13 +143,89 @@ fn unescape(s: &str) -> String {
|
||||
.replace("'", "'")
|
||||
}
|
||||
|
||||
/// Turn an operator topic into an arXiv `search_query`.
|
||||
///
|
||||
/// A bare topic is NOT a search. Passed through unfielded, arXiv matched
|
||||
/// essentially nothing and `sortBy=submittedDate` then returned the newest
|
||||
/// submissions across the whole archive — so a run for "speculative decoding"
|
||||
/// shelved Galois extensions, a quantum black hole microstate, and blazar dark
|
||||
/// matter in IceCube. Measured against the live API:
|
||||
///
|
||||
/// ```text
|
||||
/// speculative decoding -> pixel-space diffusion, simplicial actions
|
||||
/// all:"speculative decoding" -> S2-MoE self-speculative decoding, DARTree
|
||||
/// ```
|
||||
///
|
||||
/// So the phrase is quoted into `all:` (title, abstract, authors, comments) and
|
||||
/// constrained to `cat:cs.*` — this library exists to serve software projects,
|
||||
/// and without the category bound the archive's physics and maths volume
|
||||
/// dominates every recency-sorted result.
|
||||
///
|
||||
/// A topic that already looks fielded (`cat:`, `ti:`, `abs:`, `all:`) is passed
|
||||
/// through untouched, so an operator who knows arXiv's syntax keeps full control.
|
||||
pub fn arxiv_query(topic: &str) -> String {
|
||||
let t = topic.trim();
|
||||
const FIELDED: &[&str] = &["all:", "ti:", "abs:", "au:", "cat:", "co:", "jr:"];
|
||||
// Only a topic that STARTS with a field prefix is treated as hand-written
|
||||
// arXiv syntax. Also accepting anything containing " AND "/" OR " was the
|
||||
// first version, and a test caught it immediately: `agent" OR cat:hep-th`
|
||||
// passed straight through, so a topic string could escape the phrase and
|
||||
// rewrite the category bound. A natural-language topic may legitimately
|
||||
// contain the word "and" too.
|
||||
if FIELDED.iter().any(|p| t.starts_with(p)) {
|
||||
return t.to_string();
|
||||
}
|
||||
// Quotes make it a phrase; without them "vector index pruning" matches any
|
||||
// paper containing all three words anywhere, which is most of cs.
|
||||
let escaped = t.replace('"', "");
|
||||
format!("all:\"{escaped}\" AND cat:cs.*")
|
||||
}
|
||||
|
||||
/// The looser form of a topic: every term required, but not adjacent.
|
||||
///
|
||||
/// A quoted phrase is precise and brittle. "hybrid retrieval BM25 dense" is a
|
||||
/// perfectly good topic and appears verbatim in no paper on arXiv — measured, 0
|
||||
/// hits — while requiring the same four terms anywhere returns exactly the
|
||||
/// hybrid-retrieval evaluations the topic was asking for. Used only when the
|
||||
/// phrase finds nothing, so an exact match still wins when one exists.
|
||||
pub fn arxiv_query_broad(topic: &str) -> String {
|
||||
let terms: Vec<String> = topic
|
||||
.split_whitespace()
|
||||
.map(|w| w.trim_matches(|c: char| !c.is_alphanumeric() && c != '-'))
|
||||
.filter(|w| !w.is_empty())
|
||||
.map(|w| format!("all:{w}"))
|
||||
.collect();
|
||||
if terms.is_empty() {
|
||||
return arxiv_query(topic);
|
||||
}
|
||||
format!("{} AND cat:cs.*", terms.join(" AND "))
|
||||
}
|
||||
|
||||
/// Search arXiv. `max_results` is capped to keep one run bounded.
|
||||
pub async fn search(query: &str, max_results: usize) -> Result<Vec<Paper>, String> {
|
||||
let found = search_with(&arxiv_query(query), max_results).await?;
|
||||
if !found.is_empty() {
|
||||
return Ok(found);
|
||||
}
|
||||
// The phrase matched nothing. Before reporting a quiet day — which the whole
|
||||
// pipeline treats as a real and legitimate outcome — try the same terms
|
||||
// unquoted. A topic the operator writes as prose often is not a literal
|
||||
// phrase in any title, and silently harvesting zero because of punctuation
|
||||
// would be indistinguishable from a genuinely quiet field.
|
||||
let broad = arxiv_query_broad(query);
|
||||
if broad == arxiv_query(query) {
|
||||
return Ok(found);
|
||||
}
|
||||
eprintln!("papers: no exact phrase match for {query:?} — retrying as {broad}");
|
||||
search_with(&broad, max_results).await
|
||||
}
|
||||
|
||||
async fn search_with(search_query: &str, max_results: usize) -> Result<Vec<Paper>, String> {
|
||||
let max = max_results.clamp(1, 50);
|
||||
let url = format!(
|
||||
"https://export.arxiv.org/api/query?search_query={}&start=0&max_results={max}\
|
||||
&sortBy=submittedDate&sortOrder=descending",
|
||||
urlencoding(query)
|
||||
urlencoding(search_query)
|
||||
);
|
||||
let body = reqwest::Client::new()
|
||||
.get(&url)
|
||||
@@ -251,6 +327,69 @@ fn urlencoding(s: &str) -> String {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
/// A bare topic must become a PHRASE search bound to cs — unfielded, arXiv
|
||||
/// matched nothing and recency-sort returned the whole archive, so a run
|
||||
/// for "speculative decoding" shelved blazar dark matter in IceCube.
|
||||
#[test]
|
||||
fn a_bare_topic_becomes_a_fielded_phrase_query() {
|
||||
let q = arxiv_query("speculative decoding");
|
||||
assert_eq!(q, "all:\"speculative decoding\" AND cat:cs.*");
|
||||
assert!(q.contains('"'), "unquoted, the words match separately");
|
||||
assert!(q.contains("cat:cs.*"), "without a category bound physics wins");
|
||||
}
|
||||
|
||||
/// An operator who writes arXiv syntax keeps control — wrapping their query
|
||||
/// in another `all:"..."` would search for the literal text of their query.
|
||||
#[test]
|
||||
fn an_already_fielded_topic_is_left_alone() {
|
||||
for q in [
|
||||
"cat:cs.IR AND all:\"dense retrieval\"",
|
||||
"ti:\"world model\"",
|
||||
"abs:hnsw OR abs:\"vector index\"",
|
||||
] {
|
||||
assert_eq!(arxiv_query(q), q, "{q} must pass through untouched");
|
||||
}
|
||||
}
|
||||
|
||||
/// The broad form requires every term but not adjacency. Measured: the
|
||||
/// phrase "hybrid retrieval BM25 dense" has 0 hits on arXiv; the same four
|
||||
/// terms unquoted return the hybrid-retrieval evaluations that were asked
|
||||
/// for. Without the fallback that topic silently harvests nothing, which is
|
||||
/// indistinguishable from a genuinely quiet day.
|
||||
#[test]
|
||||
fn the_broad_form_requires_every_term_without_adjacency() {
|
||||
let q = arxiv_query_broad("hybrid retrieval BM25 dense");
|
||||
assert_eq!(
|
||||
q,
|
||||
"all:hybrid AND all:retrieval AND all:BM25 AND all:dense AND cat:cs.*"
|
||||
);
|
||||
assert!(!q.contains('"'), "the broad form must not be a phrase: {q}");
|
||||
assert!(q.contains("cat:cs.*"), "still category-bound: {q}");
|
||||
}
|
||||
|
||||
/// Punctuation must not leak into a term and must not empty the query.
|
||||
#[test]
|
||||
fn the_broad_form_strips_punctuation_and_never_empties() {
|
||||
assert_eq!(
|
||||
arxiv_query_broad("retrieval-augmented, generation!"),
|
||||
"all:retrieval-augmented AND all:generation AND cat:cs.*",
|
||||
"hyphens are part of a term; trailing punctuation is not"
|
||||
);
|
||||
// Nothing usable left: fall back to the phrase form rather than
|
||||
// emitting a bare `cat:cs.*`, which would match all of computer science.
|
||||
let q = arxiv_query_broad("!!!");
|
||||
assert!(q.contains("all:"), "must never degrade to a bare category: {q}");
|
||||
}
|
||||
|
||||
/// Quotes in a topic would terminate the phrase early and corrupt the query.
|
||||
#[test]
|
||||
fn quotes_in_a_topic_cannot_break_out_of_the_phrase() {
|
||||
let q = arxiv_query("agent\" OR cat:hep-th");
|
||||
assert_eq!(q.matches('"').count(), 2, "exactly one balanced phrase: {q}");
|
||||
assert!(q.ends_with("cat:cs.*"), "{q}");
|
||||
}
|
||||
|
||||
use super::*;
|
||||
|
||||
/// A revision must not read as a new paper.
|
||||
|
||||
@@ -53,6 +53,44 @@ pub const KNOWN_KEYS: &[KnownKey] = &[
|
||||
phase that changes no files still completes; also vm_stop_gate::\
|
||||
StopGate::for_phase, where it drops the in-loop delivery check",
|
||||
},
|
||||
KnownKey {
|
||||
key: "agent_tools",
|
||||
read_by: "vm_tool_gate::TaskPolicy::for_phase — the tools this phase's \
|
||||
agents may use at all, enforced by the PreToolUse gate. \
|
||||
Absent means the default work surface (files, commands, \
|
||||
search, web, delegation); the platform-control tools are \
|
||||
never in it. DISTINCT from `tools` below, which is \
|
||||
security_scan's scanner list — two keys, two meanings, and \
|
||||
they are next to each other here so nobody conflates them. \
|
||||
Shadow unless CLAWMATES_TASK_PERMISSION=enforce.",
|
||||
},
|
||||
KnownKey {
|
||||
key: "tools",
|
||||
read_by: "security_scan::run — gates which of cargo_audit / gitleaks / \
|
||||
trivy_fs / semgrep run against the phase's checkout; absent \
|
||||
means all four. Listed here as NOT IMPLEMENTED while wired, \
|
||||
which understated the recipe: the key was real, what was \
|
||||
missing was anything that FIRED the scan outside an operator \
|
||||
button — now phase_runner::scan_finished_security_phases",
|
||||
},
|
||||
KnownKey {
|
||||
key: "harness",
|
||||
read_by: "benchmark_runner::harness_from_config — selects criterion / \
|
||||
cargo_bench / vitest_bench / pytest_bench / shell, with \
|
||||
`bench_name` (criterion) and `cmd` (shell) as its arguments. \
|
||||
phase_runner's benchmark sweep runs the baseline through it. \
|
||||
This key was listed as NOT IMPLEMENTED while being fully \
|
||||
wired, which is worse than an unread key: the registry exists \
|
||||
so an operator can trust what a recipe does, and it was wrong",
|
||||
},
|
||||
KnownKey {
|
||||
key: "bench_name",
|
||||
read_by: "benchmark_runner::harness_from_config — the criterion bench target",
|
||||
},
|
||||
KnownKey {
|
||||
key: "cmd",
|
||||
read_by: "benchmark_runner::harness_from_config — the shell harness command line",
|
||||
},
|
||||
KnownKey {
|
||||
key: "done_when_check",
|
||||
read_by: "vm_stop_gate::StopGate::for_phase — a shell command the agent's \
|
||||
@@ -84,14 +122,6 @@ pub const DECLARED_BUT_UNREAD: &[KnownKey] = &[
|
||||
key: "mode",
|
||||
read_by: "NOT IMPLEMENTED — benchmark/refactor mode selection",
|
||||
},
|
||||
KnownKey {
|
||||
key: "harness",
|
||||
read_by: "NOT IMPLEMENTED — benchmark harness selection",
|
||||
},
|
||||
KnownKey {
|
||||
key: "tools",
|
||||
read_by: "NOT IMPLEMENTED — per-phase tool selection",
|
||||
},
|
||||
KnownKey {
|
||||
key: "benchmark",
|
||||
read_by: "NOT IMPLEMENTED — nested benchmark settings",
|
||||
@@ -104,11 +134,6 @@ pub const DECLARED_BUT_UNREAD: &[KnownKey] = &[
|
||||
setting this per phase changes nothing: security_hardening.toml \
|
||||
asks for gitea_forge + security_scan and its phase gets neither",
|
||||
},
|
||||
KnownKey {
|
||||
key: "test_command",
|
||||
read_by: "NOT IMPLEMENTED — mission_delivery::discover_test_command infers \
|
||||
from the repo and does not consult config",
|
||||
},
|
||||
];
|
||||
|
||||
fn is_listed(key: &str, list: &[KnownKey]) -> bool {
|
||||
@@ -240,18 +265,31 @@ mod tests {
|
||||
"order_idx",
|
||||
"requires_repo",
|
||||
"default_team_template",
|
||||
"default_phase_teams",
|
||||
"default_topology",
|
||||
"phases",
|
||||
"description",
|
||||
];
|
||||
// `[default_phase_teams]` maps a phase PURPOSE to a team template key,
|
||||
// so its keys are not config keys and must not be checked as such.
|
||||
// They are checked against the purposes `phase_runner::purposes_for`
|
||||
// can actually emit instead — a typo'd purpose matches no phase and
|
||||
// that phase silently falls back to the mission-wide team, which is
|
||||
// exactly the kind of quiet wrong staffing this table exists to end.
|
||||
const PURPOSES: &[&str] = &["research", "coding", "security", "mission"];
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.extension().and_then(|e| e.to_str()) != Some("toml") {
|
||||
continue;
|
||||
}
|
||||
let body = std::fs::read_to_string(&path).unwrap();
|
||||
let mut table = String::new();
|
||||
for line in body.lines() {
|
||||
let line = line.trim();
|
||||
if line.starts_with('[') {
|
||||
table = line.trim_matches(['[', ']'].as_slice()).to_string();
|
||||
continue;
|
||||
}
|
||||
if line.starts_with('#') || !line.contains('=') {
|
||||
continue;
|
||||
}
|
||||
@@ -259,6 +297,16 @@ mod tests {
|
||||
if key.is_empty() || key.contains(' ') || key.contains('[') {
|
||||
continue;
|
||||
}
|
||||
if table == "default_phase_teams" {
|
||||
assert!(
|
||||
PURPOSES.contains(&key),
|
||||
"{} staffs purpose `{key}`, which `purposes_for` never emits — \
|
||||
that phase would fall back to the mission-wide team with \
|
||||
nothing reporting it",
|
||||
path.display()
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let accounted = ENVELOPE.contains(&key)
|
||||
|| is_listed(key, KNOWN_KEYS)
|
||||
|| is_listed(key, DECLARED_BUT_UNREAD);
|
||||
|
||||
+1322
-75
File diff suppressed because it is too large
Load Diff
@@ -22,12 +22,34 @@ use sqlx::Row;
|
||||
use std::time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEFAULT_MODEL: &str = "claude-opus-4-8";
|
||||
const DEFAULT_MODEL: &str = "claude-opus-5";
|
||||
const POLL_INTERVAL: Duration = Duration::from_secs(30);
|
||||
/// Cap the raw material we send to the model. Missions can produce
|
||||
/// hundreds of KB of agent output; we slice by turn and by phase
|
||||
/// artifact but still bound the total prompt.
|
||||
const MAX_OUTPUT_BYTES: usize = 60_000;
|
||||
const MAX_OUTPUT_BYTES: usize = 120_000;
|
||||
|
||||
/// The longest prefix of `s` that is at most `max_bytes` and ends on a
|
||||
/// character boundary.
|
||||
///
|
||||
/// `&s[..max_bytes]` PANICS when the cut lands inside a multi-byte character,
|
||||
/// and `s` here is agent-authored turn output — arbitrary UTF-8, routinely
|
||||
/// containing arrows, box-drawing and emoji. The panic would take down the
|
||||
/// evaluation sweep for a phase whose only crime was writing a long enough
|
||||
/// line with a non-ASCII character at the wrong offset.
|
||||
///
|
||||
/// Exactly the bug the clawhdf5 agents found and fixed in
|
||||
/// `clawhdf5-migrate/src/validate.rs` this week, in our own code.
|
||||
fn clamp_to_char_boundary(s: &str, max_bytes: usize) -> &str {
|
||||
if s.len() <= max_bytes {
|
||||
return s;
|
||||
}
|
||||
let mut end = max_bytes;
|
||||
while end > 0 && !s.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
&s[..end]
|
||||
}
|
||||
|
||||
fn model_name() -> String {
|
||||
std::env::var("CLAWMATES_SUMMARIZER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
||||
@@ -95,7 +117,7 @@ async fn summarize_one(
|
||||
mission_id,
|
||||
phase_id,
|
||||
kind,
|
||||
"claude-opus-4-8",
|
||||
"claude-opus-5",
|
||||
"This phase produced no recorded output. The agents may have failed \
|
||||
to reach their working directory or found nothing to act on.",
|
||||
&json!({
|
||||
@@ -254,7 +276,7 @@ async fn collect_material(
|
||||
concat.push_str(&format!("\n\n── turn {} ──\n", i + 1));
|
||||
let remaining = MAX_OUTPUT_BYTES.saturating_sub(concat.len());
|
||||
if s.len() > remaining {
|
||||
concat.push_str(&s[..remaining]);
|
||||
concat.push_str(clamp_to_char_boundary(&s, remaining));
|
||||
concat.push_str("\n… (truncated)");
|
||||
} else {
|
||||
concat.push_str(&s);
|
||||
@@ -578,3 +600,39 @@ async fn record_error(
|
||||
.map_err(|e| format!("record error: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Agent output is arbitrary UTF-8. A byte-offset cut that lands inside a
|
||||
/// multi-byte character must not panic — that panic would take down the
|
||||
/// evaluation sweep for the phase, and the only trigger is an agent
|
||||
/// happening to write a long enough line containing a non-ASCII character.
|
||||
#[test]
|
||||
fn truncation_never_splits_a_multibyte_character() {
|
||||
// 4-byte characters, so every offset not a multiple of 4 is
|
||||
// mid-character and would panic a naive `&s[..cut]`.
|
||||
let s = "😀".repeat(10);
|
||||
for cut in 0..=s.len() {
|
||||
let out = clamp_to_char_boundary(&s, cut);
|
||||
assert!(out.len() <= cut, "must respect the budget at cut={cut}");
|
||||
assert!(s.starts_with(out), "must stay a prefix at cut={cut}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Mixed-width text: the cut must land on a boundary, never inside `é`.
|
||||
#[test]
|
||||
fn truncation_handles_mixed_width_text() {
|
||||
let s = "héllo wörld";
|
||||
for cut in 0..=s.len() {
|
||||
let out = clamp_to_char_boundary(s, cut);
|
||||
assert!(s.starts_with(out));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncation_returns_everything_when_it_fits() {
|
||||
assert_eq!(clamp_to_char_boundary("héllo", 100), "héllo");
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -173,6 +173,7 @@ impl TurnExecutor for SubTopologyExecutor {
|
||||
output: record.final_output,
|
||||
tokens: record.totals.tokens,
|
||||
gated,
|
||||
spend: Default::default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,6 +56,27 @@ async fn decide(
|
||||
workspace_approval(&state, &user, id).await?;
|
||||
let approval = approvals::decide(&state.pool, id, user.user_id, decision).await?;
|
||||
|
||||
// A held DOOR action has no chat run to resume: the tool itself is what
|
||||
// was waiting. Approve executes it now, with the grant decide just
|
||||
// minted; reject leaves the audit trail decide already wrote.
|
||||
if approval
|
||||
.session_key
|
||||
.starts_with(crate::mcp_door::HELD_SESSION_KEY_PREFIX)
|
||||
{
|
||||
let executed = if decision == Decision::Approve {
|
||||
Some(crate::mcp_door::execute_held(&state, &approval).await)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
return Ok(Json(json!({
|
||||
"id": approval.id,
|
||||
"status": approval.status,
|
||||
"door_action": approval.action_type,
|
||||
"executed": executed.as_ref().map(|r| r.is_ok()),
|
||||
"error": executed.and_then(|r| r.err()),
|
||||
})));
|
||||
}
|
||||
|
||||
// Kick the resume before returning. resume_run's awaited portion is only
|
||||
// the setup (claim + checkpoint load + open the broadcast channel); it
|
||||
// spawns the actual multi-step work internally, so this doesn't block the
|
||||
|
||||
@@ -1407,3 +1407,58 @@ pub async fn settings_full(
|
||||
"managed_by_name": manager.display_name,
|
||||
})))
|
||||
}
|
||||
|
||||
/// `GET /api/claws/lifecycle` — the agent census.
|
||||
///
|
||||
/// Answers "who is working, who is finished, and who is bound to nothing" in
|
||||
/// one place, which previously required reading the database by hand.
|
||||
pub async fn lifecycle_census(
|
||||
State(state): State<AppState>,
|
||||
Authed(user): Authed,
|
||||
) -> Result<axum::Json<serde_json::Value>, ApiError> {
|
||||
let rows = crate::agent_lifecycle::census(&state.pool, user.workspace_id.as_uuid())
|
||||
.await
|
||||
.map_err(|e| {
|
||||
eprintln!("claws::lifecycle_census: {e}");
|
||||
ApiError::Internal
|
||||
})?;
|
||||
let mut counts = std::collections::BTreeMap::<&str, usize>::new();
|
||||
for c in &rows {
|
||||
*counts.entry(c.state.as_str()).or_default() += 1;
|
||||
}
|
||||
Ok(axum::Json(serde_json::json!({
|
||||
"counts": counts,
|
||||
"agents": rows.iter().map(|c| serde_json::json!({
|
||||
"id": c.id,
|
||||
"name": c.name,
|
||||
"state": c.state.as_str(),
|
||||
"reapable": c.state.reapable(),
|
||||
"finished_hours_ago": c.finished_hours_ago,
|
||||
})).collect::<Vec<_>>(),
|
||||
})))
|
||||
}
|
||||
|
||||
/// `POST /api/claws/lifecycle/sweep` — run the reap now.
|
||||
///
|
||||
/// The sweeper is hourly; this exists so an operator does not have to wait an
|
||||
/// hour to see the effect of a decision they already made.
|
||||
pub async fn lifecycle_sweep(
|
||||
State(state): State<AppState>,
|
||||
Authed(_user): Authed,
|
||||
) -> Result<axum::Json<serde_json::Value>, ApiError> {
|
||||
let swept = crate::agent_lifecycle::sweep(
|
||||
&state.pool,
|
||||
&state.runtime,
|
||||
crate::agent_lifecycle::COMPLETED_GRACE_HOURS,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
eprintln!("claws::lifecycle_sweep: {e}");
|
||||
ApiError::Internal
|
||||
})?;
|
||||
Ok(axum::Json(serde_json::json!({
|
||||
"reaped": swept.reaped,
|
||||
"failed": swept.failed,
|
||||
"kept_in_grace": swept.kept_in_grace,
|
||||
})))
|
||||
}
|
||||
|
||||
@@ -14,8 +14,8 @@ use crate::{ApiError, AppState, Authed};
|
||||
/// Default corpus + repo. Single-operator deployment, so these are constants
|
||||
/// rather than another table to keep in sync; a second library becomes a
|
||||
/// request field the day one exists.
|
||||
const DEFAULT_CORPUS: &str = "valhalla-vault";
|
||||
const DEFAULT_VAULT_URL: &str = "https://git.redclaw.dev/redclaw/valhalla-vault.git";
|
||||
pub const DEFAULT_CORPUS: &str = "valhalla-vault";
|
||||
pub const DEFAULT_VAULT_URL: &str = "https://git.redclaw.dev/redclaw/valhalla-vault.git";
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct RunRequest {
|
||||
|
||||
@@ -48,7 +48,12 @@ async fn repo_digest(pool: &sqlx::PgPool, mission_id: uuid::Uuid) -> String {
|
||||
return "(this mission has no repository)".to_string();
|
||||
};
|
||||
let branch = branch.unwrap_or_else(|| "main".to_string());
|
||||
// Distinguish "no credential" from "the forge said no". Both used to
|
||||
// arrive as the same "(could not be read)" string, so an unconfigured
|
||||
// deployment looked identical to a private repo — and the planner, told
|
||||
// only that the read failed, cannot say which.
|
||||
let token = std::env::var("GITEA_TOKEN").unwrap_or_default();
|
||||
let unauthenticated = token.trim().is_empty();
|
||||
let Ok(client) = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(20))
|
||||
.build()
|
||||
@@ -70,6 +75,11 @@ async fn repo_digest(pool: &sqlx::PgPool, mission_id: uuid::Uuid) -> String {
|
||||
);
|
||||
let tree: serde_json::Value = match auth(client.get(&tree_url)).send().await {
|
||||
Ok(r) if r.status().is_success() => r.json().await.unwrap_or_default(),
|
||||
_ if unauthenticated => {
|
||||
return "(the repository tree could not be read: GITEA_TOKEN is unset, \
|
||||
so this read was unauthenticated)"
|
||||
.to_string()
|
||||
}
|
||||
_ => return "(the repository tree could not be read)".to_string(),
|
||||
};
|
||||
let entries: Vec<crate::repo_digest::FileEntry> = tree
|
||||
@@ -304,8 +314,11 @@ pub async fn decide(
|
||||
.map_err(|_| ApiError::Internal)?
|
||||
.ok_or(ApiError::NotFound)?;
|
||||
if mission.status != "draft" {
|
||||
eprintln!("mission {id}: plan approval refused — mission is {}", mission.status);
|
||||
return Err(ApiError::BadRequest);
|
||||
return Err(ApiError::Refused(format!(
|
||||
"this mission is {} — a {} can only be approved while it is a draft, \
|
||||
because approving one rewrites how the mission will run",
|
||||
mission.status, "plan"
|
||||
)));
|
||||
}
|
||||
|
||||
let plan: Plan = serde_json::from_value(proposal.plan.clone()).map_err(|e| {
|
||||
@@ -317,6 +330,7 @@ pub async fn decide(
|
||||
// before a deploy could name a kind this build no longer dispatches.
|
||||
if let Err(why) = plan.validate() {
|
||||
eprintln!("mission {id}: plan {pid} is no longer runnable: {why}");
|
||||
let reason = why.to_string();
|
||||
let _ = cm_db::repo::mission_plan_proposals::decide(
|
||||
&state.pool,
|
||||
pid,
|
||||
@@ -326,7 +340,12 @@ pub async fn decide(
|
||||
Some(user.user_id.as_uuid().to_owned()),
|
||||
)
|
||||
.await;
|
||||
return Err(ApiError::BadRequest);
|
||||
// `Refusal` is already written as human-readable copy — it names the
|
||||
// constraint and why it exists. It was going to stderr only.
|
||||
return Err(ApiError::Refused(format!(
|
||||
"this plan is no longer runnable on the current build, so it was \
|
||||
rejected: {reason}"
|
||||
)));
|
||||
}
|
||||
|
||||
let phases = plan.phases();
|
||||
|
||||
@@ -253,8 +253,11 @@ pub async fn decide(
|
||||
.map_err(|_| ApiError::Internal)?
|
||||
.ok_or(ApiError::NotFound)?;
|
||||
if mission.status != "draft" {
|
||||
eprintln!("mission {id}: roster approval refused — mission is {}", mission.status);
|
||||
return Err(ApiError::BadRequest);
|
||||
return Err(ApiError::Refused(format!(
|
||||
"this mission is {} — a {} can only be approved while it is a draft, \
|
||||
because approving one rewrites how the mission will run",
|
||||
mission.status, "roster"
|
||||
)));
|
||||
}
|
||||
|
||||
let roster: Roster = serde_json::from_value(proposal.roster.clone()).map_err(|e| {
|
||||
@@ -269,6 +272,7 @@ pub async fn decide(
|
||||
.map_err(|_| ApiError::Internal)?;
|
||||
if let Err(why) = roster.validate(&available) {
|
||||
eprintln!("mission {id}: roster {pid} is no longer applicable: {why}");
|
||||
let reason = why.to_string();
|
||||
let _ = cm_db::repo::mission_team_proposals::decide(
|
||||
&state.pool,
|
||||
pid,
|
||||
@@ -278,7 +282,13 @@ pub async fn decide(
|
||||
Some(user.user_id.as_uuid().to_owned()),
|
||||
)
|
||||
.await;
|
||||
return Err(ApiError::BadRequest);
|
||||
// The proposal has just been auto-rejected, so the caller is about to
|
||||
// re-read a list where it says "rejected" with no visible cause. The
|
||||
// reason is the whole content of this response.
|
||||
return Err(ApiError::Refused(format!(
|
||||
"this roster no longer applies to the fleet as it is now, so it was \
|
||||
rejected: {reason}"
|
||||
)));
|
||||
}
|
||||
|
||||
let graph = roster.graph().map_err(|e| {
|
||||
|
||||
@@ -206,7 +206,21 @@ fn phases_for_create(
|
||||
})
|
||||
.map(|rp| rp.config.clone())
|
||||
.unwrap_or(Value::Null);
|
||||
// Decided from the caller's config BEFORE the merge: afterwards a
|
||||
// recipe task and a caller task are indistinguishable.
|
||||
let caller_task = p
|
||||
.config
|
||||
.get("task")
|
||||
.and_then(|t| t.as_str())
|
||||
.is_some_and(|s| !s.trim().is_empty());
|
||||
let caller_condition =
|
||||
p.config.get("done_when").is_some() || p.config.get("done_when_check").is_some();
|
||||
let config = merge_config(base, p.config);
|
||||
let config = if caller_task && !caller_condition {
|
||||
drop_orphaned_condition(config, &p.kind, p.order_idx)
|
||||
} else {
|
||||
config
|
||||
};
|
||||
// Say what this phase asked for that will not happen. A config key
|
||||
// nothing reads is silent by construction — `task` sat unread
|
||||
// through every mission until two phases with different tasks
|
||||
@@ -221,6 +235,36 @@ fn phases_for_create(
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A recipe's completion condition is a condition on the recipe's own task.
|
||||
/// When the caller supplied a different `task` and no condition of its own,
|
||||
/// keeping the recipe's `done_when` judges the phase against work it was
|
||||
/// never asked to do — `research_and_code`'s coding phase inherits "an
|
||||
/// implementation for each INT-XX item in IMPLEMENTATION_BRIEF", and a
|
||||
/// phase asked to write CHAIN.md fails on it, honestly, every time (missions
|
||||
/// 01a0c20d, 01a0c493). The condition and the check travel with the task
|
||||
/// they were written for; a phase with its own task gets its own, or none.
|
||||
///
|
||||
/// Called only when the caller supplied a task and no condition; the
|
||||
/// decision is made before the merge, where the two are still telling apart.
|
||||
fn drop_orphaned_condition(config: Value, kind: &str, order_idx: i32) -> Value {
|
||||
let Value::Object(mut c) = config else { return config };
|
||||
let dropped: Vec<&str> = ["done_when", "done_when_check"]
|
||||
.into_iter()
|
||||
.filter(|k| c.contains_key(*k))
|
||||
.collect();
|
||||
if !dropped.is_empty() {
|
||||
eprintln!(
|
||||
"phase {kind}[{order_idx}]: caller supplied its own task and no completion \
|
||||
condition — the recipe's {} is NOT inherited (it describes the recipe's task)",
|
||||
dropped.join("/")
|
||||
);
|
||||
for k in dropped {
|
||||
c.remove(k);
|
||||
}
|
||||
}
|
||||
Value::Object(c)
|
||||
}
|
||||
|
||||
/// Shallow-merge `over` onto `base`, key by key.
|
||||
///
|
||||
/// Shallow is deliberate: phase config is a flat settings bag, and a caller
|
||||
@@ -279,16 +323,100 @@ pub async fn create(
|
||||
_ => return Err(ApiError::BadRequest),
|
||||
}
|
||||
|
||||
// Honour the recipe's `default_team_template`.
|
||||
//
|
||||
// Every recipe declares one and NOTHING read it: the field was parsed into
|
||||
// `WorkflowRecipe` and then ignored, so a mission created from a card with
|
||||
// no explicit team was rejected at launch with "no team_id, no
|
||||
// team_template_id, no config.phase_teams" — a card that cannot be launched
|
||||
// by clicking it. Only resolved when the caller named no team of any kind,
|
||||
// so an explicit choice always wins.
|
||||
let recipe = crate::workflow_registry::get(body.template_kind.trim());
|
||||
let mut team_template_id = body.team_template_id;
|
||||
let has_phase_teams = body
|
||||
.config
|
||||
.get("phase_teams")
|
||||
.and_then(|v| v.as_object())
|
||||
.is_some_and(|o| o.values().any(|v| v.as_array().is_some_and(|a| !a.is_empty())));
|
||||
// Per-purpose defaults first: a multi-phase recipe does not have one job,
|
||||
// and staffing every phase from one team is what put a coder, a tester and
|
||||
// a committer on a repo-less markdown mission. Only applied when the caller
|
||||
// named no team of any kind, so an explicit choice always wins.
|
||||
let mut config = body.config;
|
||||
if team_template_id.is_none() && body.team_id.is_none() && !has_phase_teams {
|
||||
if let Some(r) = recipe {
|
||||
let mut resolved = serde_json::Map::new();
|
||||
for (purpose, key) in &r.default_phase_teams {
|
||||
match cm_db::repo::team_templates::get_by_key(&state.pool, key).await {
|
||||
Ok(Some(t)) => {
|
||||
resolved.insert(
|
||||
purpose.clone(),
|
||||
serde_json::json!([t.id.to_string()]),
|
||||
);
|
||||
}
|
||||
// Loud, and it does NOT fall back silently: a recipe naming
|
||||
// a template that is not loaded would otherwise stage the
|
||||
// wrong crew and look deliberate.
|
||||
Ok(None) => eprintln!(
|
||||
"missions: recipe {} maps purpose {purpose:?} to team template \
|
||||
{key:?}, which is not loaded — that phase will fall back to the \
|
||||
mission-wide default",
|
||||
body.template_kind.trim()
|
||||
),
|
||||
Err(e) => eprintln!("missions: looking up team template {key:?}: {e}"),
|
||||
}
|
||||
}
|
||||
if !resolved.is_empty() {
|
||||
eprintln!(
|
||||
"missions: {} staffs {} phase purpose(s) from the recipe",
|
||||
body.template_kind.trim(),
|
||||
resolved.len()
|
||||
);
|
||||
if let Some(obj) = config.as_object_mut() {
|
||||
obj.insert("phase_teams".into(), serde_json::Value::Object(resolved));
|
||||
} else {
|
||||
config = serde_json::json!({ "phase_teams": resolved });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let has_phase_teams = has_phase_teams
|
||||
|| config
|
||||
.get("phase_teams")
|
||||
.and_then(|v| v.as_object())
|
||||
.is_some_and(|o| o.values().any(|v| v.as_array().is_some_and(|a| !a.is_empty())));
|
||||
|
||||
if team_template_id.is_none() && body.team_id.is_none() && !has_phase_teams {
|
||||
if let Some(key) = recipe.and_then(|r| r.default_team_template.as_deref()) {
|
||||
match cm_db::repo::team_templates::get_by_key(&state.pool, key).await {
|
||||
Ok(Some(t)) => {
|
||||
eprintln!(
|
||||
"missions: {} defaults to team template {key}",
|
||||
body.template_kind.trim()
|
||||
);
|
||||
team_template_id = Some(t.id);
|
||||
}
|
||||
// Loud: a recipe naming a template that is not loaded would
|
||||
// otherwise fail at launch, one step removed from the cause.
|
||||
Ok(None) => eprintln!(
|
||||
"missions: recipe {} names default_team_template {key:?}, which is not loaded — the mission will have no team",
|
||||
body.template_kind.trim()
|
||||
),
|
||||
Err(e) => eprintln!("missions: looking up team template {key:?}: {e}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let new = NewMission {
|
||||
workspace_id: user.workspace_id.as_uuid(),
|
||||
title: body.title.trim(),
|
||||
template_kind: body.template_kind.trim(),
|
||||
team_id: body.team_id,
|
||||
team_template_id: body.team_template_id,
|
||||
team_template_id,
|
||||
repo_id: body.repo_id,
|
||||
schedule: body.schedule,
|
||||
description: body.description.as_deref(),
|
||||
config: body.config,
|
||||
config,
|
||||
runtime_kind: Some(runtime_kind),
|
||||
target_node_id: body.target_node_id,
|
||||
backend: body.backend.as_deref(),
|
||||
@@ -414,7 +542,9 @@ pub async fn artifact_download(
|
||||
[
|
||||
(
|
||||
axum::http::header::CONTENT_TYPE,
|
||||
artifact.mime.unwrap_or_else(|| "application/octet-stream".into()),
|
||||
artifact
|
||||
.mime
|
||||
.unwrap_or_else(|| "application/octet-stream".into()),
|
||||
),
|
||||
(
|
||||
axum::http::header::CONTENT_DISPOSITION,
|
||||
@@ -898,6 +1028,14 @@ async fn reap_mission_resources(state: &AppState, mission_id: Uuid) {
|
||||
// all DB rows. Shared with the batch-delete reaper so this path cannot
|
||||
// drift back into skipping the container teardown.
|
||||
let provisioner = crate::runtime_provision::RuntimeProvisioner::from_env();
|
||||
// Counted, not assumed. The summary below used to report `claw_ids.len()`,
|
||||
// which is how many claws were FOUND — including every one skipped as still
|
||||
// employed and every one whose purge failed. So "reaped 4 claw(s)" was
|
||||
// printed by a delete that purged none, which is exactly the log you would
|
||||
// read while wondering why the agents are still there.
|
||||
let mut purged = 0usize;
|
||||
let mut kept = 0usize;
|
||||
let mut failed = 0usize;
|
||||
for cid in &claw_ids {
|
||||
// Only claws this mission is the LAST holder of.
|
||||
//
|
||||
@@ -921,6 +1059,7 @@ async fn reap_mission_resources(state: &AppState, mission_id: Uuid) {
|
||||
eprintln!(
|
||||
"missions::delete: keeping claw {cid} — {shared} other mission(s) still employ it"
|
||||
);
|
||||
kept += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -931,10 +1070,14 @@ async fn reap_mission_resources(state: &AppState, mission_id: Uuid) {
|
||||
cm_domain::AgentId::from(*cid),
|
||||
)
|
||||
.await;
|
||||
if let Err(e) = report.counts {
|
||||
match report.counts {
|
||||
Ok(_) => purged += 1,
|
||||
Err(e) => {
|
||||
failed += 1;
|
||||
eprintln!("missions::delete: hard_purge claw {cid} failed (continuing): {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Delete the (permanent-lifecycle) teams — no mission FK cascades them.
|
||||
// team_members cascades from teams.
|
||||
@@ -969,10 +1112,32 @@ async fn reap_mission_resources(state: &AppState, mission_id: Uuid) {
|
||||
}
|
||||
}
|
||||
|
||||
// 6. The captured outputs. `mission_gc` keeps `_outputs/<id>` for 90 days
|
||||
// because they are artifacts a user can still open — but once the
|
||||
// mission row is gone so are its `mission_artifacts`, and nothing can
|
||||
// open them. Found on 2026-09-14 as 163 orphaned directories on prod,
|
||||
// the newest belonging to a mission deleted twenty minutes earlier.
|
||||
let outputs = crate::mission_workspace::missions_root()
|
||||
.join("_outputs")
|
||||
.join(mission_id.to_string());
|
||||
if outputs.is_dir() {
|
||||
if let Err(e) = tokio::fs::remove_dir_all(&outputs).await {
|
||||
eprintln!(
|
||||
"missions::delete: reaped {} claw(s), {} team(s) for mission {mission_id}",
|
||||
claw_ids.len(),
|
||||
team_ids.len()
|
||||
"missions::delete: remove {} failed (continuing; mission_gc will reap it in \
|
||||
90 days): {e}",
|
||||
outputs.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Say what actually happened. `failed > 0` means the mission row is about to
|
||||
// be deleted while its claws survive with nothing left pointing at them —
|
||||
// the orphan case, and the only way to notice it after the fact.
|
||||
eprintln!(
|
||||
"missions::delete: mission {mission_id}: {purged} claw(s) purged, {kept} kept (still \
|
||||
employed), {failed} FAILED, {} team(s) deleted, {} claw(s) considered",
|
||||
team_ids.len(),
|
||||
claw_ids.len()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1116,6 +1281,33 @@ pub async fn retry_phase(
|
||||
.execute(&mut *tx)
|
||||
.await?
|
||||
.rows_affected();
|
||||
// Drop the previous attempt's capture, or this retry's work is DESTROYED.
|
||||
//
|
||||
// `capture_finished_coding_phases` skips any phase that already has a
|
||||
// `code_diff` artifact (`NOT EXISTS`, phase_runner.rs). That guard is right
|
||||
// for a phase that ran once, and catastrophic for a retried one: the stale
|
||||
// artifact from the failed attempt suppresses capture of the new attempt
|
||||
// forever, the container is then reaped on its normal grace, and everything
|
||||
// the agents committed inside it is gone. The UI meanwhile shows the OLD
|
||||
// diff, so the mission reads as delivered.
|
||||
//
|
||||
// That is exactly what happened to mission 01a00538: it completed both
|
||||
// phases on the retry, 11 agent commits and all, and delivered a patch
|
||||
// dated the previous day. Deleting here is what makes the doc comment above
|
||||
// ("the phase card starts fresh on the retry") true of the artifacts too.
|
||||
let cleared = sqlx::query(
|
||||
"DELETE FROM mission_artifacts a
|
||||
USING mission_phases mp
|
||||
WHERE a.phase_id = mp.id
|
||||
AND a.mission_id = $1
|
||||
AND mp.status = 'pending'
|
||||
AND a.kind = 'code_diff'",
|
||||
)
|
||||
.bind(id)
|
||||
.execute(&mut *tx)
|
||||
.await?
|
||||
.rows_affected();
|
||||
|
||||
// And put the mission back to running, or nothing sweeps the phase: every
|
||||
// launcher and closer keys off `missions.status = 'running'`.
|
||||
sqlx::query(
|
||||
@@ -1127,7 +1319,7 @@ pub async fn retry_phase(
|
||||
.await?;
|
||||
tx.commit().await?;
|
||||
Ok(Json(
|
||||
serde_json::json!({ "reset": true, "reopened_phases": reopened }),
|
||||
serde_json::json!({ "reset": true, "reopened_phases": reopened, "cleared_captures": cleared }),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -1141,6 +1333,42 @@ pub async fn retry_phase(
|
||||
/// One row per pass. The `reason` is the operator-facing explanation of why a
|
||||
/// phase iterated (or stopped), and is the same text fed back to the agents as
|
||||
/// guidance for the following pass.
|
||||
/// Skill-Use scores for a mission: did the skills we delivered change what the
|
||||
/// agent did?
|
||||
///
|
||||
/// Reads only what was recorded — the prompts the agent received and the
|
||||
/// narratives it returned. An empty result means the evidence is gone (events
|
||||
/// are reaped after 7 days unless `retain_events_until` is set), NOT that no
|
||||
/// skill was followed, and the caller has to present it that way.
|
||||
pub async fn skill_use(
|
||||
State(state): State<AppState>,
|
||||
Authed(user): Authed,
|
||||
Path(id): Path<Uuid>,
|
||||
) -> Result<Json<Value>, ApiError> {
|
||||
let _ = cm_db::repo::missions::get(&state.pool, id, user.workspace_id.as_uuid())
|
||||
.await?
|
||||
.ok_or(ApiError::NotFound)?;
|
||||
let scores = crate::skill_use::score_mission(&state.pool, id)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
eprintln!("skill_use: scoring mission {id} failed: {e}");
|
||||
ApiError::Internal
|
||||
})?;
|
||||
Ok(Json(serde_json::json!({
|
||||
"mission_id": id,
|
||||
"skills": scores,
|
||||
// Said in the payload rather than left for the reader to infer: an
|
||||
// empty list has two very different causes and they must not look the
|
||||
// same to whoever consumes this.
|
||||
"evidence": if scores.is_empty() {
|
||||
"no delivered skills found in the recorded prompts — either none \
|
||||
were delivered, or the events have been reaped"
|
||||
} else {
|
||||
"scored from recorded prompt.composed and reasoning events"
|
||||
},
|
||||
})))
|
||||
}
|
||||
|
||||
pub async fn list_phase_evaluations(
|
||||
State(state): State<AppState>,
|
||||
Authed(user): Authed,
|
||||
@@ -1152,7 +1380,7 @@ pub async fn list_phase_evaluations(
|
||||
.ok_or(ApiError::NotFound)?;
|
||||
use sqlx::Row;
|
||||
let rows = sqlx::query(
|
||||
"SELECT iteration, met, reason, model, error, created_at, checks
|
||||
"SELECT iteration, met, reason, model, error, created_at, checks, expectation
|
||||
FROM mission_phase_evaluations
|
||||
WHERE mission_id = $1 AND phase_id = $2
|
||||
ORDER BY iteration DESC",
|
||||
@@ -1175,6 +1403,10 @@ pub async fn list_phase_evaluations(
|
||||
// empty list means the verdict rests on agent claims
|
||||
// alone, which an operator should be able to see.
|
||||
"checks": r.get::<serde_json::Value, _>("checks"),
|
||||
// What the judge said it would check BEFORE it read the
|
||||
// evidence; set beside `checks` so an operator can see
|
||||
// whether it kept to its plan.
|
||||
"expectation": r.get::<Option<String>, _>("expectation"),
|
||||
"created_at": created_at
|
||||
.format(&time::format_description::well_known::Rfc3339)
|
||||
.unwrap_or_default(),
|
||||
@@ -1295,6 +1527,7 @@ pub async fn set_status(
|
||||
user.user_id,
|
||||
id,
|
||||
Some(state.node_hub.clone()),
|
||||
state.blobs.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -1305,6 +1538,11 @@ pub async fn set_status(
|
||||
|
||||
cm_db::repo::missions::set_status(&state.pool, id, user.workspace_id.as_uuid(), &body.status)
|
||||
.await?;
|
||||
// An operator's stop is a terminal transition too, and the runner's
|
||||
// close never sees it: revoke here as well.
|
||||
if matches!(body.status.as_str(), "completed" | "failed" | "cancelled") {
|
||||
crate::mission_orchestrator::revoke_mission_credentials(&state.pool, id).await;
|
||||
}
|
||||
|
||||
let mission = cm_db::repo::missions::get(&state.pool, id, user.workspace_id.as_uuid())
|
||||
.await?
|
||||
@@ -1504,6 +1742,34 @@ mod tests {
|
||||
/// per-phase settings are read from at run time. Before this, every
|
||||
/// wizard-created mission stored a null config and every recipe setting
|
||||
/// was inert.
|
||||
/// Every shipped recipe must name a team template that is actually
|
||||
/// authored. `default_team_template` was parsed and never read, so a
|
||||
/// mismatch here used to surface as "launch rejected — no team_id" on a
|
||||
/// card the user simply clicked.
|
||||
#[test]
|
||||
fn every_recipe_names_a_team_template_that_exists() {
|
||||
let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../../templates/teams");
|
||||
let authored: std::collections::HashSet<String> = std::fs::read_dir(dir)
|
||||
.expect("templates/teams is readable")
|
||||
.filter_map(Result::ok)
|
||||
.filter_map(|e| {
|
||||
let n = e.file_name().to_string_lossy().to_string();
|
||||
n.strip_suffix(".toml").map(str::to_string)
|
||||
})
|
||||
.collect();
|
||||
for r in crate::workflow_registry::load() {
|
||||
let Some(key) = r.default_team_template.as_deref() else {
|
||||
continue;
|
||||
};
|
||||
assert!(
|
||||
authored.contains(key),
|
||||
"recipe {:?} defaults to team template {key:?}, which has no \
|
||||
templates/teams/{key}.toml — the card would be unlaunchable",
|
||||
r.key
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn phase_config_is_backfilled_from_the_recipe() {
|
||||
let recipe = test_recipe();
|
||||
@@ -1533,6 +1799,44 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// A caller's own task does not inherit the recipe's condition; a
|
||||
/// caller's own condition is kept; a phase with neither keeps the
|
||||
/// recipe's pair as before.
|
||||
#[test]
|
||||
fn a_custom_task_does_not_inherit_the_recipes_done_when() {
|
||||
let recipe = test_recipe();
|
||||
let coding_has_condition = recipe
|
||||
.phases
|
||||
.iter()
|
||||
.any(|p| p.kind == "coding" && p.config.get("done_when").is_some());
|
||||
assert!(coding_has_condition, "the fixture must carry a recipe condition");
|
||||
let phases = phases_for_create(
|
||||
Some(&recipe),
|
||||
vec![
|
||||
PhaseSpec {
|
||||
kind: "coding".into(),
|
||||
order_idx: 1,
|
||||
config: serde_json::json!({"task": "write CHAIN.md"}),
|
||||
},
|
||||
PhaseSpec {
|
||||
kind: "coding".into(),
|
||||
order_idx: 2,
|
||||
config: serde_json::json!({"task": "write X", "done_when": "X exists"}),
|
||||
},
|
||||
PhaseSpec {
|
||||
kind: "coding".into(),
|
||||
order_idx: 3,
|
||||
config: Value::Null,
|
||||
},
|
||||
],
|
||||
);
|
||||
assert!(phases[0].config.get("done_when").is_none(), "{:?}", phases[0].config);
|
||||
assert_eq!(phases[1].config["done_when"], "X exists");
|
||||
assert!(phases[2].config.get("done_when").is_some(), "{:?}", phases[2].config);
|
||||
// The rest of the recipe's config still backfills the custom-task phase.
|
||||
assert!(phases[0].config.get("loop").is_some());
|
||||
}
|
||||
|
||||
/// Omitting phases entirely takes the recipe's list wholesale.
|
||||
#[test]
|
||||
fn phases_default_to_the_recipe() {
|
||||
@@ -1614,6 +1918,12 @@ mod tests {
|
||||
blurb: String::new(),
|
||||
requires_repo: true,
|
||||
default_team_template: Some("rust_sdlc".into()),
|
||||
default_phase_teams: [
|
||||
("research".to_string(), "topic_research".to_string()),
|
||||
("coding".to_string(), "rust_sdlc".to_string()),
|
||||
]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
phases: vec![
|
||||
crate::workflow_registry::WorkflowPhase {
|
||||
kind: "research".into(),
|
||||
@@ -1625,7 +1935,10 @@ mod tests {
|
||||
order_idx: 1,
|
||||
config: serde_json::json!({
|
||||
"loop": "until_no_more_int_items",
|
||||
"commit_policy": "on_green_tests"
|
||||
"commit_policy": "on_green_tests",
|
||||
// The recipe's own task and the condition written for it.
|
||||
"task": "implement every INT-XX item",
|
||||
"done_when": "an implementation exists for each INT-XX item"
|
||||
}),
|
||||
},
|
||||
],
|
||||
@@ -1727,7 +2040,8 @@ pub async fn workforce(
|
||||
.await?;
|
||||
|
||||
let mut missions: Vec<Value> = Vec::new();
|
||||
let mut seen_mission: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
|
||||
let mut seen_mission: std::collections::HashMap<String, usize> =
|
||||
std::collections::HashMap::new();
|
||||
for r in rows {
|
||||
let mid: String = r.get("mission_id");
|
||||
let idx = match seen_mission.get(&mid) {
|
||||
@@ -1755,7 +2069,9 @@ pub async fn workforce(
|
||||
"status": r.get::<String, _>("agent_status"),
|
||||
});
|
||||
// A claw bound to two NODES of the same mission is still one colleague.
|
||||
let list = missions[idx]["agents"].as_array_mut().expect("agents array");
|
||||
let list = missions[idx]["agents"]
|
||||
.as_array_mut()
|
||||
.expect("agents array");
|
||||
let id = agent["id"].clone();
|
||||
if !list.iter().any(|a| a["id"] == id) {
|
||||
list.push(agent);
|
||||
@@ -1863,7 +2179,8 @@ mod artifact_tests {
|
||||
"one resolver"
|
||||
);
|
||||
assert_eq!(
|
||||
src.matches(concat!("resolve_", "artifact_path(&artifact.path)")).count(),
|
||||
src.matches(concat!("resolve_", "artifact_path(&artifact.path)"))
|
||||
.count(),
|
||||
2,
|
||||
"and both routes must go through it"
|
||||
);
|
||||
|
||||
@@ -17,6 +17,7 @@ pub mod library;
|
||||
pub mod mission_plan;
|
||||
pub mod mission_roster;
|
||||
pub mod missions;
|
||||
pub mod podcast;
|
||||
pub mod nodes;
|
||||
pub mod oauth;
|
||||
pub mod orgs;
|
||||
|
||||
@@ -514,3 +514,14 @@ fn backend_label(id: &str) -> String {
|
||||
other => other.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `GET /api/judge/quota` — the judge providers' plan usage as last polled by
|
||||
/// `judge_quota`, and the thresholds that act on it. Read-only; no workspace
|
||||
/// data. The readings are per deployment, since the keys are.
|
||||
pub async fn judge_quota(Authed(_user): Authed) -> Json<Value> {
|
||||
Json(serde_json::json!({
|
||||
"warn_at_pct": crate::judge_quota::WARN_AT,
|
||||
"switch_at_pct": crate::judge_quota::SWITCH_AT,
|
||||
"readings": crate::judge_quota::snapshot(),
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -0,0 +1,302 @@
|
||||
//! The private podcast feed.
|
||||
//!
|
||||
//! A podcast app is the right client for this: it downloads overnight, plays
|
||||
//! offline, remembers position, and has lock-screen controls — none of which a
|
||||
//! file in a folder gives you at the gym.
|
||||
//!
|
||||
//! Auth is a token in the query string, not a bearer header, because no podcast
|
||||
//! app lets you set headers. That is a real trade: the token is in the URL and
|
||||
//! therefore in the app's database and any proxy log it passes. It is scoped to
|
||||
//! reading this feed and nothing else, and can be rotated by reissuing it.
|
||||
|
||||
use axum::extract::{Path, Query, State};
|
||||
use axum::http::{header, StatusCode};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use serde::Deserialize;
|
||||
use sqlx::Row;
|
||||
|
||||
use crate::{ApiError, AppState};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct FeedAuth {
|
||||
pub token: String,
|
||||
}
|
||||
|
||||
/// The audio endpoint accepts a token either way — see `episode_audio`.
|
||||
#[derive(Deserialize)]
|
||||
pub struct OptionalAuth {
|
||||
#[serde(default)]
|
||||
pub token: Option<String>,
|
||||
}
|
||||
|
||||
/// Resolve a feed token to the workspace it may read.
|
||||
///
|
||||
/// Reuses the normal API token table, so revoking a token revokes the feed with
|
||||
/// it — a second secret store for podcasts would be one more thing to forget to
|
||||
/// rotate.
|
||||
async fn workspace_for(state: &AppState, token: &str) -> Result<uuid::Uuid, ApiError> {
|
||||
let user = state
|
||||
.auth
|
||||
.authenticate(token)
|
||||
.await
|
||||
.map_err(|_| ApiError::Unauthorized)?;
|
||||
Ok(user.workspace_id.as_uuid())
|
||||
}
|
||||
|
||||
fn xml_escape(s: &str) -> String {
|
||||
s.replace('&', "&")
|
||||
.replace('<', "<")
|
||||
.replace('>', ">")
|
||||
.replace('"', """)
|
||||
}
|
||||
|
||||
fn rfc2822(ts: time::OffsetDateTime) -> String {
|
||||
// Podcast clients are strict about pubDate. `time`'s RFC2822 is exactly it.
|
||||
ts.format(&time::format_description::well_known::Rfc2822)
|
||||
.unwrap_or_else(|_| "Thu, 01 Jan 1970 00:00:00 +0000".into())
|
||||
}
|
||||
|
||||
/// `GET /api/podcast/feed.xml?token=…`
|
||||
pub async fn feed(
|
||||
State(state): State<AppState>,
|
||||
Query(auth): Query<FeedAuth>,
|
||||
) -> Result<Response, ApiError> {
|
||||
let workspace_id = workspace_for(&state, &auth.token).await?;
|
||||
let rows = sqlx::query(
|
||||
"SELECT id, episode_date, title, bytes, duration_secs, created_at
|
||||
FROM podcast_episodes
|
||||
WHERE workspace_id = $1
|
||||
-- Skip markers for missions whose script was reaped before the
|
||||
-- render sweep reached them: a zero-byte enclosure makes a podcast
|
||||
-- app show a broken episode rather than simply not showing one.
|
||||
AND bytes > 0
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 100",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.fetch_all(&state.pool)
|
||||
.await?;
|
||||
|
||||
let base = std::env::var("CLAWMATES_PUBLIC_URL")
|
||||
.unwrap_or_else(|_| "http://localhost:8080".to_string());
|
||||
let base = base.trim_end_matches('/');
|
||||
|
||||
let mut items = String::new();
|
||||
for r in &rows {
|
||||
let id: uuid::Uuid = r.get("id");
|
||||
let title: String = r.get("title");
|
||||
let date: String = r.get("episode_date");
|
||||
let bytes: i64 = r.get("bytes");
|
||||
let secs: i32 = r.get("duration_secs");
|
||||
let created: time::OffsetDateTime = r.get("created_at");
|
||||
// The token rides on the enclosure too: the app fetches the audio in a
|
||||
// separate request that carries none of the feed's context.
|
||||
let url = format!("{base}/api/podcast/episodes/{id}.mp3?token={}", auth.token);
|
||||
items.push_str(&format!(
|
||||
r#" <item>
|
||||
<title>{t}</title>
|
||||
<description>Research digest for {d}</description>
|
||||
<pubDate>{p}</pubDate>
|
||||
<guid isPermaLink="false">{id}</guid>
|
||||
<enclosure url="{u}" length="{len}" type="audio/mpeg"/>
|
||||
<itunes:duration>{secs}</itunes:duration>
|
||||
</item>
|
||||
"#,
|
||||
t = xml_escape(&title),
|
||||
d = xml_escape(&date),
|
||||
p = rfc2822(created),
|
||||
u = xml_escape(&url),
|
||||
len = bytes,
|
||||
));
|
||||
}
|
||||
|
||||
let xml = format!(
|
||||
r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd">
|
||||
<channel>
|
||||
<title>ClawMates Research</title>
|
||||
<link>{base}</link>
|
||||
<description>Papers read against your projects, every morning.</description>
|
||||
<language>en-us</language>
|
||||
<itunes:explicit>false</itunes:explicit>
|
||||
{items} </channel>
|
||||
</rss>
|
||||
"#
|
||||
);
|
||||
Ok((
|
||||
StatusCode::OK,
|
||||
[(header::CONTENT_TYPE, "application/rss+xml; charset=utf-8")],
|
||||
xml,
|
||||
)
|
||||
.into_response())
|
||||
}
|
||||
|
||||
/// `GET /api/podcast/episodes/{id}.mp3?token=…`
|
||||
pub async fn episode_audio(
|
||||
State(state): State<AppState>,
|
||||
Path(file): Path<String>,
|
||||
Query(auth): Query<OptionalAuth>,
|
||||
headers: axum::http::HeaderMap,
|
||||
) -> Result<Response, ApiError> {
|
||||
// A podcast app fetches this with the token in the URL, because it cannot
|
||||
// set headers. The browser plays it through the same-origin proxy, which
|
||||
// supplies a bearer and no query token. Both are the same session; refusing
|
||||
// either would break one of the two ways this is listened to.
|
||||
let token = auth
|
||||
.token
|
||||
.or_else(|| {
|
||||
headers
|
||||
.get(axum::http::header::AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.strip_prefix("Bearer "))
|
||||
.map(str::to_string)
|
||||
})
|
||||
.ok_or(ApiError::Unauthorized)?;
|
||||
let workspace_id = workspace_for(&state, &token).await?;
|
||||
let id = file
|
||||
.strip_suffix(".mp3")
|
||||
.and_then(|s| uuid::Uuid::parse_str(s).ok())
|
||||
.ok_or(ApiError::NotFound)?;
|
||||
|
||||
let row = sqlx::query(
|
||||
"SELECT blob_key, bytes FROM podcast_episodes WHERE id = $1 AND workspace_id = $2",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(workspace_id)
|
||||
.fetch_optional(&state.pool)
|
||||
.await?
|
||||
.ok_or(ApiError::NotFound)?;
|
||||
|
||||
let key: String = row.get("blob_key");
|
||||
let blobs = state.blobs.clone().ok_or(ApiError::Internal)?;
|
||||
let bytes = blobs.get(&key).await.map_err(|e| {
|
||||
eprintln!("podcast: reading {key}: {e}");
|
||||
ApiError::Internal
|
||||
})?;
|
||||
|
||||
Ok((
|
||||
StatusCode::OK,
|
||||
[
|
||||
(header::CONTENT_TYPE, "audio/mpeg".to_string()),
|
||||
(header::CONTENT_LENGTH, bytes.len().to_string()),
|
||||
// Podcast apps re-fetch on every refresh otherwise.
|
||||
(header::CACHE_CONTROL, "private, max-age=86400".to_string()),
|
||||
],
|
||||
bytes,
|
||||
)
|
||||
.into_response())
|
||||
}
|
||||
|
||||
/// `GET /api/podcast/subscription` — the URL to paste into a podcast app.
|
||||
///
|
||||
/// Minted here rather than in the browser because the session lives in an
|
||||
/// httpOnly cookie that JavaScript cannot read, and the same-origin proxy that
|
||||
/// normally supplies the bearer is not available to a podcast app on a phone.
|
||||
/// So the caller's own token is echoed back inside a URL that points DIRECTLY
|
||||
/// at this backend.
|
||||
pub async fn subscription(
|
||||
State(state): State<AppState>,
|
||||
headers: axum::http::HeaderMap,
|
||||
crate::extract::Authed(_user): crate::extract::Authed,
|
||||
) -> Result<axum::Json<serde_json::Value>, ApiError> {
|
||||
let token = headers
|
||||
.get(axum::http::header::AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.strip_prefix("Bearer "))
|
||||
.ok_or(ApiError::Unauthorized)?;
|
||||
|
||||
let base = std::env::var("CLAWMATES_PUBLIC_URL")
|
||||
.unwrap_or_else(|_| "http://localhost:8080".to_string());
|
||||
let base = base.trim_end_matches('/');
|
||||
let _ = &state;
|
||||
Ok(axum::Json(serde_json::json!({
|
||||
"feedUrl": format!("{base}/api/podcast/feed.xml?token={token}"),
|
||||
// The panel warns when this is still localhost: a phone cannot reach it,
|
||||
// and a feed that only works on the machine that made it is a feed that
|
||||
// silently never syncs.
|
||||
"reachable": !base.contains("localhost") && !base.contains("127.0.0.1"),
|
||||
})))
|
||||
}
|
||||
|
||||
/// `GET /api/podcast/episodes` — the list behind the UI panel.
|
||||
///
|
||||
/// Normal bearer auth, unlike the feed: this is the app talking to its own API,
|
||||
/// where a header is available and a token in a URL would be needless exposure.
|
||||
pub async fn list_episodes(
|
||||
State(state): State<AppState>,
|
||||
crate::extract::Authed(user): crate::extract::Authed,
|
||||
) -> Result<axum::Json<serde_json::Value>, ApiError> {
|
||||
let rows = sqlx::query(
|
||||
"SELECT e.id, e.episode_date, e.title, e.bytes, e.duration_secs,
|
||||
e.rendered_by, e.created_at, e.mission_id, m.title AS mission_title
|
||||
FROM podcast_episodes e
|
||||
-- LEFT: an episode outlives its mission (migration 0080). An inner
|
||||
-- join would hide exactly the back-catalogue that change protects.
|
||||
LEFT JOIN missions m ON m.id = e.mission_id
|
||||
WHERE e.workspace_id = $1 AND e.bytes > 0
|
||||
ORDER BY e.created_at DESC
|
||||
LIMIT 50",
|
||||
)
|
||||
.bind(user.workspace_id.as_uuid())
|
||||
.fetch_all(&state.pool)
|
||||
.await?;
|
||||
|
||||
let episodes: Vec<serde_json::Value> = rows
|
||||
.iter()
|
||||
.map(|r| {
|
||||
let secs: i32 = r.get("duration_secs");
|
||||
let created: time::OffsetDateTime = r.get("created_at");
|
||||
serde_json::json!({
|
||||
"id": r.get::<uuid::Uuid, _>("id"),
|
||||
"missionId": r.get::<Option<uuid::Uuid>, _>("mission_id"),
|
||||
"missionTitle": r
|
||||
.get::<Option<String>, _>("mission_title")
|
||||
.unwrap_or_else(|| "(mission deleted)".to_string()),
|
||||
"title": r.get::<String, _>("title"),
|
||||
"date": r.get::<String, _>("episode_date"),
|
||||
"bytes": r.get::<i64, _>("bytes"),
|
||||
"durationSecs": secs,
|
||||
"renderedBy": r.get::<String, _>("rendered_by"),
|
||||
"createdAt": created.unix_timestamp(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
// How many missions produced no audio, so the panel can say so rather than
|
||||
// leaving a silent gap the operator has to notice for themselves.
|
||||
let unrenderable: i64 = sqlx::query_scalar(
|
||||
"SELECT count(*) FROM podcast_episodes WHERE workspace_id = $1 AND bytes = 0",
|
||||
)
|
||||
.bind(user.workspace_id.as_uuid())
|
||||
.fetch_one(&state.pool)
|
||||
.await
|
||||
.unwrap_or(0);
|
||||
|
||||
Ok(axum::Json(serde_json::json!({
|
||||
"episodes": episodes,
|
||||
"unrenderable": unrenderable,
|
||||
})))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A title with an ampersand must not produce invalid XML — a single bad
|
||||
/// character makes a podcast app reject the WHOLE feed, not one episode.
|
||||
#[test]
|
||||
fn titles_are_xml_escaped() {
|
||||
let out = xml_escape(r#"BM25 & <dense> "hybrid""#);
|
||||
assert_eq!(out, "BM25 & <dense> "hybrid"");
|
||||
assert!(!out.contains(" & "), "raw ampersand breaks the feed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pubdate_is_rfc2822() {
|
||||
let t = time::OffsetDateTime::from_unix_timestamp(1_755_000_000).unwrap();
|
||||
let s = rfc2822(t);
|
||||
// "Mon, 12 Aug 2025 ..." — clients parse this strictly.
|
||||
assert!(s.contains(", "), "{s}");
|
||||
assert!(s.ends_with("+0000"), "{s}");
|
||||
}
|
||||
}
|
||||
@@ -430,13 +430,30 @@ async fn sync_gitea(
|
||||
Some(owner) => format!("{api_base}/orgs/{owner}/repos?limit={per_page}&page={page}"),
|
||||
None => format!("{api_base}/repos/search?limit={per_page}&page={page}"),
|
||||
};
|
||||
let (status, body) = broker
|
||||
let (mut status, mut body) = broker
|
||||
.fetch_authorized(secret_ref, &url)
|
||||
.await
|
||||
.map_err(|e| format!("broker fetch: {e}"))?;
|
||||
// A Gitea owner is either an ORG or a USER, and they live on different
|
||||
// endpoints. Scoping a connection to a personal namespace — `osobh`,
|
||||
// where clawmates itself lives — 404s on /orgs and reported "not found
|
||||
// or PAT lacks access", which points at permissions when the account is
|
||||
// simply not an org. Retry as a user before giving up.
|
||||
if status == 404 {
|
||||
if let Some(owner) = conn.owner.as_deref() {
|
||||
let user_url =
|
||||
format!("{api_base}/users/{owner}/repos?limit={per_page}&page={page}");
|
||||
let (s2, b2) = broker
|
||||
.fetch_authorized(secret_ref, &user_url)
|
||||
.await
|
||||
.map_err(|e| format!("broker fetch: {e}"))?;
|
||||
status = s2;
|
||||
body = b2;
|
||||
}
|
||||
}
|
||||
if status == 404 && conn.owner.is_some() {
|
||||
return Err(format!(
|
||||
"org '{}' not found or PAT lacks access",
|
||||
"'{}' matched neither an org nor a user, or the PAT lacks access",
|
||||
conn.owner.as_deref().unwrap_or("")
|
||||
));
|
||||
}
|
||||
|
||||
@@ -100,7 +100,11 @@ pub async fn leaderboard(
|
||||
COUNT(u.id)::BIGINT AS "runs!"
|
||||
FROM agents a
|
||||
LEFT JOIN usage_events u ON u.agent_id = a.id
|
||||
WHERE a.workspace_id = $1
|
||||
-- deleted_at: a soft-deleted agent is gone everywhere else, so
|
||||
-- listing it here made deletion look like a no-op — the operator
|
||||
-- deletes it, the board still shows it, and deleting again does
|
||||
-- nothing because the row is already marked.
|
||||
WHERE a.workspace_id = $1 AND a.deleted_at IS NULL
|
||||
GROUP BY a.id, a.name, a.accent
|
||||
ORDER BY "credits!" DESC, "tokens!" DESC, a.name"#,
|
||||
user.workspace_id.as_uuid(),
|
||||
|
||||
@@ -83,6 +83,21 @@ pub(crate) async fn build_team(
|
||||
.await
|
||||
}
|
||||
|
||||
/// MCP bundles for a team built by the wizard or the planner rather than from a
|
||||
/// team template.
|
||||
///
|
||||
/// These teams have no template, so there is no `mcp_bundles` list to inherit —
|
||||
/// which previously meant they were provisioned with the door alone and could
|
||||
/// not reach the skills catalogue at all. `mcp_skills` scopes what it lists to
|
||||
/// the caller's workspace, so an agent with no template link still sees the
|
||||
/// global skills, which is the useful half for an ad-hoc team.
|
||||
fn adhoc_bundles() -> Vec<String> {
|
||||
vec![
|
||||
"clawmates_door".to_string(),
|
||||
"clawmates_skills".to_string(),
|
||||
]
|
||||
}
|
||||
|
||||
/// Same as `build_team` but with an explicit `lifecycle` (`permanent` |
|
||||
/// `ephemeral`). Ephemeral teams are torn down by the topology_worker after
|
||||
/// their last run terminates — used by the Scheduled + Triggered planner modes.
|
||||
@@ -140,7 +155,7 @@ pub(crate) async fn build_team_with_lifecycle(
|
||||
// Ad-hoc team-wizard teams aren't mission-bound, so they use the
|
||||
// default per-agent workspace under <install>/agents/<alias>/workspace/.
|
||||
provisioner
|
||||
.provision_claw(claw_id, &m.model, risk)
|
||||
.provision_claw(claw_id, &m.model, risk, &adhoc_bundles())
|
||||
.await
|
||||
.map_err(|e| {
|
||||
eprintln!("teams: provision claw {claw_id} failed: {e}");
|
||||
@@ -574,7 +589,7 @@ pub struct AutoProvisionRequest {
|
||||
#[serde(default)]
|
||||
pub risk_profile: Option<String>,
|
||||
/// MCP bundle aliases — same fall-back rule applies (always
|
||||
/// clawmates_door; gitea_forge when a repo is bound; deep-research
|
||||
/// clawmates_door + clawmates_skills; deep-research
|
||||
/// skill for research profiles).
|
||||
#[serde(default)]
|
||||
pub mcp_bundles: Vec<String>,
|
||||
@@ -656,11 +671,13 @@ pub async fn auto_provision(
|
||||
let mut mcp_bundles = body.mcp_bundles.clone();
|
||||
if mcp_bundles.is_empty() {
|
||||
mcp_bundles.push("clawmates_door".to_string());
|
||||
// gitea_forge is scoped to teams that will touch repos; the
|
||||
// wizard's downstream repo-binding step is what earns it.
|
||||
// Always safe to add now — the MCP layer no-ops when the token
|
||||
// isn't present in the container env.
|
||||
mcp_bundles.push("gitea_forge".to_string());
|
||||
// No `gitea_forge`: it was named in nine places and defined in none,
|
||||
// and agents reach the forge through `git` over HTTPS with the ambient
|
||||
// GITEA_TOKEN (mission_workspace::with_ambient_auth) — which is why
|
||||
// nothing ever broke. It was harmless while provision_claw ignored the
|
||||
// bundle list; now that the list is honoured, an undefined name is a
|
||||
// capability an agent is told it has and does not.
|
||||
mcp_bundles.push("clawmates_skills".to_string());
|
||||
}
|
||||
|
||||
// 1) LLM plan pass → roster JSON.
|
||||
|
||||
@@ -109,14 +109,17 @@ pub async fn compare_topologies(
|
||||
Json(req): Json<CompareRequest>,
|
||||
) -> Result<Json<Comparison>, ApiError> {
|
||||
// Execution turns run on the exec model (default = configured model, e.g.
|
||||
// sonnet); the judge uses the judge model (default claude-opus-4-8). Either
|
||||
// sonnet); the judge uses the judge model (cm_runtime::judge_model). Either
|
||||
// can name a registry provider as "<name>:<model>" (e.g. "glm:glm-4.6",
|
||||
// "kimi:kimi-k2") to run on GLM/Kimi instead.
|
||||
let exec_spec = std::env::var("CLAWMATES_TOPOLOGY_EXEC_MODEL")
|
||||
.unwrap_or_else(|_| state.runtime.model().to_string());
|
||||
let (exec_provider, exec_model) = state.runtime.resolve_provider(&exec_spec);
|
||||
let judge_spec =
|
||||
std::env::var("CLAWMATES_JUDGE_MODEL").unwrap_or_else(|_| "claude-opus-4-8".to_string());
|
||||
// `cm_runtime::judge_model()`, not a second read of the same variable: this
|
||||
// line and that function disagreed on the default (opus-4-8 vs opus-5), so
|
||||
// an unconfigured deployment scored topology comparisons on a different
|
||||
// model than the door governor and nothing recorded which.
|
||||
let judge_spec = cm_runtime::judge_model();
|
||||
let (judge_provider, judge_model) = state.runtime.resolve_provider(&judge_spec);
|
||||
let executor = ProviderExecutor::new(exec_provider, exec_model, state.runtime.max_tokens());
|
||||
let scorer = JudgeScorer::new(judge_provider, judge_model, 16);
|
||||
|
||||
@@ -36,7 +36,7 @@ async fn working_agents(pool: &PgPool, ws: WorkspaceId) -> HashSet<String> {
|
||||
let rows = sqlx::query(
|
||||
"SELECT DISTINCT a.id::text AS id
|
||||
FROM agents a JOIN agent_containers ac ON ac.agent_id = a.id
|
||||
WHERE a.workspace_id = $1",
|
||||
WHERE a.workspace_id = $1 AND a.deleted_at IS NULL",
|
||||
)
|
||||
.bind(ws.as_uuid())
|
||||
.fetch_all(pool)
|
||||
@@ -77,11 +77,7 @@ struct MissionRow {
|
||||
agent_id: Option<String>,
|
||||
}
|
||||
|
||||
async fn world_missions(
|
||||
pool: &PgPool,
|
||||
ws: WorkspaceId,
|
||||
only: Option<Uuid>,
|
||||
) -> Vec<MissionRow> {
|
||||
async fn world_missions(pool: &PgPool, ws: WorkspaceId, only: Option<Uuid>) -> Vec<MissionRow> {
|
||||
let rows = sqlx::query(
|
||||
"SELECT m.id::text AS mission_id,
|
||||
m.title AS title,
|
||||
@@ -416,11 +412,7 @@ fn benchmark_note(delta: &serde_json::Value) -> Option<String> {
|
||||
|
||||
/// Agents that execute a phase of this kind, via the purposes the phase runner
|
||||
/// itself uses. Returns empty for a teamless (microVM) mission.
|
||||
async fn phase_agents(
|
||||
pool: &PgPool,
|
||||
mission_id: &str,
|
||||
kind: &str,
|
||||
) -> Vec<String> {
|
||||
async fn phase_agents(pool: &PgPool, mission_id: &str, kind: &str) -> Vec<String> {
|
||||
let Ok(mid) = Uuid::parse_str(mission_id) else {
|
||||
return Vec::new();
|
||||
};
|
||||
@@ -702,6 +694,96 @@ async fn agent_telemetry(
|
||||
m
|
||||
}
|
||||
|
||||
/// What an agent did on its most recent FINISHED mission.
|
||||
///
|
||||
/// The metric band is a live command centre: tokens over the last minute,
|
||||
/// credits over the last hour, active routines, pending approvals. Every one of
|
||||
/// those is correctly zero for an agent whose mission ended, so the page an
|
||||
/// operator opens to ask "what did this agent do" answers with six zeros.
|
||||
///
|
||||
/// This is the other half, and it is deliberately a SEPARATE event rather than
|
||||
/// a fallback folded into `telemetry`. `agent.task.update` already refuses to
|
||||
/// emit for a finished mission so that "idle" stays truthful, and quietly
|
||||
/// substituting a two-day-old number into a live tile would undo exactly that.
|
||||
/// The client decides how to label it; the protocol keeps them apart.
|
||||
struct AgentLastRun {
|
||||
mission_id: uuid::Uuid,
|
||||
title: String,
|
||||
status: String,
|
||||
ended_at: Option<time::OffsetDateTime>,
|
||||
tokens: i64,
|
||||
credits: f64,
|
||||
tool_calls: i64,
|
||||
}
|
||||
|
||||
/// Every agent's last finished mission, in ONE query rather than N.
|
||||
///
|
||||
/// `usage_events` carries no mission id, so its rows are attributed by time
|
||||
/// window — the mission's own span, plus a small tail because a turn's usage is
|
||||
/// recorded as the turn settles rather than before the mission is marked
|
||||
/// complete. `mission_events` needs no such guess: it carries `mission_id` and
|
||||
/// `agent_id` directly, which is why the tool count is the trustworthy half of
|
||||
/// this row and the token figure is the approximate one.
|
||||
async fn agent_last_run(
|
||||
pool: &PgPool,
|
||||
ws: WorkspaceId,
|
||||
) -> std::collections::HashMap<String, AgentLastRun> {
|
||||
let mut m = std::collections::HashMap::new();
|
||||
let rows = sqlx::query(
|
||||
"WITH latest AS (
|
||||
SELECT DISTINCT ON (tm.claw_id)
|
||||
tm.claw_id AS agent_id, m.id AS mission_id, m.title, m.status,
|
||||
COALESCE(m.completed_at, m.updated_at) AS ended_at,
|
||||
m.created_at AS started_at
|
||||
FROM team_members tm
|
||||
JOIN mission_teams mt ON mt.team_id = tm.team_id
|
||||
JOIN missions m ON m.id = mt.mission_id
|
||||
WHERE m.workspace_id = $1
|
||||
AND m.status IN ('completed', 'failed')
|
||||
ORDER BY tm.claw_id, COALESCE(m.completed_at, m.updated_at) DESC
|
||||
)
|
||||
SELECT l.agent_id, l.mission_id, l.title, l.status, l.ended_at,
|
||||
COALESCE(u.tokens, 0) AS tokens,
|
||||
COALESCE(u.credits, 0) AS credits,
|
||||
COALESCE(t.tool_calls, 0) AS tool_calls
|
||||
FROM latest l
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT SUM(tokens_in + tokens_out)::bigint AS tokens,
|
||||
SUM(credits)::float8 AS credits
|
||||
FROM usage_events ue
|
||||
WHERE ue.agent_id = l.agent_id
|
||||
AND ue.created_at BETWEEN l.started_at AND l.ended_at + interval '5 minutes'
|
||||
) u ON TRUE
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT count(*)::bigint AS tool_calls
|
||||
FROM mission_events me
|
||||
WHERE me.mission_id = l.mission_id
|
||||
AND me.agent_id = l.agent_id
|
||||
AND me.kind = 'tool.call'
|
||||
) t ON TRUE",
|
||||
)
|
||||
.bind(ws.as_uuid())
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
for r in rows {
|
||||
let agent_id: uuid::Uuid = r.get("agent_id");
|
||||
m.insert(
|
||||
agent_id.to_string(),
|
||||
AgentLastRun {
|
||||
mission_id: r.get("mission_id"),
|
||||
title: r.get("title"),
|
||||
status: r.get("status"),
|
||||
ended_at: r.get("ended_at"),
|
||||
tokens: r.get("tokens"),
|
||||
credits: r.get("credits"),
|
||||
tool_calls: r.get("tool_calls"),
|
||||
},
|
||||
);
|
||||
}
|
||||
m
|
||||
}
|
||||
|
||||
/// Query for `GET /api/world/live`.
|
||||
#[derive(serde::Deserialize)]
|
||||
pub struct LiveQuery {
|
||||
@@ -723,6 +805,11 @@ pub async fn world_live(
|
||||
|
||||
let stream = async_stream::stream! {
|
||||
let mut first = true;
|
||||
// How many polls since the last-run summary was refreshed. Historical
|
||||
// by definition, so it does not belong on the 2s cadence — but it must
|
||||
// not be seed-only either, or a mission finishing mid-session leaves
|
||||
// the card reading whatever it read before.
|
||||
let mut polls: u32 = 0;
|
||||
// Remember last status per agent so we only push deltas after the seed.
|
||||
let mut last: std::collections::HashMap<String, String> = std::collections::HashMap::new();
|
||||
// Per-run journal cursor so we stream only NEW run_events each poll.
|
||||
@@ -751,11 +838,19 @@ pub async fn world_live(
|
||||
// Audit-log cursor for edge-initiated inter-agent events (delegation,
|
||||
// A2A) that bypass the run loop. -1 until seeded on the first pass.
|
||||
let mut audit_cursor: i64 = -1;
|
||||
// Cursor over mission_events for the per-agent LIVE column. Starts at
|
||||
// -1 and jumps to the current max on first sight, so a page load
|
||||
// streams forward instead of replaying every past turn.
|
||||
let mut agent_ev_cursor: i64 = -1;
|
||||
// Track the brain-file size we last announced per agent so we only
|
||||
// emit `agent.memory` when the file has actually grown (or shrunk).
|
||||
// On first seed we still emit — the World engine needs the initial
|
||||
// scale for every pawn.
|
||||
let mut last_bytes: std::collections::HashMap<String, u64> = std::collections::HashMap::new();
|
||||
// Push channel for frames that cannot wait for the 2s poll — an agent's
|
||||
// reasoning arrives token by token. Subscribed BEFORE the first poll so
|
||||
// nothing produced during the initial queries is missed.
|
||||
let mut live_rx = crate::live_bus::global().subscribe();
|
||||
loop {
|
||||
let roster = match cm_db::repo::agents::roster(&pool, ws).await {
|
||||
Ok(r) => r,
|
||||
@@ -1106,6 +1201,153 @@ pub async fn world_live(
|
||||
}));
|
||||
}
|
||||
|
||||
// The last finished mission, refreshed on the seed and then once a
|
||||
// minute. Stateful on the client, so a late subscriber paints it
|
||||
// immediately instead of waiting for the next refresh.
|
||||
if first || polls % 30 == 0 {
|
||||
let last_runs = agent_last_run(&pool, ws).await;
|
||||
for a in &roster {
|
||||
let id = a.id.to_string();
|
||||
let Some(lr) = last_runs.get(&id) else { continue };
|
||||
yield sse("agent.last_run", json!({
|
||||
"agentId": id,
|
||||
"missionId": lr.mission_id.to_string(),
|
||||
"title": lr.title,
|
||||
"status": lr.status,
|
||||
"endedAt": lr.ended_at.map(|t| t.unix_timestamp()),
|
||||
"tokens": lr.tokens,
|
||||
"credits": lr.credits,
|
||||
"toolCalls": lr.tool_calls,
|
||||
}));
|
||||
}
|
||||
}
|
||||
polls = polls.wrapping_add(1);
|
||||
|
||||
// Per-agent LIVE column: REASONING STREAM + tool lines.
|
||||
//
|
||||
// These two taxonomy types were declared and listened for since the
|
||||
// command centre shipped, and NOTHING ever emitted them — the cards
|
||||
// could not populate no matter what an agent did. mission_events is
|
||||
// the durable source: `reasoning` rows carry the agent's own step
|
||||
// output, `tool.call` rows its actions.
|
||||
//
|
||||
// This used to say the container tier "legitimately stays empty —
|
||||
// those agents are tool-free". That was wrong, and it was wrong in
|
||||
// the most expensive way: it explained the silence, so nobody
|
||||
// looked. Those agents call `Bash` and `Write` constantly; the
|
||||
// calls happen inside claude's own subprocess and so never reached
|
||||
// ZeroClaw's executor. `container_tool_hooks` records them now, and
|
||||
// `tool.call` is populated on both tiers.
|
||||
if agent_ev_cursor < 0 {
|
||||
agent_ev_cursor = sqlx::query_scalar(
|
||||
"SELECT coalesce(max(id), 0) FROM mission_events",
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap_or(0);
|
||||
} else {
|
||||
let rows = sqlx::query(
|
||||
"SELECT e.id, e.agent_id, e.kind, e.target, e.detail
|
||||
FROM mission_events e
|
||||
JOIN missions m ON m.id = e.mission_id
|
||||
WHERE m.workspace_id = $1
|
||||
AND e.id > $2
|
||||
AND e.agent_id IS NOT NULL
|
||||
-- 'reasoning' is NOT read here: live_bus pushes those
|
||||
-- as the runtime emits them, and emitting from both
|
||||
-- delivered every turn twice — once pushed, once polled
|
||||
-- ~2s later. The row is still written, as the durable
|
||||
-- record; this feed just is not its second mouth.
|
||||
AND e.kind = 'tool.call'
|
||||
ORDER BY e.id
|
||||
LIMIT 200",
|
||||
)
|
||||
.bind(ws.as_uuid())
|
||||
.bind(agent_ev_cursor)
|
||||
.fetch_all(&pool)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
for r in &rows {
|
||||
let eid: i64 = r.get("id");
|
||||
agent_ev_cursor = agent_ev_cursor.max(eid);
|
||||
let agent_id: Option<uuid::Uuid> = r.get("agent_id");
|
||||
let Some(agent_id) = agent_id else { continue };
|
||||
let kind: String = r.get("kind");
|
||||
let detail: serde_json::Value = r.get("detail");
|
||||
let target: Option<String> = r.get("target");
|
||||
debug_assert_eq!(kind, "tool.call", "query filters to tool.call");
|
||||
let _ = kind;
|
||||
yield sse("agent.tool.call", json!({
|
||||
"agentId": agent_id.to_string(),
|
||||
"tool": target.clone().unwrap_or_default(),
|
||||
"target": detail.get("path").and_then(|v| v.as_str()),
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
// WORKING ON NOW — the third card that was declared, listened for,
|
||||
// and never emitted. `agent.task.update` has no producer anywhere in
|
||||
// the backend, so the card read "idle" for an agent mid-turn.
|
||||
//
|
||||
// Derived rather than newly instrumented: an agent is working on its
|
||||
// crew's RUNNING mission, and that mission's phases are the steps.
|
||||
// Nothing is emitted for an agent with no running mission, so "idle"
|
||||
// stays truthful instead of becoming a stale last-known task.
|
||||
{
|
||||
let rows = sqlx::query(
|
||||
"SELECT tm.claw_id AS agent_id, m.id AS mission_id, m.title,
|
||||
p.kind, p.status, p.order_idx
|
||||
FROM team_members tm
|
||||
JOIN mission_teams mt ON mt.team_id = tm.team_id
|
||||
JOIN missions m ON m.id = mt.mission_id
|
||||
JOIN mission_phases p ON p.mission_id = m.id
|
||||
WHERE m.workspace_id = $1 AND m.status = 'running'
|
||||
ORDER BY tm.claw_id, p.order_idx",
|
||||
)
|
||||
.bind(ws.as_uuid())
|
||||
.fetch_all(&pool)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let mut cur: Option<(uuid::Uuid, uuid::Uuid, String)> = None;
|
||||
let mut steps: Vec<serde_json::Value> = Vec::new();
|
||||
let mut flush = |cur: &Option<(uuid::Uuid, uuid::Uuid, String)>,
|
||||
steps: &Vec<serde_json::Value>|
|
||||
-> Option<serde_json::Value> {
|
||||
let (agent_id, mission_id, title) = cur.as_ref()?;
|
||||
Some(json!({
|
||||
"agentId": agent_id.to_string(),
|
||||
"taskId": mission_id.to_string(),
|
||||
"title": title,
|
||||
"steps": steps,
|
||||
}))
|
||||
};
|
||||
for r in &rows {
|
||||
let agent_id: uuid::Uuid = r.get("agent_id");
|
||||
let mission_id: uuid::Uuid = r.get("mission_id");
|
||||
let title: String = r.get("title");
|
||||
if cur.as_ref().map(|c| c.0) != Some(agent_id) {
|
||||
if let Some(v) = flush(&cur, &steps) {
|
||||
yield sse("agent.task.update", v);
|
||||
}
|
||||
steps = Vec::new();
|
||||
cur = Some((agent_id, mission_id, title));
|
||||
}
|
||||
let status: String = r.get("status");
|
||||
let kind: String = r.get("kind");
|
||||
steps.push(json!({
|
||||
"label": kind,
|
||||
"state": match status.as_str() {
|
||||
"completed" | "skipped" => "done",
|
||||
"running" | "evaluating" => "active",
|
||||
_ => "pending",
|
||||
},
|
||||
}));
|
||||
}
|
||||
if let Some(v) = flush(&cur, &steps) {
|
||||
yield sse("agent.task.update", v);
|
||||
}
|
||||
}
|
||||
|
||||
// Workspace-wide telemetry (top-bar pills / Observe system strip).
|
||||
yield sse(
|
||||
"telemetry",
|
||||
@@ -1169,7 +1411,30 @@ pub async fn world_live(
|
||||
}
|
||||
|
||||
first = false;
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
// Forward pushed frames until the next poll is due, instead of
|
||||
// sleeping through them. This is what makes the reasoning stream
|
||||
// token-level: a chunk reaches the browser as the runtime emits it,
|
||||
// while everything queryable keeps its 2s cadence.
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(2);
|
||||
loop {
|
||||
let left = deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
if left.is_zero() {
|
||||
break;
|
||||
}
|
||||
match tokio::time::timeout(left, live_rx.recv()).await {
|
||||
Ok(Ok(ev)) => {
|
||||
if ev.workspace_id == ws.as_uuid() {
|
||||
yield sse(&ev.kind, ev.data);
|
||||
}
|
||||
}
|
||||
// Lagged: this subscriber fell behind and frames were
|
||||
// dropped for it. Keep going — a live view that skips is
|
||||
// right, and blocking the producer would be wrong.
|
||||
Ok(Err(tokio::sync::broadcast::error::RecvError::Lagged(_))) => continue,
|
||||
Ok(Err(_)) => break,
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1274,7 +1539,10 @@ mod mission_feed_tests {
|
||||
// finds. Written whole, the test fails on itself — which it did, and
|
||||
// which is the same self-match that makes `pkill -f <pattern>` kill the
|
||||
// shell carrying the pattern.
|
||||
let needle = concat!("SELECT checkpoint FROM topology_", "runs WHERE id = $1::uuid");
|
||||
let needle = concat!(
|
||||
"SELECT checkpoint FROM topology_",
|
||||
"runs WHERE id = $1::uuid"
|
||||
);
|
||||
assert!(
|
||||
!src.contains(needle),
|
||||
"the checkpoint tail is keyed on an agent_runs id and cannot match a \
|
||||
@@ -1359,3 +1627,181 @@ mod mission_feed_tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Does the agent command centre have anything to say about a finished mission?
|
||||
///
|
||||
/// Its metric cards read a LIVE feed: tokens in the last minute, credits in the
|
||||
/// last hour, active routines, pending approvals. Every one is correctly zero
|
||||
/// once a mission ends, so an operator opening the page to ask "what did this
|
||||
/// agent do" was answered with six zeros and nothing saying the question had
|
||||
/// been understood differently than they meant it.
|
||||
///
|
||||
/// The data was never missing — `usage_events` carries a row per turn and
|
||||
/// `mission_events` every attributed tool call. Nothing queried them. That is
|
||||
/// why this is a test: the failure produced no error anywhere, every query ran,
|
||||
/// and the answer was honestly nothing.
|
||||
#[cfg(test)]
|
||||
mod last_run_tests {
|
||||
use cm_domain::WorkspaceId;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// One finished mission, a crew of one, two turns of usage and four tool
|
||||
/// calls — of which one belongs to nobody.
|
||||
///
|
||||
/// Raw SQL on purpose: this asserts on the SHAPE of the join
|
||||
/// (team_members → mission_teams → missions), and going through helpers
|
||||
/// that already assume that shape would be testing itself.
|
||||
async fn seed(pool: &sqlx::PgPool) -> (WorkspaceId, Uuid) {
|
||||
let ws = WorkspaceId::new();
|
||||
let agent = Uuid::now_v7();
|
||||
let team = Uuid::now_v7();
|
||||
let mission = Uuid::now_v7();
|
||||
|
||||
let user = Uuid::now_v7();
|
||||
sqlx::query("INSERT INTO workspaces (id, name, plan) VALUES ($1,'w','free')")
|
||||
.bind(ws.as_uuid())
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("workspace");
|
||||
// `agents.managed_by` is NOT NULL and a real FK, so an owner has to
|
||||
// exist before any agent does.
|
||||
sqlx::query(
|
||||
"INSERT INTO users (id, workspace_id, email, role, display_name)
|
||||
VALUES ($1,$2,$3,'owner','Owner')",
|
||||
)
|
||||
.bind(user)
|
||||
.bind(ws.as_uuid())
|
||||
.bind(format!("o-{}@example.test", &user.to_string()[..8]))
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("user");
|
||||
sqlx::query(
|
||||
"INSERT INTO agents
|
||||
(id, workspace_id, name, job_title, system_prompt, avatar, accent,
|
||||
wallpaper, managed_by, status)
|
||||
VALUES ($1,$2,'Tomasz','researcher','','','#fff','',$3,'online')",
|
||||
)
|
||||
.bind(agent)
|
||||
.bind(ws.as_uuid())
|
||||
.bind(user)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("agent");
|
||||
sqlx::query(
|
||||
"INSERT INTO teams (id, workspace_id, name, kind, graph, status, lifecycle, mcp_bundles)
|
||||
VALUES ($1,$2,'crew','crew','{}'::jsonb,'active','permanent','{}')",
|
||||
)
|
||||
.bind(team)
|
||||
.bind(ws.as_uuid())
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("team");
|
||||
sqlx::query(
|
||||
"INSERT INTO team_members (team_id, node_id, claw_id, role)
|
||||
VALUES ($1,'n1',$2,'researcher')",
|
||||
)
|
||||
.bind(team)
|
||||
.bind(agent)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("member");
|
||||
sqlx::query(
|
||||
"INSERT INTO missions
|
||||
(id, workspace_id, title, template_kind, schedule, status, config,
|
||||
runtime_kind, created_at, updated_at, completed_at)
|
||||
VALUES ($1,$2,'JEPA Research','research_only','{}'::jsonb,'completed',
|
||||
'{}'::jsonb,'zeroclaw',
|
||||
now() - interval '2 days',
|
||||
now() - interval '2 days',
|
||||
now() - interval '2 days' + interval '30 minutes')",
|
||||
)
|
||||
.bind(mission)
|
||||
.bind(ws.as_uuid())
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("mission");
|
||||
sqlx::query(
|
||||
"INSERT INTO mission_teams (mission_id, team_id, purpose) VALUES ($1,$2,'crew')",
|
||||
)
|
||||
.bind(mission)
|
||||
.bind(team)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("mission_team");
|
||||
|
||||
// Usage rows land INSIDE the mission's window, because the window is
|
||||
// how they are attributed — `usage_events` carries no mission id.
|
||||
for (tin, tout, credits) in [(100_i32, 900_i32, 4.0_f64), (200, 800, 5.0)] {
|
||||
sqlx::query(
|
||||
"INSERT INTO usage_events
|
||||
(workspace_id, agent_id, kind, tokens_in, tokens_out, credits, created_at)
|
||||
VALUES ($1,$2,'llm_tokens',$3,$4,$5,
|
||||
now() - interval '2 days' + interval '10 minutes')",
|
||||
)
|
||||
.bind(ws.as_uuid())
|
||||
.bind(agent)
|
||||
.bind(tin)
|
||||
.bind(tout)
|
||||
.bind(credits)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("usage");
|
||||
}
|
||||
for owner in [Some(agent), Some(agent), Some(agent), None] {
|
||||
sqlx::query(
|
||||
"INSERT INTO mission_events (mission_id, agent_id, kind, target, detail)
|
||||
VALUES ($1,$2,'tool.call','Bash','{}'::jsonb)",
|
||||
)
|
||||
.bind(mission)
|
||||
.bind(owner)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("event");
|
||||
}
|
||||
(ws, agent)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_finished_mission_still_answers_what_the_agent_did() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (ws, agent) = seed(&pool).await;
|
||||
|
||||
let got = super::agent_last_run(&pool, ws).await;
|
||||
let lr = got
|
||||
.get(&agent.to_string())
|
||||
.expect("the agent's last finished mission must be found");
|
||||
|
||||
assert_eq!(lr.title, "JEPA Research");
|
||||
assert_eq!(lr.status, "completed");
|
||||
assert_eq!(lr.tokens, 2000, "tokens_in + tokens_out over both turns");
|
||||
assert_eq!(lr.credits, 9.0);
|
||||
assert_eq!(
|
||||
lr.tool_calls, 3,
|
||||
"only this agent's calls — the unattributed row belongs to no one \
|
||||
and must not be credited to them"
|
||||
);
|
||||
}
|
||||
|
||||
/// A running mission is the live feed's business. Reporting it here would
|
||||
/// put a current number behind a card the UI labels "last run".
|
||||
#[tokio::test]
|
||||
async fn a_mission_still_running_is_not_reported_as_a_last_run() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (ws, agent) = seed(&pool).await;
|
||||
sqlx::query(
|
||||
"UPDATE missions SET status='running', completed_at=NULL WHERE workspace_id=$1",
|
||||
)
|
||||
.bind(ws.as_uuid())
|
||||
.execute(&pool)
|
||||
.await
|
||||
.expect("update");
|
||||
|
||||
assert!(
|
||||
super::agent_last_run(&pool, ws)
|
||||
.await
|
||||
.get(&agent.to_string())
|
||||
.is_none(),
|
||||
"only completed and failed missions are history"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,24 @@
|
||||
use uuid::Uuid;
|
||||
|
||||
/// The runtime agent alias for a claw id.
|
||||
/// The bundles an agent is provisioned with: whatever the template asked for,
|
||||
/// plus `clawmates_door`, always.
|
||||
///
|
||||
/// The door is not optional. It carries the §15 approval gate, so an agent
|
||||
/// provisioned without it is not a restricted agent, it is an ungated one —
|
||||
/// and a template that simply forgot to list it would silently get that.
|
||||
fn with_door(bundles: &[String]) -> Vec<String> {
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
out.push("clawmates_door".to_string());
|
||||
for b in bundles {
|
||||
let b = b.trim();
|
||||
if !b.is_empty() && !out.iter().any(|x| x == b) {
|
||||
out.push(b.to_string());
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub fn claw_alias(claw_id: Uuid) -> String {
|
||||
format!("claw_{}", claw_id.simple())
|
||||
}
|
||||
@@ -168,6 +186,36 @@ impl RuntimeProvisioner {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Point `claude_cli.default` at a settings document, so the hooks written
|
||||
/// into the container are actually read.
|
||||
///
|
||||
/// Without this the gate and the tap exist on disk and claude never loads
|
||||
/// them — installed, inert, and indistinguishable from working. The alias
|
||||
/// is `claude_cli.default` because that is what `provider_alias_for` binds
|
||||
/// every claude model to.
|
||||
pub async fn set_claude_cli_settings(&self, path: &str) -> Result<(), String> {
|
||||
self.set_prop(
|
||||
"providers.models.claude_cli.default.settings",
|
||||
serde_json::json!(path),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Point `claude -p` at an MCP configuration.
|
||||
///
|
||||
/// The counterpart to [`set_claude_cli_settings`](Self::set_claude_cli_settings):
|
||||
/// writing the document into the container and telling the daemon about it
|
||||
/// are two halves of one thing, and doing one without the other leaves a
|
||||
/// door that is installed and unreachable — which looks exactly like a door
|
||||
/// nobody walked through.
|
||||
pub async fn set_claude_cli_mcp_config(&self, path: &str) -> Result<(), String> {
|
||||
self.set_prop(
|
||||
"providers.models.claude_cli.default.mcp_config",
|
||||
serde_json::json!(path),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Rebind an existing claw's model without touching its risk_profile
|
||||
/// or mcp_bundles. Used by the "change model" UI on the Agents page
|
||||
/// so we don't accidentally demote a coding_readwrite claw back to
|
||||
@@ -232,8 +280,21 @@ impl RuntimeProvisioner {
|
||||
/// agent gets: `toolfree` = nothing, `research_readonly` = file_read +
|
||||
/// content_search + glob_search, `coding_readwrite` = adds file_edit +
|
||||
/// git_operations + shell, etc.; see the `[risk_profiles.*]` allowlists
|
||||
/// in `deploy/clawmates-runtime/agent.config.example.toml`), and the
|
||||
/// `clawmates_door` MCP bundle.
|
||||
/// in `deploy/clawmates-runtime/agent.config.example.toml`), and the MCP
|
||||
/// bundles the team template asked for.
|
||||
///
|
||||
/// `bundles` used to be the constant `["clawmates_door"]`, which is how
|
||||
/// every skill in the catalogue became unreachable from a mission. The
|
||||
/// skills are delivered by ONE channel — the `clawmates_skills` MCP server
|
||||
/// (`mcp_skills.rs`) — a template that does not receive that bundle cannot
|
||||
/// list or read a single skill, and 5 of 11 templates ask for it. Two
|
||||
/// separate doc comments in `cm-runtime` describe the mission path as
|
||||
/// already having this, which is why nobody looked: the belief was written
|
||||
/// down twice and checked zero times.
|
||||
///
|
||||
/// `clawmates_door` is always included regardless of what is passed. It
|
||||
/// carries the §15 approval gate, and an agent provisioned without it does
|
||||
/// not become safer, it becomes ungated.
|
||||
///
|
||||
/// NOTE ON WORKSPACE PINNING: `[agents.<alias>.workspace.path]` is an
|
||||
/// `Option<PathBuf>` field that the ZeroClaw config prop-schema does NOT
|
||||
@@ -251,6 +312,7 @@ impl RuntimeProvisioner {
|
||||
claw_id: Uuid,
|
||||
model: &str,
|
||||
risk_profile: &str,
|
||||
bundles: &[String],
|
||||
) -> Result<String, String> {
|
||||
let alias = claw_alias(claw_id);
|
||||
let model_alias = provider_alias_for(model);
|
||||
@@ -285,7 +347,7 @@ impl RuntimeProvisioner {
|
||||
.await?;
|
||||
self.set_prop(
|
||||
&format!("agents.{alias}.mcp_bundles"),
|
||||
serde_json::json!(["clawmates_door"]),
|
||||
serde_json::json!(with_door(bundles)),
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -36,6 +36,9 @@ use uuid::Uuid;
|
||||
|
||||
use cm_db::repo::missions::UpsertTask;
|
||||
|
||||
/// `external_id` of the marker row proving a scan ran against a phase.
|
||||
pub const SCAN_MARKER: &str = "security_scan:complete";
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Finding {
|
||||
pub external_id: String,
|
||||
@@ -92,6 +95,32 @@ pub async fn run(pool: &PgPool, mission_id: Uuid, phase_id: Uuid) -> Result<usiz
|
||||
}
|
||||
}
|
||||
|
||||
// A completion marker, always written — including for a scan that found
|
||||
// nothing. Without it "we scanned and the repo is clean" and "no scan ever
|
||||
// ran" are both zero rows, and the sweep in `phase_runner` that fires this
|
||||
// would have no way to tell whether it had already run: a clean phase would
|
||||
// be rescanned on every tick, forever. It is also the answer to the
|
||||
// question an operator actually asks, which is not "how many findings"
|
||||
// but "was this looked at, by what, and when".
|
||||
let scanned_with = tools.join(", ");
|
||||
cm_db::repo::missions::upsert_task(
|
||||
pool,
|
||||
UpsertTask {
|
||||
mission_id,
|
||||
phase_id,
|
||||
external_id: SCAN_MARKER,
|
||||
title: &format!(
|
||||
"security scan complete — ran [{scanned_with}], {} finding(s)",
|
||||
all_findings.len()
|
||||
),
|
||||
assigned_agent_id: None,
|
||||
status: "created",
|
||||
run_id: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("upsert scan marker: {e}"))?;
|
||||
|
||||
for f in &all_findings {
|
||||
cm_db::repo::missions::upsert_task(
|
||||
pool,
|
||||
|
||||
@@ -0,0 +1,520 @@
|
||||
//! How a mission agent receives the skills bound to it.
|
||||
//!
|
||||
//! Two arms, and this module exists to hold them side by side rather than to
|
||||
//! replace one with the other:
|
||||
//!
|
||||
//! - [`Mode::Inline`] — every pinned skill's full body is appended to the turn
|
||||
//! prompt. What production has always done.
|
||||
//! - [`Mode::Index`] — the prompt carries each skill's name, description and
|
||||
//! `when_to_use` plus the URI that returns its body, and the agent fetches
|
||||
//! the ones it judges relevant through the MCP door.
|
||||
//! - [`Mode::Files`] — the same entry with a file path where the URI was; the
|
||||
//! bodies are written into the container and the agent `Read`s them. Added
|
||||
//! after `Index` measured 1 retrieval in 9 across three matched runs — see
|
||||
//! [`FILES_PREAMBLE`] for why.
|
||||
//!
|
||||
//! # Why this is an A/B and not a switch
|
||||
//!
|
||||
//! Trigger — did the agent reach for the skill when it applied? — is
|
||||
//! unmeasurable under `Inline` by construction. Nothing was reached for; the
|
||||
//! text was handed over. `skill_use` reports `NotObservable` for exactly that
|
||||
//! reason, and it is right to.
|
||||
//!
|
||||
//! `Index` makes Trigger observable, because retrieval is a recorded
|
||||
//! `ReadMcpResourceTool` call. But it can only *cost* Compliance: under
|
||||
//! `Inline` the procedure is in front of the model whether or not it noticed
|
||||
//! it applied, and under `Index` a missed judgement means the body is never
|
||||
//! read at all. Trading a measured axis for an unmeasured regression in
|
||||
//! another is not an improvement, so the arm is selected per mission and
|
||||
//! recorded on the mission row, and both arms stay runnable.
|
||||
//!
|
||||
//! # `Index` requires the door, and degrades rather than lying
|
||||
//!
|
||||
//! An index names a body and tells the agent how to fetch it. If the
|
||||
//! `clawmates_skills` MCP server is not reachable from the container, that is
|
||||
//! an index of procedures the agent cannot obtain — strictly worse than
|
||||
//! `Inline`, and it fails as an agent that ignored its skills rather than as a
|
||||
//! missing config. [`resolve`] therefore takes the door's install result and
|
||||
//! refuses `Index` without it. This is the same failure the old
|
||||
//! `pinned_skills_text` doc comment warned about; what changed is that the
|
||||
//! door now exists, not that the warning stopped applying.
|
||||
|
||||
/// Where the index tells agents to fetch a skill body from.
|
||||
///
|
||||
/// Must match the server name in
|
||||
/// [`crate::container_tool_hooks::mcp_document`] — the agent passes it
|
||||
/// straight to `ReadMcpResourceTool`.
|
||||
pub const MCP_SERVER: &str = "clawmates_skills";
|
||||
|
||||
/// Selects the arm. Unset means [`DEFAULT`]; unrecognised means [`Mode::Inline`].
|
||||
pub const ENV_VAR: &str = "CLAWMATES_SKILL_DELIVERY";
|
||||
|
||||
/// The arm a deployment runs when nothing selects one.
|
||||
///
|
||||
/// `Files` since 2026-09-13. It was `Inline` — the control arm of an A/B has
|
||||
/// to be the thing already running — until the A/B produced its answer: the
|
||||
/// MCP-door arm retrieved 1 skill in 9 across three matched production runs,
|
||||
/// and the file arm retrieved 3 of 3 on the fourth (`01a098dd`), with the
|
||||
/// judge loop closing on the same run. That is a signal and not a rate, but
|
||||
/// 0, 1, 0 → 3 on an otherwise identical task is not noise, and a default that
|
||||
/// hands agents procedures they demonstrably read beats one that hands them
|
||||
/// bodies they were never asked to look for.
|
||||
///
|
||||
/// A code default and not an env var on one server, because a setting that
|
||||
/// exists only in one deployment is a setting nobody can find — the exact
|
||||
/// shape `always_inject` had before it moved into the skill files.
|
||||
pub const DEFAULT: Mode = Mode::Files;
|
||||
|
||||
/// The `# Your skills` preamble under [`Mode::Inline`].
|
||||
///
|
||||
/// **Byte-identical to what production has always sent.** The A arm of an A/B
|
||||
/// has to be the thing already running, or the comparison measures this edit
|
||||
/// as well as the change under test.
|
||||
pub const INLINE_PREAMBLE: &str = "These are procedures you are expected to follow for \
|
||||
this kind of work. Where one applies to what you are about to do, follow it.";
|
||||
|
||||
/// The `# Your skills` preamble under [`Mode::Index`] as first shipped.
|
||||
///
|
||||
/// Kept because [`mode_in_prompt`] reads the arm off a RECORDED prompt, and
|
||||
/// prompts composed before the tool-loading sentence was added are still being
|
||||
/// scored — `retain_events_until` holds them for 90 days. Dropping this
|
||||
/// constant would silently re-label every stored `index` run as `inline` and
|
||||
/// report Trigger against the wrong arm.
|
||||
///
|
||||
/// Never send this one. It is a reader, not a writer.
|
||||
pub const INDEX_PREAMBLE_V1: &str = "These procedures are AVAILABLE to you; their bodies are \
|
||||
not included below. Each entry names one, says when it applies, and gives the uri that \
|
||||
returns it. Where an entry applies to what you are about to do, read it FIRST and then \
|
||||
follow it.";
|
||||
|
||||
/// The `# Your skills` preamble under [`Mode::Index`].
|
||||
///
|
||||
/// Written and matched in one place ([`mode_in_prompt`]) so the reader cannot
|
||||
/// drift from the writer — the same rule `SKILL_MARKER` is under, and for the
|
||||
/// same reason: a scorer that misreads the arm reports the wrong axis.
|
||||
///
|
||||
/// # Why the last sentence exists
|
||||
///
|
||||
/// `ReadMcpResourceTool` is a DEFERRED tool: it is not on the agent's default
|
||||
/// tool list and cannot be called until `ToolSearch` loads its schema. Naming
|
||||
/// it — which [`READ_IT`] already did — is therefore not enough, and the
|
||||
/// difference is measurable. Prod mission `01a07812` made 76 tool calls,
|
||||
/// searched for two other tools, never searched for this one, and retrieved
|
||||
/// ZERO skills. `01a0842e`, same recipe and same offered uris, ran
|
||||
/// `ToolSearch(select:ReadMcpResourceTool)` and then fetched. One agent worked
|
||||
/// the extra step out on its own; the other did not, and a capability that
|
||||
/// depends on the model guessing that a tool is loadable is not delivered.
|
||||
pub const INDEX_PREAMBLE: &str = "These procedures are AVAILABLE to you; their bodies are \
|
||||
not included below. Each entry names one, says when it applies, and gives the uri that \
|
||||
returns it. Where an entry applies to what you are about to do, read it FIRST and then \
|
||||
follow it. ReadMcpResourceTool may not be loaded in this session: if you do not already \
|
||||
have it, run ToolSearch with the query select:ReadMcpResourceTool before your first read.";
|
||||
|
||||
/// Where the `files` arm puts skill bodies inside the mission container.
|
||||
///
|
||||
/// Under `/mission` because that is the one directory every container-tier
|
||||
/// mission has ([`crate::mission_fs::CONTAINER_MISSION_DIR`]), and beside
|
||||
/// `repo/` rather than inside it so a skill never shows up in a diff or a
|
||||
/// delivery.
|
||||
pub const SKILLS_DIR: &str = "/mission/skills";
|
||||
|
||||
/// The file a skill's body is written to under the `files` arm, and the path
|
||||
/// the index entry tells the agent to `Read`. One function for both, so the
|
||||
/// writer and the reader cannot spell it differently.
|
||||
pub fn skill_file_path(name: &str) -> String {
|
||||
format!("{SKILLS_DIR}/{name}.md")
|
||||
}
|
||||
|
||||
/// The skill a `Read` of this path is a retrieval of, if it is one.
|
||||
///
|
||||
/// The scorer's half of [`skill_file_path`]. Anything outside [`SKILLS_DIR`]
|
||||
/// is an ordinary file read and returns `None`.
|
||||
pub fn skill_from_file_path(path: &str) -> Option<String> {
|
||||
let rest = path.strip_prefix(SKILLS_DIR)?.strip_prefix('/')?;
|
||||
let name = rest.strip_suffix(".md")?;
|
||||
if name.is_empty() || name.contains('/') {
|
||||
return None;
|
||||
}
|
||||
Some(name.to_string())
|
||||
}
|
||||
|
||||
/// The `# Your skills` preamble under [`Mode::Files`].
|
||||
///
|
||||
/// # Why a third arm
|
||||
///
|
||||
/// `Index` retrieves through `ReadMcpResourceTool`, which is a DEFERRED tool:
|
||||
/// absent from the agent's default list until `ToolSearch` loads it. Measured
|
||||
/// across three matched production runs (`01a07812`, `01a0842e`, `01a09877` —
|
||||
/// same recipe, same task, same three offered uris), that path retrieved
|
||||
/// **1 skill in 9 chances**, and telling the agent in the preamble to load
|
||||
/// the tool first changed nothing: the third run's three reasoning narratives
|
||||
/// never mention skills at all. The section was not declined; it was never
|
||||
/// engaged with.
|
||||
///
|
||||
/// `Read` is a core tool. It is never deferred, and every one of those agents
|
||||
/// used it. So this arm keeps progressive disclosure exactly as `Index` has it
|
||||
/// — name, `when_to_use`, and a pointer the agent has to follow — and changes
|
||||
/// only what the pointer is: a file path instead of an MCP uri. A `Read` of
|
||||
/// that path is a tapped tool call, so Trigger stays as observable as before.
|
||||
pub const FILES_PREAMBLE: &str = "These procedures are AVAILABLE to you; their bodies are \
|
||||
not included below. Each entry names one, says when it applies, and gives the path of the \
|
||||
file that holds it. Where an entry applies to what you are about to do, Read that file FIRST \
|
||||
and then follow it.";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Mode {
|
||||
Inline,
|
||||
Index,
|
||||
Files,
|
||||
}
|
||||
|
||||
impl Mode {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Mode::Inline => "inline",
|
||||
Mode::Index => "index",
|
||||
Mode::Files => "files",
|
||||
}
|
||||
}
|
||||
|
||||
/// Does this arm hand the agent a pointer rather than a body?
|
||||
///
|
||||
/// The two retrieval arms share every rule that follows from that — the
|
||||
/// scorer's Trigger axis, the `always_inject` override, the fallback when
|
||||
/// nothing was installed — and branching on this rather than on `Index`
|
||||
/// is what keeps a third arm from silently inheriting `Inline`'s answers.
|
||||
pub fn is_retrieval(self) -> bool {
|
||||
matches!(self, Mode::Index | Mode::Files)
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a recorded or configured arm. Unrecognised input is `None`, and every
|
||||
/// caller resolves that to `Inline` — an unreadable value must not silently
|
||||
/// select the arm that needs a door.
|
||||
pub fn parse(s: &str) -> Option<Mode> {
|
||||
match s.trim().to_ascii_lowercase().as_str() {
|
||||
"inline" => Some(Mode::Inline),
|
||||
"index" | "progressive" => Some(Mode::Index),
|
||||
"files" | "file" => Some(Mode::Files),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The arm this deployment asks for, before the door is taken into account.
|
||||
pub fn requested() -> Mode {
|
||||
let Ok(raw) = std::env::var(ENV_VAR) else {
|
||||
return DEFAULT;
|
||||
};
|
||||
if raw.trim().is_empty() {
|
||||
return DEFAULT;
|
||||
}
|
||||
match parse(&raw) {
|
||||
Some(m) => m,
|
||||
// Garbage falls to `Inline`, not to `DEFAULT`: an unreadable value must
|
||||
// not silently select an arm that needs something installed.
|
||||
None => {
|
||||
eprintln!(
|
||||
"skill_delivery: {ENV_VAR}={raw:?} is not `inline`, `index` or `files` — \
|
||||
delivering skills inline"
|
||||
);
|
||||
Mode::Inline
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The arm for one mission: `config.skill_delivery` if it names one, otherwise
|
||||
/// the deployment default.
|
||||
///
|
||||
/// Per-mission and not only per-deployment because the alternative is
|
||||
/// restarting the server between arms, and an A/B whose two halves ran against
|
||||
/// different server processes has a confound in it that nothing in the numbers
|
||||
/// will show. This way both arms run against one binary, interleaved.
|
||||
pub fn requested_for(config: &serde_json::Value) -> Mode {
|
||||
let Some(raw) = config.get("skill_delivery").and_then(|v| v.as_str()) else {
|
||||
return requested();
|
||||
};
|
||||
match parse(raw) {
|
||||
Some(m) => m,
|
||||
None => {
|
||||
eprintln!(
|
||||
"skill_delivery: config.skill_delivery={raw:?} is not `inline`, `index` \
|
||||
or `files` — falling back to the deployment default"
|
||||
);
|
||||
requested()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The arm a mission will actually run, given whether what it retrieves from
|
||||
/// was installed — the MCP door for `index`, the skill files for `files`.
|
||||
pub fn resolve(requested: Mode, installed: bool) -> Mode {
|
||||
match (requested, installed) {
|
||||
(m, true) if m.is_retrieval() => m,
|
||||
(m, false) if m.is_retrieval() => {
|
||||
eprintln!(
|
||||
"skill_delivery: `{}` was asked for but this mission has nothing to \
|
||||
retrieve from — falling back to `inline`, because an index the agent \
|
||||
cannot fetch from is worse than no index",
|
||||
m.as_str()
|
||||
);
|
||||
Mode::Inline
|
||||
}
|
||||
_ => Mode::Inline,
|
||||
}
|
||||
}
|
||||
|
||||
/// The `# Your skills` section heading for an arm.
|
||||
pub fn preamble(mode: Mode) -> &'static str {
|
||||
match mode {
|
||||
Mode::Inline => INLINE_PREAMBLE,
|
||||
Mode::Index => INDEX_PREAMBLE,
|
||||
Mode::Files => FILES_PREAMBLE,
|
||||
}
|
||||
}
|
||||
|
||||
/// Which arm produced a recorded prompt.
|
||||
///
|
||||
/// Read back from the prompt rather than from the mission row on purpose: the
|
||||
/// row says what the mission was configured to do *now*, and a score is being
|
||||
/// computed against a prompt that was composed then. The recorded prompt is
|
||||
/// the only artefact that cannot have changed since the turn ran.
|
||||
///
|
||||
/// Matched as a whole line. A skill body that quotes the preamble mid-sentence
|
||||
/// is prose; this is the same rule `skill_names_in` learned the hard way.
|
||||
pub fn mode_in_prompt(prompt: &str) -> Mode {
|
||||
// Both spellings, because this reads prompts composed by older builds as
|
||||
// well as the current one. A stored measurement that changes arm when the
|
||||
// writer is edited is not a measurement.
|
||||
for l in prompt.lines() {
|
||||
let l = l.trim();
|
||||
if l == INDEX_PREAMBLE || l == INDEX_PREAMBLE_V1 {
|
||||
return Mode::Index;
|
||||
}
|
||||
if l == FILES_PREAMBLE {
|
||||
return Mode::Files;
|
||||
}
|
||||
}
|
||||
Mode::Inline
|
||||
}
|
||||
|
||||
/// One index entry's text — everything under the `--- SKILL: <name> ---`
|
||||
/// marker, which [`crate::topology_exec::render_pinned_skill`] writes.
|
||||
///
|
||||
/// `when_to_use` is the load-bearing field: it is the only thing the agent has
|
||||
/// to judge relevance from, so a skill with none says so rather than omitting
|
||||
/// the line and leaving the model to infer from the description alone.
|
||||
pub fn index_entry(description: &str, when_to_use: Option<&str>, uri: &str) -> String {
|
||||
let when = when_to_use
|
||||
.map(str::trim)
|
||||
.filter(|w| !w.is_empty())
|
||||
.unwrap_or("not stated — judge from the description");
|
||||
format!(
|
||||
"{}\nWhen to use: {}\n{READ_IT}server=\"{}\", uri=\"{}\")",
|
||||
description.trim(),
|
||||
when,
|
||||
MCP_SERVER,
|
||||
uri,
|
||||
)
|
||||
}
|
||||
|
||||
/// The line that makes an index entry recognisable as one.
|
||||
///
|
||||
/// Shared by the renderer and [`skill_was_indexed`] so the scorer cannot drift
|
||||
/// from the delivery — two spellings of one marker is how a detector quietly
|
||||
/// stops detecting.
|
||||
pub const READ_IT: &str = "Read it: ReadMcpResourceTool(";
|
||||
|
||||
/// [`READ_IT`]'s counterpart for the `files` arm. Same rule: one constant,
|
||||
/// written by [`file_entry`] and read by [`skill_was_indexed`].
|
||||
pub const READ_FILE_IT: &str = "Read it: Read(file_path=\"";
|
||||
|
||||
/// One `files`-arm entry — [`index_entry`] with a path where the uri was.
|
||||
pub fn file_entry(description: &str, when_to_use: Option<&str>, path: &str) -> String {
|
||||
let when = when_to_use
|
||||
.map(str::trim)
|
||||
.filter(|w| !w.is_empty())
|
||||
.unwrap_or("not stated — judge from the description");
|
||||
format!(
|
||||
"{}\nWhen to use: {}\n{READ_FILE_IT}{}\")",
|
||||
description.trim(),
|
||||
when,
|
||||
path,
|
||||
)
|
||||
}
|
||||
|
||||
/// How was THIS skill delivered, regardless of the arm the prompt announces?
|
||||
///
|
||||
/// `Some(true)` — an index entry: named, described, and left to be fetched.
|
||||
/// `Some(false)` — the body itself, which under `Index` means the skill is
|
||||
/// marked `always_inject`.
|
||||
/// `None` — not in the prompt at all (it was retrieved, or never delivered).
|
||||
///
|
||||
/// The arm is a property of the PROMPT; `always_inject` is a property of the
|
||||
/// SKILL. Scoring the arm alone would report a Trigger failure against a skill
|
||||
/// the agent was handed and was never asked to fetch.
|
||||
pub fn skill_was_indexed(prompt: &str, skill: &str) -> Option<bool> {
|
||||
let marker = crate::topology_exec::SKILL_MARKER;
|
||||
let mut lines = prompt.lines();
|
||||
// Find this skill's section...
|
||||
lines.find(|l| {
|
||||
l.trim()
|
||||
.strip_prefix(marker)
|
||||
.map(|rest| rest.trim_end_matches(" ---").trim() == skill)
|
||||
.unwrap_or(false)
|
||||
})?;
|
||||
// ...and read to the next one.
|
||||
for l in lines {
|
||||
if l.trim().starts_with(marker) {
|
||||
break;
|
||||
}
|
||||
if l.contains(READ_IT) || l.contains(READ_FILE_IT) {
|
||||
return Some(true);
|
||||
}
|
||||
}
|
||||
Some(false)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn an_unreadable_arm_never_selects_the_one_that_needs_a_door() {
|
||||
assert_eq!(parse("nonsense"), None);
|
||||
assert_eq!(parse("INDEX"), Some(Mode::Index));
|
||||
assert_eq!(parse(" inline "), Some(Mode::Inline));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_mission_can_name_its_own_arm() {
|
||||
assert_eq!(
|
||||
requested_for(&serde_json::json!({ "skill_delivery": "index" })),
|
||||
Mode::Index
|
||||
);
|
||||
// Unreadable values and absent ones both defer to the deployment
|
||||
// default, which is `Inline` unless the environment says otherwise.
|
||||
assert_eq!(
|
||||
requested_for(&serde_json::json!({ "skill_delivery": "sideways" })),
|
||||
requested()
|
||||
);
|
||||
assert_eq!(requested_for(&serde_json::json!({})), requested());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn index_without_a_door_falls_back() {
|
||||
assert_eq!(resolve(Mode::Index, false), Mode::Inline);
|
||||
assert_eq!(resolve(Mode::Index, true), Mode::Index);
|
||||
assert_eq!(resolve(Mode::Inline, true), Mode::Inline);
|
||||
}
|
||||
|
||||
/// The deployment default is a measured decision; changing it should fail
|
||||
/// a test so it is made on purpose, with the numbers in front of you.
|
||||
#[test]
|
||||
fn the_default_arm_is_files_and_garbage_still_falls_to_inline() {
|
||||
assert_eq!(DEFAULT, Mode::Files);
|
||||
assert_eq!(requested_for(&serde_json::json!({})), requested());
|
||||
assert_eq!(
|
||||
requested_for(&serde_json::json!({ "skill_delivery": "sideways" })),
|
||||
requested(),
|
||||
"an unreadable per-mission value defers to the deployment, as before"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_files_arm_parses_resolves_and_reads_back() {
|
||||
assert_eq!(parse("files"), Some(Mode::Files));
|
||||
assert_eq!(resolve(Mode::Files, true), Mode::Files);
|
||||
assert_eq!(
|
||||
resolve(Mode::Files, false),
|
||||
Mode::Inline,
|
||||
"files that were never written must not be advertised"
|
||||
);
|
||||
let prompt = format!("Task: x\n\n# Your skills\n\n{FILES_PREAMBLE}\n\nentry");
|
||||
assert_eq!(mode_in_prompt(&prompt), Mode::Files);
|
||||
assert_eq!(preamble(Mode::Files), FILES_PREAMBLE);
|
||||
}
|
||||
|
||||
/// The writer and the reader of a skill path are one pair of functions.
|
||||
#[test]
|
||||
fn a_skill_path_round_trips_and_nothing_else_parses_as_one() {
|
||||
let p = skill_file_path("web-search-triage");
|
||||
assert_eq!(p, "/mission/skills/web-search-triage.md");
|
||||
assert_eq!(skill_from_file_path(&p).as_deref(), Some("web-search-triage"));
|
||||
for not_a_skill in [
|
||||
"/mission/repo/skills/x.md",
|
||||
"/mission/skills/x.txt",
|
||||
"/mission/skills/.md",
|
||||
"/mission/skills/a/b.md",
|
||||
"/mission/skills",
|
||||
"mission/skills/x.md",
|
||||
] {
|
||||
assert_eq!(skill_from_file_path(not_a_skill), None, "{not_a_skill}");
|
||||
}
|
||||
}
|
||||
|
||||
/// `skill_was_indexed` is how the scorer tells a pointer from a body. A
|
||||
/// file entry must read as a pointer, or `always_inject` logic would treat
|
||||
/// every `files`-arm skill as handed over.
|
||||
#[test]
|
||||
fn a_file_entry_reads_as_indexed_not_inlined() {
|
||||
let entry = file_entry("Summarise.", Some("when asked"), &skill_file_path("x"));
|
||||
assert!(entry.contains(READ_FILE_IT), "{entry}");
|
||||
let prompt = format!(
|
||||
"Task\n\n{}x ---\n{entry}\n",
|
||||
crate::topology_exec::SKILL_MARKER
|
||||
);
|
||||
assert_eq!(skill_was_indexed(&prompt, "x"), Some(true));
|
||||
}
|
||||
|
||||
/// A prompt composed before the tool-loading sentence existed must still
|
||||
/// score as `Index`. Stored prompts are held for 90 days and re-scored
|
||||
/// when the scorer changes; if this regressed, every one of them would
|
||||
/// quietly become an `inline` run and Trigger would be reported against an
|
||||
/// arm that never ran.
|
||||
#[test]
|
||||
fn an_older_index_prompt_still_reads_as_index() {
|
||||
let old = format!("Task: x\n\n# Your skills\n\n{INDEX_PREAMBLE_V1}\n\nentry");
|
||||
assert_eq!(mode_in_prompt(&old), Mode::Index);
|
||||
let new = format!("Task: x\n\n# Your skills\n\n{INDEX_PREAMBLE}\n\nentry");
|
||||
assert_eq!(mode_in_prompt(&new), Mode::Index);
|
||||
}
|
||||
|
||||
/// The two spellings must stay one text plus an addition, not two texts.
|
||||
/// Written out in full because `concat!` cannot take a const, so nothing
|
||||
/// but this test stops them drifting apart.
|
||||
#[test]
|
||||
fn the_current_preamble_extends_the_original() {
|
||||
assert!(
|
||||
INDEX_PREAMBLE.starts_with(INDEX_PREAMBLE_V1),
|
||||
"the v1 preamble must remain a prefix, or old prompts stop matching"
|
||||
);
|
||||
assert!(INDEX_PREAMBLE.contains("select:ReadMcpResourceTool"));
|
||||
}
|
||||
|
||||
/// The scorer reads the arm off the prompt, so the writer and this reader
|
||||
/// have to agree for every arm — including the one that writes no marker.
|
||||
#[test]
|
||||
fn the_arm_is_recoverable_from_the_prompt_that_was_sent() {
|
||||
let inline = format!("Task: x\n\n# Your skills\n\n{INLINE_PREAMBLE}\n\nbody");
|
||||
let index = format!("Task: x\n\n# Your skills\n\n{INDEX_PREAMBLE}\n\nentry");
|
||||
assert_eq!(mode_in_prompt(&inline), Mode::Inline);
|
||||
assert_eq!(mode_in_prompt(&index), Mode::Index);
|
||||
assert_eq!(mode_in_prompt("Task: x"), Mode::Inline);
|
||||
}
|
||||
|
||||
/// A body quoting the preamble must not re-label the arm — the same
|
||||
/// failure `SKILL_MARKER` had when a heading inside a body counted.
|
||||
#[test]
|
||||
fn a_body_quoting_the_preamble_does_not_change_the_arm() {
|
||||
let body = format!("The index arm opens with \"{INDEX_PREAMBLE}\" and then lists.");
|
||||
let prompt = format!("Task: x\n\n# Your skills\n\n{INLINE_PREAMBLE}\n\n{body}");
|
||||
assert_eq!(mode_in_prompt(&prompt), Mode::Inline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_entry_states_a_missing_when_to_use_rather_than_dropping_the_line() {
|
||||
let e = index_entry("Summarise a paper.", None, "skill:global/x");
|
||||
assert!(e.contains("When to use: not stated"), "{e}");
|
||||
assert!(e.contains("ReadMcpResourceTool(server=\"clawmates_skills\""), "{e}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
//! Applies agents' own skill drafts, with no human decision.
|
||||
//!
|
||||
//! `level_up` has generated complete skill drafts from a model since it
|
||||
//! shipped; the only thing between a draft and the catalogue was an operator
|
||||
//! ticking a checkbox in `LevelUpDrawer`. This worker removes the checkbox —
|
||||
//! when switched on. It is OFF by default since 2026-09-20; see
|
||||
//! `level_up::self_authoring_enabled` for why.
|
||||
//!
|
||||
//! What is deliberately NOT removed is the record. Every write stays
|
||||
//! workspace-scoped and versioned, cannot take the name of a hand-authored
|
||||
//! skill, and lands with `approved_by = NULL` — so "an agent decided this" is
|
||||
//! distinguishable from "a person decided this" forever after, which is the
|
||||
//! property that makes the change reversible instead of merely fast.
|
||||
//!
|
||||
//! Only `skill_candidate` items apply here. `identity_refinement` and
|
||||
//! `brain_consolidation` still wait for a human: they change what an agent IS
|
||||
//! rather than adding a procedure it can consult.
|
||||
|
||||
use sqlx::{PgPool, Row};
|
||||
use std::time::Duration;
|
||||
|
||||
/// How often to sweep for pending drafts.
|
||||
///
|
||||
/// Proposals arrive when someone runs a level-up, not continuously, so this is
|
||||
/// slow on purpose — the work is bounded by how often an agent reflects, and
|
||||
/// polling faster would only add load.
|
||||
const SWEEP_INTERVAL: Duration = Duration::from_secs(120);
|
||||
|
||||
/// Start the sweep, unless self-authoring is switched off.
|
||||
pub fn spawn(pool: PgPool) {
|
||||
if !crate::level_up::self_authoring_enabled() {
|
||||
eprintln!(
|
||||
"skill_self_authoring: DISABLED (the default since 2026-09-20) — \
|
||||
agent skill drafts wait for a human in the level-up drawer. \
|
||||
Set CLAWMATES_SKILL_SELF_AUTHORING=1 to let agents apply their own."
|
||||
);
|
||||
return;
|
||||
}
|
||||
eprintln!(
|
||||
"skill_self_authoring: ENABLED — agents apply their own skill drafts \
|
||||
without human approval. Writes are workspace-scoped, versioned, and \
|
||||
cannot take a hand-authored skill's name; each lands with no approver \
|
||||
recorded. Unset CLAWMATES_SKILL_SELF_AUTHORING to restore the gate."
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
if let Err(e) = sweep(&pool).await {
|
||||
eprintln!("skill_self_authoring: sweep failed: {e}");
|
||||
}
|
||||
tokio::time::sleep(SWEEP_INTERVAL).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Apply every pending proposal's skill candidates. Returns how many skills landed.
|
||||
pub async fn sweep(pool: &PgPool) -> Result<usize, String> {
|
||||
// Bounded per pass: a backlog drains over several sweeps rather than
|
||||
// holding the pool for as long as it takes to apply all of it.
|
||||
let rows = sqlx::query(
|
||||
"SELECT id, workspace_id FROM level_up_proposals
|
||||
WHERE status = 'pending'
|
||||
ORDER BY created_at
|
||||
LIMIT 20",
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| format!("select pending proposals: {e}"))?;
|
||||
|
||||
let mut applied = 0usize;
|
||||
for row in &rows {
|
||||
let id: uuid::Uuid = row.get("id");
|
||||
let workspace_id: uuid::Uuid = row.get("workspace_id");
|
||||
match crate::level_up::apply_autonomous(
|
||||
pool,
|
||||
cm_domain::WorkspaceId::from(workspace_id),
|
||||
id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(items) if !items.is_empty() => {
|
||||
applied += items.len();
|
||||
eprintln!(
|
||||
"skill_self_authoring: applied {} skill draft(s) from proposal {id} \
|
||||
with no human approval",
|
||||
items.len()
|
||||
);
|
||||
}
|
||||
// A proposal with no skill candidates is left pending on purpose —
|
||||
// its identity/memory items still belong to the human gate.
|
||||
Ok(_) => {}
|
||||
Err(e) => eprintln!("skill_self_authoring: proposal {id}: {e}"),
|
||||
}
|
||||
}
|
||||
Ok(applied)
|
||||
}
|
||||
@@ -0,0 +1,131 @@
|
||||
//! Skill triage, in shadow: which of the visible skills a phase's task calls
|
||||
//! for, by a calibrated decision model, recorded beside what the agent then
|
||||
//! actually read.
|
||||
//!
|
||||
//! SRA-Bench (arXiv 2604.24594) found agents load skills at the same rate
|
||||
//! whether or not one applies — the bottleneck is knowing WHEN, and the
|
||||
//! agent's only signal today is the `when_to_use` line in its own prompt. A
|
||||
//! host-side oracle that answers the same question in 200 ms is the thing
|
||||
//! to measure against that. `cm_decide::jev` scored AUROC 0.989 on the
|
||||
//! labelled set (`crates/cm-decide/eval`); this records its answer per phase
|
||||
//! as a `skill.triage` event and the Skill-Use scorer reads it back next to
|
||||
//! the agent's Trigger. It selects nothing: the files arm still installs
|
||||
//! every visible skill. Promotion to a real selector is a later, measured
|
||||
//! step, once the agreement numbers from real missions say what the
|
||||
//! oracle's misses cost.
|
||||
//!
|
||||
//! One call per phase launch, spawned so the launch never waits on it, and
|
||||
//! silent when `TYPESAFE_API_KEY` is unset. The key never leaves the server.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use cm_decide::{Answer, Decider};
|
||||
use sqlx::PgPool;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const EVENT: &str = "skill.triage";
|
||||
|
||||
/// How long a shadow decision may take before it is dropped. Jev measures
|
||||
/// ~200 ms; a backend that takes ten seconds is not the one to shadow.
|
||||
const TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
||||
/// Fire the triage for one phase and record it. Best-effort throughout: a
|
||||
/// missing key, a failed call, or a timeout leaves no event and one log line.
|
||||
pub fn spawn(pool: PgPool, mission_id: Uuid, phase_id: Uuid, workspace_id: Uuid, task: String) {
|
||||
let Some(jev) = cm_decide::jev::Jev::from_env() else {
|
||||
return;
|
||||
};
|
||||
tokio::spawn(async move {
|
||||
let skills = match cm_db::repo::skills_catalog::list_visible(&pool, workspace_id).await {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
eprintln!("skill_triage: could not list skills for {mission_id}: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let questions: BTreeMap<String, cm_decide::Question> = skills
|
||||
.iter()
|
||||
.map(|s| {
|
||||
(
|
||||
s.name.clone(),
|
||||
cm_decide::triage::question(&s.name, s.when_to_use.as_deref().unwrap_or(&s.description)),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
if questions.is_empty() {
|
||||
return;
|
||||
}
|
||||
let decision = match tokio::time::timeout(TIMEOUT, jev.decide(&task, &questions)).await {
|
||||
Ok(Ok(d)) => d,
|
||||
Ok(Err(e)) => {
|
||||
eprintln!("skill_triage: {} failed for phase {phase_id}: {e}", jev.name());
|
||||
return;
|
||||
}
|
||||
Err(_) => {
|
||||
eprintln!("skill_triage: {} timed out for phase {phase_id}", jev.name());
|
||||
return;
|
||||
}
|
||||
};
|
||||
let probabilities: BTreeMap<&str, f64> = decision
|
||||
.answers
|
||||
.iter()
|
||||
.filter_map(|(k, a)| match a {
|
||||
Answer::Noul { noul } => Some((k.as_str(), *noul)),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
let applies = probabilities
|
||||
.iter()
|
||||
.filter(|(_, p)| **p >= cm_decide::triage::APPLIES_AT)
|
||||
.count();
|
||||
eprintln!(
|
||||
"skill_triage: phase {phase_id} — {} says {applies} of {} skills apply ({} ms, {} tokens)",
|
||||
decision.model,
|
||||
probabilities.len(),
|
||||
decision.latency.as_millis(),
|
||||
decision.usage.map(|u| u.input_tokens).unwrap_or(0),
|
||||
);
|
||||
crate::mission_events::record(
|
||||
&pool,
|
||||
crate::mission_events::MissionEvent::new(mission_id, EVENT)
|
||||
.phase(phase_id)
|
||||
.detail(serde_json::json!({
|
||||
"backend": jev.name(),
|
||||
"model": decision.model,
|
||||
"wording": cm_decide::triage::WORDING,
|
||||
"latency_ms": decision.latency.as_millis() as u64,
|
||||
"input_tokens": decision.usage.map(|u| u.input_tokens),
|
||||
"applies_at": cm_decide::triage::APPLIES_AT,
|
||||
"skills": probabilities,
|
||||
})),
|
||||
)
|
||||
.await;
|
||||
});
|
||||
}
|
||||
|
||||
/// The recorded triage for a mission: skill → highest probability any phase
|
||||
/// gave it. Empty when no event was recorded (no key, or before this existed).
|
||||
pub async fn recorded(pool: &PgPool, mission_id: Uuid) -> BTreeMap<String, f64> {
|
||||
let rows: Vec<(serde_json::Value,)> = sqlx::query_as(
|
||||
"SELECT detail FROM mission_events WHERE mission_id = $1 AND kind = $2 ORDER BY id",
|
||||
)
|
||||
.bind(mission_id)
|
||||
.bind(EVENT)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
let mut out: BTreeMap<String, f64> = BTreeMap::new();
|
||||
for (detail,) in rows {
|
||||
if let Some(map) = detail.get("skills").and_then(|s| s.as_object()) {
|
||||
for (name, p) in map {
|
||||
if let Some(p) = p.as_f64() {
|
||||
let e = out.entry(name.clone()).or_insert(0.0);
|
||||
if p > *e {
|
||||
*e = p;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -7,6 +7,13 @@
|
||||
//! description: <one-line, shown to the LLM in resources/list>
|
||||
//! when_to_use: <trigger sentence, appended to description>
|
||||
//! tags: [foundation, rust, ...]
|
||||
//! always_inject: true # optional, default false
|
||||
//!
|
||||
//! `always_inject` makes the body reach the agent in full even under the
|
||||
//! `index` (progressive-disclosure) arm. It is for a CROSS-CUTTING procedure —
|
||||
//! one that applies to everyone who writes, and so reads to each agent as
|
||||
//! nobody's in particular, which is how `workspace-repo-commit-protocol`
|
||||
//! scored Trigger=FAIL beside a passing boundary check.
|
||||
//!
|
||||
//! The body is the rest of the file. Both are upserted idempotently:
|
||||
//! `skills_catalog::upsert_builtin` bumps the version + appends to
|
||||
@@ -27,6 +34,8 @@ struct Frontmatter {
|
||||
when_to_use: Option<String>,
|
||||
#[serde(default)]
|
||||
tags: Vec<String>,
|
||||
#[serde(default)]
|
||||
always_inject: bool,
|
||||
}
|
||||
|
||||
fn skills_dir() -> PathBuf {
|
||||
@@ -120,6 +129,7 @@ async fn load_one(pool: &PgPool, path: &std::path::Path) -> Result<String, Strin
|
||||
when_to_use: fm.when_to_use.as_deref(),
|
||||
tags: fm.tags.clone(),
|
||||
body,
|
||||
always_inject: fm.always_inject,
|
||||
};
|
||||
upsert_builtin(pool, skill)
|
||||
.await
|
||||
@@ -158,6 +168,36 @@ mod tests {
|
||||
assert!(split_frontmatter("# plain md\n").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn always_inject_is_opt_in_and_parses() {
|
||||
let off: Frontmatter = serde_yaml::from_str("name: a\ndescription: b\n").unwrap();
|
||||
assert!(
|
||||
!off.always_inject,
|
||||
"full delivery must be opted INTO — defaulting true would abolish the index arm"
|
||||
);
|
||||
let on: Frontmatter =
|
||||
serde_yaml::from_str("name: a\ndescription: b\nalways_inject: true\n").unwrap();
|
||||
assert!(on.always_inject);
|
||||
}
|
||||
|
||||
/// The flag reached production as a hand-run UPDATE first, which a rebuilt
|
||||
/// database would have silently dropped. This asserts the repo carries it,
|
||||
/// so the cross-cutting skill cannot go back to being deliverable only by
|
||||
/// an agent noticing it applies — the exact failure it was measured on.
|
||||
#[test]
|
||||
fn the_commit_protocol_ships_marked_for_full_delivery() {
|
||||
let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../skills/foundation/workspace-repo-commit-protocol.md");
|
||||
let text = std::fs::read_to_string(&path).expect("read the commit-protocol skill");
|
||||
let (yaml, _) = split_frontmatter(&text).expect("frontmatter");
|
||||
let fm: Frontmatter = serde_yaml::from_str(yaml).expect("parse frontmatter");
|
||||
assert!(
|
||||
fm.always_inject,
|
||||
"workspace-repo-commit-protocol must be always_inject: it applies to everyone \
|
||||
who writes, and under the index arm it scored Trigger=FAIL unread"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builtin_id_stable() {
|
||||
assert_eq!(
|
||||
@@ -170,3 +210,266 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod contradiction_tests {
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn repo_root(rel: &str) -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../..")
|
||||
.join(rel)
|
||||
.canonicalize()
|
||||
.unwrap_or_else(|e| panic!("{rel}: {e}"))
|
||||
}
|
||||
|
||||
fn walk_ext(dir: &std::path::Path, ext: &str, out: &mut Vec<(String, String)>) {
|
||||
for e in std::fs::read_dir(dir).expect("read dir") {
|
||||
let p = e.expect("entry").path();
|
||||
if p.is_dir() {
|
||||
walk_ext(&p, ext, out);
|
||||
} else if p.extension().and_then(|x| x.to_str()) == Some(ext) {
|
||||
out.push((
|
||||
p.file_name().unwrap().to_string_lossy().to_string(),
|
||||
std::fs::read_to_string(&p).expect("read file"),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Skill bodies alone.
|
||||
fn all_skills() -> Vec<(String, String)> {
|
||||
let mut out = Vec::new();
|
||||
walk_ext(&repo_root("skills"), "md", &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
/// **Everything we ship that becomes prompt text an agent reads.**
|
||||
///
|
||||
/// Skills and team-template role prompts, in one corpus, because the rules
|
||||
/// below are properties of *what an agent is told* — not of which file it
|
||||
/// happened to be written in.
|
||||
///
|
||||
/// This function is the finding. The `/workspace/repo` guard was written on
|
||||
/// 2026-08-19 against `skills/` only, and the same wrong path had been
|
||||
/// sitting in **four team templates** the whole time — including
|
||||
/// `rust_sdlc`, the default for five of the six workflow recipes, whose
|
||||
/// coder was told "your working directory is /workspace/repo" and whose
|
||||
/// committer was told to `cd` there. A guard that covers one corpus and not
|
||||
/// the other reads exactly like a guard that covers the problem.
|
||||
fn all_shipped_prompts() -> Vec<(String, String)> {
|
||||
let mut out = all_skills();
|
||||
walk_ext(&repo_root("templates/teams"), "toml", &mut out);
|
||||
walk_ext(&repo_root("templates/workflows"), "toml", &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
/// Nothing we ship may teach a workspace path the platform does not mount.
|
||||
///
|
||||
/// `workspace-repo-commit-protocol` told agents that `/workspace/repo` was
|
||||
/// "the ONLY path where source-modifying edits belong". The platform mounts
|
||||
/// and advertises `/mission/repo` — in 26 places — and `/workspace/repo`
|
||||
/// appears nowhere in the code. The skill is pinned on 29 role bindings and
|
||||
/// was delivered twice in a single measured run, so agents received the
|
||||
/// platform's real path and a skill contradicting it in the SAME prompt.
|
||||
#[test]
|
||||
fn nothing_we_ship_teaches_a_repo_path_the_platform_does_not_mount() {
|
||||
let mut offenders = Vec::new();
|
||||
for (name, body) in all_shipped_prompts() {
|
||||
if body.contains("/workspace/repo") {
|
||||
offenders.push(name);
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
offenders.is_empty(),
|
||||
"{} shipped prompt file(s) name /workspace/repo; the mission \
|
||||
checkout is /mission/repo, so an agent following them writes \
|
||||
somewhere that is never delivered: {}",
|
||||
offenders.len(),
|
||||
offenders.join(", ")
|
||||
);
|
||||
}
|
||||
|
||||
/// No skill may instruct an agent to call a tool it does not have.
|
||||
///
|
||||
/// Every mission turn ends in `claude -p`, so the tools are Claude Code's
|
||||
/// (`Read`/`Edit`/`Write`/`Bash`/`Glob`/`Grep`). `phase_task_text` used to
|
||||
/// advertise ZeroClaw's names and was fixed after five agents spent 7.4k
|
||||
/// tokens on one mission describing the mismatch instead of working — and
|
||||
/// the same wrong names survived inside a pinned skill.
|
||||
///
|
||||
/// Matched as a backticked instruction, not as bare words: a skill may
|
||||
/// legitimately DISCUSS these names, as this one now does when warning
|
||||
/// against them.
|
||||
#[test]
|
||||
fn nothing_we_ship_instructs_an_agent_to_call_a_zeroclaw_tool() {
|
||||
const ZEROCLAW_TOOLS: &[&str] = &[
|
||||
"`file_read`",
|
||||
"`file_write`",
|
||||
"`file_edit`",
|
||||
"`content_search`",
|
||||
"`glob_search`",
|
||||
];
|
||||
let mut offenders = Vec::new();
|
||||
for (name, body) in all_shipped_prompts() {
|
||||
// The line has to READ as an instruction. "Do not reach for
|
||||
// `file_read`" is the correction, not the defect.
|
||||
for line in body.lines() {
|
||||
let l = line.to_ascii_lowercase();
|
||||
if l.contains("do not")
|
||||
|| l.contains("never")
|
||||
|| l.contains("instead of")
|
||||
|| l.contains("not what")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if ZEROCLAW_TOOLS.iter().any(|t| line.contains(t)) {
|
||||
offenders.push(format!("{name}: {}", line.trim()));
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
offenders.is_empty(),
|
||||
"{} shipped prompt line(s) tell an agent to use a tool its \
|
||||
subprocess does not expose:\n {}",
|
||||
offenders.len(),
|
||||
offenders.join("\n ")
|
||||
);
|
||||
}
|
||||
|
||||
/// No skill may show a marker the real parser rejects.
|
||||
///
|
||||
/// Checked by running `task_card_parser::parse` itself, never a copy of its
|
||||
/// rules — a second implementation of the contract drifts, and then the
|
||||
/// test passes while the mission loop stalls.
|
||||
///
|
||||
/// This is the third instance of one class: the skills were written
|
||||
/// alongside the platform and then never compared to it again. The first
|
||||
/// was a repo path the platform does not mount; the second a tool the agent
|
||||
/// does not have; this one is `PLAN_COMPLETE: INT-01..05` in
|
||||
/// `decompose-int-items`, which a live planner emitted verbatim. Ids are
|
||||
/// strictly `INT-<digits>`, so the range form parses to nothing — the plan
|
||||
/// pass records no completion at all while every item stays open.
|
||||
///
|
||||
/// Scoped to fenced code blocks, which is where a skill puts the text it
|
||||
/// tells an agent to EMIT. A marker named in a sentence is prose.
|
||||
#[test]
|
||||
fn no_skill_shows_a_marker_the_parser_would_reject() {
|
||||
// The templates. `INT-NN` is a placeholder an agent substitutes, not a
|
||||
// literal it emits, so it is not a contradiction.
|
||||
const PLACEHOLDERS: &[&str] = &["INT-NN", "INT-XX", "INT-N", "INT-nn"];
|
||||
let mut offenders = Vec::new();
|
||||
for (name, body) in all_skills() {
|
||||
let mut fenced = false;
|
||||
for line in body.lines() {
|
||||
if line.trim_start().starts_with("```") {
|
||||
fenced = !fenced;
|
||||
continue;
|
||||
}
|
||||
let t = line.trim();
|
||||
if !fenced || !t.contains("INT-") || !t.contains(':') {
|
||||
continue;
|
||||
}
|
||||
let Some((kind, _)) = t.split_once(':') else {
|
||||
continue;
|
||||
};
|
||||
if !MARKER_KINDS.contains(&kind.trim()) {
|
||||
continue;
|
||||
}
|
||||
if PLACEHOLDERS.iter().any(|p| t.contains(p)) {
|
||||
continue;
|
||||
}
|
||||
if crate::task_card_parser::parse(t).is_empty() {
|
||||
offenders.push(format!("{name}: {t}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
offenders.is_empty(),
|
||||
"{} skill line(s) show a marker the parser rejects — an agent that \
|
||||
follows them exactly is silently ignored:\n {}",
|
||||
offenders.len(),
|
||||
offenders.join("\n ")
|
||||
);
|
||||
}
|
||||
|
||||
/// Every team a recipe names must be a team that exists.
|
||||
///
|
||||
/// `create()` logs and carries on when a recipe names a template that is
|
||||
/// not loaded, because failing mission creation over it would be worse.
|
||||
/// That makes a typo here invisible in exactly the way that matters: the
|
||||
/// mission is staffed by the fallback crew and looks deliberate. `research_only`
|
||||
/// pointed at `rust_sdlc` for months and nothing said a word.
|
||||
#[test]
|
||||
fn every_team_a_recipe_names_exists() {
|
||||
let mut keys = std::collections::HashSet::new();
|
||||
for (_, body) in {
|
||||
let mut v = Vec::new();
|
||||
walk_ext(&repo_root("templates/teams"), "toml", &mut v);
|
||||
v
|
||||
} {
|
||||
for line in body.lines() {
|
||||
if let Some(rest) = line.trim().strip_prefix("key") {
|
||||
if let Some((_, val)) = rest.split_once('=') {
|
||||
keys.insert(val.trim().trim_matches('"').to_string());
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(!keys.is_empty(), "no team templates found at all");
|
||||
|
||||
let mut recipes = Vec::new();
|
||||
walk_ext(&repo_root("templates/workflows"), "toml", &mut recipes);
|
||||
let mut missing = Vec::new();
|
||||
for (name, body) in recipes {
|
||||
let mut table = String::new();
|
||||
for line in body.lines() {
|
||||
let line = line.trim();
|
||||
if line.starts_with('[') {
|
||||
table = line.trim_matches(['[', ']'].as_slice()).to_string();
|
||||
continue;
|
||||
}
|
||||
if line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
let named = if let Some((_, v)) = line.split_once('=') {
|
||||
if line.starts_with("default_team_template")
|
||||
|| table == "default_phase_teams"
|
||||
{
|
||||
Some(v.trim().trim_matches('"').to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(k) = named {
|
||||
if !keys.contains(&k) {
|
||||
missing.push(format!("{name} -> {k}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
missing.is_empty(),
|
||||
"{} recipe(s) name a team template that does not exist, so the mission \
|
||||
is staffed by the fallback crew and looks deliberate: {}",
|
||||
missing.len(),
|
||||
missing.join(", ")
|
||||
);
|
||||
}
|
||||
|
||||
/// The marker kinds, as the parser spells them.
|
||||
const MARKER_KINDS: &[&str] = &[
|
||||
"TASK",
|
||||
"PLAN_COMPLETE",
|
||||
"WORK",
|
||||
"HANDOFF",
|
||||
"TEST_PASS",
|
||||
"TEST_FAIL",
|
||||
"REVIEW_APPROVE",
|
||||
"REVIEW_BLOCK",
|
||||
"COMPLETED",
|
||||
];
|
||||
}
|
||||
|
||||
@@ -153,7 +153,12 @@ fn is_transient(e: &cm_llm::LlmError) -> bool {
|
||||
///
|
||||
/// Override with `CLAWMATES_MODEL_FALLBACK` (comma-separated). An empty value
|
||||
/// disables fallback and restores plain "503 and wait".
|
||||
const DEFAULT_FALLBACK: &str = "claude-sonnet-4-6,claude-haiku-4-5-20251001,\
|
||||
///
|
||||
/// Ordered by the operator's model policy: sonnet-5 is the working tier, and
|
||||
/// haiku sits BELOW it as a last-resort Anthropic link rather than as a peer —
|
||||
/// a degraded answer beats a 503, but it must never be reached while a capable
|
||||
/// model has capacity.
|
||||
const DEFAULT_FALLBACK: &str = "claude-sonnet-5,claude-haiku-4-5-20251001,\
|
||||
kimi:kimi-k2.7-code,glm:glm-4.7,local:ornith-fleet:9b";
|
||||
|
||||
/// The chain to walk after `requested`, with `requested` itself removed so a
|
||||
@@ -320,7 +325,7 @@ pub async fn preflight(runtime: &cm_runtime::Runtime, head: &str) -> Vec<(String
|
||||
pub fn report_at_boot(runtime: cm_runtime::Runtime) {
|
||||
tokio::spawn(async move {
|
||||
let head = std::env::var("CLAWMATES_PREFLIGHT_HEAD")
|
||||
.unwrap_or_else(|_| "claude-opus-4-8".to_string());
|
||||
.unwrap_or_else(|_| "claude-opus-5".to_string());
|
||||
let links = preflight(&runtime, &head).await;
|
||||
let bad: Vec<_> = links.iter().filter(|(_, s)| !s.usable()).collect();
|
||||
eprintln!(
|
||||
@@ -524,6 +529,15 @@ mod tests {
|
||||
if path.ends_with("subscription.rs") {
|
||||
continue;
|
||||
}
|
||||
// The LLM proxy never ORIGINATES a model call: it relays a mission
|
||||
// container's own Claude Code request byte for byte and swaps in
|
||||
// the credential. Routing it through `complete_or` would re-build
|
||||
// (and could re-route) what the agent asked for. Credential choice
|
||||
// for relayed calls is `llm_proxy::upstream`, and it reads the same
|
||||
// env and auth mode (`runtime_auth_mode`) as everything else.
|
||||
if path.ends_with("llm_proxy.rs") {
|
||||
continue;
|
||||
}
|
||||
let src = std::fs::read_to_string(&path).expect("readable source");
|
||||
for needle in ["api.anthropic.com", "\"x-api-key\""] {
|
||||
assert!(
|
||||
@@ -593,11 +607,11 @@ mod tests {
|
||||
#[test]
|
||||
fn the_chain_excludes_the_model_that_just_failed() {
|
||||
// No env override in scope: this asserts the SHIPPED default.
|
||||
let chain = fallback_chain("claude-opus-4-8");
|
||||
let chain = fallback_chain("claude-opus-5");
|
||||
assert_eq!(
|
||||
chain,
|
||||
vec![
|
||||
"claude-sonnet-4-6",
|
||||
"claude-sonnet-5",
|
||||
"claude-haiku-4-5-20251001",
|
||||
"kimi:kimi-k2.7-code",
|
||||
"glm:glm-4.7",
|
||||
|
||||
@@ -86,6 +86,7 @@ fn step(
|
||||
output: output.into(),
|
||||
gated: Vec::new(),
|
||||
tokens: 0,
|
||||
spend: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -146,7 +147,7 @@ pub async fn run_swarm_job(
|
||||
runtime,
|
||||
PLAN_SYSTEM,
|
||||
&plan_user,
|
||||
"claude-opus-4-8",
|
||||
"claude-opus-5",
|
||||
4000,
|
||||
false,
|
||||
)
|
||||
@@ -207,7 +208,7 @@ pub async fn run_swarm_job(
|
||||
runtime,
|
||||
&vsys,
|
||||
&vuser,
|
||||
"claude-opus-4-8",
|
||||
"claude-opus-5",
|
||||
1200,
|
||||
true,
|
||||
)
|
||||
|
||||
@@ -39,6 +39,7 @@ pub struct Marker {
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum MarkerKind {
|
||||
Task,
|
||||
PlanComplete,
|
||||
Work,
|
||||
Handoff,
|
||||
TestPass,
|
||||
@@ -53,7 +54,11 @@ impl MarkerKind {
|
||||
/// motion — the UPSERT layer may still overwrite prior states.
|
||||
pub fn status(&self) -> &'static str {
|
||||
match self {
|
||||
MarkerKind::Task => "created",
|
||||
// The planner finished specifying; no work has started, so the item
|
||||
// is in the same state a fresh TASK leaves it in. A distinct status
|
||||
// would need a column value the UI does not render, and inventing
|
||||
// one to look complete is how a status stops meaning anything.
|
||||
MarkerKind::Task | MarkerKind::PlanComplete => "created",
|
||||
MarkerKind::Work => "working",
|
||||
MarkerKind::Handoff | MarkerKind::TestPass | MarkerKind::ReviewApprove => "validating",
|
||||
MarkerKind::TestFail | MarkerKind::ReviewBlock => "failed",
|
||||
@@ -75,12 +80,27 @@ pub fn parse(text: &str) -> Vec<Marker> {
|
||||
out
|
||||
}
|
||||
|
||||
/// `INT-` followed by at least one digit and nothing else.
|
||||
fn is_int_id(id: &str) -> bool {
|
||||
match id.strip_prefix("INT-") {
|
||||
Some(rest) => !rest.is_empty() && rest.chars().all(|c| c.is_ascii_digit()),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_line(line: &str) -> Option<Marker> {
|
||||
// Match `<KIND>: INT-NN` (rest optional). Strict on the colon and
|
||||
// the INT- prefix — anything laxer starts matching prose.
|
||||
let (kind_str, rest) = line.split_once(':')?;
|
||||
let kind = match kind_str.trim() {
|
||||
"TASK" => MarkerKind::Task,
|
||||
// Documented in `skills/foundation/int-xx-marker-protocol.md` since the
|
||||
// skill was written, and never implemented here. Agents that followed
|
||||
// the skill exactly emitted it and were silently ignored — observed on
|
||||
// a live mission, found by the Skill-Use measurement. Implemented
|
||||
// rather than removed from the skill: the planner needs a way to say
|
||||
// it is done specifying, and agents already emit this one.
|
||||
"PLAN_COMPLETE" => MarkerKind::PlanComplete,
|
||||
"WORK" => MarkerKind::Work,
|
||||
"HANDOFF" => MarkerKind::Handoff,
|
||||
"TEST_PASS" => MarkerKind::TestPass,
|
||||
@@ -95,10 +115,16 @@ fn parse_line(line: &str) -> Option<Marker> {
|
||||
Some((a, b)) => (a, Some(b.trim())),
|
||||
None => (rest, None),
|
||||
};
|
||||
if !id_tok.starts_with("INT-") {
|
||||
let int_id = id_tok.trim_end_matches(&[',', ';', '.'][..]).to_string();
|
||||
// Strictly `INT-<digits>`. `starts_with("INT-")` alone accepted range forms
|
||||
// like `INT-01..02`, which parse into an id matching no real item — so a
|
||||
// task card appeared for something that did not exist while the two items
|
||||
// it was meant to cover stayed open. Observed live. Rejecting is right:
|
||||
// the marker is ignored, which is visible, instead of creating a plausible
|
||||
// row, which is not.
|
||||
if !is_int_id(&int_id) {
|
||||
return None;
|
||||
}
|
||||
let int_id = id_tok.trim_end_matches(&[',', ';', '.'][..]).to_string();
|
||||
// Title: after the id + any of ` — / – / - ` separators
|
||||
let title = tail.and_then(|t| {
|
||||
let t = t.trim_start_matches(['—', '–', '-', ':'].as_slice()).trim();
|
||||
|
||||
@@ -306,6 +306,34 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// EVERY referenced name must resolve to an authored skill.
|
||||
///
|
||||
/// The other direction, and the one that was missing. Both existing tests
|
||||
/// assert `authored ⊆ referenced` — true of all 30 authored skills, so both
|
||||
/// passed while 55 of 85 bindings resolved to nothing and ten roles ran with
|
||||
/// an empty context bundle.
|
||||
///
|
||||
/// The old comment on the test below called the gap "deliberately
|
||||
/// aspirational". An aspirational binding is indistinguishable at runtime
|
||||
/// from a typo: `get_by_name` returns Ok(None), the loader logs a line
|
||||
/// nobody reads, and the role ships without the instructions its prompt
|
||||
/// assumes it has. If a skill is worth naming it is worth authoring, and if
|
||||
/// it is not, the name should not be in the template.
|
||||
#[test]
|
||||
fn every_referenced_skill_resolves_to_an_authored_one() {
|
||||
let mut authored = HashSet::new();
|
||||
authored_skill_names(&repo_root().join("skills"), &mut authored);
|
||||
let referenced = referenced_skill_names();
|
||||
let mut missing: Vec<_> = referenced.difference(&authored).cloned().collect();
|
||||
missing.sort();
|
||||
assert!(
|
||||
missing.is_empty(),
|
||||
"{} referenced skill(s) bind to nothing — the role gets no instructions \
|
||||
and nothing errors: {missing:#?}",
|
||||
missing.len()
|
||||
);
|
||||
}
|
||||
|
||||
/// A referenced name that matches no authored skill binds to nothing. Some
|
||||
/// are deliberately aspirational, so this asserts the *resolvable* ones
|
||||
/// stay resolvable rather than demanding every name exist.
|
||||
@@ -322,3 +350,86 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod bundle_tests {
|
||||
use std::collections::HashSet;
|
||||
|
||||
fn repo() -> std::path::PathBuf {
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../..")
|
||||
.canonicalize()
|
||||
.expect("repo root")
|
||||
}
|
||||
|
||||
/// Every `mcp_bundles` name a template asks for must be one the runtime
|
||||
/// config actually defines.
|
||||
///
|
||||
/// This was harmless while `provision_claw` wrote a constant bundle list
|
||||
/// and ignored the templates. It is not harmless now that the list is
|
||||
/// honoured: an undefined name is a capability the agent is told it has and
|
||||
/// does not, which is the same failure as an unresolved skill binding one
|
||||
/// layer down. `gitea_forge` was named by seven team templates, one
|
||||
/// workflow recipe, the auto-provision path and a user-selectable dropdown,
|
||||
/// and defined nowhere.
|
||||
#[test]
|
||||
fn every_named_mcp_bundle_is_defined_by_the_runtime_config() {
|
||||
let cfg = std::fs::read_to_string(
|
||||
repo().join("deploy/clawmates-runtime/agent.config.example.toml"),
|
||||
)
|
||||
.expect("runtime config");
|
||||
let defined: HashSet<String> = cfg
|
||||
.lines()
|
||||
.filter_map(|l| l.trim().strip_prefix("[mcp_bundles."))
|
||||
.filter_map(|r| r.strip_suffix(']'))
|
||||
.map(|s| s.to_string())
|
||||
.collect();
|
||||
assert!(
|
||||
defined.contains("clawmates_door"),
|
||||
"parsed no bundles from the runtime config — the parser, not the \
|
||||
templates, is what broke"
|
||||
);
|
||||
|
||||
let mut missing: Vec<String> = Vec::new();
|
||||
for dir in ["templates/teams", "templates/workflows"] {
|
||||
for entry in std::fs::read_dir(repo().join(dir)).expect("template dir") {
|
||||
let path = entry.expect("entry").path();
|
||||
if path.extension().and_then(|e| e.to_str()) != Some("toml") {
|
||||
continue;
|
||||
}
|
||||
let body = std::fs::read_to_string(&path).expect("read template");
|
||||
for line in body.lines() {
|
||||
let t = line.trim();
|
||||
// Skip comments: several deliberately NAME a bundle while
|
||||
// explaining that it is not delivered.
|
||||
if t.starts_with('#') || !t.starts_with("mcp_bundles") {
|
||||
continue;
|
||||
}
|
||||
let Some(inner) = t.split_once('[').and_then(|(_, r)| r.rsplit_once(']'))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
for name in inner.0.split(',') {
|
||||
let name = name.trim().trim_matches('"');
|
||||
if !name.is_empty() && !defined.contains(name) {
|
||||
missing.push(format!(
|
||||
"{}: {name}",
|
||||
path.file_name().unwrap().to_string_lossy()
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
missing.sort();
|
||||
missing.dedup();
|
||||
assert!(
|
||||
missing.is_empty(),
|
||||
"{} template(s) name an MCP bundle the runtime does not define, so \
|
||||
the agent is provisioned with a capability that resolves to \
|
||||
nothing:\n {}",
|
||||
missing.len(),
|
||||
missing.join("\n ")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,11 +7,22 @@
|
||||
//! gateway, opens `/ws/chat?agent=<alias>`, sends the role+task+context prompt,
|
||||
//! and streams the turn's events back into a [`TurnOutcome`].
|
||||
//!
|
||||
//! **§15 by construction:** the agents are provisioned tool-free (every
|
||||
//! sensitive capability is a gated Clawmates MCP tool — the "door"), so a turn
|
||||
//! takes no sandbox-leaving action here. If the gateway nonetheless emits an
|
||||
//! `approval_request`, we record it as a **blocked** `GatedAction` and end the
|
||||
//! turn — we never auto-approve.
|
||||
//! **These agents are NOT tool-free.** That claim stood here for months and is
|
||||
//! false — see `docs/TOOL-CALL-ARCHITECTURE.md`. It was inferred from a frame
|
||||
//! stream that carried no tool events, and the emptiness has a different cause:
|
||||
//! `claude_cli` runs `claude -p --output-format json`, which returns a single
|
||||
//! final result object, and the provider hardcodes `tool_calls: Vec::new()`.
|
||||
//! The agent calls Claude Code's own tools; the transport discards them.
|
||||
//! `--output-format stream-json` emits `tool_use`/`tool_result` blocks —
|
||||
//! verified against the deployed Claude Code 2.1.228.
|
||||
//!
|
||||
//! The door-shaped provider that WOULD make this true (`--mcp-config` +
|
||||
//! `--disallowedTools` on the natives) is built and documented in
|
||||
//! `agent.config.example.toml`, and is not deployed: mission claws bind to
|
||||
//! `claude_cli.default`, which sets none of it.
|
||||
//!
|
||||
//! If the gateway emits an `approval_request` we still record it as a
|
||||
//! **blocked** `GatedAction` and end the turn — we never auto-approve.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
@@ -42,6 +53,42 @@ use tokio_tungstenite::tungstenite::Message;
|
||||
const TURN_TIMEOUT: Duration = Duration::from_secs(3600);
|
||||
|
||||
/// Drives ZeroClaw role-agents (in one container) to execute topology turns.
|
||||
/// Cap on the pinned-skill text injected into one mission turn.
|
||||
///
|
||||
/// Skill bodies average ~3.5 KB and pinning is `idx < 2 || foundation`, so a
|
||||
/// role lands near 7-10 KB. The cap exists for the role that grows a long
|
||||
/// foundation set, and it is stated in the prompt when it fires.
|
||||
pub(crate) const MAX_PINNED_SKILL_BYTES: usize = 24_000;
|
||||
|
||||
/// The line that introduces each skill in a prompt.
|
||||
///
|
||||
/// NOT a markdown heading. The first version used `## <name>`, and skill bodies
|
||||
/// are markdown that contain their own `##` headings — so anything reading the
|
||||
/// prompt back counted every section of every body as a separate skill. A live
|
||||
/// mission scored "Sizing heuristic" and "The output shape" as skills, which is
|
||||
/// what surfaced it.
|
||||
///
|
||||
/// This marker cannot occur inside a body, so the prompt stays parseable by
|
||||
/// whatever reads it later. Skills are written by one function
|
||||
/// ([`render_pinned_skill`]) for the same reason: two renderers would drift and
|
||||
/// the reader would silently match only one.
|
||||
pub const SKILL_MARKER: &str = "--- SKILL: ";
|
||||
|
||||
/// One skill, rendered for a prompt.
|
||||
pub fn render_pinned_skill(name: &str, body: &str) -> String {
|
||||
format!("\n{SKILL_MARKER}{name} ---\n{body}\n")
|
||||
}
|
||||
|
||||
/// The skill names a rendered prompt delivered.
|
||||
pub fn skill_names_in(prompt: &str) -> Vec<String> {
|
||||
prompt
|
||||
.lines()
|
||||
.filter_map(|l| l.trim().strip_prefix(SKILL_MARKER))
|
||||
.map(|rest| rest.trim_end_matches(" ---").trim().to_string())
|
||||
.filter(|n| !n.is_empty())
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub struct ZeroClawDriveExecutor {
|
||||
/// Gateway base URL, e.g. `http://127.0.0.1:42617`.
|
||||
gateway_url: String,
|
||||
@@ -67,6 +114,9 @@ pub struct ZeroClawDriveExecutor {
|
||||
/// put mission concepts into every tier that has no missions.
|
||||
pub struct MissionTap {
|
||||
pub pool: sqlx::PgPool,
|
||||
/// Which workspace's live feed these frames belong to. Every subscriber is
|
||||
/// workspace-scoped, so a frame without this could not be routed.
|
||||
pub workspace_id: uuid::Uuid,
|
||||
pub mission_id: uuid::Uuid,
|
||||
pub phase_id: Option<uuid::Uuid>,
|
||||
pub run_id: Option<uuid::Uuid>,
|
||||
@@ -97,9 +147,11 @@ pub struct ToolTrace {
|
||||
/// `chunk`, `done` and `session_start` and no tool frames at all. That is
|
||||
/// not a protocol mismatch — `tool_call` is in the deployed binary
|
||||
/// (`zeroclaw-gateway/src/ws.rs` emits `{"type":"tool_call","id","name",
|
||||
/// "args"}`) — it is §15: these agents are provisioned tool-free behind the
|
||||
/// MCP door, so they call nothing. The histogram is what let us tell those
|
||||
/// two apart, which was its whole purpose.
|
||||
/// "args"}`) — and it is NOT that the agents are tool-free, which is what
|
||||
/// this comment used to say. `claude_cli` asks for `--output-format json`,
|
||||
/// so the subprocess's tool calls never reach the gateway to be framed.
|
||||
/// The histogram still does its job: it distinguishes "no frames" from
|
||||
/// "frames we do not recognise", and the answer was the former.
|
||||
pub unmatched: std::collections::BTreeMap<String, u32>,
|
||||
}
|
||||
|
||||
@@ -175,6 +227,21 @@ impl ZeroClawDriveExecutor {
|
||||
/// new one-time code at startup. The env-derived ZEROCLAW_TOKEN
|
||||
/// is ignored (belongs to the shared runtime) so the lazy pair
|
||||
/// path runs and issues a bearer for this specific gateway.
|
||||
/// Reuse a token that was already paired and persisted.
|
||||
///
|
||||
/// The pairing code is single-use, so a restarted server cannot pair again:
|
||||
/// it gets 403 and the mission is unrecoverable. Seeding the cache from
|
||||
/// `missions.runtime_token` is what makes a mission survive a restart.
|
||||
pub fn with_token(self, token: Option<String>) -> Self {
|
||||
if let Some(t) = token.filter(|t| !t.trim().is_empty()) {
|
||||
// try_lock: this runs at construction, before any turn holds it.
|
||||
if let Ok(mut g) = self.token.try_lock() {
|
||||
*g = Some(t);
|
||||
}
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
pub fn from_env_for_gateway_with_code(
|
||||
gateway_url: String,
|
||||
pairing_code: String,
|
||||
@@ -236,9 +303,145 @@ impl ZeroClawDriveExecutor {
|
||||
.ok_or_else(|| OrchestratorError::Executor("pair response had no token".into()))?
|
||||
.to_string();
|
||||
*guard = Some(token.clone());
|
||||
// Persist it. The code we just spent cannot be used again, so if this
|
||||
// token only ever lives in memory the next server process has no way
|
||||
// back in — that is the 403 that killed a 93k-token research phase.
|
||||
// Best-effort: failing to save must not fail a turn that just paired
|
||||
// successfully; the cost is that a restart before the next write
|
||||
// re-opens the original hole.
|
||||
if let Some(tap) = self.tap.as_ref() {
|
||||
if let Err(e) = sqlx::query("UPDATE missions SET runtime_token = $1 WHERE id = $2")
|
||||
.bind(&token)
|
||||
.bind(tap.mission_id)
|
||||
.execute(&tap.pool)
|
||||
.await
|
||||
{
|
||||
eprintln!(
|
||||
"topology_exec: could not persist runtime token for mission {}: {e}",
|
||||
tap.mission_id
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
/// The pinned skills for the claw behind `alias`, rendered for the prompt.
|
||||
///
|
||||
/// Missions had NO path to a skill. The catalogue's only delivery channel
|
||||
/// is the `clawmates_skills` MCP server, and a mission agent cannot reach
|
||||
/// it for three independent reasons: `provision_claw` wrote a constant
|
||||
/// bundle list, the runtime config defines no such bundle, and mission
|
||||
/// claws run on `claude_cli`, which is text-only and cannot surface a tool
|
||||
/// call at all. Two doc comments in `cm-runtime` describe the mission path
|
||||
/// as already having this contract. It never did — so every skill authored
|
||||
/// for a mission role was unreachable prose, and no measurement of whether
|
||||
/// skills fire could have returned anything but zero.
|
||||
///
|
||||
/// Bodies or an index, depending on the mission's arm — see
|
||||
/// [`crate::skill_delivery`]. Bodies were once the only honest option:
|
||||
/// there was no tool on the mission path that could fetch one, so an index
|
||||
/// would have advertised a capability that did not exist. The skills door
|
||||
/// changed that, and the arm is now recorded per mission so both can run.
|
||||
///
|
||||
/// Pinned only (`pin_in_context`) in either arm, because everything else
|
||||
/// would go in unbounded and unread.
|
||||
pub async fn pinned_skills_text(&self, alias: &str) -> Option<String> {
|
||||
let mode = self.skill_delivery_mode().await;
|
||||
self.pinned_skills_in_mode(alias, mode).await
|
||||
}
|
||||
|
||||
/// The arm this mission was launched with.
|
||||
///
|
||||
/// Read per turn rather than cached on the executor: the executor is
|
||||
/// constructed from the environment by `topology_worker`, which knows
|
||||
/// nothing about a mission, and the arm is decided at launch by the code
|
||||
/// that also learns whether the door installed.
|
||||
///
|
||||
/// Anything unreadable — no tap, no row, an unrecognised value — resolves
|
||||
/// to `Inline`, which is the arm that needs nothing to be true.
|
||||
pub(crate) async fn skill_delivery_mode(&self) -> crate::skill_delivery::Mode {
|
||||
let Some(tap) = self.tap.as_ref() else {
|
||||
return crate::skill_delivery::Mode::Inline;
|
||||
};
|
||||
sqlx::query_scalar::<_, Option<String>>(
|
||||
"SELECT skill_delivery FROM missions WHERE id = $1",
|
||||
)
|
||||
.bind(tap.mission_id)
|
||||
.fetch_optional(&tap.pool)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.flatten()
|
||||
.and_then(|s| crate::skill_delivery::parse(&s))
|
||||
.unwrap_or(crate::skill_delivery::Mode::Inline)
|
||||
}
|
||||
|
||||
pub(crate) async fn pinned_skills_in_mode(
|
||||
&self,
|
||||
alias: &str,
|
||||
mode: crate::skill_delivery::Mode,
|
||||
) -> Option<String> {
|
||||
let tap = self.tap.as_ref()?;
|
||||
let agent_id = crate::runtime_provision::claw_from_alias(alias)?;
|
||||
let link = cm_db::repo::agent_template_link::get(&tap.pool, agent_id)
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
let (tpl_id, slot) = link
|
||||
.as_ref()
|
||||
.map(|l| (Some(l.template_id), Some(l.role_slot.as_str())))
|
||||
.unwrap_or((None, None));
|
||||
let bindings =
|
||||
cm_db::repo::skills_catalog::effective_for_agent(&tap.pool, agent_id, tpl_id, slot)
|
||||
.await
|
||||
.ok()?;
|
||||
|
||||
let mut out = String::new();
|
||||
let mut n = 0usize;
|
||||
for b in bindings.iter().filter(|b| b.pin_in_context) {
|
||||
// `always_inject` overrides the arm. Progressive disclosure asks
|
||||
// the agent to recognise that a procedure applies before fetching
|
||||
// it, and a CROSS-CUTTING procedure is the case that breaks: the
|
||||
// first A/B pair had `workspace-repo-commit-protocol` scored
|
||||
// Trigger=FAIL beside a passing boundary check, because a rule that
|
||||
// applies to everyone who writes reads as nobody's in particular.
|
||||
let text = match mode {
|
||||
crate::skill_delivery::Mode::Inline => b.skill.body.clone(),
|
||||
m if m.is_retrieval() && b.skill.always_inject => b.skill.body.clone(),
|
||||
// An entry is a few hundred bytes whatever the body weighs, so
|
||||
// the retrieval arms cannot hit the cap that follows. That is
|
||||
// the point of them, and the reason the cap is checked against
|
||||
// the rendered text rather than against the body.
|
||||
crate::skill_delivery::Mode::Index => crate::skill_delivery::index_entry(
|
||||
&b.skill.description,
|
||||
b.skill.when_to_use.as_deref(),
|
||||
&crate::mcp_skills::skill_uri(b.skill.workspace_id, &b.skill.name),
|
||||
),
|
||||
crate::skill_delivery::Mode::Files => crate::skill_delivery::file_entry(
|
||||
&b.skill.description,
|
||||
b.skill.when_to_use.as_deref(),
|
||||
&crate::skill_delivery::skill_file_path(&b.skill.name),
|
||||
),
|
||||
};
|
||||
// Bounded, and truncation is STATED. A silently clipped procedure
|
||||
// is worse than an absent one: the agent follows the half it can
|
||||
// see and reports success against a rule it never read.
|
||||
if out.len() + text.len() > MAX_PINNED_SKILL_BYTES {
|
||||
out.push_str(&format!(
|
||||
"\n[skill \"{}\" omitted — the pinned set exceeded {} bytes]\n",
|
||||
b.skill.name, MAX_PINNED_SKILL_BYTES
|
||||
));
|
||||
continue;
|
||||
}
|
||||
out.push_str(&render_pinned_skill(&b.skill.name, &text));
|
||||
n += 1;
|
||||
}
|
||||
if n == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Mirror of `ProviderExecutor`'s prompt, flattened to one `content` string
|
||||
/// (the gateway `message` envelope carries a single content field).
|
||||
fn build_prompt(req: &TurnRequest) -> String {
|
||||
@@ -294,7 +497,16 @@ impl ZeroClawDriveExecutor {
|
||||
.await
|
||||
.map_err(|e| OrchestratorError::Executor(format!("ws send failed: {e}")))?;
|
||||
|
||||
let (outcome, trace) = match tokio::time::timeout(TURN_TIMEOUT, Self::drain(&mut ws)).await
|
||||
let (outcome, trace) = match tokio::time::timeout(
|
||||
TURN_TIMEOUT,
|
||||
Self::drain(
|
||||
&mut ws,
|
||||
self.tap.as_ref().and_then(|t| {
|
||||
crate::live_bus::agent_id_from_alias(alias).map(|a| (t.workspace_id, a))
|
||||
}),
|
||||
),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(res) => res?,
|
||||
Err(_) => {
|
||||
@@ -393,19 +605,19 @@ impl ZeroClawDriveExecutor {
|
||||
}
|
||||
|
||||
/// Use a runtime agent as a governance judge: drive `alias` with the judge
|
||||
/// prompt and parse the verdict (`DENY` anywhere ⇒ deny, else allow). This
|
||||
/// prompt and parse the verdict with [`cm_runtime::governor_allows`]. This
|
||||
/// lets a **subscription-only** model (e.g. Kimi via `kimi_cli`) be the judge
|
||||
/// with no platform API key — the registry/SDK path GLM and Kimi can't take.
|
||||
/// Fail-open (returns `(true, …)`) so a judge outage never halts agents.
|
||||
/// Fail-closed: an unreachable judge denies, for the reason given on
|
||||
/// [`cm_runtime::Runtime::judge`].
|
||||
pub async fn judge(&self, alias: &str, system: &str, user: &str) -> (bool, String) {
|
||||
let prompt = format!("{system}\n\n{user}");
|
||||
match self.drive(alias, &prompt).await {
|
||||
Ok(outcome) => {
|
||||
let text = outcome.output.trim().to_string();
|
||||
let allow = !text.to_uppercase().contains("DENY");
|
||||
(allow, text)
|
||||
(cm_runtime::governor_allows(&text), text)
|
||||
}
|
||||
Err(e) => (true, format!("governor unreachable (fail-open): {e}")),
|
||||
Err(e) => (false, format!("governor unreachable (fail-closed): {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -457,7 +669,13 @@ impl ZeroClawDriveExecutor {
|
||||
/// trace is separate from [`TurnOutcome`] deliberately: that type is the
|
||||
/// shared orchestrator contract used by every tier, and tool telemetry is a
|
||||
/// mission concern.
|
||||
async fn drain<S>(ws: &mut S) -> Result<(TurnOutcome, ToolTrace), OrchestratorError>
|
||||
/// `live` is the push target for this turn: `Some((workspace, agent))` when
|
||||
/// the turn belongs to a mission AND runs under a claw alias. `None` for the
|
||||
/// governor/door/evaluator, whose output belongs to no agent.
|
||||
async fn drain<S>(
|
||||
ws: &mut S,
|
||||
live: Option<(uuid::Uuid, uuid::Uuid)>,
|
||||
) -> Result<(TurnOutcome, ToolTrace), OrchestratorError>
|
||||
where
|
||||
S: StreamExt<Item = Result<Message, tokio_tungstenite::tungstenite::Error>>
|
||||
+ SinkExt<Message>
|
||||
@@ -465,6 +683,7 @@ impl ZeroClawDriveExecutor {
|
||||
{
|
||||
let mut output = String::new();
|
||||
let mut tokens: u64 = 0;
|
||||
let mut spend = cm_orchestrator::Spend::default();
|
||||
let mut gated: Vec<GatedAction> = Vec::new();
|
||||
let mut trace = ToolTrace::default();
|
||||
|
||||
@@ -478,12 +697,47 @@ impl ZeroClawDriveExecutor {
|
||||
"chunk" => {
|
||||
if let Some(c) = v.get("content").and_then(|c| c.as_str()) {
|
||||
output.push_str(c);
|
||||
// Push, don't wait for the poll. This is the
|
||||
// whole point of the bus: the reasoning card
|
||||
// previously showed a step's text only after the
|
||||
// step ended and the row was written, so an
|
||||
// agent mid-thought looked idle for seconds.
|
||||
if let Some((ws_id, agent_id)) = live {
|
||||
if !c.trim().is_empty() {
|
||||
crate::live_bus::global().publish(
|
||||
ws_id,
|
||||
"agent.reasoning.delta",
|
||||
serde_json::json!({
|
||||
"agentId": agent_id.to_string(),
|
||||
"text": c,
|
||||
"channel": "say",
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"done" => {
|
||||
let input = v.get("input_tokens").and_then(|n| n.as_u64()).unwrap_or(0);
|
||||
let out = v.get("output_tokens").and_then(|n| n.as_u64()).unwrap_or(0);
|
||||
tokens = input + out;
|
||||
// The frame has always carried these; only
|
||||
// `tokens` was read, so every agent turn was
|
||||
// charged with no record of who was paid.
|
||||
spend = cm_orchestrator::Spend {
|
||||
input_tokens: input,
|
||||
output_tokens: out,
|
||||
provider: v
|
||||
.get("provider")
|
||||
.and_then(|p| p.as_str())
|
||||
.filter(|p| !p.is_empty())
|
||||
.map(str::to_string),
|
||||
model: v
|
||||
.get("model")
|
||||
.and_then(|m| m.as_str())
|
||||
.filter(|m| !m.is_empty())
|
||||
.map(str::to_string),
|
||||
};
|
||||
break;
|
||||
}
|
||||
"approval_request" => {
|
||||
@@ -571,6 +825,7 @@ impl ZeroClawDriveExecutor {
|
||||
output: output.trim().to_string(),
|
||||
tokens,
|
||||
gated,
|
||||
spend,
|
||||
},
|
||||
trace,
|
||||
))
|
||||
@@ -604,11 +859,105 @@ impl TurnExecutor for ZeroClawDriveExecutor {
|
||||
);
|
||||
fallback
|
||||
});
|
||||
let prompt = Self::build_prompt(&req);
|
||||
// One lookup, used for the section, its preamble and the record.
|
||||
// Deriving it three times would let a mission compose an index under
|
||||
// an inline heading if the row changed mid-run.
|
||||
let mode = self.skill_delivery_mode().await;
|
||||
let prompt = compose_turn_prompt(
|
||||
&Self::build_prompt(&req),
|
||||
self.pinned_skills_in_mode(&alias, mode).await.as_deref(),
|
||||
mode,
|
||||
);
|
||||
// Record what this agent is ACTUALLY about to receive, before driving.
|
||||
// Re-deriving it later would re-run the skill lookup against a
|
||||
// catalogue that may have changed — and once agents author their own
|
||||
// skills, it certainly will have.
|
||||
if let Some(tap) = self.tap.as_ref() {
|
||||
let mut ev = crate::mission_events::MissionEvent::new(
|
||||
tap.mission_id,
|
||||
crate::mission_events::PROMPT_COMPOSED,
|
||||
);
|
||||
ev.phase_id = tap.phase_id;
|
||||
ev.run_id = tap.run_id;
|
||||
ev.agent_id = crate::runtime_provision::claw_from_alias(&alias);
|
||||
ev.target = Some(req.role.clone());
|
||||
ev.detail = serde_json::json!({
|
||||
"text": prompt,
|
||||
"tier": "container",
|
||||
// The A/B arm, alongside the prompt it produced. `skill_use`
|
||||
// recovers this from the prompt text itself, so this field is
|
||||
// for reporting and for catching the two disagreeing.
|
||||
"skill_delivery": mode.as_str(),
|
||||
});
|
||||
crate::mission_events::record(&tap.pool, ev).await;
|
||||
}
|
||||
self.drive(&alias, &prompt).await
|
||||
}
|
||||
}
|
||||
|
||||
/// The base turn prompt with the agent's pinned skills appended, if it has any.
|
||||
///
|
||||
/// Split out from `run_turn` so the wiring is testable: `pinned_skills_text`
|
||||
/// working and `run_turn` actually calling it are different claims, and the
|
||||
/// second is the one that was false for every skill in the catalogue.
|
||||
pub fn compose_turn_prompt(
|
||||
base: &str,
|
||||
skills: Option<&str>,
|
||||
mode: crate::skill_delivery::Mode,
|
||||
) -> String {
|
||||
let Some(skills) = skills.map(str::trim).filter(|s| !s.is_empty()) else {
|
||||
// No heading when there is nothing under it. An empty "Your skills"
|
||||
// section tells the model it has skills and then shows it none, which
|
||||
// is worse than silence.
|
||||
return base.to_string();
|
||||
};
|
||||
// The preamble differs per arm and lives in `skill_delivery`, because it
|
||||
// is also what the scorer reads the arm back from. Two copies of this
|
||||
// sentence is two chances for the reader to stop recognising the writer.
|
||||
let preamble = crate::skill_delivery::preamble(mode);
|
||||
let section = format!("# Your skills\n\n{preamble}\n\n{skills}");
|
||||
// After the identity paragraph and BEFORE the task. This section is the
|
||||
// one part of the prompt that asks the agent to do something before it
|
||||
// starts — read a procedure — and until 2026-09-18 it was appended last,
|
||||
// after the task, the tool list, the workspace rules and the marker
|
||||
// contract, sitting 87–90% of the way into a 6 KB prompt. On the three
|
||||
// `index`-arm runs the narratives never mentioned it at all. Position is
|
||||
// the untested lever; this is the test.
|
||||
match base.split_once("\n\n") {
|
||||
Some((identity, rest)) if identity.starts_with("You are ") => {
|
||||
format!("{identity}\n\n{section}\n\n{rest}")
|
||||
}
|
||||
_ => format!("{section}\n\n{base}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod prompt_order_tests {
|
||||
use super::compose_turn_prompt;
|
||||
use crate::skill_delivery::Mode;
|
||||
|
||||
/// The section comes right after the identity paragraph, before the task —
|
||||
/// not appended after everything else.
|
||||
#[test]
|
||||
fn skills_come_after_identity_and_before_the_task() {
|
||||
let base = "You are the \"x\" agent. Do your part.\n\nTask: MISSION: y\n\nTOOLS AVAILABLE";
|
||||
let p = compose_turn_prompt(base, Some("--- SKILL: a ---\nbody"), Mode::Files);
|
||||
let i_id = p.find("You are the").unwrap();
|
||||
let i_sk = p.find("# Your skills").unwrap();
|
||||
let i_task = p.find("Task: MISSION").unwrap();
|
||||
assert!(i_id < i_sk && i_sk < i_task, "order was identity={i_id} skills={i_sk} task={i_task}\n{p}");
|
||||
assert!(p.ends_with("TOOLS AVAILABLE"), "the base's tail is untouched");
|
||||
}
|
||||
|
||||
/// A base with no identity paragraph still gets the section first.
|
||||
#[test]
|
||||
fn skills_lead_when_there_is_no_identity_paragraph() {
|
||||
let p = compose_turn_prompt("Task: y", Some("--- SKILL: a ---\nbody"), Mode::Files);
|
||||
assert!(p.starts_with("# Your skills"), "{p}");
|
||||
assert!(p.ends_with("Task: y"));
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse `role=alias,role=alias` into a map (blank/malformed entries skipped).
|
||||
fn parse_agent_map(s: &str) -> HashMap<String, String> {
|
||||
s.split(',')
|
||||
@@ -634,22 +983,31 @@ mod tests {
|
||||
/// second error on top of the first.
|
||||
#[test]
|
||||
fn the_container_name_is_derived_or_absent_never_wrong() {
|
||||
let ex = |url: &str| ZeroClawDriveExecutor::new(
|
||||
let ex = |url: &str| {
|
||||
ZeroClawDriveExecutor::new(
|
||||
url.to_string(),
|
||||
String::new(),
|
||||
std::collections::HashMap::new(),
|
||||
"scout".into(),
|
||||
);
|
||||
)
|
||||
};
|
||||
assert_eq!(
|
||||
ex("http://cm-runtime-mission-019fec2d596f:42617").container_name().as_deref(),
|
||||
ex("http://cm-runtime-mission-019fec2d596f:42617")
|
||||
.container_name()
|
||||
.as_deref(),
|
||||
Some("cm-runtime-mission-019fec2d596f")
|
||||
);
|
||||
assert_eq!(
|
||||
ex("https://host.example:8443/base").container_name().as_deref(),
|
||||
ex("https://host.example:8443/base")
|
||||
.container_name()
|
||||
.as_deref(),
|
||||
Some("host.example")
|
||||
);
|
||||
// No scheme is still a host.
|
||||
assert_eq!(ex("clawmates-runtime:42617").container_name().as_deref(), Some("clawmates-runtime"));
|
||||
assert_eq!(
|
||||
ex("clawmates-runtime:42617").container_name().as_deref(),
|
||||
Some("clawmates-runtime")
|
||||
);
|
||||
assert_eq!(ex("").container_name(), None);
|
||||
}
|
||||
|
||||
@@ -669,7 +1027,10 @@ mod tests {
|
||||
json!({"type": "session_start", "session_id": "s1", "resumed": false}),
|
||||
json!({"type": "chunk", "content": "hel"}),
|
||||
json!({"type": "chunk", "content": "lo"}),
|
||||
json!({"type": "done", "input_tokens": 5, "output_tokens": 7}),
|
||||
// The real frame carries model and provider; the executor read
|
||||
// only the two token counts until 2026-09-14.
|
||||
json!({"type": "done", "input_tokens": 5, "output_tokens": 7,
|
||||
"model": "claude-sonnet-5", "provider": "anthropic"}),
|
||||
]
|
||||
}
|
||||
|
||||
@@ -769,6 +1130,16 @@ mod tests {
|
||||
let out = exec.run_turn(req()).await.unwrap();
|
||||
assert_eq!(out.output, "hello");
|
||||
assert_eq!(out.tokens, 12);
|
||||
assert_eq!(
|
||||
out.spend,
|
||||
cm_orchestrator::Spend {
|
||||
input_tokens: 5,
|
||||
output_tokens: 7,
|
||||
provider: Some("anthropic".into()),
|
||||
model: Some("claude-sonnet-5".into()),
|
||||
},
|
||||
"the split and the provider must survive the done frame, not just the sum"
|
||||
);
|
||||
assert!(out.gated.is_empty());
|
||||
}
|
||||
|
||||
@@ -792,18 +1163,31 @@ mod tests {
|
||||
assert_eq!(
|
||||
trace.calls,
|
||||
vec![
|
||||
ToolCall { tool: "Read".into(), path: Some("/mission/repo/src/a.rs".into()) },
|
||||
ToolCall { tool: "Bash".into(), path: None },
|
||||
ToolCall {
|
||||
tool: "Read".into(),
|
||||
path: Some("/mission/repo/src/a.rs".into())
|
||||
},
|
||||
ToolCall {
|
||||
tool: "Bash".into(),
|
||||
path: None
|
||||
},
|
||||
// `arguments_summary` said "src/main.rs". It is prose, so it is
|
||||
// not a file touch — a path scraped from a sentence would put
|
||||
// files on the map that no agent opened.
|
||||
ToolCall { tool: "Grep".into(), path: None },
|
||||
ToolCall {
|
||||
tool: "Grep".into(),
|
||||
path: None
|
||||
},
|
||||
]
|
||||
);
|
||||
assert_eq!(trace.unmatched.get("a_frame_we_have_never_seen"), Some(&2));
|
||||
assert_eq!(trace.unmatched.get("session_start"), Some(&1));
|
||||
// `done` terminates the drain and is not an unmatched frame.
|
||||
assert!(!trace.unmatched.contains_key("done"), "{:?}", trace.unmatched);
|
||||
assert!(
|
||||
!trace.unmatched.contains_key("done"),
|
||||
"{:?}",
|
||||
trace.unmatched
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -26,10 +26,24 @@ use crate::topology_exec::ZeroClawDriveExecutor;
|
||||
const STALE_AFTER_SECS: f64 = 180.0;
|
||||
|
||||
/// Maximum age a `running` run may spend WITHOUT journaling any step
|
||||
/// records before the reaper kills its container and fails it. 15 min
|
||||
/// is generous: healthy first-step latency is typically 5–60s; anything
|
||||
/// past this is a stuck container (usually a wedged provider CLI).
|
||||
const REAP_STUCK_AFTER_SECS: i64 = 15 * 60;
|
||||
/// records before the reaper kills its container and fails it.
|
||||
///
|
||||
/// **This must stay LONGER than the runtime's per-turn timeout.** A run
|
||||
/// journals its first record when its first step COMPLETES, so any turn still
|
||||
/// legitimately in flight looks identical to a wedged container. The runtime
|
||||
/// grants a turn `timeout_secs = 3000` (50 min), so a shorter reaper window
|
||||
/// does not detect stuck runs — it kills healthy slow ones.
|
||||
///
|
||||
/// This was 15 minutes, chosen when "healthy first-step latency is typically
|
||||
/// 5–60s" was true of the model in use. It was, on haiku. Moving the mission
|
||||
/// agents to sonnet-5 made first turns longer than the window, and mission
|
||||
/// 01a00c41's research phase was reaped at 900s having already written 402
|
||||
/// lines across 13 files — work the delivery path then captured and pushed,
|
||||
/// which is the only reason we could tell the run was healthy at all.
|
||||
///
|
||||
/// The lesson generalises past this constant: a liveness timeout calibrated
|
||||
/// against one model silently becomes a correctness bug when the model changes.
|
||||
const REAP_STUCK_AFTER_SECS: i64 = 60 * 60;
|
||||
|
||||
/// Spawn the durable topology job worker. Polls for queued jobs every `poll`
|
||||
/// interval; runs each to completion (or failure), checkpointing per step.
|
||||
@@ -185,9 +199,16 @@ async fn run_job(
|
||||
// the missions row; else fall back to the shared env-derived
|
||||
// gateway (pre-C3 missions + non-mission runs). This is what
|
||||
// isolates agents' workspace filesystem to that mission's repo.
|
||||
type MissionBinding = (Option<String>, Option<String>, Uuid, Option<Uuid>);
|
||||
type MissionBinding = (
|
||||
Option<String>,
|
||||
Option<String>,
|
||||
Uuid,
|
||||
Option<Uuid>,
|
||||
Option<String>,
|
||||
);
|
||||
let mission_binding: Option<MissionBinding> = sqlx::query_as::<_, MissionBinding>(
|
||||
"SELECT m.runtime_endpoint, m.runtime_pairing_code, m.id, r.mission_phase_id
|
||||
"SELECT m.runtime_endpoint, m.runtime_pairing_code, m.id, r.mission_phase_id,
|
||||
m.runtime_token
|
||||
FROM topology_runs r
|
||||
JOIN missions m ON m.id = r.mission_id
|
||||
WHERE r.id = $1",
|
||||
@@ -201,17 +222,27 @@ async fn run_job(
|
||||
// to no mission — a bare topology run has no phase to hang tool calls on.
|
||||
let tap = mission_binding
|
||||
.as_ref()
|
||||
.map(|(_, _, mission_id, phase_id)| crate::topology_exec::MissionTap {
|
||||
.map(
|
||||
|(_, _, mission_id, phase_id, _)| crate::topology_exec::MissionTap {
|
||||
pool: pool.clone(),
|
||||
workspace_id: job.workspace_id,
|
||||
mission_id: *mission_id,
|
||||
phase_id: *phase_id,
|
||||
run_id: Some(id),
|
||||
});
|
||||
},
|
||||
);
|
||||
let leaf_result = match mission_binding {
|
||||
Some((Some(url), Some(code), _, _)) => {
|
||||
// Seed the cached bearer from `runtime_token` when we have one: the
|
||||
// pairing code is single-use, so after a restart it is the only way in.
|
||||
Some((Some(url), Some(code), _, _, tok)) => {
|
||||
ZeroClawDriveExecutor::from_env_for_gateway_with_code(url, code)
|
||||
.map(|e| e.with_token(tok))
|
||||
}
|
||||
// No pairing code (pre-C3 missions): the persisted token is the only
|
||||
// credential, so seed it here too.
|
||||
Some((Some(url), None, _, _, tok)) => {
|
||||
ZeroClawDriveExecutor::from_env_for_gateway(url).map(|e| e.with_token(tok))
|
||||
}
|
||||
Some((Some(url), None, _, _)) => ZeroClawDriveExecutor::from_env_for_gateway(url),
|
||||
_ => ZeroClawDriveExecutor::from_env(),
|
||||
};
|
||||
let leaf = match leaf_result {
|
||||
@@ -246,9 +277,29 @@ async fn run_job(
|
||||
id,
|
||||
Arc::new(leaf),
|
||||
);
|
||||
drive(pool, id, &graph, &job.task, progress, &exec).await
|
||||
drive(
|
||||
pool,
|
||||
id,
|
||||
job.workspace_id,
|
||||
&graph,
|
||||
&job.task,
|
||||
progress,
|
||||
&exec,
|
||||
)
|
||||
.await
|
||||
}
|
||||
_ => {
|
||||
drive(
|
||||
pool,
|
||||
id,
|
||||
job.workspace_id,
|
||||
&graph,
|
||||
&job.task,
|
||||
progress,
|
||||
&leaf,
|
||||
)
|
||||
.await
|
||||
}
|
||||
_ => drive(pool, id, &graph, &job.task, progress, &leaf).await,
|
||||
};
|
||||
|
||||
finish(pool, id, result).await;
|
||||
@@ -303,8 +354,7 @@ async fn run_composed(
|
||||
OrchestratorError::Executor("a composed run must belong to a mission phase".into())
|
||||
})?;
|
||||
|
||||
let mission: (Option<Uuid>, Option<String>, Option<String>, bool) =
|
||||
sqlx::query_as(
|
||||
let mission: (Option<Uuid>, Option<String>, Option<String>, bool) = sqlx::query_as(
|
||||
"SELECT target_node_id, backend, team_engine, (repo_id IS NOT NULL) \
|
||||
FROM missions WHERE id = $1",
|
||||
)
|
||||
@@ -347,7 +397,16 @@ async fn run_composed(
|
||||
completed_steps: progress.completed as u32,
|
||||
},
|
||||
);
|
||||
drive(pool, job.id, graph, &job.task, progress, &exec).await
|
||||
drive(
|
||||
pool,
|
||||
job.id,
|
||||
job.workspace_id,
|
||||
graph,
|
||||
&job.task,
|
||||
progress,
|
||||
&exec,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Post-terminal hook: if this run's team is `ephemeral` and no siblings are
|
||||
@@ -406,14 +465,30 @@ async fn maybe_teardown_ephemeral_team(pool: &PgPool, runtime: &cm_runtime::Runt
|
||||
async fn drive<E: TurnExecutor>(
|
||||
pool: &PgPool,
|
||||
id: Uuid,
|
||||
workspace_id: Uuid,
|
||||
graph: &TopologyGraph,
|
||||
task: &str,
|
||||
progress: RunProgress,
|
||||
executor: &E,
|
||||
) -> Result<RunRecord, OrchestratorError> {
|
||||
let pool_cb = pool.clone();
|
||||
// node_id -> agent id, resolved once. The binding lives in the node's
|
||||
// attrs (`agent = claw_<uuid>`), which is also what the runtime dispatches
|
||||
// on — so usage is attributed to exactly the claw that did the work.
|
||||
let agent_of: std::sync::Arc<std::collections::HashMap<String, Uuid>> = std::sync::Arc::new(
|
||||
graph
|
||||
.nodes
|
||||
.iter()
|
||||
.filter_map(|n| {
|
||||
let alias = n.attrs.get("agent")?;
|
||||
let uuid = alias.strip_prefix("claw_")?;
|
||||
Some((n.id.clone(), Uuid::parse_str(uuid).ok()?))
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
execute_resumable(graph, task, executor, progress, move |snap| {
|
||||
let pool = pool_cb.clone();
|
||||
let agent_of = agent_of.clone();
|
||||
async move {
|
||||
// 2026-07-15: verbose per-step trace so `docker logs
|
||||
// clawmates_server_1` shows which topology node just fired,
|
||||
@@ -439,6 +514,84 @@ async fn drive<E: TurnExecutor>(
|
||||
last.tokens,
|
||||
last.gated.len(),
|
||||
);
|
||||
// Per-agent usage. Without this the command centre's SPEND,
|
||||
// ACTIVITY and THROUGHPUT cards read `usage_events`, which
|
||||
// nothing on the mission path ever wrote — so they showed 0 for
|
||||
// an agent that had just burned 15k tokens.
|
||||
//
|
||||
// `charge` also decrements credit lots, which is the point: a
|
||||
// mission turn costs what it costs. It clamps at the available
|
||||
// balance and still records the full obligation, so an empty
|
||||
// wallet cannot fail a turn.
|
||||
// The agent's own words, for the REASONING STREAM card. The
|
||||
// world feed is a DB poll, not a push bus, so a live card can
|
||||
// only show what was persisted — this is the step output the
|
||||
// worker already has in hand, attributed to the claw that
|
||||
// produced it. Truncated because the card renders a tail, not a
|
||||
// transcript, and mission_events is capped per phase.
|
||||
if let Some(agent_id) = agent_of.get(&last.node_id).copied() {
|
||||
let text: String = last.output.chars().take(600).collect();
|
||||
if !text.trim().is_empty() {
|
||||
if let Some(mission_id) = sqlx::query_scalar::<_, Option<Uuid>>(
|
||||
"SELECT mission_id FROM topology_runs WHERE id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(&pool)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.flatten()
|
||||
{
|
||||
let mut ev = crate::mission_events::MissionEvent::new(
|
||||
mission_id,
|
||||
"reasoning",
|
||||
);
|
||||
ev.agent_id = Some(agent_id);
|
||||
ev.run_id = Some(id);
|
||||
ev.target = Some(last.role.clone());
|
||||
ev.detail = serde_json::json!({ "text": text });
|
||||
crate::mission_events::record(&pool, ev).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(agent_id) = agent_of.get(&last.node_id).copied() {
|
||||
if last.tokens > 0 {
|
||||
// The split and the provider come from the runtime's
|
||||
// `done` frame via `StepRecord.spend`. An executor
|
||||
// that reports only a total leaves the split at 0/0
|
||||
// and the total goes on the output side, as before.
|
||||
let (tin, tout) = if last.spend.input_tokens + last.spend.output_tokens > 0
|
||||
{
|
||||
(last.spend.input_tokens, last.spend.output_tokens)
|
||||
} else {
|
||||
(0, last.tokens as u64)
|
||||
};
|
||||
let mission_id: Option<Uuid> = sqlx::query_scalar::<_, Option<Uuid>>(
|
||||
"SELECT mission_id FROM topology_runs WHERE id = $1",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_optional(&pool)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.flatten();
|
||||
if let Err(e) = cm_billing::charge(
|
||||
&pool,
|
||||
cm_domain::WorkspaceId::from(workspace_id),
|
||||
cm_domain::AgentId::from(agent_id),
|
||||
None,
|
||||
tin,
|
||||
tout,
|
||||
last.spend.provider.as_deref(),
|
||||
last.spend.model.as_deref(),
|
||||
mission_id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
eprintln!("topology_worker: usage for {agent_id} failed: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Best-effort checkpoint: a failed write just means we re-run the
|
||||
// step on resume (idempotent — topology turns are pure reads here).
|
||||
|
||||
@@ -464,7 +464,7 @@ mod tests {
|
||||
assert!(!install.contains(write), "{install}");
|
||||
}
|
||||
assert_eq!(
|
||||
crate::vm_tool_tap::guest_settings(Some(GATE_DIR), None)["hooks"]["Stop"][0]["hooks"]
|
||||
crate::vm_tool_tap::guest_settings(Some(GATE_DIR), None, None)["hooks"]["Stop"][0]["hooks"]
|
||||
[0]["command"],
|
||||
json!("/root/gate/stop-gate.sh")
|
||||
);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -71,8 +71,186 @@ pub struct Observed {
|
||||
pub tool: String,
|
||||
/// The path the tool's **input** named, if any. From JSON, never prose.
|
||||
pub path: Option<String>,
|
||||
/// Claude Code's session id for the `claude -p` invocation this call
|
||||
/// happened inside.
|
||||
///
|
||||
/// One invocation is one turn is one agent, so this is the only thing in
|
||||
/// the payload that separates one agent's actions from another's. The tap
|
||||
/// is per-CONTAINER and every role in a phase shares one, so without this
|
||||
/// the whole phase arrives as an undifferentiated stream.
|
||||
pub session: Option<String>,
|
||||
/// What a **command** produced, bounded by [`bounded_response`].
|
||||
///
|
||||
/// Only for tools that run something. `Read`'s response is the file it just
|
||||
/// read and `Write`'s is a restatement of what was written — both are
|
||||
/// already knowable from the arguments and the delivered diff, and storing
|
||||
/// them would double the largest write path in the system for nothing.
|
||||
///
|
||||
/// A command's OUTCOME is different: it is the only place a failing test
|
||||
/// run is visible. Without it "did this phase go red before it went green"
|
||||
/// cannot be answered from anything — not from tool order (in Rust the
|
||||
/// unit test lives in the file under test, so one `Edit` adds both), and
|
||||
/// not from the repository either, because `tdd-red-green-refactor` says
|
||||
/// in so many words to "commit the RED-to-GREEN pair as one commit".
|
||||
pub response: Value,
|
||||
/// The subagent that made this call, when it was not the turn's own agent.
|
||||
///
|
||||
/// Claude Code's `Agent` tool spawns a subagent that runs its own tools,
|
||||
/// and those calls DO reach this hook — measured against the real binary,
|
||||
/// which is the good news, because it means nothing is invisible. What they
|
||||
/// carry is the PARENT's `session_id`, so [`Observed::session`] cannot tell
|
||||
/// them apart and attribution silently credits the parent for work a
|
||||
/// subagent did.
|
||||
///
|
||||
/// The payload has always said so: `agent_type` and `agent_id` are present
|
||||
/// on a subagent's call and absent on the parent's. This parser read past
|
||||
/// them. Two production missions spawned twelve subagents to fetch web
|
||||
/// pages, and every tool call they made was recorded as the parent's with
|
||||
/// nothing anywhere reporting the difference.
|
||||
pub subagent: Option<String>,
|
||||
/// Which subagent instance, so several running under one turn stay apart.
|
||||
pub subagent_id: Option<String>,
|
||||
/// The tool's arguments, bounded by [`bounded_input`].
|
||||
///
|
||||
/// Kept because the tool NAME alone answers almost nothing. A phase that
|
||||
/// recorded `Bash × 6` is indistinguishable from one that ran the test
|
||||
/// suite six times, one that pushed to a branch it was told not to, and
|
||||
/// one that queried an API a skill forbids. The argument is where the
|
||||
/// behaviour is, and until now this parser read it, took the path out of
|
||||
/// it, and dropped the rest on the floor.
|
||||
pub input: Value,
|
||||
}
|
||||
|
||||
/// How much of one argument string is worth keeping.
|
||||
///
|
||||
/// A shell command longer than this is a heredoc or a generated payload; its
|
||||
/// first half still carries the verb, which is what any check reads.
|
||||
const MAX_ARG_LEN: usize = 512;
|
||||
|
||||
/// Argument keys whose value is a file BODY rather than a description of an
|
||||
/// action.
|
||||
///
|
||||
/// Dropped to a byte count rather than truncated. These carry whole source
|
||||
/// files — `mission_events` is already the largest write path on a coding
|
||||
/// phase, and storing every `Write` twice (once in the event, once in the
|
||||
/// delivered diff) buys nothing: no check reads the body, and the diff is the
|
||||
/// authority on what was written anyway.
|
||||
const BODY_KEYS: [&str; 4] = ["content", "new_string", "old_string", "edits"];
|
||||
|
||||
/// Tools whose response is an outcome rather than a restatement.
|
||||
const RESPONSE_TOOLS: [&str; 1] = ["Bash"];
|
||||
|
||||
/// How much of a command's output to keep.
|
||||
const MAX_OUTPUT_LEN: usize = 600;
|
||||
|
||||
/// Shrink a command's response, keeping the **end** of its output.
|
||||
///
|
||||
/// The opposite of [`bounded_input`], and deliberately so. An argument's
|
||||
/// meaning is at the start — the verb of the command. A command's meaning is at
|
||||
/// the END: `cargo test` prints hundreds of lines and then `test result: ok` or
|
||||
/// `test result: FAILED`, and a head-biased truncation would keep the noise and
|
||||
/// throw away the verdict, which is the one thing being stored for.
|
||||
pub fn bounded_response(tool: &str, response: &Value) -> Value {
|
||||
if !RESPONSE_TOOLS.contains(&tool) {
|
||||
return Value::Null;
|
||||
}
|
||||
let Some(obj) = response.as_object() else {
|
||||
return Value::Null;
|
||||
};
|
||||
let mut out = serde_json::Map::new();
|
||||
for key in ["stdout", "stderr", "interrupted"] {
|
||||
match obj.get(key) {
|
||||
Some(Value::String(s)) if s.len() > MAX_OUTPUT_LEN => {
|
||||
let start = s
|
||||
.char_indices()
|
||||
.map(|(i, _)| i)
|
||||
.find(|i| *i >= s.len().saturating_sub(MAX_OUTPUT_LEN))
|
||||
.unwrap_or(0);
|
||||
out.insert(key.into(), Value::String(format!("[truncated]…{}", &s[start..])));
|
||||
}
|
||||
Some(v) => {
|
||||
out.insert(key.into(), v.clone());
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
Value::Object(out)
|
||||
}
|
||||
|
||||
/// Shrink a tool's arguments to something safe to store on every call.
|
||||
///
|
||||
/// Bounded rather than whitelisted on purpose. A whitelist of "interesting"
|
||||
/// keys silently drops the one argument that matters the first time a tool
|
||||
/// grows a new field, and the loss is invisible — the event still looks
|
||||
/// complete. Bounding keeps every key and says, in the record itself, where it
|
||||
/// stopped.
|
||||
pub fn bounded_input(input: &Value) -> Value {
|
||||
let Some(obj) = input.as_object() else {
|
||||
return Value::Null;
|
||||
};
|
||||
let mut out = serde_json::Map::new();
|
||||
for (k, v) in obj {
|
||||
if BODY_KEYS.contains(&k.as_str()) {
|
||||
let bytes = match v {
|
||||
Value::String(s) => s.len(),
|
||||
other => other.to_string().len(),
|
||||
};
|
||||
out.insert(k.clone(), json!({ "omitted_bytes": bytes }));
|
||||
continue;
|
||||
}
|
||||
match v {
|
||||
Value::String(s) if s.len() > MAX_ARG_LEN => {
|
||||
let cut = s
|
||||
.char_indices()
|
||||
.map(|(i, _)| i)
|
||||
.take_while(|i| *i <= MAX_ARG_LEN)
|
||||
.last()
|
||||
.unwrap_or(0);
|
||||
out.insert(k.clone(), Value::String(format!("{}…[truncated]", &s[..cut])));
|
||||
}
|
||||
other => {
|
||||
out.insert(k.clone(), other.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
Value::Object(out)
|
||||
}
|
||||
|
||||
/// Hosts named in content the agent FETCHED, one per line, beside the tap.
|
||||
///
|
||||
/// Stage 1 of argument provenance (docs/TASK-PERMISSION-AND-TAINT.md): the
|
||||
/// invariant is "no outbound action whose target was derived from untrusted
|
||||
/// content", and this file is the "derived from untrusted content" half. It is
|
||||
/// written in the guest because the gate that will read it runs in the guest,
|
||||
/// and it lives in the tap's directory because the gate's `hook-files` rule
|
||||
/// already refuses every write there, so the agent it governs cannot erase it.
|
||||
///
|
||||
/// Nothing reads it to refuse anything yet. It is drained into
|
||||
/// `taint.hosts` so what it actually collects can be inspected on real
|
||||
/// missions before a rule is built on it.
|
||||
pub const TAINT_FILE: &str = "untrusted-hosts.txt";
|
||||
|
||||
/// How many hosts the file may hold. A page with thousands of links must not
|
||||
/// turn the tap into the largest write in the guest; past the cap, new hosts
|
||||
/// are dropped, and the host-side record says how many it saw.
|
||||
pub const MAX_TAINT_HOSTS: usize = 500;
|
||||
|
||||
/// Reads one `PostToolUse` event on stdin and prints the hosts its RESPONSE
|
||||
/// names that its INPUT did not, one per line.
|
||||
///
|
||||
/// Only fetching calls count: `WebFetch`, `WebSearch`, and a `Bash` command
|
||||
/// with `curl` or `wget` in COMMAND position — at the start, or after `;`,
|
||||
/// `&`, `|`, `(`, a backtick or `$(`. Anywhere else it is an argument:
|
||||
/// `grep -r curl docs` searches for the word, and counting it as a fetch was
|
||||
/// the first thing the shell test caught. Hosts, not strings — tainting arbitrary
|
||||
/// text and matching it against later commands fires on ordinary research
|
||||
/// immediately, the cardinal failure for this module. A host the agent itself
|
||||
/// put in the command is its own choice, not the page's, and is excluded.
|
||||
///
|
||||
/// Silent on any error, like the gate's extractor: the caller ignores output
|
||||
/// it cannot use, and the tap never exits non-zero.
|
||||
pub const NODE_TAINT: &str = r#"let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{try{const j=JSON.parse(s);const n=String(j.tool_name||"");const t=j.tool_input||{};const cmd=String(t.command||"");const fetching=n==="WebFetch"||n==="WebSearch"||(n==="Bash"&&/(^|[;&|(`]|\$\()\s*(curl|wget)\s/.test(cmd));if(!fetching)return;const r=j.tool_response;const text=typeof r==="string"?r:(r&&typeof r==="object"&&n==="Bash")?String(r.stdout||"")+"\n"+String(r.stderr||""):JSON.stringify(r||"");const own=(cmd+" "+String(t.url||"")+" "+String(t.query||"")).toLowerCase();const seen=new Set();const re=/\bhttps?:\/\/([a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?(?:\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?)+)/gi;let m;while((m=re.exec(text))!==null){const h=m[1].toLowerCase();if(!own.includes(h))seen.add(h)}for(const h of seen)process.stdout.write(h+"\n")}catch(e){}})"#;
|
||||
|
||||
/// The hook script. Copies stdin verbatim to the tap file and gets out of the
|
||||
/// way.
|
||||
///
|
||||
@@ -80,20 +258,55 @@ pub struct Observed {
|
||||
/// guest would need the tool's JSON schema baked into a shell script, inside an
|
||||
/// image we do not rebuild for a parser change, with no way to tell a parse
|
||||
/// failure from a quiet turn.
|
||||
///
|
||||
/// One exception, and it is guest-side because its reader will be: the taint
|
||||
/// step ([`TAINT_FILE`]). It runs only when the payload could be a fetch — a
|
||||
/// `node` spawn on every `Read` would tax the hottest path in the guest for
|
||||
/// nothing — and every failure in it is swallowed, so the tap still exits 0.
|
||||
pub fn hook_script(dir: &str) -> String {
|
||||
format!(
|
||||
"#!/bin/sh\n\
|
||||
# The tool tap. See cm-api/src/vm_tool_tap.rs.\n\
|
||||
mkdir -p {dir} 2>/dev/null\n\
|
||||
# `cat` of stdin, appended whole. One JSON object per line, because\n\
|
||||
# Claude Code hands the hook one event per invocation.\n\
|
||||
cat >> {dir}/tools.jsonl 2>/dev/null\n\
|
||||
printf '\\n' >> {dir}/tools.jsonl 2>/dev/null\n\
|
||||
# Held once, because stdin can be read once and two things need it.\n\
|
||||
payload=$(cat)\n\
|
||||
# Appended whole. One JSON object per line, because Claude Code hands\n\
|
||||
# the hook one event per invocation.\n\
|
||||
printf '%s\\n\\n' \"$payload\" >> {dir}/tools.jsonl 2>/dev/null\n\
|
||||
# Taint: hosts a FETCHED page named. Cheap prefilter first.\n\
|
||||
case \"$payload\" in\n\
|
||||
\x20 *'\"WebFetch\"'*|*'\"WebSearch\"'*|*curl*|*wget*)\n\
|
||||
\x20 printf '%s' \"$payload\" | node -e {taint} 2>/dev/null | while IFS= read -r h; do\n\
|
||||
\x20 [ -n \"$h\" ] || continue\n\
|
||||
\x20 grep -qxF \"$h\" {dir}/{file} 2>/dev/null && continue\n\
|
||||
\x20 [ \"$(cat {dir}/{file} 2>/dev/null | grep -c '')\" -lt {cap} ] || break\n\
|
||||
\x20 printf '%s\\n' \"$h\" >> {dir}/{file} 2>/dev/null\n\
|
||||
\x20 done\n\
|
||||
\x20 ;;\n\
|
||||
esac\n\
|
||||
# ALWAYS zero. A non-zero PostToolUse hook talks back to the model.\n\
|
||||
exit 0\n"
|
||||
exit 0\n",
|
||||
taint = shell_quote(NODE_TAINT),
|
||||
file = TAINT_FILE,
|
||||
cap = MAX_TAINT_HOSTS,
|
||||
)
|
||||
}
|
||||
|
||||
/// Read the taint file. Not cleared: it is state the gate will consult for
|
||||
/// the rest of the mission, not a log to be consumed.
|
||||
pub fn taint_probe(dir: &str) -> String {
|
||||
format!("cat {dir}/{TAINT_FILE} 2>/dev/null || true")
|
||||
}
|
||||
|
||||
/// Parse a drained taint file into hosts, dropping blanks.
|
||||
pub fn parse_taint(raw: &str) -> Vec<String> {
|
||||
raw.lines()
|
||||
.map(str::trim)
|
||||
.filter(|l| !l.is_empty())
|
||||
.map(str::to_string)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The settings document for the guest, carrying **every** hook at once.
|
||||
///
|
||||
/// This function exists because the alternative — each feature writing its own
|
||||
@@ -103,8 +316,12 @@ pub fn hook_script(dir: &str) -> String {
|
||||
/// gate exists to catch. One writer, one document, one test that both hooks
|
||||
/// survive it.
|
||||
///
|
||||
/// `None` for either half means that hook is simply absent.
|
||||
pub fn guest_settings(gate_dir: Option<&str>, tap_dir: Option<&str>) -> Value {
|
||||
/// `None` for any part means that hook is simply absent.
|
||||
pub fn guest_settings(
|
||||
gate_dir: Option<&str>,
|
||||
tap_dir: Option<&str>,
|
||||
tool_gate_dir: Option<&str>,
|
||||
) -> Value {
|
||||
let mut hooks = serde_json::Map::new();
|
||||
if let Some(dir) = gate_dir {
|
||||
hooks.insert(
|
||||
@@ -118,6 +335,12 @@ pub fn guest_settings(gate_dir: Option<&str>, tap_dir: Option<&str>) -> Value {
|
||||
json!([{ "hooks": [{ "type": "command", "command": format!("{dir}/tap.sh") }] }]),
|
||||
);
|
||||
}
|
||||
if let Some(dir) = tool_gate_dir {
|
||||
// PRE-execution, unlike the tap above. Composed here rather than
|
||||
// written by `vm_tool_gate` itself for the same reason everything else
|
||||
// is: one writer, one document.
|
||||
hooks.insert("PreToolUse".into(), crate::vm_tool_gate::settings_hook(dir));
|
||||
}
|
||||
json!({ "hooks": Value::Object(hooks) })
|
||||
}
|
||||
|
||||
@@ -127,16 +350,17 @@ pub fn guest_settings(gate_dir: Option<&str>, tap_dir: Option<&str>) -> Value {
|
||||
/// tar: the tar lands in `/mission/repo`, which is exactly where this must not.
|
||||
pub fn install_command(dir: &str) -> String {
|
||||
format!(
|
||||
"mkdir -p {dir} && rm -f {dir}/tools.jsonl \
|
||||
"mkdir -p {dir} && rm -f {dir}/tools.jsonl {dir}/{taint} \
|
||||
&& printf '%s' {script} > {dir}/tap.sh && chmod +x {dir}/tap.sh",
|
||||
dir = dir,
|
||||
script = q(&hook_script(dir)),
|
||||
taint = TAINT_FILE,
|
||||
script = shell_quote(&hook_script(dir)),
|
||||
)
|
||||
}
|
||||
|
||||
/// Write the composed settings document.
|
||||
pub fn settings_command(path: &str, settings: &Value) -> String {
|
||||
format!("printf '%s' {} > {path}", q(&settings.to_string()))
|
||||
format!("printf '%s' {} > {path}", shell_quote(&settings.to_string()))
|
||||
}
|
||||
|
||||
/// Parse a drained tap.
|
||||
@@ -168,17 +392,44 @@ pub fn parse(raw: &str) -> Vec<Observed> {
|
||||
.or_else(|| v.get("toolInput"))
|
||||
.cloned()
|
||||
.unwrap_or(Value::Null);
|
||||
let response = v
|
||||
.get("tool_response")
|
||||
.or_else(|| v.get("toolResponse"))
|
||||
.cloned()
|
||||
.unwrap_or(Value::Null);
|
||||
Some(Observed {
|
||||
path: crate::mission_events::tool_path(&input),
|
||||
input: bounded_input(&input),
|
||||
response: bounded_response(&tool, &response),
|
||||
session: v
|
||||
.get("session_id")
|
||||
.or_else(|| v.get("sessionId"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string),
|
||||
// Absent on the turn agent's own calls, present on a
|
||||
// subagent's. That absence IS the signal, so an empty string
|
||||
// must read as "not a subagent" rather than as one named "".
|
||||
subagent: non_empty(&v, "agent_type", "agentType"),
|
||||
subagent_id: non_empty(&v, "agent_id", "agentId"),
|
||||
tool,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A string field under either spelling, treating empty as missing.
|
||||
fn non_empty(v: &Value, snake: &str, camel: &str) -> Option<String> {
|
||||
v.get(snake)
|
||||
.or_else(|| v.get(camel))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(str::to_string)
|
||||
}
|
||||
|
||||
/// Single-quote for `sh`. Local copy, same rule as the stop gate's — these two
|
||||
/// modules deliberately share no code, so neither can break the other.
|
||||
fn q(s: &str) -> String {
|
||||
pub fn shell_quote(s: &str) -> String {
|
||||
format!("'{}'", s.replace('\'', r"'\''"))
|
||||
}
|
||||
|
||||
@@ -195,7 +446,7 @@ mod tests {
|
||||
/// failure the gate was built to catch.
|
||||
#[test]
|
||||
fn both_hooks_survive_one_settings_document() {
|
||||
let s = guest_settings(Some("/root/gate"), Some(TAP_DIR));
|
||||
let s = guest_settings(Some("/root/gate"), Some(TAP_DIR), None);
|
||||
let hooks = s.get("hooks").expect("hooks");
|
||||
assert_eq!(
|
||||
hooks["Stop"][0]["hooks"][0]["command"],
|
||||
@@ -211,11 +462,11 @@ mod tests {
|
||||
/// Either half absent leaves the other exactly as it was.
|
||||
#[test]
|
||||
fn one_hook_alone_is_a_valid_document() {
|
||||
let gate_only = guest_settings(Some("/root/gate"), None);
|
||||
let gate_only = guest_settings(Some("/root/gate"), None, None);
|
||||
assert!(gate_only["hooks"].get("Stop").is_some());
|
||||
assert!(gate_only["hooks"].get("PostToolUse").is_none());
|
||||
|
||||
let tap_only = guest_settings(None, Some(TAP_DIR));
|
||||
let tap_only = guest_settings(None, Some(TAP_DIR), None);
|
||||
assert!(tap_only["hooks"].get("Stop").is_none());
|
||||
assert!(tap_only["hooks"].get("PostToolUse").is_some());
|
||||
}
|
||||
@@ -258,12 +509,121 @@ mod tests {
|
||||
assert_eq!(
|
||||
parse(raw),
|
||||
vec![
|
||||
Observed { tool: "Edit".into(), path: Some("/mission/repo/src/a.rs".into()) },
|
||||
Observed { tool: "Bash".into(), path: None },
|
||||
Observed {
|
||||
tool: "Edit".into(),
|
||||
path: Some("/mission/repo/src/a.rs".into()),
|
||||
input: json!({"file_path": "/mission/repo/src/a.rs"}),
|
||||
session: None,
|
||||
subagent: None,
|
||||
subagent_id: None,
|
||||
response: Value::Null,
|
||||
},
|
||||
Observed {
|
||||
tool: "Bash".into(),
|
||||
path: None,
|
||||
input: json!({"command": "ls"}),
|
||||
session: None,
|
||||
subagent: None,
|
||||
subagent_id: None,
|
||||
response: Value::Null,
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// A subagent's tool calls reach this hook carrying the PARENT's session
|
||||
/// id, so the only thing that separates them is `agent_type`/`agent_id`.
|
||||
///
|
||||
/// Measured against claude 2.1.246: spawning one subagent and having it run
|
||||
/// `echo SUB` produced three `PostToolUse` events on one session id — the
|
||||
/// parent's `Agent` call, the parent's own `Bash`, and the subagent's
|
||||
/// `Bash` — and only the last carried an `agent_type`. Reading past those
|
||||
/// fields is what made twelve production subagent spawns indistinguishable
|
||||
/// from the work of the agents that spawned them.
|
||||
#[test]
|
||||
fn a_subagents_call_is_told_apart_from_its_parents() {
|
||||
let raw = concat!(
|
||||
r#"{"tool_name":"Agent","session_id":"s1","tool_input":{"prompt":"fetch it"}}"#,
|
||||
"\n",
|
||||
r#"{"tool_name":"Bash","session_id":"s1","tool_input":{"command":"echo PARENT"}}"#,
|
||||
"\n",
|
||||
r#"{"tool_name":"Bash","session_id":"s1","agent_id":"a0b8","agent_type":"general-purpose","#,
|
||||
r#""tool_input":{"command":"echo SUB"}}"#,
|
||||
);
|
||||
let got = parse(raw);
|
||||
assert_eq!(got.len(), 3);
|
||||
assert!(
|
||||
got.iter().all(|o| o.session.as_deref() == Some("s1")),
|
||||
"a subagent shares its parent's session id — that is the whole problem"
|
||||
);
|
||||
assert_eq!(got[0].subagent, None, "the parent spawned it; it did not run inside it");
|
||||
assert_eq!(got[1].subagent, None);
|
||||
assert_eq!(got[2].subagent.as_deref(), Some("general-purpose"));
|
||||
assert_eq!(got[2].subagent_id.as_deref(), Some("a0b8"));
|
||||
}
|
||||
|
||||
/// An empty `agent_type` must read as "the turn's own agent", not as a
|
||||
/// subagent whose name happens to be blank.
|
||||
#[test]
|
||||
fn a_blank_agent_type_is_not_a_subagent() {
|
||||
let raw = r#"{"tool_name":"Bash","session_id":"s1","agent_type":" ","tool_input":{"command":"ls"}}"#;
|
||||
assert_eq!(parse(raw)[0].subagent, None);
|
||||
}
|
||||
|
||||
/// The command survives the parse.
|
||||
///
|
||||
/// The regression this guards is the one that made the first container-tier
|
||||
/// measurement unusable: six `Bash` calls were recorded and not one of them
|
||||
/// said what it ran, so every behavioural question — did it run the tests,
|
||||
/// did it commit, did it call the API a skill forbids — was unanswerable
|
||||
/// from a record that looked complete.
|
||||
#[test]
|
||||
fn the_argument_is_what_carries_the_behaviour() {
|
||||
let raw = concat!(
|
||||
r#"{"tool_name":"Bash","tool_input":{"command":"cargo nextest run -p cm-api"}}"#,
|
||||
"\n",
|
||||
);
|
||||
let got = parse(raw);
|
||||
assert_eq!(got[0].input["command"], json!("cargo nextest run -p cm-api"));
|
||||
}
|
||||
|
||||
/// A file body is counted, not stored; everything else survives bounded.
|
||||
#[test]
|
||||
fn bodies_are_dropped_and_long_arguments_are_marked() {
|
||||
let long = "x".repeat(MAX_ARG_LEN + 50);
|
||||
let got = bounded_input(&json!({
|
||||
"file_path": "/mission/repo/src/a.rs",
|
||||
"content": "fn main() {}",
|
||||
"command": long,
|
||||
}));
|
||||
assert_eq!(got["file_path"], json!("/mission/repo/src/a.rs"));
|
||||
assert_eq!(
|
||||
got["content"],
|
||||
json!({"omitted_bytes": 12}),
|
||||
"a file body is stored in the delivered diff already; the event only \
|
||||
needs to say how big it was"
|
||||
);
|
||||
let cmd = got["command"].as_str().expect("command kept");
|
||||
assert!(cmd.ends_with("…[truncated]"), "{cmd}");
|
||||
assert!(
|
||||
cmd.len() < MAX_ARG_LEN + 40,
|
||||
"a bounded argument must actually be bounded: {}",
|
||||
cmd.len()
|
||||
);
|
||||
}
|
||||
|
||||
/// Truncation must not split a multi-byte character.
|
||||
///
|
||||
/// `&s[..cut]` on a byte index inside a UTF-8 sequence panics, and the
|
||||
/// panic would land in the drain — losing a whole phase's tap to a command
|
||||
/// that happened to contain an emoji or an em dash.
|
||||
#[test]
|
||||
fn truncation_respects_character_boundaries() {
|
||||
let long = "é".repeat(MAX_ARG_LEN);
|
||||
let got = bounded_input(&json!({ "command": long }));
|
||||
assert!(got["command"].as_str().unwrap().ends_with("…[truncated]"));
|
||||
}
|
||||
|
||||
/// Exactly one place in the tree writes the guest settings document.
|
||||
///
|
||||
/// The unit test above proves `guest_settings` composes correctly; it says
|
||||
@@ -329,3 +689,143 @@ mod tests {
|
||||
assert!(parse(r#"{"tool_name":" ","tool_input":{}}"#).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod taint_tests {
|
||||
use super::*;
|
||||
|
||||
/// Run the GENERATED hook, with the real `node`, on each payload in turn.
|
||||
/// Returns the taint file and the number of tap events, or `None` where
|
||||
/// `sh`/`node` are missing (the gate's shell tests skip the same way).
|
||||
fn run_hook(payloads: &[Value]) -> Option<(Vec<String>, usize)> {
|
||||
let has = |bin: &str| {
|
||||
std::process::Command::new(bin).arg("--version").output().is_ok_and(|o| o.status.success())
|
||||
};
|
||||
if !has("node") {
|
||||
eprintln!("node not found; skipping the taint shell test");
|
||||
return None;
|
||||
}
|
||||
let dir = std::env::temp_dir().join(format!("cm-taint-{}", uuid::Uuid::now_v7()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let d = dir.to_str().unwrap();
|
||||
let script = dir.join("tap.sh");
|
||||
std::fs::write(&script, hook_script(d)).unwrap();
|
||||
for p in payloads {
|
||||
use std::io::Write;
|
||||
let mut child = std::process::Command::new("sh")
|
||||
.arg(&script)
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
child.stdin.take().unwrap().write_all(p.to_string().as_bytes()).unwrap();
|
||||
let out = child.wait_with_output().unwrap();
|
||||
assert_eq!(out.status.code(), Some(0), "the tap must always exit 0");
|
||||
}
|
||||
let hosts = parse_taint(&std::fs::read_to_string(dir.join(TAINT_FILE)).unwrap_or_default());
|
||||
let events = parse(&std::fs::read_to_string(dir.join("tools.jsonl")).unwrap_or_default()).len();
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
Some((hosts, events))
|
||||
}
|
||||
|
||||
fn bash(cmd: &str, stdout: &str) -> Value {
|
||||
json!({"tool_name":"Bash","tool_input":{"command":cmd},
|
||||
"tool_response":{"stdout":stdout,"stderr":"","interrupted":false}})
|
||||
}
|
||||
|
||||
/// The whole of stage 1 in one run: fetched content taints the hosts it
|
||||
/// names, the agent's own target does not, reading a file does not, and
|
||||
/// the tap records every event exactly as before.
|
||||
#[test]
|
||||
fn fetched_content_taints_the_hosts_it_names() {
|
||||
let Some((hosts, events)) = run_hook(&[
|
||||
// A fetched page names two hosts; one is the page's own.
|
||||
bash(
|
||||
"curl -s https://api.github.com/repos/x/y",
|
||||
"see https://evil.example/collect and https://api.github.com/other",
|
||||
),
|
||||
// The same host again: deduplicated.
|
||||
bash("curl -sL https://api.github.com/z", "mirror at https://EVIL.example/x"),
|
||||
// Reading a file that names a host is not a fetch.
|
||||
json!({"tool_name":"Read","tool_input":{"file_path":"/mission/repo/README.md"},
|
||||
"tool_response":"docs at https://readme-host.example/"}),
|
||||
// A command that merely mentions curl in its OUTPUT is not a fetch.
|
||||
bash("grep -r curl docs", "https://grep-host.example/ curl"),
|
||||
// WebFetch: the url is the agent's choice, the links in the result are not.
|
||||
json!({"tool_name":"WebFetch","tool_input":{"url":"https://docs.rs/serde"},
|
||||
"tool_response":"published on https://crates.io/crates/serde (see https://docs.rs/x)"}),
|
||||
// Garbage in: still exit 0, still recorded as nothing.
|
||||
json!("not an event"),
|
||||
]) else {
|
||||
return;
|
||||
};
|
||||
assert_eq!(hosts, vec!["evil.example".to_string(), "crates.io".to_string()], "{hosts:?}");
|
||||
assert_eq!(events, 5, "the tap must still record every tool event");
|
||||
}
|
||||
|
||||
/// The cap holds, and past it the file stops growing rather than failing.
|
||||
#[test]
|
||||
fn the_taint_file_is_capped() {
|
||||
let many: String = (0..MAX_TAINT_HOSTS + 20)
|
||||
.map(|i| format!("https://h{i}.example/ "))
|
||||
.collect();
|
||||
let Some((hosts, _)) = run_hook(&[bash("curl https://index.example/", &many)]) else {
|
||||
return;
|
||||
};
|
||||
assert_eq!(hosts.len(), MAX_TAINT_HOSTS);
|
||||
}
|
||||
|
||||
/// The taint file sits where the gate's `hook-files` rule already refuses
|
||||
/// writes, on both tiers — or the agent it governs could erase it.
|
||||
#[test]
|
||||
fn the_taint_file_is_protected_on_both_tiers() {
|
||||
for dir in [TAP_DIR, crate::container_tool_hooks::TAP_DIR] {
|
||||
let path = format!("{dir}/{TAINT_FILE}");
|
||||
let d = crate::vm_tool_gate::decide("Bash", &format!("rm -f {path}"), None, None)
|
||||
.unwrap_or_else(|| panic!("{path} is writable by the agent"));
|
||||
assert_eq!(d.rule, "hook-files");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod three_hook_tests {
|
||||
use super::*;
|
||||
|
||||
/// All three hooks must survive one document.
|
||||
///
|
||||
/// The tap and the stop gate already shared it; the pre-execution gate is
|
||||
/// the third, and the clobber this function exists to prevent gets more
|
||||
/// likely with each one. A missing `Stop` lets a phase finish having
|
||||
/// written nothing; a missing `PreToolUse` runs every command unchecked.
|
||||
#[test]
|
||||
fn the_document_carries_the_stop_gate_the_tap_and_the_pre_execution_gate() {
|
||||
let s = guest_settings(
|
||||
Some("/root/gate"),
|
||||
Some(TAP_DIR),
|
||||
Some(crate::vm_tool_gate::GUEST_DIR),
|
||||
);
|
||||
let hooks = s["hooks"].as_object().expect("hooks object");
|
||||
assert!(hooks.contains_key("Stop"), "stop gate lost");
|
||||
assert!(hooks.contains_key("PostToolUse"), "tap lost");
|
||||
assert!(hooks.contains_key("PreToolUse"), "pre-execution gate lost");
|
||||
assert_eq!(hooks.len(), 3, "an unexpected hook appeared: {hooks:?}");
|
||||
|
||||
// And the pre-execution hook points at the gate's own script, not the
|
||||
// tap's — pointing PreToolUse at tap.sh would exit 0 on everything and
|
||||
// read as a gate that allows all.
|
||||
let cmd = s["hooks"]["PreToolUse"][0]["hooks"][0]["command"]
|
||||
.as_str()
|
||||
.expect("command");
|
||||
assert!(cmd.ends_with("tool-gate.sh"), "wrong script: {cmd}");
|
||||
}
|
||||
|
||||
/// The gate alone must still produce a usable document.
|
||||
#[test]
|
||||
fn the_gate_can_be_installed_without_the_others() {
|
||||
let s = guest_settings(None, None, Some(crate::vm_tool_gate::GUEST_DIR));
|
||||
assert_eq!(s["hooks"].as_object().unwrap().len(), 1);
|
||||
assert!(s["hooks"]["PreToolUse"].is_array());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,6 +34,20 @@ pub struct WorkflowRecipe {
|
||||
pub phases: Vec<WorkflowPhase>,
|
||||
#[serde(default)]
|
||||
pub default_team_template: Option<String>,
|
||||
/// Default team **per phase purpose**, by template key:
|
||||
/// `{ research = "topic_research", coding = "rust_sdlc" }`.
|
||||
///
|
||||
/// `default_team_template` names ONE team for a whole mission, and a
|
||||
/// multi-phase recipe does not have one job. `research_and_code` staffs a
|
||||
/// research phase and a coding phase from the same `rust_sdlc` crew, which
|
||||
/// is why its research phase has to spend a paragraph of `task` telling
|
||||
/// coders not to code — a workaround for staffing, written into the prompt.
|
||||
///
|
||||
/// Resolved to `config.phase_teams` at mission-create, which the
|
||||
/// orchestrator and `composed_graph` already read. Purposes come from
|
||||
/// `phase_runner::purposes_for`.
|
||||
#[serde(default)]
|
||||
pub default_phase_teams: std::collections::BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
|
||||
BIN
Binary file not shown.
@@ -128,7 +128,7 @@ async fn on_launch_materializes_team_from_template() {
|
||||
let template_id = seed_test_template(&pool).await;
|
||||
let mission_id = seed_mission(&pool, ws, template_id, "Test Mission").await;
|
||||
|
||||
let team_id = mission_orchestrator::on_launch(&pool, ws, owner, mission_id, None)
|
||||
let team_id = mission_orchestrator::on_launch(&pool, ws, owner, mission_id, None, None)
|
||||
.await
|
||||
.expect("on_launch succeeds")
|
||||
.expect("returns a team id");
|
||||
@@ -205,11 +205,11 @@ async fn on_launch_is_idempotent() {
|
||||
let template_id = seed_test_template(&pool).await;
|
||||
let mission_id = seed_mission(&pool, ws, template_id, "Idempotency Mission").await;
|
||||
|
||||
let team_a = mission_orchestrator::on_launch(&pool, ws, owner, mission_id, None)
|
||||
let team_a = mission_orchestrator::on_launch(&pool, ws, owner, mission_id, None, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let team_b = mission_orchestrator::on_launch(&pool, ws, owner, mission_id, None)
|
||||
let team_b = mission_orchestrator::on_launch(&pool, ws, owner, mission_id, None, None)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
@@ -248,7 +248,7 @@ async fn on_launch_no_template_hard_fails() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = mission_orchestrator::on_launch(&pool, ws, owner, mission_id, None).await;
|
||||
let result = mission_orchestrator::on_launch(&pool, ws, owner, mission_id, None, None).await;
|
||||
let err = result.expect_err("no template + no team must be a hard error");
|
||||
assert!(
|
||||
err.contains("no team_template_id") && err.contains("config.phase_teams"),
|
||||
@@ -344,7 +344,7 @@ async fn a_template_role_may_run_on_its_own_model() {
|
||||
let template_id = seed_test_template(&pool).await;
|
||||
let mission = seed_mission(&pool, ws, template_id, "per-role models").await;
|
||||
|
||||
mission_orchestrator::on_launch(&pool, ws, user, mission, None)
|
||||
mission_orchestrator::on_launch(&pool, ws, user, mission, None, None)
|
||||
.await
|
||||
.expect("launch");
|
||||
|
||||
|
||||
@@ -0,0 +1,409 @@
|
||||
//! Do a mission agent's pinned skills actually reach its prompt?
|
||||
//!
|
||||
//! Before this test the honest answer was no, for every skill and every role.
|
||||
//! The catalogue's only delivery channel was the `clawmates_skills` MCP server,
|
||||
//! and a mission claw could not reach it: `provision_claw` wrote a constant
|
||||
//! bundle list, the runtime config defines no such bundle, and mission claws
|
||||
//! run on `claude_cli`, which is text-only and cannot surface a tool call.
|
||||
//!
|
||||
//! So the skills were authored, bound, listed in the boot log as bound — and
|
||||
//! structurally unreadable. That is why this is a test and not a comment: the
|
||||
//! failure produced no error anywhere, and every layer reported success.
|
||||
|
||||
use cm_api::skill_delivery::Mode;
|
||||
use cm_api::topology_exec::{MissionTap, ZeroClawDriveExecutor};
|
||||
use cm_domain::{Agent, AgentId, AgentStatus, Role, User, UserId, Workspace, WorkspaceId};
|
||||
|
||||
/// `agents.managed_by` is a real FK, so the owner has to exist.
|
||||
async fn seed_user(pool: &sqlx::PgPool, ws: WorkspaceId) -> UserId {
|
||||
let user = User {
|
||||
id: UserId::new(),
|
||||
workspace_id: ws,
|
||||
email: format!("owner-{}@example.com", Uuid::now_v7().simple()),
|
||||
role: Role::Owner,
|
||||
display_name: "Owner".into(),
|
||||
created_at: time::OffsetDateTime::UNIX_EPOCH,
|
||||
};
|
||||
cm_db::repo::users::insert(pool, &user).await.unwrap();
|
||||
user.id
|
||||
}
|
||||
use std::collections::HashMap;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// A claw with one template-bound, pinned skill. Returns its runtime alias.
|
||||
async fn seed_claw_with_pinned_skill(pool: &sqlx::PgPool, body: &str) -> (String, WorkspaceId) {
|
||||
let ws = Workspace {
|
||||
id: WorkspaceId::new(),
|
||||
name: "Skill Delivery Test".into(),
|
||||
plan: "team".into(),
|
||||
};
|
||||
cm_db::repo::workspaces::insert(pool, &ws).await.unwrap();
|
||||
|
||||
let user = seed_user(pool, ws.id).await;
|
||||
let agent = Agent {
|
||||
id: AgentId::new(),
|
||||
workspace_id: ws.id,
|
||||
name: "Scout".into(),
|
||||
job_title: "researcher".into(),
|
||||
system_prompt: String::new(),
|
||||
avatar: String::new(),
|
||||
accent: String::new(),
|
||||
wallpaper: String::new(),
|
||||
managed_by: user,
|
||||
status: AgentStatus::Online,
|
||||
};
|
||||
cm_db::repo::agents::insert(pool, &agent, &cm_domain::AccessPolicy::default())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// A template with one role, and a skill pinned to it.
|
||||
let template_id = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO team_templates (id, key, name, description, category, stack,
|
||||
default_topology, risk_profile, mcp_bundles, version)
|
||||
VALUES ($1, $2, 'Delivery Test', 'test', 'research', '{}',
|
||||
'pipeline', 'research_readonly', '{}', 1)",
|
||||
)
|
||||
.bind(template_id)
|
||||
.bind(format!("delivery_test_{}", template_id.simple()))
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
sqlx::query(
|
||||
"INSERT INTO template_roles (template_id, slot, order_idx, system_prompt)
|
||||
VALUES ($1, 'researcher', 0, 'you research')",
|
||||
)
|
||||
.bind(template_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let skill_id = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO skills
|
||||
(id, workspace_id, name, title, author, description, when_to_use,
|
||||
tags, source_kind, current_version, body)
|
||||
VALUES ($1, NULL, $2, $2, 'system',
|
||||
'a procedure the agent must follow', 'always', '{}',
|
||||
'builtin', 1, $3)",
|
||||
)
|
||||
.bind(skill_id)
|
||||
.bind(format!("delivery-test-skill-{}", skill_id.simple()))
|
||||
.bind(body)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
sqlx::query(
|
||||
"INSERT INTO template_role_skills (template_id, slot, skill_id, pin_in_context, order_idx)
|
||||
VALUES ($1, 'researcher', $2, true, 0)",
|
||||
)
|
||||
.bind(template_id)
|
||||
.bind(skill_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
cm_db::repo::agent_template_link::upsert(
|
||||
pool,
|
||||
agent.id.as_uuid(),
|
||||
template_id,
|
||||
1,
|
||||
"researcher",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
(
|
||||
cm_api::runtime_provision::claw_alias(agent.id.as_uuid()),
|
||||
ws.id,
|
||||
)
|
||||
}
|
||||
|
||||
fn executor(pool: &sqlx::PgPool, workspace_id: WorkspaceId) -> ZeroClawDriveExecutor {
|
||||
executor_for(pool, workspace_id, Uuid::now_v7())
|
||||
}
|
||||
|
||||
fn executor_for(
|
||||
pool: &sqlx::PgPool,
|
||||
workspace_id: WorkspaceId,
|
||||
mission_id: Uuid,
|
||||
) -> ZeroClawDriveExecutor {
|
||||
ZeroClawDriveExecutor::new(
|
||||
"http://127.0.0.1:1".into(),
|
||||
"unused".into(),
|
||||
HashMap::new(),
|
||||
"default".into(),
|
||||
)
|
||||
.with_tap(MissionTap {
|
||||
pool: pool.clone(),
|
||||
workspace_id: workspace_id.as_uuid(),
|
||||
mission_id,
|
||||
phase_id: None,
|
||||
run_id: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// A mission row carrying an explicit delivery arm.
|
||||
async fn seed_mission_on_arm(pool: &sqlx::PgPool, ws: WorkspaceId, arm: Option<&str>) -> Uuid {
|
||||
let mission = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO missions (id, workspace_id, title, template_kind, status, skill_delivery)
|
||||
VALUES ($1, $2, 'arm', 'research_only', 'running', $3)",
|
||||
)
|
||||
.bind(mission)
|
||||
.bind(ws.as_uuid())
|
||||
.bind(arm)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
mission
|
||||
}
|
||||
|
||||
// ── The index arm ───────────────────────────────────────────────────
|
||||
//
|
||||
// Trigger — did the agent reach for the skill? — cannot exist while every body
|
||||
// is handed over unasked. These tests cover the arm that makes it a question,
|
||||
// and the one guarantee the control arm needs: that it did not change.
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_index_arm_sends_the_uri_and_withholds_the_body() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
const MARKER: &str = "Never review a paper from its title alone.";
|
||||
let (alias, ws) = seed_claw_with_pinned_skill(&pool, MARKER).await;
|
||||
let mission = seed_mission_on_arm(&pool, ws, Some("index")).await;
|
||||
|
||||
let text = executor_for(&pool, ws, mission)
|
||||
.pinned_skills_text(&alias)
|
||||
.await
|
||||
.expect("an indexed skill is still delivered — as an entry, not a body");
|
||||
|
||||
assert!(
|
||||
!text.contains(MARKER),
|
||||
"the BODY is what the index withholds; leaving it in delivers both \
|
||||
arms at once and measures neither. Got:\n{text}"
|
||||
);
|
||||
assert!(
|
||||
text.contains("uri=\"skill:global/delivery-test-skill-"),
|
||||
"an entry without a fetchable uri names a procedure the agent cannot \
|
||||
obtain — worse than inlining it. Got:\n{text}"
|
||||
);
|
||||
assert!(
|
||||
text.contains("When to use: always"),
|
||||
"`when_to_use` is the only thing the agent can judge relevance from, \
|
||||
and judging relevance is the entire axis. Got:\n{text}"
|
||||
);
|
||||
// The scorer counts delivered skills by the marker, in both arms.
|
||||
assert_eq!(
|
||||
cm_api::topology_exec::skill_names_in(&text).len(),
|
||||
1,
|
||||
"an indexed skill must still count as delivered:\n{text}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_unrecorded_arm_delivers_bodies() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
const MARKER: &str = "Never review a paper from its title alone.";
|
||||
let (alias, ws) = seed_claw_with_pinned_skill(&pool, MARKER).await;
|
||||
|
||||
// Every mission that ran before the column existed, plus any row whose
|
||||
// value is unreadable, plus a turn with no mission row at all. All three
|
||||
// resolve to the arm that needs nothing installed to work.
|
||||
for arm in [None, Some("nonsense")] {
|
||||
let mission = seed_mission_on_arm(&pool, ws, arm).await;
|
||||
let text = executor_for(&pool, ws, mission)
|
||||
.pinned_skills_text(&alias)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
text.contains(MARKER),
|
||||
"skill_delivery={arm:?} must deliver the body — an index arm \
|
||||
selected by accident hands out uris behind a door that may not \
|
||||
be installed. Got:\n{text}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_pinned_skill_body_reaches_the_turn_prompt() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
const MARKER: &str = "Never review a paper from its title alone.";
|
||||
let (alias, ws) = seed_claw_with_pinned_skill(&pool, MARKER).await;
|
||||
|
||||
let text = executor(&pool, ws)
|
||||
.pinned_skills_text(&alias)
|
||||
.await
|
||||
.expect("a claw with a pinned template skill must produce skill text");
|
||||
|
||||
assert!(
|
||||
text.contains(MARKER),
|
||||
"the default arm delivers the BODY. It is the control in the delivery \
|
||||
A/B, so it has to stay what production has always sent; the index arm \
|
||||
is selected per mission and tested separately. Got:\n{text}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_agent_with_no_pinned_skills_adds_nothing() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let ws = Workspace {
|
||||
id: WorkspaceId::new(),
|
||||
name: "No Skills".into(),
|
||||
plan: "team".into(),
|
||||
};
|
||||
cm_db::repo::workspaces::insert(&pool, &ws).await.unwrap();
|
||||
let user = seed_user(&pool, ws.id).await;
|
||||
let agent = Agent {
|
||||
id: AgentId::new(),
|
||||
workspace_id: ws.id,
|
||||
name: "Bare".into(),
|
||||
job_title: "researcher".into(),
|
||||
system_prompt: String::new(),
|
||||
avatar: String::new(),
|
||||
accent: String::new(),
|
||||
wallpaper: String::new(),
|
||||
managed_by: user,
|
||||
status: AgentStatus::Online,
|
||||
};
|
||||
cm_db::repo::agents::insert(&pool, &agent, &cm_domain::AccessPolicy::default())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let alias = cm_api::runtime_provision::claw_alias(agent.id.as_uuid());
|
||||
assert!(
|
||||
executor(&pool, ws.id)
|
||||
.pinned_skills_text(&alias)
|
||||
.await
|
||||
.is_none(),
|
||||
"an agent with no bound skills must add no section at all — an empty \
|
||||
`# Your skills` heading tells the model it has skills and then shows \
|
||||
it none"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_composed_prompt_carries_the_skill_and_omits_the_heading_when_empty() {
|
||||
let base = "You are the \"researcher\" agent.\n\nTask: read the papers";
|
||||
|
||||
let with = cm_api::topology_exec::compose_turn_prompt(base, Some("## arxiv-daily\nDo not re-search."), Mode::Inline);
|
||||
assert!(with.contains("Task: read the papers"), "the base turn must survive");
|
||||
assert!(with.contains("Do not re-search."), "the skill body must be in the prompt");
|
||||
assert!(with.contains("# Your skills"), "the section needs a heading");
|
||||
|
||||
for empty in [None, Some(""), Some(" \n ")] {
|
||||
let without = cm_api::topology_exec::compose_turn_prompt(base, empty, Mode::Inline);
|
||||
assert_eq!(
|
||||
without, base,
|
||||
"with no skills the prompt must be byte-identical to the base — an \
|
||||
empty heading announces skills the agent does not have"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── The other three tiers ───────────────────────────────────────────
|
||||
//
|
||||
// `topology_exec` injects per TURN and covers only the container tier. The
|
||||
// composed-microVM, solo-microVM and direct-session paths in `phase_runner`
|
||||
// share one task string built by `phase_task_text`, and until now that string
|
||||
// carried no skill at all — so a mission on any of those tiers ran with the
|
||||
// catalogue unreachable, exactly as the container tier did before e4942ce.
|
||||
|
||||
/// Bind a claw to a mission's crew so `phase_skills_text` can find it.
|
||||
async fn seed_mission_with_crew(pool: &sqlx::PgPool, ws: WorkspaceId, agent: AgentId) -> Uuid {
|
||||
let mission = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO missions (id, workspace_id, title, template_kind, status)
|
||||
VALUES ($1, $2, 'skill delivery', 'research_only', 'running')",
|
||||
)
|
||||
.bind(mission)
|
||||
.bind(ws.as_uuid())
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let team = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO teams (id, workspace_id, name, kind, lifecycle, graph)
|
||||
VALUES ($1, $2, 'crew', 'pipeline', 'permanent', '{}'::jsonb)",
|
||||
)
|
||||
.bind(team)
|
||||
.bind(ws.as_uuid())
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO team_members (team_id, claw_id, node_id, role) VALUES ($1, $2, 'researcher', 'researcher')")
|
||||
.bind(team)
|
||||
.bind(agent.as_uuid())
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO mission_teams (mission_id, team_id, purpose) VALUES ($1, $2, 'mission')")
|
||||
.bind(mission)
|
||||
.bind(team)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
mission
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_microvm_and_session_tiers_get_the_skill_in_their_task_text() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
const MARKER: &str = "Never review a paper from its title alone.";
|
||||
let (alias, ws) = seed_claw_with_pinned_skill(&pool, MARKER).await;
|
||||
let agent = AgentId::from(cm_api::runtime_provision::claw_from_alias(&alias).unwrap());
|
||||
let mission = seed_mission_with_crew(&pool, ws, agent).await;
|
||||
|
||||
let skills = cm_api::phase_runner::phase_skills_text(&pool, mission)
|
||||
.await
|
||||
.expect("a mission whose crew holds a pinned skill must produce skill text");
|
||||
assert!(
|
||||
skills.contains(MARKER),
|
||||
"the pinned BODY must reach the phase task — these tiers run one \
|
||||
`claude -p` session with no per-turn injection, so this string is the \
|
||||
agent's only route to the procedure. Got:\n{skills}"
|
||||
);
|
||||
|
||||
// All three tiers share this composition, so testing it once covers them.
|
||||
let composed = cm_api::topology_exec::compose_turn_prompt("Task: read the papers", Some(&skills), Mode::Inline);
|
||||
assert!(composed.contains(MARKER));
|
||||
assert!(composed.contains("Task: read the papers"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_mission_whose_crew_has_no_skills_adds_nothing() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let ws = Workspace {
|
||||
id: WorkspaceId::new(),
|
||||
name: "Bare Crew".into(),
|
||||
plan: "team".into(),
|
||||
};
|
||||
cm_db::repo::workspaces::insert(&pool, &ws).await.unwrap();
|
||||
let user = seed_user(&pool, ws.id).await;
|
||||
let agent = Agent {
|
||||
id: AgentId::new(),
|
||||
workspace_id: ws.id,
|
||||
name: "Bare".into(),
|
||||
job_title: "researcher".into(),
|
||||
system_prompt: String::new(),
|
||||
avatar: String::new(),
|
||||
accent: String::new(),
|
||||
wallpaper: String::new(),
|
||||
managed_by: user,
|
||||
status: AgentStatus::Online,
|
||||
};
|
||||
cm_db::repo::agents::insert(&pool, &agent, &cm_domain::AccessPolicy::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let mission = seed_mission_with_crew(&pool, ws.id, agent.id).await;
|
||||
|
||||
assert!(
|
||||
cm_api::phase_runner::phase_skills_text(&pool, mission)
|
||||
.await
|
||||
.is_none(),
|
||||
"a crew with no pinned skills must add no section — the empty-heading \
|
||||
rule has to hold on this path too"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,352 @@
|
||||
//! The orphan sweep's two Docker-touching seams, against real containers.
|
||||
//!
|
||||
//! `sweep_orphans` force-removes containers. Its decision logic is pure and
|
||||
//! unit-tested in `mission_runtime`, but the two calls that talk to Docker —
|
||||
//! "which containers exist" and "does this checkout hold work no remote has" —
|
||||
//! had never run against a daemon. Those are exactly the ones worth exercising
|
||||
//! for real: the first decides what is considered at all, and the second is the
|
||||
//! only thing standing between a reaper and ten unpushed commits.
|
||||
//!
|
||||
//! That is not hypothetical. The orphan that motivated this sweep held
|
||||
//! +3451/-30 across 30 files on a branch that existed nowhere else.
|
||||
//!
|
||||
//! Skips cleanly when there is no Docker, so a machine or CI runner without one
|
||||
//! reports "not run" rather than failing.
|
||||
|
||||
use cm_api::mission_runtime::{container_name, MissionRuntimeProvisioner, UnpushedWork};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// The image is already local on any machine that runs missions, and it has
|
||||
/// `git`, which the probe needs.
|
||||
const FIXTURE_IMAGE: &str = "clawmates-runtime:hooks";
|
||||
|
||||
/// These tests must not run at the same time as each other.
|
||||
///
|
||||
/// `sweep_orphans` is global: it reaps EVERY orphaned `cm-runtime-mission-*`
|
||||
/// container on the daemon, which on a parallel test runner includes the
|
||||
/// fixtures another test in this file just started. That is not a flaw in the
|
||||
/// sweep — it is what a sweep is — but it means anything here that creates a
|
||||
/// mission-shaped container has to hold this lock.
|
||||
///
|
||||
/// Found the honest way: the reap test deleted the listing test's fixture
|
||||
/// mid-run and the listing test reported a container it could not see.
|
||||
static FIXTURES: std::sync::LazyLock<tokio::sync::Mutex<()>> =
|
||||
std::sync::LazyLock::new(|| tokio::sync::Mutex::new(()));
|
||||
|
||||
fn docker_available() -> bool {
|
||||
std::process::Command::new("docker")
|
||||
.args(["image", "inspect", FIXTURE_IMAGE])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Start a fixture container named like a mission runtime, running a shell
|
||||
/// script that leaves `/mission/repo` in a known state.
|
||||
fn start_fixture(id: Uuid, setup: &str) -> String {
|
||||
let name = container_name(id);
|
||||
let _ = std::process::Command::new("docker")
|
||||
.args(["rm", "-f", &name])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status();
|
||||
let script = format!("{setup}\nsleep 3600");
|
||||
let out = std::process::Command::new("docker")
|
||||
.args([
|
||||
"run", "-d", "--name", &name, "--entrypoint", "sh", FIXTURE_IMAGE, "-c", &script,
|
||||
])
|
||||
.output()
|
||||
.expect("docker run");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"could not start fixture {name}: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
// The script has to have finished its git work before the probe runs.
|
||||
std::thread::sleep(std::time::Duration::from_secs(3));
|
||||
name
|
||||
}
|
||||
|
||||
fn remove(name: &str) {
|
||||
let _ = std::process::Command::new("docker")
|
||||
.args(["rm", "-f", name])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status();
|
||||
}
|
||||
|
||||
const GIT_INIT: &str = "set -e
|
||||
mkdir -p /mission/repo && cd /mission/repo
|
||||
git init -q .
|
||||
git config user.email t@t && git config user.name t
|
||||
echo hello > a.txt && git add a.txt && git commit -qm 'work nobody else has'";
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_sweep_can_see_containers_and_refuses_the_ones_holding_work() {
|
||||
if !docker_available() {
|
||||
eprintln!("orphan_sweep: no docker or no {FIXTURE_IMAGE} — not run");
|
||||
return;
|
||||
}
|
||||
let Some(prov) = MissionRuntimeProvisioner::from_env() else {
|
||||
eprintln!("orphan_sweep: no docker connection — not run");
|
||||
return;
|
||||
};
|
||||
let _serial = FIXTURES.lock().await;
|
||||
|
||||
let dirty_id = Uuid::now_v7();
|
||||
let clean_id = Uuid::now_v7();
|
||||
let empty_id = Uuid::now_v7();
|
||||
|
||||
// Commits, and no remote ref anywhere: this is the container that must
|
||||
// survive. It is the one the real orphan looked like.
|
||||
let dirty = start_fixture(dirty_id, GIT_INIT);
|
||||
// The same repo, but every commit is reachable from a remote-tracking ref,
|
||||
// which is what "already pushed" looks like to `git rev-list --not
|
||||
// --remotes`.
|
||||
let clean = start_fixture(
|
||||
clean_id,
|
||||
&format!("{GIT_INIT}\ngit update-ref refs/remotes/origin/main HEAD"),
|
||||
);
|
||||
// No checkout at all — nothing to lose.
|
||||
let empty = start_fixture(empty_id, "set -e\nmkdir -p /root");
|
||||
|
||||
let result = async {
|
||||
let names = prov.list_mission_containers().await?;
|
||||
let found: Vec<&String> = names.iter().map(|(n, _)| n).collect();
|
||||
for expected in [&dirty, &clean, &empty] {
|
||||
assert!(
|
||||
found.iter().any(|n| *n == expected),
|
||||
"the sweep cannot see {expected}; a container it cannot list is \
|
||||
one it can never reap, which is the whole defect this closes. \
|
||||
saw: {found:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// Docker's own creation timestamp must come back, or the sweep declines
|
||||
// to reap for want of an age.
|
||||
let (_, created) = names
|
||||
.iter()
|
||||
.find(|(n, _)| n == &dirty)
|
||||
.expect("dirty in listing");
|
||||
assert!(
|
||||
MissionRuntimeProvisioner::container_age(*created).is_some(),
|
||||
"a container docker will not date is never reaped, so an absent \
|
||||
timestamp here would silently disable the sweep"
|
||||
);
|
||||
|
||||
match prov.unpushed_commits(&dirty).await {
|
||||
UnpushedWork::SomeOrUnknown(why) => {
|
||||
assert!(why.contains("no remote"), "{why}");
|
||||
}
|
||||
UnpushedWork::None => panic!(
|
||||
"the probe said a checkout with an unpushed commit holds nothing — \
|
||||
this is the exact answer that destroys work"
|
||||
),
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
prov.unpushed_commits(&clean).await,
|
||||
UnpushedWork::None,
|
||||
"every commit is reachable from a remote ref, so there is nothing to lose"
|
||||
);
|
||||
assert_eq!(
|
||||
prov.unpushed_commits(&empty).await,
|
||||
UnpushedWork::None,
|
||||
"no /mission/repo at all means nothing to lose"
|
||||
);
|
||||
Ok::<(), String>(())
|
||||
}
|
||||
.await;
|
||||
|
||||
remove(&dirty);
|
||||
remove(&clean);
|
||||
remove(&empty);
|
||||
result.expect("orphan sweep probes");
|
||||
}
|
||||
|
||||
/// A container we cannot question is not a container we may delete.
|
||||
#[tokio::test]
|
||||
async fn a_container_that_is_gone_reads_as_holding_work() {
|
||||
if !docker_available() {
|
||||
eprintln!("orphan_sweep: no docker — not run");
|
||||
return;
|
||||
}
|
||||
let Some(prov) = MissionRuntimeProvisioner::from_env() else {
|
||||
return;
|
||||
};
|
||||
match prov
|
||||
.unpushed_commits("cm-runtime-mission-does-not-exist-at-all")
|
||||
.await
|
||||
{
|
||||
UnpushedWork::SomeOrUnknown(_) => {}
|
||||
UnpushedWork::None => panic!(
|
||||
"an unanswerable probe must never read as 'safe to delete' — every \
|
||||
failure path in this check is one-sided for that reason"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set to run the destructive sweep test.
|
||||
///
|
||||
/// The other tests in this file only create fixtures and read them. This one
|
||||
/// calls `sweep_orphans`, which REMOVES containers — and CI runs
|
||||
/// `cargo test --workspace` inside a container with `/var/run/docker.sock`
|
||||
/// mounted, on gw04, which is the host that runs production missions.
|
||||
///
|
||||
/// `adopt_existing` protects everything already present, but it cannot protect
|
||||
/// a mission container created in the seconds between that call and the sweep.
|
||||
/// On a developer machine that race is nothing; on the production host it is a
|
||||
/// mission. So the destructive test is opt-in, and CI simply does not run it.
|
||||
const RUN_DESTRUCTIVE: &str = "CM_TEST_ORPHAN_SWEEP";
|
||||
|
||||
/// The reap decision itself, against real containers.
|
||||
///
|
||||
/// The probes above are the inputs; this is the act. A clean orphan past its
|
||||
/// grace must go, an orphan holding unpushed work must stay, and a young one
|
||||
/// must stay regardless — and all three have to be true of the same sweep, in
|
||||
/// one pass, because that is how it runs.
|
||||
#[tokio::test]
|
||||
async fn the_sweep_reaps_the_clean_orphan_and_spares_the_others() {
|
||||
if !docker_available() {
|
||||
eprintln!("orphan_sweep: no docker — not run");
|
||||
return;
|
||||
}
|
||||
if std::env::var(RUN_DESTRUCTIVE).is_err() {
|
||||
eprintln!(
|
||||
"orphan_sweep: not run — this test removes containers, and the CI \
|
||||
runner shares a docker daemon with production. Set \
|
||||
{RUN_DESTRUCTIVE}=1 to run it."
|
||||
);
|
||||
return;
|
||||
}
|
||||
if MissionRuntimeProvisioner::from_env().is_none() {
|
||||
return;
|
||||
}
|
||||
let _serial = FIXTURES.lock().await;
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
|
||||
// Adopt every mission container that already exists on this daemon.
|
||||
//
|
||||
// The sweep asks the DATABASE whether a container is known, and a fresh
|
||||
// test database knows nothing — so on a developer machine the sweep
|
||||
// classifies the live local stack's mission containers as orphans and
|
||||
// reaps them. It did exactly that on the first run of this test, deleting
|
||||
// two real mission containers.
|
||||
//
|
||||
// Giving each a row makes the test safe AND covers the case the other
|
||||
// assertions do not: a container the platform still knows about is never
|
||||
// touched, whatever its checkout looks like.
|
||||
let adopted = adopt_existing(&pool).await;
|
||||
|
||||
let clean_id = Uuid::now_v7();
|
||||
let dirty_id = Uuid::now_v7();
|
||||
let young_id = Uuid::now_v7();
|
||||
|
||||
let alive = |name: &str| {
|
||||
std::process::Command::new("docker")
|
||||
.args(["inspect", name])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false)
|
||||
};
|
||||
|
||||
let clean = start_fixture(
|
||||
clean_id,
|
||||
&format!("{GIT_INIT}\ngit update-ref refs/remotes/origin/main HEAD"),
|
||||
);
|
||||
let dirty = start_fixture(dirty_id, GIT_INIT);
|
||||
|
||||
// Pass one, no grace: everything present is past its window, so the
|
||||
// decision is made purely on whether the checkout holds work.
|
||||
let swept = cm_api::mission_runtime::sweep_orphans(&pool, std::time::Duration::ZERO).await;
|
||||
let clean_gone = !alive(&clean);
|
||||
let dirty_alive = alive(&dirty);
|
||||
|
||||
// Pass two, a real grace, on a container minted seconds ago. It is clean
|
||||
// and orphaned — reapable on every axis except its age — so if the grace is
|
||||
// decorative this is where that shows.
|
||||
//
|
||||
// Started AFTER the first pass on purpose: a grace applies to every
|
||||
// container in the sweep, so a fixture created before a zero-grace pass is
|
||||
// reaped by that pass and proves nothing about the window. The first
|
||||
// version of this test made exactly that mistake and failed itself.
|
||||
let young = start_fixture(
|
||||
young_id,
|
||||
&format!("{GIT_INIT}\ngit update-ref refs/remotes/origin/main HEAD"),
|
||||
);
|
||||
let swept2 =
|
||||
cm_api::mission_runtime::sweep_orphans(&pool, std::time::Duration::from_secs(3600)).await;
|
||||
let young_alive = alive(&young);
|
||||
|
||||
remove(&clean);
|
||||
remove(&dirty);
|
||||
remove(&young);
|
||||
|
||||
for name in &adopted {
|
||||
assert!(
|
||||
alive(name),
|
||||
"the sweep reaped {name}, which HAS a mission row — a container the \
|
||||
platform still knows about must never be touched"
|
||||
);
|
||||
}
|
||||
|
||||
swept.expect("first sweep");
|
||||
swept2.expect("second sweep");
|
||||
assert!(
|
||||
clean_gone,
|
||||
"a clean orphan past its grace is exactly what this sweep exists to \
|
||||
reclaim; leaving it means the disk leak is still open"
|
||||
);
|
||||
assert!(
|
||||
dirty_alive,
|
||||
"an orphan holding commits no remote has MUST survive — the container \
|
||||
that motivated this held ten of them"
|
||||
);
|
||||
assert!(
|
||||
young_alive,
|
||||
"a container inside the grace window must be left alone even when it is \
|
||||
otherwise reapable, or the grace is decorative"
|
||||
);
|
||||
}
|
||||
|
||||
/// Give every mission container already on this daemon a row, so the sweep
|
||||
/// treats it as known and leaves it alone.
|
||||
///
|
||||
/// Returns the names, which then double as an assertion: none of them may be
|
||||
/// reaped.
|
||||
async fn adopt_existing(pool: &sqlx::PgPool) -> Vec<String> {
|
||||
let Some(prov) = MissionRuntimeProvisioner::from_env() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let Ok(existing) = prov.list_mission_containers().await else {
|
||||
return Vec::new();
|
||||
};
|
||||
let ws = Uuid::now_v7();
|
||||
sqlx::query("INSERT INTO workspaces (id, name, plan) VALUES ($1, 'orphan-test', 'free')")
|
||||
.bind(ws)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("seed workspace");
|
||||
let mut names = Vec::new();
|
||||
for (name, _) in existing {
|
||||
let Some(id) = cm_api::mission_runtime::mission_id_from_container(&name) else {
|
||||
continue;
|
||||
};
|
||||
sqlx::query(
|
||||
"INSERT INTO missions (id, workspace_id, title, template_kind)
|
||||
VALUES ($1, $2, 'adopted by orphan_sweep test', 'research_only')
|
||||
ON CONFLICT (id) DO NOTHING",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(ws)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("adopt container");
|
||||
names.push(name);
|
||||
}
|
||||
names
|
||||
}
|
||||
@@ -279,6 +279,8 @@ async fn evaluations_are_unique_per_iteration_and_upsert() {
|
||||
error: None,
|
||||
checks: Vec::new(),
|
||||
independent: false,
|
||||
usage: Default::default(),
|
||||
expectation: None,
|
||||
};
|
||||
cm_api::evaluator::record(&pool, mission, phase, 0, &first)
|
||||
.await
|
||||
@@ -298,6 +300,8 @@ async fn evaluations_are_unique_per_iteration_and_upsert() {
|
||||
evidence: "exit status: 0".into(),
|
||||
}],
|
||||
independent: false,
|
||||
usage: Default::default(),
|
||||
expectation: Some("rg 'brief' docs/".into()),
|
||||
};
|
||||
cm_api::evaluator::record(&pool, mission, phase, 0, &second)
|
||||
.await
|
||||
@@ -328,6 +332,15 @@ async fn evaluations_are_unique_per_iteration_and_upsert() {
|
||||
.unwrap()
|
||||
.get("reason");
|
||||
assert_eq!(reason, "brief written");
|
||||
// The commit-first plan is stored beside the verdict it governed.
|
||||
let expectation: Option<String> =
|
||||
sqlx::query("SELECT expectation FROM mission_phase_evaluations WHERE phase_id = $1")
|
||||
.bind(phase)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap()
|
||||
.get("expectation");
|
||||
assert_eq!(expectation.as_deref(), Some("rg 'brief' docs/"));
|
||||
}
|
||||
|
||||
/// `latest` must return the newest pass, which is what feeds guidance into the
|
||||
@@ -356,6 +369,8 @@ async fn latest_returns_the_most_recent_iteration() {
|
||||
error: None,
|
||||
checks: Vec::new(),
|
||||
independent: false,
|
||||
usage: Default::default(),
|
||||
expectation: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
//! What an agent received, and what it said it did, must survive the phase.
|
||||
//!
|
||||
//! `docs/PROVENANCE-ASSESSMENT.md` records "why did the agent say X?" as
|
||||
//! unanswerable. The first two things it needs are the prompt and the
|
||||
//! narrative. Before this, the prompt was never stored at all, and the
|
||||
//! narrative was stored and then read by nothing — both database readers in
|
||||
//! `routes/world.rs` filter to `tool.call`/`file.touch`, and the only other
|
||||
//! statement touching the table is the GC that deletes it.
|
||||
|
||||
use cm_api::mission_events::{self, MissionEvent, PER_PHASE_CAP, PROMPT_COMPOSED, REASONING, TOOL_CALL};
|
||||
use cm_domain::{Workspace, WorkspaceId};
|
||||
use uuid::Uuid;
|
||||
|
||||
async fn seed_phase(pool: &sqlx::PgPool) -> (Uuid, Uuid) {
|
||||
let ws = Workspace {
|
||||
id: WorkspaceId::new(),
|
||||
name: "Provenance Test".into(),
|
||||
plan: "team".into(),
|
||||
};
|
||||
cm_db::repo::workspaces::insert(pool, &ws).await.unwrap();
|
||||
let mission = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO missions (id, workspace_id, title, template_kind, status)
|
||||
VALUES ($1, $2, 'provenance', 'research_only', 'running')",
|
||||
)
|
||||
.bind(mission)
|
||||
.bind(ws.id.as_uuid())
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let phase = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO mission_phases (id, mission_id, kind, order_idx, status)
|
||||
VALUES ($1, $2, 'research', 0, 'running')",
|
||||
)
|
||||
.bind(phase)
|
||||
.bind(mission)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
(mission, phase)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_prompt_and_the_narrative_are_both_readable_after_the_fact() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (mission, phase) = seed_phase(&pool).await;
|
||||
|
||||
let mut prompt = MissionEvent::new(mission, PROMPT_COMPOSED);
|
||||
prompt.phase_id = Some(phase);
|
||||
prompt.target = Some("researcher".into());
|
||||
prompt.detail = serde_json::json!({ "text": "Task: read the papers\n## arxiv-daily\nDo not re-search." });
|
||||
mission_events::record(&pool, prompt).await;
|
||||
|
||||
let mut said = MissionEvent::new(mission, REASONING);
|
||||
said.phase_id = Some(phase);
|
||||
said.detail = serde_json::json!({ "text": "I read the manifest and wrote analysis.md." });
|
||||
mission_events::record(&pool, said).await;
|
||||
|
||||
let narrative = mission_events::narrative_for_mission(&pool, mission)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let prompt_text = narrative
|
||||
.iter()
|
||||
.find(|(kind, ..)| kind == PROMPT_COMPOSED)
|
||||
.map(|(.., text)| text.clone())
|
||||
.expect("the prompt must be recoverable — re-deriving it later re-runs \
|
||||
the skill lookup against a catalogue that will have changed");
|
||||
assert!(prompt_text.contains("Do not re-search."));
|
||||
assert!(
|
||||
prompt_text.contains("Task: read the papers"),
|
||||
"the whole composed prompt, not just the skills half"
|
||||
);
|
||||
|
||||
assert!(
|
||||
narrative
|
||||
.iter()
|
||||
.any(|(kind, .., text)| kind == REASONING && text.contains("analysis.md")),
|
||||
"the agent's own account must come back out of the database"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_busy_phase_cannot_push_out_its_own_provenance() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (mission, phase) = seed_phase(&pool).await;
|
||||
|
||||
// Fill the phase past the cap with the kind the cap exists to bound.
|
||||
for i in 0..(PER_PHASE_CAP + 20) {
|
||||
let mut ev = MissionEvent::new(mission, TOOL_CALL);
|
||||
ev.phase_id = Some(phase);
|
||||
ev.target = Some(format!("tool_{i}"));
|
||||
mission_events::record(&pool, ev).await;
|
||||
}
|
||||
|
||||
let tool_rows: i64 = sqlx::query_scalar(
|
||||
"SELECT count(*) FROM mission_events WHERE phase_id = $1 AND kind = $2",
|
||||
)
|
||||
.bind(phase)
|
||||
.bind(TOOL_CALL)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
tool_rows, PER_PHASE_CAP,
|
||||
"the cap must still bound the kind it was written for"
|
||||
);
|
||||
|
||||
// The prompt arrives AFTER the flood, which is the real ordering: a coding
|
||||
// phase calls its tools and then the next turn is composed.
|
||||
let mut prompt = MissionEvent::new(mission, PROMPT_COMPOSED);
|
||||
prompt.phase_id = Some(phase);
|
||||
prompt.detail = serde_json::json!({ "text": "the next turn's prompt" });
|
||||
mission_events::record(&pool, prompt).await;
|
||||
|
||||
let narrative = mission_events::narrative_for_mission(&pool, mission)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
narrative.iter().any(|(.., text)| text == "the next turn's prompt"),
|
||||
"a phase that called {} tools dropped its own prompt — the cap counted \
|
||||
provenance against a budget meant for the two unbounded kinds, so the \
|
||||
busier the phase, the less of it is explainable",
|
||||
PER_PHASE_CAP + 20
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_mission_under_measurement_keeps_its_events_past_the_window() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (kept, kept_phase) = seed_phase(&pool).await;
|
||||
let (reaped, reaped_phase) = seed_phase(&pool).await;
|
||||
|
||||
// Only one of them is held.
|
||||
sqlx::query("UPDATE missions SET retain_events_until = now() + interval '30 days' WHERE id = $1")
|
||||
.bind(kept)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
for (mission, phase) in [(kept, kept_phase), (reaped, reaped_phase)] {
|
||||
let mut ev = MissionEvent::new(mission, PROMPT_COMPOSED);
|
||||
ev.phase_id = Some(phase);
|
||||
ev.detail = serde_json::json!({ "text": "the prompt" });
|
||||
mission_events::record(&pool, ev).await;
|
||||
}
|
||||
// Age both beyond the global window.
|
||||
sqlx::query("UPDATE mission_events SET created_at = now() - interval '90 days'")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut out = cm_api::mission_gc::Reclaimed::default();
|
||||
cm_api::mission_gc::reap_mission_events(&pool, &mut out).await;
|
||||
|
||||
assert!(
|
||||
!mission_events::narrative_for_mission(&pool, kept)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty(),
|
||||
"a mission held for measurement lost its events — the evidence expires \
|
||||
while the question is still open, and 'no events' reads exactly like \
|
||||
'nothing happened'"
|
||||
);
|
||||
assert!(
|
||||
mission_events::narrative_for_mission(&pool, reaped)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty(),
|
||||
"an unheld mission must still be reaped — an exemption that applies to \
|
||||
everything is not an exemption, it is a raised global bound"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
//! The guard on the security-scan sweep.
|
||||
//!
|
||||
//! `phase_runner::scan_finished_security_phases` fires `security_scan::run`
|
||||
//! for finished `security_scan` phases. The scan needs Docker; the part that
|
||||
//! decides whether it runs twice, once, or never is pure SQL, and it is the
|
||||
//! part that fails silently in both directions — rescanning forever, or never
|
||||
//! scanning at all and leaving a phase that looks identical to a clean repo.
|
||||
|
||||
use cm_domain::{Workspace, WorkspaceId};
|
||||
use uuid::Uuid;
|
||||
|
||||
async fn seed_mission(pool: &sqlx::PgPool) -> Uuid {
|
||||
let ws = Workspace {
|
||||
id: WorkspaceId::new(),
|
||||
name: "Security Sweep Test".into(),
|
||||
plan: "team".into(),
|
||||
};
|
||||
cm_db::repo::workspaces::insert(pool, &ws).await.unwrap();
|
||||
let id = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO missions (id, workspace_id, title, template_kind, status)
|
||||
VALUES ($1, $2, 'security test', 'security_hardening', 'running')",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(ws.id.as_uuid())
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
id
|
||||
}
|
||||
|
||||
async fn seed_phase(
|
||||
pool: &sqlx::PgPool,
|
||||
mission_id: Uuid,
|
||||
kind: &str,
|
||||
status: &str,
|
||||
order_idx: i32,
|
||||
) -> Uuid {
|
||||
let id = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO mission_phases (id, mission_id, kind, order_idx, status, completed_at)
|
||||
VALUES ($1, $2, $3, $4, $5, now())",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(mission_id)
|
||||
.bind(kind)
|
||||
.bind(order_idx)
|
||||
.bind(status)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
id
|
||||
}
|
||||
|
||||
async fn mark_scanned(pool: &sqlx::PgPool, mission_id: Uuid, phase_id: Uuid) {
|
||||
cm_db::repo::missions::upsert_task(
|
||||
pool,
|
||||
cm_db::repo::missions::UpsertTask {
|
||||
mission_id,
|
||||
phase_id,
|
||||
external_id: cm_api::security_scan::SCAN_MARKER,
|
||||
title: "security scan complete — ran [gitleaks], 0 finding(s)",
|
||||
assigned_agent_id: None,
|
||||
status: "created",
|
||||
run_id: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_finished_security_phase_is_selected_until_it_carries_a_scan_marker() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let mission = seed_mission(&pool).await;
|
||||
let phase = seed_phase(&pool, mission, "security_scan", "completed", 0).await;
|
||||
|
||||
let selected = cm_api::phase_runner::unscanned_security_phases(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
selected.iter().any(|(p, _)| *p == phase),
|
||||
"a finished security_scan phase with no marker must be selected — \
|
||||
otherwise the scan never runs and the phase reports completed having \
|
||||
scanned nothing"
|
||||
);
|
||||
|
||||
// A clean scan writes NO findings, so the marker is the only evidence the
|
||||
// scan happened. This is the case that would otherwise rescan forever.
|
||||
mark_scanned(&pool, mission, phase).await;
|
||||
|
||||
let selected = cm_api::phase_runner::unscanned_security_phases(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
!selected.iter().any(|(p, _)| *p == phase),
|
||||
"a phase carrying the scan marker must not be selected again, even \
|
||||
though it has zero findings"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn only_finished_security_phases_are_selected() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let mission = seed_mission(&pool).await;
|
||||
|
||||
let running = seed_phase(&pool, mission, "security_scan", "running", 0).await;
|
||||
let coding = seed_phase(&pool, mission, "coding", "completed", 1).await;
|
||||
let failed = seed_phase(&pool, mission, "security_scan", "failed", 2).await;
|
||||
|
||||
let selected: Vec<Uuid> = cm_api::phase_runner::unscanned_security_phases(&pool)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|(p, _)| p)
|
||||
.collect();
|
||||
|
||||
assert!(
|
||||
!selected.contains(&running),
|
||||
"scanning a phase still running would scan a half-written checkout"
|
||||
);
|
||||
assert!(!selected.contains(&coding), "only security_scan phases scan");
|
||||
assert!(
|
||||
selected.contains(&failed),
|
||||
"a FAILED security phase is exactly the one worth scanning — the \
|
||||
scanners are how we find out what state it left behind"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
//! Every skill a team template names must survive the trip through the
|
||||
//! database and come back out as a real binding.
|
||||
//!
|
||||
//! `team_template_loader`'s unit test checks names against the files in
|
||||
//! `skills/`. That is not the same question: bindings are resolved by
|
||||
//! `skills_catalog::get_by_name` against rows that `skills_loader` wrote, so a
|
||||
//! skill file that exists but fails to ingest (bad frontmatter, a name that
|
||||
//! does not match its filename) still leaves the role with an empty bundle —
|
||||
//! the exact silent-empty failure the loader's own comment describes.
|
||||
//!
|
||||
//! This runs both loaders in boot order and asserts the bindings landed.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
fn repo_root() -> std::path::PathBuf {
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../..")
|
||||
.canonicalize()
|
||||
.expect("canonicalize repo root")
|
||||
}
|
||||
|
||||
/// Every `skills = [...]` name in every team template, deduplicated per role
|
||||
/// the same way the loader binds them (slot + name).
|
||||
fn referenced_bindings() -> HashSet<(String, String, String)> {
|
||||
let mut out = HashSet::new();
|
||||
let dir = repo_root().join("templates/teams");
|
||||
for entry in std::fs::read_dir(&dir).expect("read templates/teams") {
|
||||
let path = entry.expect("dir entry").path();
|
||||
if path.extension().and_then(|e| e.to_str()) != Some("toml") {
|
||||
continue;
|
||||
}
|
||||
let key = path.file_stem().unwrap().to_string_lossy().to_string();
|
||||
let body = std::fs::read_to_string(&path).expect("read template");
|
||||
let mut slot = String::new();
|
||||
for line in body.lines() {
|
||||
let t = line.trim();
|
||||
if let Some(rest) = t.strip_prefix("slot") {
|
||||
if let Some(v) = rest.split('"').nth(1) {
|
||||
slot = v.to_string();
|
||||
}
|
||||
}
|
||||
if t.starts_with("skills") {
|
||||
if let Some(inner) = t.split_once('[').and_then(|(_, r)| r.rsplit_once(']')) {
|
||||
for name in inner.0.split(',') {
|
||||
let name = name.trim().trim_matches('"');
|
||||
if !name.is_empty() {
|
||||
out.insert((key.clone(), slot.clone(), name.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn every_template_role_skill_binds_through_the_database() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
|
||||
// Both loaders resolve their directory relative to the process cwd, which
|
||||
// for an integration test is the crate, not the repo.
|
||||
let root = repo_root();
|
||||
std::env::set_var("CLAWMATES_SKILLS_DIR", root.join("skills"));
|
||||
std::env::set_var("CLAWMATES_TEAM_TEMPLATES_DIR", root.join("templates/teams"));
|
||||
|
||||
// Boot order, from clawmates-server/src/main.rs: skills first, then the
|
||||
// templates that reference them.
|
||||
let n_skills = cm_api::skills_loader::load_builtins(&pool).await;
|
||||
assert!(n_skills > 0, "skills_loader ingested nothing");
|
||||
cm_api::team_template_loader::load_builtins(&pool).await;
|
||||
|
||||
let bound: HashSet<(String, String, String)> = sqlx::query_as::<_, (String, String, String)>(
|
||||
"SELECT t.key, trs.slot, s.name \
|
||||
FROM template_role_skills trs \
|
||||
JOIN team_templates t ON t.id = trs.template_id \
|
||||
JOIN skills s ON s.id = trs.skill_id",
|
||||
)
|
||||
.fetch_all(&pool)
|
||||
.await
|
||||
.expect("read template_role_skills")
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
let referenced = referenced_bindings();
|
||||
let mut missing: Vec<String> = referenced
|
||||
.difference(&bound)
|
||||
.map(|(k, slot, name)| format!("{k}.{slot} → {name}"))
|
||||
.collect();
|
||||
missing.sort();
|
||||
|
||||
assert!(
|
||||
missing.is_empty(),
|
||||
"{} template role skill binding(s) named in TOML never reached the \
|
||||
database — those roles run with a smaller context bundle than their \
|
||||
prompt assumes:\n {}",
|
||||
missing.len(),
|
||||
missing.join("\n ")
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
//! Agents author their own skills, with no human in the loop.
|
||||
//!
|
||||
//! The operator's decision. The machinery already existed — `level_up` has
|
||||
//! generated full skill drafts from a model since it shipped — and the only
|
||||
//! thing between propose and apply was an operator ticking a checkbox.
|
||||
//!
|
||||
//! What replaces that checkbox is not another gate but three properties, and
|
||||
//! these tests are what hold them: the write is workspace-scoped and can never
|
||||
//! take a hand-authored skill's name, every change appends a version so it can
|
||||
//! be read back and reverted, and a proposal applied with no human carries no
|
||||
//! human's name in its approval trail.
|
||||
|
||||
use cm_domain::{Agent, AgentId, AgentStatus, Role, User, UserId, Workspace, WorkspaceId};
|
||||
use serde_json::json;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// A workspace with one agent. `level_up_target_one` requires a proposal to
|
||||
/// name exactly one of agent_id / team_id, so the agent is not optional here.
|
||||
async fn seed_workspace(pool: &sqlx::PgPool) -> (WorkspaceId, AgentId) {
|
||||
let ws = Workspace {
|
||||
id: WorkspaceId::new(),
|
||||
name: "Self Authoring".into(),
|
||||
plan: "team".into(),
|
||||
};
|
||||
cm_db::repo::workspaces::insert(pool, &ws).await.unwrap();
|
||||
|
||||
let user = User {
|
||||
id: UserId::new(),
|
||||
workspace_id: ws.id,
|
||||
email: format!("owner-{}@example.com", Uuid::now_v7().simple()),
|
||||
role: Role::Owner,
|
||||
display_name: "Owner".into(),
|
||||
created_at: time::OffsetDateTime::UNIX_EPOCH,
|
||||
};
|
||||
cm_db::repo::users::insert(pool, &user).await.unwrap();
|
||||
|
||||
let agent = Agent {
|
||||
id: AgentId::new(),
|
||||
workspace_id: ws.id,
|
||||
name: "Scribe".into(),
|
||||
job_title: "researcher".into(),
|
||||
system_prompt: String::new(),
|
||||
avatar: String::new(),
|
||||
accent: String::new(),
|
||||
wallpaper: String::new(),
|
||||
managed_by: user.id,
|
||||
status: AgentStatus::Online,
|
||||
};
|
||||
cm_db::repo::agents::insert(pool, &agent, &cm_domain::AccessPolicy::default())
|
||||
.await
|
||||
.unwrap();
|
||||
(ws.id, agent.id)
|
||||
}
|
||||
|
||||
/// A pending proposal carrying one `skill_candidate` draft.
|
||||
async fn seed_proposal(
|
||||
pool: &sqlx::PgPool,
|
||||
ws: WorkspaceId,
|
||||
agent: AgentId,
|
||||
name: &str,
|
||||
body: &str,
|
||||
) -> Uuid {
|
||||
let payload = json!({
|
||||
"suggested_items": [{
|
||||
"id": "item-1",
|
||||
"kind": "skill_candidate",
|
||||
"draft": {
|
||||
"name": name,
|
||||
"description": "a procedure the agent wrote for itself",
|
||||
"when_to_use": "when the situation arises",
|
||||
"body": body,
|
||||
"tags": ["self-authored"],
|
||||
}
|
||||
}]
|
||||
});
|
||||
cm_db::repo::level_up::insert(
|
||||
pool,
|
||||
cm_db::repo::level_up::NewProposal {
|
||||
workspace_id: ws.as_uuid(),
|
||||
agent_id: Some(agent.as_uuid()),
|
||||
team_id: None,
|
||||
payload: &payload,
|
||||
model: Some("glm:glm-4.7"),
|
||||
created_by: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_agent_applies_its_own_skill_with_no_human_and_it_is_versioned() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (ws, agent) = seed_workspace(&pool).await;
|
||||
|
||||
let p1 = seed_proposal(&pool, ws, agent, "vault-note-shape", "First: write the date line.").await;
|
||||
let applied = cm_api::level_up::apply_autonomous(&pool, ws, p1)
|
||||
.await
|
||||
.expect("autonomous apply must succeed");
|
||||
assert_eq!(applied, vec!["item-1".to_string()]);
|
||||
|
||||
let (id, source_kind, workspace, version): (Uuid, String, Option<Uuid>, i32) = sqlx::query_as(
|
||||
"SELECT id, source_kind, workspace_id, current_version FROM skills WHERE name = $1",
|
||||
)
|
||||
.bind("vault-note-shape")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("the skill must exist with no human approval");
|
||||
assert_eq!(source_kind, "promoted_from_brain", "self-authored skills must stay distinguishable from builtins in one query");
|
||||
assert_eq!(workspace, Some(ws.as_uuid()), "must be workspace-scoped, never global");
|
||||
assert_eq!(version, 1);
|
||||
|
||||
// The approval trail must not name a human who did not approve.
|
||||
let approved_by: Option<Uuid> =
|
||||
sqlx::query_scalar("SELECT approved_by FROM level_up_proposals WHERE id = $1")
|
||||
.bind(p1)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
approved_by.is_none(),
|
||||
"an autonomously applied proposal must record NO approver — putting a \
|
||||
user id here would attribute a decision to someone who never made it"
|
||||
);
|
||||
|
||||
// A revision bumps the version and keeps the old body readable.
|
||||
let p2 = seed_proposal(&pool, ws, agent, "vault-note-shape", "First: write the date line. Then the source.").await;
|
||||
cm_api::level_up::apply_autonomous(&pool, ws, p2).await.unwrap();
|
||||
|
||||
let versions: Vec<(i32, String)> =
|
||||
sqlx::query_as("SELECT version, body_md FROM skill_versions WHERE skill_id = $1 ORDER BY version")
|
||||
.bind(id)
|
||||
.fetch_all(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
versions.len(),
|
||||
2,
|
||||
"each self-authored revision must append a version — without history \
|
||||
there is no revert, and no way to read back which text a past run was \
|
||||
actually judged under"
|
||||
);
|
||||
assert_eq!(versions[0].1, "First: write the date line.");
|
||||
assert!(versions[1].1.contains("Then the source."));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_draft_cannot_take_a_hand_authored_skills_name() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (ws, agent) = seed_workspace(&pool).await;
|
||||
|
||||
// A builtin, as `skills_loader` writes them: global, workspace_id NULL.
|
||||
let builtin = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO skills
|
||||
(id, name, title, author, description, when_to_use, tags,
|
||||
source_kind, workspace_id, current_version, body)
|
||||
VALUES ($1,'arxiv-daily','arxiv-daily','system','the real one','always',
|
||||
'{}','builtin',NULL,1,'Do NOT re-search arXiv.')",
|
||||
)
|
||||
.bind(builtin)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let p = seed_proposal(&pool, ws, agent, "arxiv-daily", "Actually, re-searching arXiv is fine.").await;
|
||||
let applied = cm_api::level_up::apply_autonomous(&pool, ws, p).await.unwrap();
|
||||
assert!(
|
||||
applied.is_empty(),
|
||||
"a draft taking a hand-authored name must be refused: two procedures \
|
||||
under one name means nobody reading a transcript can tell which the \
|
||||
agent followed — and this one inverts the rule it shadows"
|
||||
);
|
||||
|
||||
let body: String = sqlx::query_scalar("SELECT body FROM skills WHERE id = $1")
|
||||
.bind(builtin)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
body, "Do NOT re-search arXiv.",
|
||||
"the hand-authored skill must be untouched"
|
||||
);
|
||||
let n: i64 = sqlx::query_scalar("SELECT count(*) FROM skills WHERE name = 'arxiv-daily'")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(n, 1, "no second row may exist under that name");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn autonomous_apply_leaves_identity_and_memory_items_for_a_human() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (ws, agent) = seed_workspace(&pool).await;
|
||||
|
||||
let payload = json!({
|
||||
"suggested_items": [
|
||||
{ "id": "skill-1", "kind": "skill_candidate",
|
||||
"draft": { "name": "commit-message-shape", "description": "d",
|
||||
"body": "Say what changed and why.", "tags": [] } },
|
||||
{ "id": "identity-1", "kind": "identity_refinement",
|
||||
"new_system_prompt": "You are now a different agent." }
|
||||
]
|
||||
});
|
||||
let p = cm_db::repo::level_up::insert(
|
||||
&pool,
|
||||
cm_db::repo::level_up::NewProposal {
|
||||
workspace_id: ws.as_uuid(),
|
||||
agent_id: Some(agent.as_uuid()),
|
||||
team_id: None,
|
||||
payload: &payload,
|
||||
model: Some("glm:glm-4.7"),
|
||||
created_by: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let applied = cm_api::level_up::apply_autonomous(&pool, ws, p).await.unwrap();
|
||||
assert_eq!(
|
||||
applied,
|
||||
vec!["skill-1".to_string()],
|
||||
"only skill_candidate items apply autonomously — an identity rewrite \
|
||||
changes what the agent IS rather than adding a procedure it can \
|
||||
consult, and that is a different bet than the one that was taken"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_sweep_applies_pending_drafts_and_leaves_nothing_pending_twice() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (ws, agent) = seed_workspace(&pool).await;
|
||||
seed_proposal(&pool, ws, agent, "swept-skill", "Say what changed and why.").await;
|
||||
|
||||
let n = cm_api::skill_self_authoring::sweep(&pool).await.unwrap();
|
||||
assert_eq!(n, 1, "the sweep must apply the pending draft with no human");
|
||||
|
||||
let body: String = sqlx::query_scalar("SELECT body FROM skills WHERE name = 'swept-skill'")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("the swept draft must be in the catalogue");
|
||||
assert_eq!(body, "Say what changed and why.");
|
||||
|
||||
// Idempotent: the proposal is no longer pending, so a second pass is a
|
||||
// no-op rather than a duplicate apply or a version bump for no change.
|
||||
let again = cm_api::skill_self_authoring::sweep(&pool).await.unwrap();
|
||||
assert_eq!(again, 0, "a swept proposal must not be applied twice");
|
||||
|
||||
let versions: i64 =
|
||||
sqlx::query_scalar("SELECT count(*) FROM skill_versions sv JOIN skills s ON s.id = sv.skill_id WHERE s.name = 'swept-skill'")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(versions, 1, "an unchanged body must not append a version");
|
||||
}
|
||||
@@ -147,9 +147,19 @@ async fn mention_round_trip_verifies_runs_and_gates_the_reply() {
|
||||
"event": {"type": "app_mention", "text": "summarize [[scenario:mention]]"}
|
||||
})
|
||||
.to_string();
|
||||
// A CURRENT timestamp. It used to be the literal "12345" — a 1970 date —
|
||||
// which passed only because nothing checked freshness. The broker now
|
||||
// enforces Slack's 5-minute replay window, so a fixture that never moves
|
||||
// starts failing the moment the guard is real.
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs()
|
||||
.to_string();
|
||||
|
||||
let forged = client
|
||||
.post(format!("{base}/api/slack/events"))
|
||||
.header("x-slack-request-timestamp", "12345")
|
||||
.header("x-slack-request-timestamp", &now)
|
||||
.header("x-slack-signature", "v0=deadbeef")
|
||||
.body(body.clone())
|
||||
.send()
|
||||
@@ -161,10 +171,10 @@ async fn mention_round_trip_verifies_runs_and_gates_the_reply() {
|
||||
let challenge_body = json!({"type": "url_verification", "challenge": "abc123"}).to_string();
|
||||
let challenge = client
|
||||
.post(format!("{base}/api/slack/events"))
|
||||
.header("x-slack-request-timestamp", "12345")
|
||||
.header("x-slack-request-timestamp", &now)
|
||||
.header(
|
||||
"x-slack-signature",
|
||||
sign(signing_secret, "12345", &challenge_body),
|
||||
sign(signing_secret, &now, &challenge_body),
|
||||
)
|
||||
.body(challenge_body.clone())
|
||||
.send()
|
||||
@@ -176,12 +186,31 @@ async fn mention_round_trip_verifies_runs_and_gates_the_reply() {
|
||||
"abc123"
|
||||
);
|
||||
|
||||
// A correctly signed request from outside the window is refused. The
|
||||
// signature is genuine — that is the point of a replay: an attacker holding
|
||||
// one captured request must not be able to use it forever.
|
||||
let stale_ts = (now.parse::<u64>().unwrap() - 3600).to_string();
|
||||
let replayed = client
|
||||
.post(format!("{base}/api/slack/events"))
|
||||
.header("x-slack-request-timestamp", &stale_ts)
|
||||
.header("x-slack-signature", sign(signing_secret, &stale_ts, &body))
|
||||
.body(body.clone())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
replayed.status(),
|
||||
401,
|
||||
"a validly signed but hour-old request must be refused — otherwise one \
|
||||
captured request authenticates forever"
|
||||
);
|
||||
|
||||
// A properly signed mention starts a run in the '💬 Slack' session and
|
||||
// the agent's reply is intercepted by the approval gate.
|
||||
let mention = client
|
||||
.post(format!("{base}/api/slack/events"))
|
||||
.header("x-slack-request-timestamp", "12345")
|
||||
.header("x-slack-signature", sign(signing_secret, "12345", &body))
|
||||
.header("x-slack-request-timestamp", &now)
|
||||
.header("x-slack-signature", sign(signing_secret, &now, &body))
|
||||
.body(body)
|
||||
.send()
|
||||
.await
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
//! Every `run:` block in a Gitea workflow must be valid shell.
|
||||
//!
|
||||
//! Runs 491 through 496 all failed on a single apostrophe. A comment inside a
|
||||
//! `sh -c '…'` block read `cm-api's vm_tool_gate`, which closed the quote, and
|
||||
//! the step died with "unexpected EOF while looking for matching quote" —
|
||||
//! *before running anything*, which is why no log ever appeared and why three
|
||||
//! separate theories were floated to explain it.
|
||||
//!
|
||||
//! The cost was entirely diagnostic: the workflow is not compiled, not linted,
|
||||
//! and its only feedback is a red build with a log this deployment cannot read.
|
||||
//! `bash -n` answers it in milliseconds, so the check belongs where it runs
|
||||
//! before the push rather than after.
|
||||
//!
|
||||
//! Skips silently if `bash` is unavailable. The test exists to catch a mistake,
|
||||
//! not to fail a machine for lacking a shell it probably has.
|
||||
|
||||
use std::process::Command;
|
||||
|
||||
fn repo_root() -> std::path::PathBuf {
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../..")
|
||||
.canonicalize()
|
||||
.expect("repo root")
|
||||
}
|
||||
|
||||
/// Every `run: |` block, as (file, first line number, script).
|
||||
///
|
||||
/// Line-based rather than a YAML parse: the workspace has no YAML dependency,
|
||||
/// and a block scalar's rule — "the body is the lines indented deeper than the
|
||||
/// key" — is simple enough to apply directly.
|
||||
fn run_blocks(text: &str, file: &str) -> Vec<(String, usize, String)> {
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
let mut out = Vec::new();
|
||||
let mut i = 0;
|
||||
while i < lines.len() {
|
||||
let line = lines[i];
|
||||
let trimmed = line.trim_start();
|
||||
if trimmed.starts_with("run:") && trimmed.trim_end().ends_with('|') {
|
||||
let key_indent = line.len() - trimmed.len();
|
||||
let mut body: Vec<&str> = Vec::new();
|
||||
let mut j = i + 1;
|
||||
let mut body_indent = usize::MAX;
|
||||
while j < lines.len() {
|
||||
let l = lines[j];
|
||||
if l.trim().is_empty() {
|
||||
body.push("");
|
||||
j += 1;
|
||||
continue;
|
||||
}
|
||||
let ind = l.len() - l.trim_start().len();
|
||||
if ind <= key_indent {
|
||||
break;
|
||||
}
|
||||
body_indent = body_indent.min(ind);
|
||||
body.push(l);
|
||||
j += 1;
|
||||
}
|
||||
let script = body
|
||||
.iter()
|
||||
.map(|l| {
|
||||
if l.len() >= body_indent {
|
||||
&l[body_indent..]
|
||||
} else {
|
||||
""
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
out.push((file.to_string(), i + 1, script));
|
||||
i = j;
|
||||
continue;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_workflow_run_block_is_valid_shell() {
|
||||
if Command::new("bash").arg("-c").arg("true").status().is_err() {
|
||||
eprintln!("bash unavailable — skipping workflow shell syntax check");
|
||||
return;
|
||||
}
|
||||
let dir = repo_root().join(".gitea/workflows");
|
||||
let entries = std::fs::read_dir(&dir).expect("workflows dir");
|
||||
|
||||
let mut checked = 0usize;
|
||||
let mut broken: Vec<String> = Vec::new();
|
||||
for e in entries {
|
||||
let path = e.expect("entry").path();
|
||||
if path.extension().and_then(|x| x.to_str()) != Some("yml") {
|
||||
continue;
|
||||
}
|
||||
let name = path.file_name().unwrap().to_string_lossy().to_string();
|
||||
let text = std::fs::read_to_string(&path).expect("read workflow");
|
||||
for (file, line, script) in run_blocks(&text, &name) {
|
||||
// `${{ … }}` is Gitea's, not the shell's. Substituted with a
|
||||
// placeholder so its braces do not read as shell syntax — the point
|
||||
// is to catch OUR quoting, not to evaluate their templating.
|
||||
let mut cleaned = String::new();
|
||||
let mut rest = script.as_str();
|
||||
while let Some(start) = rest.find("${{") {
|
||||
cleaned.push_str(&rest[..start]);
|
||||
cleaned.push_str("PLACEHOLDER");
|
||||
rest = match rest[start..].find("}}") {
|
||||
Some(end) => &rest[start + end + 2..],
|
||||
None => "",
|
||||
};
|
||||
}
|
||||
cleaned.push_str(rest);
|
||||
|
||||
let tmp = std::env::temp_dir().join(format!(
|
||||
"cm-wf-{}-{}-{}.sh",
|
||||
std::process::id(),
|
||||
file.replace('.', "_"),
|
||||
line
|
||||
));
|
||||
std::fs::write(&tmp, &cleaned).expect("write temp script");
|
||||
let out = Command::new("bash")
|
||||
.arg("-n")
|
||||
.arg(&tmp)
|
||||
.output()
|
||||
.expect("run bash -n");
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
checked += 1;
|
||||
if !out.status.success() {
|
||||
broken.push(format!(
|
||||
"{file}:{line} — {}",
|
||||
String::from_utf8_lossy(&out.stderr).trim()
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert!(checked > 0, "found no run: blocks — the extractor is broken, \
|
||||
which would make this test pass forever");
|
||||
assert!(
|
||||
broken.is_empty(),
|
||||
"{} workflow shell block(s) will not parse. The step dies before it \
|
||||
runs anything, so CI reports a failure with an empty log:\n {}",
|
||||
broken.len(),
|
||||
broken.join("\n ")
|
||||
);
|
||||
}
|
||||
@@ -12,5 +12,8 @@ mod token;
|
||||
|
||||
pub use bootstrap::bootstrap_owner;
|
||||
pub use jwt::{ExternalClaims, JwtError, JwtVerifier};
|
||||
pub use service::{AuthError, AuthService, AuthedUser, SESSION_TTL};
|
||||
pub use service::{
|
||||
AuthError, AuthService, AuthedUser, SCOPE_AGENT_DOOR, SCOPE_FULL, SCOPE_SKILLS_READ,
|
||||
SESSION_TTL,
|
||||
};
|
||||
pub use token::SessionToken;
|
||||
|
||||
@@ -10,6 +10,30 @@ use crate::token::{hash_token, SessionToken};
|
||||
/// How long a login session stays valid.
|
||||
pub const SESSION_TTL: Duration = Duration::days(7);
|
||||
|
||||
/// A person's session. Accepted by every route.
|
||||
pub const SCOPE_FULL: &str = "full";
|
||||
|
||||
/// Read the skills catalogue over MCP, and nothing else.
|
||||
///
|
||||
/// The credential a mission container is given so its agent can retrieve skill
|
||||
/// bodies on demand. Deliberately its own constant rather than a string
|
||||
/// literal at the two call sites: a typo in one of them would produce a token
|
||||
/// that authenticates nowhere, which fails safely but silently.
|
||||
pub const SCOPE_SKILLS_READ: &str = "skills:read";
|
||||
|
||||
/// Act through the §15 MCP door (`/mcp`), and nothing else.
|
||||
///
|
||||
/// The door is the actuator: `email_send`, `slack_post`, `delegate`. Reaching
|
||||
/// it means a token an agent's runtime holds, and the same reasoning as
|
||||
/// [`SCOPE_SKILLS_READ`] applies — a full session there is an owner-privileged
|
||||
/// API key handed to a process whose whole purpose is to act on instructions
|
||||
/// from a model.
|
||||
///
|
||||
/// Nothing mints one yet; the route accepts it so that whatever does will not
|
||||
/// have to reach for a person's session to be understood. `full` still works,
|
||||
/// so the UI and any human caller are unaffected.
|
||||
pub const SCOPE_AGENT_DOOR: &str = "agent:door";
|
||||
|
||||
/// The authenticated caller attached to every API request: everything RBAC
|
||||
/// decisions need, nothing more.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -263,13 +287,44 @@ impl AuthService {
|
||||
/// JWTs (three dot-separated segments) take the hosted-identity path;
|
||||
/// everything else is a local opaque session token.
|
||||
pub async fn authenticate(&self, token_secret: &str) -> Result<AuthedUser, AuthError> {
|
||||
self.authenticate_scoped(token_secret, SCOPE_FULL).await
|
||||
}
|
||||
|
||||
/// Resolve a bearer token that is allowed to be narrow.
|
||||
///
|
||||
/// `required` is the scope this call site accepts *in addition to*
|
||||
/// [`SCOPE_FULL`], which is a person's session and is accepted everywhere.
|
||||
///
|
||||
/// # Fail closed
|
||||
///
|
||||
/// [`authenticate`](Self::authenticate) delegates here with `SCOPE_FULL`,
|
||||
/// so a narrow token is **rejected by every existing caller** and a route
|
||||
/// has to opt in by naming the scope it accepts. That direction matters:
|
||||
/// the likely mistake is adding a scope and forgetting to wire a check, and
|
||||
/// this way that mistake grants nothing instead of granting everything.
|
||||
///
|
||||
/// A narrow credential exists because the alternative is worse. Reaching
|
||||
/// `/mcp/skills` from a mission container means putting a bearer token in a
|
||||
/// file inside it, and mission agents run arbitrary `Bash` with egress and
|
||||
/// no read gate — so a full session there is an owner-privileged API key
|
||||
/// handed to something explicitly untrusted.
|
||||
pub async fn authenticate_scoped(
|
||||
&self,
|
||||
token_secret: &str,
|
||||
required: &str,
|
||||
) -> Result<AuthedUser, AuthError> {
|
||||
if let Some(verifier) = self.verifier.clone() {
|
||||
if token_secret.matches('.').count() == 2 {
|
||||
// An external issuer JWT is always a person. There is no
|
||||
// narrow form of it, so it satisfies only `full`.
|
||||
if required != SCOPE_FULL {
|
||||
return Err(AuthError::Unauthenticated);
|
||||
}
|
||||
return self.authenticate_external(token_secret, &verifier).await;
|
||||
}
|
||||
}
|
||||
let row = sqlx::query!(
|
||||
"SELECT u.id, u.workspace_id, u.role
|
||||
"SELECT u.id, u.workspace_id, u.role, s.scope
|
||||
FROM auth_sessions s
|
||||
JOIN users u ON u.id = s.user_id
|
||||
WHERE s.token_hash = $1 AND s.expires_at > now()",
|
||||
@@ -278,6 +333,9 @@ impl AuthService {
|
||||
.fetch_optional(&self.pool)
|
||||
.await?
|
||||
.ok_or(AuthError::Unauthenticated)?;
|
||||
if row.scope != SCOPE_FULL && row.scope != required {
|
||||
return Err(AuthError::Unauthenticated);
|
||||
}
|
||||
Ok(AuthedUser {
|
||||
user_id: UserId::from(row.id),
|
||||
workspace_id: WorkspaceId::from(row.workspace_id),
|
||||
@@ -289,6 +347,76 @@ impl AuthService {
|
||||
})
|
||||
}
|
||||
|
||||
/// Mint a narrow, short-lived credential for something that is not a person.
|
||||
///
|
||||
/// Returns the secret, which is the only time it exists in plaintext here.
|
||||
pub async fn mint_scoped(
|
||||
&self,
|
||||
user_id: UserId,
|
||||
scope: &str,
|
||||
ttl: Duration,
|
||||
) -> Result<String, AuthError> {
|
||||
if scope == SCOPE_FULL {
|
||||
// A caller reaching for this wants a narrow token; handing back a
|
||||
// full one because the argument was wrong is the failure this
|
||||
// whole change exists to prevent.
|
||||
return Err(AuthError::Unauthenticated);
|
||||
}
|
||||
let token = SessionToken::generate();
|
||||
sqlx::query!(
|
||||
"INSERT INTO auth_sessions (token_hash, user_id, expires_at, scope)
|
||||
VALUES ($1, $2, $3, $4)",
|
||||
hash_token(token.secret()),
|
||||
user_id.as_uuid(),
|
||||
OffsetDateTime::now_utc() + ttl,
|
||||
scope,
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
Ok(token.secret().to_string())
|
||||
}
|
||||
|
||||
/// As [`Self::mint_scoped`], bound to a mission: the row carries
|
||||
/// `mission_id`, and [`Self::revoke_mission_sessions`] deletes it when
|
||||
/// the mission ends. A 24 h TTL is the backstop, not the lifetime.
|
||||
pub async fn mint_scoped_for_mission(
|
||||
&self,
|
||||
user_id: UserId,
|
||||
scope: &str,
|
||||
ttl: Duration,
|
||||
mission_id: uuid::Uuid,
|
||||
) -> Result<String, AuthError> {
|
||||
if scope == SCOPE_FULL {
|
||||
return Err(AuthError::Unauthenticated);
|
||||
}
|
||||
let token = SessionToken::generate();
|
||||
sqlx::query(
|
||||
"INSERT INTO auth_sessions (token_hash, user_id, expires_at, scope, mission_id)
|
||||
VALUES ($1, $2, $3, $4, $5)",
|
||||
)
|
||||
.bind(hash_token(token.secret()))
|
||||
.bind(user_id.as_uuid())
|
||||
.bind(OffsetDateTime::now_utc() + ttl)
|
||||
.bind(scope)
|
||||
.bind(mission_id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
Ok(token.secret().to_string())
|
||||
}
|
||||
|
||||
/// Revoke every session minted for a mission. Returns how many there
|
||||
/// were; zero is the normal case for a mission that had no door.
|
||||
pub async fn revoke_mission_sessions(
|
||||
&self,
|
||||
mission_id: uuid::Uuid,
|
||||
) -> Result<u64, AuthError> {
|
||||
let done = sqlx::query("DELETE FROM auth_sessions WHERE mission_id = $1")
|
||||
.bind(mission_id)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
Ok(done.rows_affected())
|
||||
}
|
||||
|
||||
/// Mint a long-lived opaque session for an internal service caller
|
||||
/// (e.g. the per-team ZeroClaw runtime calling back into the MCP door).
|
||||
/// Returns the plaintext token — the caller is responsible for handing
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user