From 93a386e706cc11272f9787915b0c144901206dd7 Mon Sep 17 00:00:00 2001 From: Omar Sobh Date: Mon, 14 Sep 2026 08:17:20 -0500 Subject: [PATCH] perf(judge): earlier check outputs shrink to a reminder before the next round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured on prod: 7 of 9 verdicts ran to the 12-check cap. Every round resends the whole history, and each check's output is bounded at 12 KB — so by the last round the judge was paying for ~144 KB of outputs it had already read, on top of up to 120 KB of evidence, and it paid that on every round. That is the quadratic term in a verdict's cost, and the reason a single blocked phase could empty a weekly plan. Before this round's results go in, every earlier tool result compacts to an 800-byte head plus a marker saying the rest was shown when the check ran. The round that just ran stays whole; a result already carrying the marker is left alone. The budget of checks is unchanged — each one is cheaper to remember, not fewer to run. Also: docs/NEXT-SESSION.md rewritten for the state as of today. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WZb5A2kfVfjpdwSochkuHz --- crates/cm-api/src/evaluator.rs | 76 +++++++++++ docs/NEXT-SESSION.md | 233 +++++++++++++++++---------------- 2 files changed, 199 insertions(+), 110 deletions(-) diff --git a/crates/cm-api/src/evaluator.rs b/crates/cm-api/src/evaluator.rs index 01fa5f5..2f05fc6 100644 --- a/crates/cm-api/src/evaluator.rs +++ b/crates/cm-api/src/evaluator.rs @@ -767,6 +767,9 @@ async fn judge_with_tools( content: Value::String(evidence), }); } + // Everything the judge has already read shrinks to a reminder before + // this round's results go in full. See `compact_earlier_results`. + compact_earlier_results(&mut messages); messages.push(ChatMessage { role: ChatRole::User, parts: results, @@ -775,6 +778,44 @@ async fn judge_with_tools( Err("evaluator exceeded its verification budget without reaching a verdict".into()) } +/// How much of an earlier check's output stays in the history. +/// +/// Enough to recognise the command and its outcome — a test summary line, a +/// grep hit, an error — not enough to re-read the whole thing, which the judge +/// already did in the round it arrived. +const KEPT_OF_EARLIER_RESULT: usize = 800; + +/// Shrink every tool result from EARLIER rounds to a short head. +/// +/// The judge's history is resent whole on every round, and each check's +/// output is bounded at `evaluator_tools::MAX_OUTPUT_BYTES` (12 KB). Measured +/// on prod, 7 of 9 verdicts ran to the 12-check cap, so by the last round the +/// history carried ~144 KB of outputs the judge had already read, on top of +/// up to 120 KB of evidence — and every round paid for all of it again. That +/// is the quadratic term in a verdict's cost, and it is why one blocked phase +/// could empty a weekly plan. +/// +/// The round that just ran keeps its results in full; only what came before +/// is compacted, and it is compacted once — a result already carrying the +/// marker is left alone. The judge's budget of checks is unchanged: this +/// makes each check cheaper to remember, not fewer to run. +fn compact_earlier_results(messages: &mut [cm_llm::ChatMessage]) { + use cm_llm::ContentPart; + const MARKER: &str = "\n[… output elided here — it was shown in full when this check ran]"; + for m in messages.iter_mut() { + for part in m.parts.iter_mut() { + if let ContentPart::ToolResult { content, .. } = part { + if let Some(text) = content.as_str() { + if text.len() > KEPT_OF_EARLIER_RESULT && !text.ends_with(MARKER) { + let kept = head(text, KEPT_OF_EARLIER_RESULT); + *content = Value::String(format!("{kept}{MARKER}")); + } + } + } + } + } +} + /// Parse the model's reply into a verdict, failing closed. fn parse_verdict(model: &str, text: &str) -> Verdict { let trimmed = text.trim(); @@ -925,6 +966,41 @@ pub async fn latest( #[cfg(test)] mod cross_provider_tests { + /// The quadratic term: earlier outputs resent whole every round. + #[test] + fn earlier_tool_results_shrink_and_the_latest_stays_whole() { + use cm_llm::{ChatMessage, ChatRole, ContentPart}; + let big = "line of output\n".repeat(900); // ~13 KB + let mut messages = vec![ + ChatMessage { + role: ChatRole::User, + parts: vec![ContentPart::text("judge this")], + }, + ChatMessage { + role: ChatRole::User, + parts: vec![ + ContentPart::ToolResult { tool_use_id: "a".into(), content: Value::String(big.clone()) }, + ContentPart::ToolResult { tool_use_id: "b".into(), content: Value::String(big.clone()) }, + ], + }, + ]; + compact_earlier_results(&mut messages); + for part in &messages[1].parts { + let ContentPart::ToolResult { content, .. } = part else { panic!() }; + let s = content.as_str().unwrap(); + assert!(s.len() < KEPT_OF_EARLIER_RESULT + 120, "not compacted: {} bytes", s.len()); + assert!(s.starts_with("line of output"), "the head survives"); + assert!(s.contains("elided"), "and says so"); + } + // Idempotent: a second pass must not shrink the reminder further. + let once: Vec = messages[1].parts.iter().map(|p| serde_json::to_string(p).unwrap()).collect(); + compact_earlier_results(&mut messages); + let twice: Vec = messages[1].parts.iter().map(|p| serde_json::to_string(p).unwrap()).collect(); + assert_eq!(once, twice); + // Plain text parts are untouched. + assert!(matches!(&messages[0].parts[0], ContentPart::Text { text } if text == "judge this")); + } + /// `LlmEvent::Usage` arrives on every provider call. It was matched by /// `Ok(_) => {}` and dropped, which is how two plan exhaustions happened /// with no row anywhere saying a judge token was spent. diff --git a/docs/NEXT-SESSION.md b/docs/NEXT-SESSION.md index 665361a..e11c184 100644 --- a/docs/NEXT-SESSION.md +++ b/docs/NEXT-SESSION.md @@ -1,138 +1,151 @@ -# Where this left off — 2026-09-05 +# Where this left off — 2026-09-14 -`MISSION-EGRESS.md` has what a mission container can reach, `UPSTREAM-SCAN.md` -the fork-vs-upstream position, `SKILL-USE-BASELINE.md` the delivery measurement. +Nine days, ~20 commits, and every item on the last handoff's open list is +closed or explained. The platform is in the best-measured state it has been +in. Read the first section, then the open list; the middle is the record. -## Read this first — the judge is back, and the subagent path is proven +## Read this first — retrieval works now, and we know why it did not -The z.ai quota reset on schedule. `glm-5.3` answers again on the same key -(`sha256[0:12] = 616fb5076696`), and it has since passed a real `done_when`, not -just a ping. **Nothing is blocked on the judge any more.** +Mission agents were not fetching their skills. Five matched production runs +— same recipe, same task, same three skills on offer — said this precisely: -Mission `01a07498-d343-7953-9ed1-8ea9ba83b140` (2026-09-05, local, `index` arm) -was the previous validation run plus one change — a task that EXPLICITLY -instructs `Agent` delegation — and it closed both items the last pass left open: +| arm | mechanism | fetched | +|---|---|---| +| `index` | `ReadMcpResourceTool` via the MCP door | **1 of 9** | +| `files` | `Read` of `/mission/skills/.md` | **7 of 9** | -| | | -|---|---| -| judge recovered | **CONFIRMED** — phase `completed` on iteration 0, first attempt. The verdict cites the actual arXiv ids it checked and the "what could not be established" section by name. A substantive read, not a rubber stamp. | -| **subagent attribution** | **OBSERVED, finally.** 87 tool calls: 43 from the main turn, **44 across 4 `general-purpose` subagents**, 4 distinct `subagent_id`s matching 4 `Agent` spawns. The field had only ever been unit-tested before this. | +The door tool is **deferred** in Claude Code: absent from the agent's default +list until `ToolSearch` loads it. Naming it in the prompt did nothing; telling +the agent to load it first did nothing (verified, `01a09877`: zero +`ToolSearch`, three narratives that never mention skills). `Read` is core, +never deferred, used in every run. So the `files` arm writes every visible +skill into the container at launch and the index points at paths. -The delegation had to be *demanded*. The earlier task invited it and got none; -naming the tool and saying "do not gather every approach yourself in a single -turn" produced four spawns. Worth remembering when designing any run that needs -to exercise the fan-out path. +**`files` is the code default now** (`skill_delivery::DEFAULT`). `index` and +`inline` stay selectable per mission (`config.skill_delivery`) so the +comparison remains runnable against one binary. The rule that came out of it: +a capability that depends on the model guessing a tool is loadable is not +delivered. + +Two more things about skills: + +- `always_inject` lives in the skill's frontmatter and the loader restores it + on boot (proved by forcing the DB column false and watching it come back). + `workspace-repo-commit-protocol` is the only one marked, and should stay the + only one: a marked skill leaves the Trigger sample. +- `web-search-triage` has a compliance check now (URLs fetched vs. a primary / + aggregator host list). All five runs pass it — including the three that + never opened the skill. The check catches the violation; it cannot tell + "followed the skill" from "would have done this anyway", and nothing + mechanical could on this skill. -The 429 that blocked the last pass was real (code `1310`, reset timestamp) and -the code was right to fail closed — `evaluator.rs:480` still refuses a -same-family fallback, and should keep refusing. If a `done_when` phase fails -again with the artifacts present and correct, check the quota before the mission. ## What shipped this pass -Seven commits on `main`, all deployed to gw-04 (server recreated -2026-08-29T02:07:48). Suite at the end: **108 binaries, 842 Rust tests, 92 -frontend tests, tsc clean.** +### The judge's cost, three ways -### The gate's exfiltration rule matched a spelling production never writes +The z.ai plan for `glm-5.3` emptied twice (08-29, 09-09) and nothing recorded +a single judge token. Three commits, each measured: -`5a2ed8f`. One rule stood between an agent and sending the checkout off the -machine, anchored to a segment STARTING with `curl -X POST`. The tap held the -answer: 166 curl invocations across two production missions, every one a GET, -and every one beginning `curl -s`. The `-s` pushes the needle off position zero. -`curl -s -X POST … -d @secret`, `-d @file`, `-F`, `-T`, `--upload-file`, -`wget --post-file` — all allowed. +1. **The retry storm** (`8d6310f`). A blocked phase re-judged on the 10s sweep + for 30 minutes — 180 attempts, each up to 13 requests. Now exponential + backoff (~10 attempts) via `mission_phases.judge_retry_after`, and a 429 + that names its own reset time fails immediately, naming it. +2. **Accounting** (`248948c`, `736b6a9`). `usage_events` gained `provider, + model, mission_id, requests`; every judge attempt writes a `kind='judge'` + row, refused requests included. The first rows read `tokens_in = 0`: z.ai + reports input in `message_delta`, Anthropic in `message_start`. Fixed. +3. **The quadratic term** (this pass). 7 of 9 verdicts ran to the 12-check + cap, and every round resent every earlier check's output (≤12 KB each) + whole. Earlier results now compact to an 800-byte head before the next + round; the round that just ran stays in full. Checks per verdict unchanged. -`Match::Carries` matches a segment that starts with the command and contains the -needle anywhere; `CarriesExact` exists for flags whose CASE is their meaning -(curl `-F` uploads, `-f` fails quietly, as in the ordinary `curl -fsSL`). -Segments are lowercased individually, not up front. All 158 recorded production -commands replay through the new script with **0 false positives**. +Ask the plan before it tells you: -### A subagent's tool calls are no longer credited to its parent +```sql +select provider, date_trunc('day', created_at), sum(requests), + sum(tokens_in), sum(tokens_out) + from usage_events where provider is not null group by 1, 2 order by 2; +``` -`fe5c7d2`. Measured against the real claude binary, not reasoned about: +### Agent-side spend is visible too (this pass) -1. Subagent tool calls DO fire both hooks. `PreToolUse` blocked a subagent's - denied curl. **`Agent` is not a gate bypass.** -2. They carry the PARENT's `session_id` — which is why `attribute_sessions` - keeps working; a subagent never adds a session. -3. Only `agent_type`/`agent_id` tell them apart, and `parse()` read past both. +The runtime's `done` frame always carried `model` and `provider`; the +executor read only the two token counts. `TurnOutcome` and `StepRecord` now +carry a `Spend` (split + provider + model), `cm_billing::charge` writes it, +and the chat runtime records its requested model (it drives one provider, +no chain, so requested is answered). Bare model names are recorded without a +guessed family. -The tap appends the raw payload, so those fields were always on disk. Host-side -fix only, no image rebuild. +### Three things that were known and written nowhere (`248948c`) -### The rest +- `gate.installed` / `gate.absent` mission events — the hook install outcome + used to go to stderr in a container that is later deleted. +- `gate.inert` — the marker the gate writes when it cannot parse now has a + production reader (`drain_inert`), not only a unit test. +- Judge `LlmEvent::Usage` was `Ok(_) => {}`. -- `bd7fd46` — a mission that cannot attach to `clawmates_edge` now FAILS to - launch. `clawmates_core` is `internal: true` with no default route, so a - discarded attach error meant a mission running with zero egress while - reporting success. **Behaviour change: it can now stop a mission starting.** -- `2f1a870` — `skills.always_inject` (migration 0083). Delivery and scoring - both, because they are different mistakes. -- `f26de3b` — `agent.last_run`, the metric band's historical half. -- `fde1341`, `563b074` — the egress measurement and the upstream scan. +### Infra -## Verified live, and one honest negative - -A validation mission ran locally on the `index` arm -(`01a04f74-564b-7cd3-9b55-9d8f29d3e2f4`): - -| | | -|---|---| -| `always_inject` | **CONFIRMED** — same prompt, same arm: `workspace-repo-commit-protocol` inlined as a full body, `scientific-writing-conventions` offered as a uri | -| retrieval still works | **CONFIRMED** — `ReadMcpResourceTool(uri="skill:global/web-search-triage")`; the flag did not collapse the arm | -| the corrected gate | **CONFIRMED installed** — new rules present in the live container, node present, no inert marker, 0 denials against 23 body-free `Bash` calls | -| attribution | 34/34, third perfect run | -| `subagent` column | written, all null — **correct**, no subagents spawned | -| **subagent path** | unexercised in THIS run; **closed on 2026-09-05** by `01a07498` — see the top of this file. | +- **ZeroClaw v0.8.5** merged into the fork and deployed. No regressions. +- **Prod was off the tailnet for a day.** Tailscale node-key expiry on + gw-01/02/04 — staggered by enrolment date, which is the tell. Re-authed, + key expiry disabled on all five Hetzner nodes, `-pub` aliases in + `~/.ssh/config` on the public IPs, vault corrected, runbook written + (`Valhalla/20 Infrastructure/30 Runbooks/tailscale-key-expiry-2026-09.md`). +- **Fleet re-enrolled**: tank + architect online. morpheus reappeared. +- `CLAWMATES_API_ORIGIN` set explicitly; `worker_glm`/`worker_glm5`/ + `worker_kimi` removed from the prod runtime template (byte-identical to + `worker`, names that promised providers they never used); map routes + `researcher`/`analyst` to `worker` directly. ## Open, in the order I would take them -1. **Confirm the last-run cards in the prod UI.** They read HISTORICAL data, so - unlike everything else this pass they should work right now against the two - 2026-08-25 missions. If blank, suspect the roster join: `agent_last_run` - reaches agents via `team_members → mission_teams → missions`, so an agent no - longer on the mission's team will not resolve. -2. **Three fork items, all behind one runtime image rebuild.** Branch - `port/config-write-lock` on the zeroclaw fork carries upstream `841f28c7f1` - (cherry-picked CLEANLY despite being 331 behind; `handle_prop_put` — the - endpoint ClawMates writes through — now takes the lock). Not merged, not - deployed. Still to take: credential-fragment redaction (`eadaee0b62`) and - `/api/pair` lockout hardening (`47adb9863e`). Tank is offline, so the rebuild - has to happen on gw-04 and push to the web-01 registry. -3. **Mission egress remediation** — written up in `MISSION-EGRESS.md`, - deliberately NOT applied on the operator's call. A mission container reaches - the entire tailnet and host SSH. -4. **The 331-commit upstream merge.** 18 overlapping files; `claude_cli.rs` - cannot conflict (zero upstream files). -5. **Two silent-discard defects found by sweep, not yet fixed.** - `container_tool_hooks::install`'s outcome is discarded at both call sites and - recorded nowhere, so "did this mission run gated?" is unanswerable once the - container is reaped — and `PreToolUse` is now a security control, not just - telemetry. `vm_tool_gate::INERT_FILE` has no reader anywhere (low severity: - node v22 is in the image, so it cannot currently trigger). +1. **The judge runs to its check cap almost every time.** Compaction made + that cheaper; it did not ask why a research verdict needs 12 commands. + Watch `requests` per verdict on the next few missions. If it stays at the + cap, the lever is the judge prompt, not the budget. +2. **microVM tier is unexercised since v0.8.5.** Fleet is online; every run + this pass was container-tier. Run one microVM mission before assuming the + upgrade left that path alone. +3. **`files` is n=3.** 7 of 9 is a signal. `structured-paper-summary` went + unread in 2 of the last 3 runs; the skills section sits 87–90% into the + prompt. Position is the untested lever. +4. **Compliance checks exist for 6 of 53 skills.** The rest score + `not_applicable` on that axis forever. +5. **The persistent `clawmates-runtime` container's own config** still names + `worker_glm`/`worker_kimi` (3 mentions). Not on the mission path; editing + it restarts the paired runtime. +6. **Prod holds our 7 test missions** with 90-day retention. They are the + evidence for everything above; wipe them when they stop being. ## State you should know about -- **Local DB has test state**: `workspace-repo-commit-protocol.always_inject = - true`, set by hand for the validation run. **Production has none set** — - `always_inject` is deployed, defaulting off, changing nothing until someone - marks a skill. -- **No production mission since 2026-08-25.** Everything above is deployed and, - apart from the last-run cards, unexercised in prod. -- **Fleet offline 17 days** — `architect`, `morpheus`, `tank`; `nodes` table is - empty. microVM tier stays blocked. -- **Postgres is named differently on each stack.** Locally it is - `clawmates-postgres-1` (dashes); on gw-04 it is `clawmates_postgres_1` - (underscores). Same for `server`/`frontend`. Using the wrong one gives - "No such container", which reads like a down stack rather than a typo. -- **`target/` is a symlink to `/Volumes/NVMeRAID`.** If it detaches mid-run, - cargo dies with SIGKILL and then `Not a directory`. It is the drive, not a - flaky test. +- **Prod: 7 missions, 6 completed** (the 7th was the quota casualty). All + research_only, all the same task — that sameness is what made the arm + comparison mean anything. +- **Judge quota** resets weekly (last: 2026-09-11 10:01 UTC). With backoff + and accounting in place a blocked phase can no longer empty it alone; a + week of missions still can. Check the query above before a batch. +- **Postgres is named differently on each stack.** Locally + `clawmates-postgres-1` (dashes); on gw-04 `clawmates_postgres_1` + (underscores). Same for `server`/`frontend`. +- **`target/` is a symlink to `/Volumes/NVMeRAID`**, and that volume went + away entirely on 2026-09-14 (SIGBUS mid-compile, then "failed to create + directory target"). Build with `CARGO_TARGET_DIR=$HOME/cargo-target-clawmates` + until it is back. It is the drive, not the code. +- **The Mac kills background processes under memory pressure** — four + watchers and Tailscale this pass. Long polls belong on gw-04 (`nohup`), not + here. +- **gw-04 is reachable two ways**: `ssh gw-04` (Tailscale) and `ssh gw-04-pub` + (public IP, `204.168.133.187`). It is NOT on the Hetzner private net; the + web-01 back door cannot reach it. ## Deliberately not done -- The mission executor swap — blockers are structural (see prior handoffs). -- A tap for the direct-session tier — dormant, `CLAWMATES_MISSION_EXECUTOR` - unset in production. -- `cm-brain` offline tests — 6 of 9 need live `clawbrainhub.com`. +- Routing any agent role to GLM. The z.ai plan is the judge's, and the judge + is the one consumer whose spend is now measured. The design for a real + `claude_cli.glm` route is in `deploy/clawmates-runtime/agent.config.example.toml`, + commented out, with the reason it does not work as a TOML sub-table. +- Marking more skills `always_inject`. See above. +- The mission executor swap; a tap for the direct-session tier; `cm-brain` + offline tests — unchanged from prior handoffs.