Files
clawmates/frontend/src/components/dashboard/PhaseGoalStrip.tsx
T
Omar SobhandClaude Opus 5 c812b714f4
ci / gates (push) Failing after 5s
ci / rust (push) Skipped
ci / frontend (push) Skipped
ci / e2e (push) Skipped
ci / publish (push) Skipped
fix(evaluator): the verification sandbox never ran a command
`evaluator_tools::Sandbox::run` shelled out to `tokio::process::Command::new
("docker")`. The server image installs `git ca-certificates chromium
fonts-liberation` and nothing else, so in production every verification
command failed to spawn.

The failure was invisible in the worst way. `Sandbox::run` deliberately turns
execution failures into evidence text rather than errors, so a judge reasons
about "that command did not run" instead of the pass collapsing. With no
`docker` binary every command returned COULD NOT RUN, the judge correctly
concluded it could not verify, and fail-closed returned "not met". The
verdicts were right. The verification never happened — and the adversarial
validation that appeared to prove the feature working proved fail-closed
working instead.

The second defect made it worse: `checks` recorded the *attempt*, pushed
before the command ran, so a verdict reached with a dead sandbox reported
"verified by 10 checks" — a stronger claim than "no checks at all", made on
weaker evidence.

- New `container_exec` routes execution through the Docker API via bollard,
  which was already a dependency and already reaches the daemon through the
  socket proxy. Captures the exit code (absent from the old helper) and keeps
  stdout and stderr apart (`LogOutput`'s Display merged them, which is why
  nothing downstream could tell JSON from a progress bar). `security_scan`
  parses stdout alone; `benchmark_runner` needs both.
- `ExecOutput::success()` requires `Some(0)`. An unreadable status is not
  success — `commit_policy = "on_green_tests"` will gate on this, and
  "unknown" reading as "green" would push untested work.
- `Sandbox::run` returns a `CheckOutcome` carrying `ran`/`refused`/
  `exit_code`. `Verdict::verified_checks()` counts executions, not attempts.
- The UI gains a third state: "could not verify (N attempted, 0 ran)" —
  precisely the case that used to render as verified.
- Regression tests reproduce the production shape: two checks recorded,
  neither executed, `was_verified() == false`; plus a failing suite (exit 101)
  still counting as verification, because that is something the judge learned
  rather than was told.

Co-Authored-By: Claude Opus 5 <[email protected]>
2026-08-01 18:33:32 -07:00

217 lines
6.7 KiB
TypeScript

"use client";
import { useCallback, useEffect, useState } from "react";
import { Target } from "lucide-react";
import {
getPhaseEvaluations,
type CheckOutcome,
type MissionPhase,
type PhaseEvaluation,
} from "@/lib/api/missions";
/**
* Whether a verdict was verified, and how honestly we can say so.
*
* Three states, not two. A judge that attempted ten commands and executed none
* — because the sandbox was unreachable — is not the same as a judge that ran
* ten, and neither is the same as a phase with no repo to check. Collapsing
* the middle case into "verified" is what the first version of this badge did.
*/
function VerificationNote({ checks }: { checks?: CheckOutcome[] }) {
const all = checks ?? [];
const ran = all.filter((c) => c.ran);
const failed = ran.filter((c) => c.exit_code !== 0);
if (ran.length > 0) {
const detail = ran.map((c) => `${c.argv.join(" ")} → exit ${c.exit_code}`).join("\n");
return (
<em style={{ color: failed.length ? "#e8b465" : "#6a8ab0" }} title={detail}>
{` — verified by ${ran.length} check${ran.length === 1 ? "" : "s"}`}
{failed.length ? ` (${failed.length} non-zero)` : ""}
</em>
);
}
if (all.length > 0) {
// Attempted but nothing executed: a broken sandbox, or every command
// refused. Say so — this is the case that used to read as "verified".
return (
<em
style={{ color: "#ff8a7a" }}
title={all.map((c) => `${c.argv.join(" ")} → ${c.refused ? "refused" : "could not run"}`).join("\n")}
>
{` — could not verify (${all.length} attempted, 0 ran)`}
</em>
);
}
return (
<em
style={{ color: "#8a7a5a" }}
title="No commands were run — this verdict rests on what the agents reported."
>
{" — from agent claims only"}
</em>
);
}
/**
* The completion condition on a phase, plus how the last pass was judged.
*
* Renders nothing for phases without a `done_when` — most missions don't have
* one, and an empty row per phase would be noise.
*
* The evaluator's `reason` is deliberately the most prominent thing here — it
* explains why the phase iterated or stopped, so it is what an operator needs
* to decide whether the condition is written well. It is *not* what the agents
* were told: they get a sanitized `guidance` that withholds the acceptance
* text, so a pass cannot be satisfied by pasting the verdict back.
*
* Whether the judge verified anything is shown alongside the verdict — see
* `VerificationNote`. An operator should never have to guess whether a verdict
* rests on executed commands or on the agents' own account of themselves.
*/
export function PhaseGoalStrip({
missionId,
phase,
}: {
missionId: string;
phase: MissionPhase;
}) {
const [evals, setEvals] = useState<PhaseEvaluation[]>([]);
const [expanded, setExpanded] = useState(false);
const load = useCallback(async () => {
try {
setEvals(await getPhaseEvaluations(missionId, phase.id));
} catch {
// A phase that has never been judged has no rows; not an error state.
}
}, [missionId, phase.id]);
useEffect(() => {
if (!phase.done_when) return;
void load();
// Poll only while there is something to wait for.
if (phase.status !== "running" && phase.status !== "evaluating") return;
const t = setInterval(() => void load(), 5000);
return () => clearInterval(t);
}, [load, phase.done_when, phase.status]);
if (!phase.done_when) return null;
const latest = evals[0];
const pass = phase.iteration + 1;
const judging = phase.status === "evaluating";
return (
<div
style={{
marginTop: 8,
padding: "8px 10px",
borderRadius: 10,
background: "#0d0d10",
border: "1px solid #1c1c22",
display: "flex",
flexDirection: "column",
gap: 6,
}}
>
<div style={{ display: "flex", alignItems: "center", gap: 6 }}>
<Target aria-hidden size={12} color="#e8b465" />
<span
style={{
fontSize: 10,
letterSpacing: 0.5,
textTransform: "uppercase",
color: "#6a6a72",
}}
>
Done when
</span>
<span style={{ marginLeft: "auto", fontSize: 10, color: "#6a6a72" }}>
pass {pass} / {phase.max_iterations}
</span>
</div>
<p style={{ margin: 0, fontSize: 12, color: "#c8c8d0", lineHeight: 1.45 }}>
{phase.done_when}
</p>
{judging && (
<span style={{ fontSize: 11, color: "#e8b465" }}>
Judging this pass…
</span>
)}
{latest && (
<div style={{ display: "flex", gap: 6, alignItems: "flex-start" }}>
<span
style={{
fontSize: 11,
fontWeight: 600,
color: latest.met ? "#5fd08a" : "#e8b465",
whiteSpace: "nowrap",
}}
>
{latest.met ? "met" : "not met"}
</span>
<span style={{ fontSize: 11, color: "#8a8a92", lineHeight: 1.45 }}>
{latest.reason}
{latest.error && (
// Distinguishes "judged incomplete" from "could not judge" —
// an evaluator outage should not read as a verdict on the work.
<em style={{ color: "#ff8a7a" }}> (evaluator error)</em>
)}
{!latest.error && <VerificationNote checks={latest.checks} />}
</span>
</div>
)}
{evals.length > 1 && (
<>
<button
type="button"
onClick={() => setExpanded((v) => !v)}
style={{
alignSelf: "flex-start",
background: "none",
border: "none",
padding: 0,
cursor: "pointer",
fontSize: 10,
color: "#6a6a72",
}}
>
{expanded ? "hide" : `show all ${evals.length} passes`}
</button>
{expanded && (
<ol
style={{
margin: 0,
paddingLeft: 16,
display: "flex",
flexDirection: "column",
gap: 4,
}}
>
{evals.map((e) => (
<li
key={e.iteration}
style={{ fontSize: 11, color: "#8a8a92", lineHeight: 1.4 }}
>
<span
style={{ color: e.met ? "#5fd08a" : "#e8b465", fontWeight: 600 }}
>
pass {e.iteration + 1} {e.met ? "met" : "not met"}
</span>{" "}
— {e.reason}
</li>
))}
</ol>
)}
</>
)}
</div>
);
}