`evaluator_tools::Sandbox::run` shelled out to `tokio::process::Command::new
("docker")`. The server image installs `git ca-certificates chromium
fonts-liberation` and nothing else, so in production every verification
command failed to spawn.
The failure was invisible in the worst way. `Sandbox::run` deliberately turns
execution failures into evidence text rather than errors, so a judge reasons
about "that command did not run" instead of the pass collapsing. With no
`docker` binary every command returned COULD NOT RUN, the judge correctly
concluded it could not verify, and fail-closed returned "not met". The
verdicts were right. The verification never happened — and the adversarial
validation that appeared to prove the feature working proved fail-closed
working instead.
The second defect made it worse: `checks` recorded the *attempt*, pushed
before the command ran, so a verdict reached with a dead sandbox reported
"verified by 10 checks" — a stronger claim than "no checks at all", made on
weaker evidence.
- New `container_exec` routes execution through the Docker API via bollard,
which was already a dependency and already reaches the daemon through the
socket proxy. Captures the exit code (absent from the old helper) and keeps
stdout and stderr apart (`LogOutput`'s Display merged them, which is why
nothing downstream could tell JSON from a progress bar). `security_scan`
parses stdout alone; `benchmark_runner` needs both.
- `ExecOutput::success()` requires `Some(0)`. An unreadable status is not
success — `commit_policy = "on_green_tests"` will gate on this, and
"unknown" reading as "green" would push untested work.
- `Sandbox::run` returns a `CheckOutcome` carrying `ran`/`refused`/
`exit_code`. `Verdict::verified_checks()` counts executions, not attempts.
- The UI gains a third state: "could not verify (N attempted, 0 ran)" —
precisely the case that used to render as verified.
- Regression tests reproduce the production shape: two checks recorded,
neither executed, `was_verified() == false`; plus a failing suite (exit 101)
still counting as verification, because that is something the judge learned
rather than was told.
Co-Authored-By: Claude Opus 5 <[email protected]>
217 lines
6.7 KiB
TypeScript
217 lines
6.7 KiB
TypeScript
"use client";
|
|
|
|
import { useCallback, useEffect, useState } from "react";
|
|
import { Target } from "lucide-react";
|
|
|
|
import {
|
|
getPhaseEvaluations,
|
|
type CheckOutcome,
|
|
type MissionPhase,
|
|
type PhaseEvaluation,
|
|
} from "@/lib/api/missions";
|
|
|
|
/**
|
|
* Whether a verdict was verified, and how honestly we can say so.
|
|
*
|
|
* Three states, not two. A judge that attempted ten commands and executed none
|
|
* — because the sandbox was unreachable — is not the same as a judge that ran
|
|
* ten, and neither is the same as a phase with no repo to check. Collapsing
|
|
* the middle case into "verified" is what the first version of this badge did.
|
|
*/
|
|
function VerificationNote({ checks }: { checks?: CheckOutcome[] }) {
|
|
const all = checks ?? [];
|
|
const ran = all.filter((c) => c.ran);
|
|
const failed = ran.filter((c) => c.exit_code !== 0);
|
|
|
|
if (ran.length > 0) {
|
|
const detail = ran.map((c) => `${c.argv.join(" ")} → exit ${c.exit_code}`).join("\n");
|
|
return (
|
|
<em style={{ color: failed.length ? "#e8b465" : "#6a8ab0" }} title={detail}>
|
|
{` — verified by ${ran.length} check${ran.length === 1 ? "" : "s"}`}
|
|
{failed.length ? ` (${failed.length} non-zero)` : ""}
|
|
</em>
|
|
);
|
|
}
|
|
if (all.length > 0) {
|
|
// Attempted but nothing executed: a broken sandbox, or every command
|
|
// refused. Say so — this is the case that used to read as "verified".
|
|
return (
|
|
<em
|
|
style={{ color: "#ff8a7a" }}
|
|
title={all.map((c) => `${c.argv.join(" ")} → ${c.refused ? "refused" : "could not run"}`).join("\n")}
|
|
>
|
|
{` — could not verify (${all.length} attempted, 0 ran)`}
|
|
</em>
|
|
);
|
|
}
|
|
return (
|
|
<em
|
|
style={{ color: "#8a7a5a" }}
|
|
title="No commands were run — this verdict rests on what the agents reported."
|
|
>
|
|
{" — from agent claims only"}
|
|
</em>
|
|
);
|
|
}
|
|
|
|
/**
|
|
* The completion condition on a phase, plus how the last pass was judged.
|
|
*
|
|
* Renders nothing for phases without a `done_when` — most missions don't have
|
|
* one, and an empty row per phase would be noise.
|
|
*
|
|
* The evaluator's `reason` is deliberately the most prominent thing here — it
|
|
* explains why the phase iterated or stopped, so it is what an operator needs
|
|
* to decide whether the condition is written well. It is *not* what the agents
|
|
* were told: they get a sanitized `guidance` that withholds the acceptance
|
|
* text, so a pass cannot be satisfied by pasting the verdict back.
|
|
*
|
|
* Whether the judge verified anything is shown alongside the verdict — see
|
|
* `VerificationNote`. An operator should never have to guess whether a verdict
|
|
* rests on executed commands or on the agents' own account of themselves.
|
|
*/
|
|
export function PhaseGoalStrip({
|
|
missionId,
|
|
phase,
|
|
}: {
|
|
missionId: string;
|
|
phase: MissionPhase;
|
|
}) {
|
|
const [evals, setEvals] = useState<PhaseEvaluation[]>([]);
|
|
const [expanded, setExpanded] = useState(false);
|
|
|
|
const load = useCallback(async () => {
|
|
try {
|
|
setEvals(await getPhaseEvaluations(missionId, phase.id));
|
|
} catch {
|
|
// A phase that has never been judged has no rows; not an error state.
|
|
}
|
|
}, [missionId, phase.id]);
|
|
|
|
useEffect(() => {
|
|
if (!phase.done_when) return;
|
|
void load();
|
|
// Poll only while there is something to wait for.
|
|
if (phase.status !== "running" && phase.status !== "evaluating") return;
|
|
const t = setInterval(() => void load(), 5000);
|
|
return () => clearInterval(t);
|
|
}, [load, phase.done_when, phase.status]);
|
|
|
|
if (!phase.done_when) return null;
|
|
|
|
const latest = evals[0];
|
|
const pass = phase.iteration + 1;
|
|
const judging = phase.status === "evaluating";
|
|
|
|
return (
|
|
<div
|
|
style={{
|
|
marginTop: 8,
|
|
padding: "8px 10px",
|
|
borderRadius: 10,
|
|
background: "#0d0d10",
|
|
border: "1px solid #1c1c22",
|
|
display: "flex",
|
|
flexDirection: "column",
|
|
gap: 6,
|
|
}}
|
|
>
|
|
<div style={{ display: "flex", alignItems: "center", gap: 6 }}>
|
|
<Target aria-hidden size={12} color="#e8b465" />
|
|
<span
|
|
style={{
|
|
fontSize: 10,
|
|
letterSpacing: 0.5,
|
|
textTransform: "uppercase",
|
|
color: "#6a6a72",
|
|
}}
|
|
>
|
|
Done when
|
|
</span>
|
|
<span style={{ marginLeft: "auto", fontSize: 10, color: "#6a6a72" }}>
|
|
pass {pass} / {phase.max_iterations}
|
|
</span>
|
|
</div>
|
|
|
|
<p style={{ margin: 0, fontSize: 12, color: "#c8c8d0", lineHeight: 1.45 }}>
|
|
{phase.done_when}
|
|
</p>
|
|
|
|
{judging && (
|
|
<span style={{ fontSize: 11, color: "#e8b465" }}>
|
|
Judging this pass…
|
|
</span>
|
|
)}
|
|
|
|
{latest && (
|
|
<div style={{ display: "flex", gap: 6, alignItems: "flex-start" }}>
|
|
<span
|
|
style={{
|
|
fontSize: 11,
|
|
fontWeight: 600,
|
|
color: latest.met ? "#5fd08a" : "#e8b465",
|
|
whiteSpace: "nowrap",
|
|
}}
|
|
>
|
|
{latest.met ? "met" : "not met"}
|
|
</span>
|
|
<span style={{ fontSize: 11, color: "#8a8a92", lineHeight: 1.45 }}>
|
|
{latest.reason}
|
|
{latest.error && (
|
|
// Distinguishes "judged incomplete" from "could not judge" —
|
|
// an evaluator outage should not read as a verdict on the work.
|
|
<em style={{ color: "#ff8a7a" }}> (evaluator error)</em>
|
|
)}
|
|
{!latest.error && <VerificationNote checks={latest.checks} />}
|
|
</span>
|
|
</div>
|
|
)}
|
|
|
|
{evals.length > 1 && (
|
|
<>
|
|
<button
|
|
type="button"
|
|
onClick={() => setExpanded((v) => !v)}
|
|
style={{
|
|
alignSelf: "flex-start",
|
|
background: "none",
|
|
border: "none",
|
|
padding: 0,
|
|
cursor: "pointer",
|
|
fontSize: 10,
|
|
color: "#6a6a72",
|
|
}}
|
|
>
|
|
{expanded ? "hide" : `show all ${evals.length} passes`}
|
|
</button>
|
|
{expanded && (
|
|
<ol
|
|
style={{
|
|
margin: 0,
|
|
paddingLeft: 16,
|
|
display: "flex",
|
|
flexDirection: "column",
|
|
gap: 4,
|
|
}}
|
|
>
|
|
{evals.map((e) => (
|
|
<li
|
|
key={e.iteration}
|
|
style={{ fontSize: 11, color: "#8a8a92", lineHeight: 1.4 }}
|
|
>
|
|
<span
|
|
style={{ color: e.met ? "#5fd08a" : "#e8b465", fontWeight: 600 }}
|
|
>
|
|
pass {e.iteration + 1} {e.met ? "met" : "not met"}
|
|
</span>{" "}
|
|
— {e.reason}
|
|
</li>
|
|
))}
|
|
</ol>
|
|
)}
|
|
</>
|
|
)}
|
|
</div>
|
|
);
|
|
}
|