fix(benchmark): the baseline runner was writing root-owned files into the checkout

Caught by the harness: `benchmark: checkout has multiple writers (uids=0,65532)`.
The previous full run passed that same check, so this was introduced by wiring
`benchmark_runner` into the sweep one commit ago.

`docker_exec` enters a container running as ROOT with the missions root
bind-mounted, and `cargo bench` writes `target/`. Run in the live tree it leaves
root-owned build output in a checkout owned by uid 65532 — the single-writer
invariant broken, and the next phase's cargo hitting permission-denied on a
directory it cannot write.

This is the SAME defect `evaluator_tools::Sandbox` was written for, found by the
same probe, and fixed the same way: benchmark a COPY. `BenchCopy` packs the
checkout through `mission_fs::pack_dir` (so it excludes exactly what the
delivered diff excludes — one exclusion list, now four consumers) into
`<missions_root>/_bench/<mission>`, a SIBLING of the per-mission dirs like
`_verify` and `_outputs`, so a mission reap cannot race a running bench. Removed
on drop, including on error paths.

The operator-triggered path (POST /api/missions/{id}/benchmark) had this bug
from the start and is fixed by the same change — it shares `run`.

Worth naming the pattern: measurement must not mutate what it measures. It
applies to the judge, to the `verifier` subagent that has no Edit or Write, and
now to the benchmark runner.

243 lib tests, zero warnings.
This commit is contained in:
Omar Sobh
2026-08-07 17:01:36 -07:00
parent 0d8db7ff0b
commit a93a4111e1
+85 -1
View File
@@ -159,8 +159,21 @@ pub async fn run(
}; };
let (container, workdir) = exec_target(pool, mission_id).await?; let (container, workdir) = exec_target(pool, mission_id).await?;
// Benchmark a COPY, never the mission's own checkout.
//
// `docker_exec` enters a container running as ROOT with the missions root
// bind-mounted, and `cargo bench` writes `target/`. Run in the live tree, it
// leaves root-owned build output in a checkout owned by uid 65532 — the
// single-writer invariant broken, and the next phase's cargo hitting
// permission-denied on a directory it cannot write.
//
// This is the SAME defect `evaluator_tools::Sandbox` exists for, found the
// same way: the harness's uid probe, reporting `uids=0,65532`. Measurement
// must not mutate what it measures — the rule this codebase already applies
// to the judge and to the `verifier` subagent.
let copy = BenchCopy::of(&workdir, &bench_copy_path(mission_id))?;
let cmd = harness.command(); let cmd = harness.command();
let raw = docker_exec(&container, &workdir, &cmd) let raw = docker_exec(&container, copy.workdir(), &cmd)
.await .await
.map_err(|e| format!("exec {cmd:?}: {e}"))?; .map_err(|e| format!("exec {cmd:?}: {e}"))?;
let metrics = parse_output(&raw, &harness); let metrics = parse_output(&raw, &harness);
@@ -236,6 +249,51 @@ async fn phase_config(pool: &PgPool, phase_id: Uuid) -> Result<Value, String> {
/// Post-task-#23: shared runtime container + per-mission working dir. /// Post-task-#23: shared runtime container + per-mission working dir.
/// See security_scan::exec_target for the same convention. /// See security_scan::exec_target for the same convention.
/// `<missions_root>/_bench/<mission>` — a sibling of the per-mission dirs, like
/// `_verify` and `_outputs`, so reaping a mission never races a running bench.
fn bench_copy_path(mission_id: Uuid) -> std::path::PathBuf {
crate::mission_workspace::missions_root()
.join("_bench")
.join(mission_id.to_string())
}
/// A throwaway copy of the checkout for one benchmark run.
///
/// Removed on drop, including on the error paths — a `target/` left behind is
/// both disk and a stale tree a later run could measure by mistake.
struct BenchCopy {
root: std::path::PathBuf,
workdir: std::path::PathBuf,
}
impl BenchCopy {
fn of(source: &std::path::Path, root: &std::path::Path) -> Result<Self, String> {
// Through the transport packer, so the copy carries exactly what the
// delivered diff carries: no `target/`, no `node_modules/`. One
// exclusion list, now four consumers.
let _ = std::fs::remove_dir_all(root);
let archive = crate::mission_fs::pack_dir(source, "repo")
.map_err(|e| format!("pack checkout for benchmark: {e}"))?;
crate::mission_fs::unpack_into(&archive, root)
.map_err(|e| format!("unpack benchmark copy: {e}"))?;
let workdir = root.join("repo");
if !workdir.is_dir() {
return Err(format!("benchmark copy missing at {}", workdir.display()));
}
Ok(Self { root: root.to_path_buf(), workdir })
}
fn workdir(&self) -> &std::path::Path {
&self.workdir
}
}
impl Drop for BenchCopy {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.root);
}
}
async fn exec_target( async fn exec_target(
pool: &PgPool, pool: &PgPool,
mission_id: Uuid, mission_id: Uuid,
@@ -401,3 +459,29 @@ fn compute_delta(before: &Value, after: &Value) -> Value {
} }
json!({ "kind": "opaque", "note": "before/after not structurally comparable" }) json!({ "kind": "opaque", "note": "before/after not structurally comparable" })
} }
#[cfg(test)]
mod bench_copy_tests {
use super::*;
/// The benchmark copy must live OUTSIDE the mission directory, and must not
/// be the checkout itself.
///
/// Running `cargo bench` in the live tree left root-owned `target/` in a
/// checkout owned by uid 65532 — caught by the harness's uid probe
/// (`uids=0,65532`) after this runner was first wired into the sweep. The
/// same rule `evaluator_tools::Sandbox` follows: measurement must not mutate
/// what it measures.
#[test]
fn a_benchmark_runs_in_a_copy_outside_the_mission_directory() {
let mission = Uuid::now_v7();
let copy = bench_copy_path(mission);
let live = crate::mission_workspace::checkout_path(mission);
assert_ne!(copy, live, "the bench copy must not be the checkout");
assert!(
!copy.starts_with(crate::mission_workspace::missions_root().join(mission.to_string())),
"{copy:?} must be a SIBLING of the mission dir, or the reaper races it"
);
assert!(copy.starts_with(crate::mission_workspace::missions_root().join("_bench")), "{copy:?}");
}
}