fix(benchmark): the baseline runner was writing root-owned files into the checkout
Caught by the harness: `benchmark: checkout has multiple writers (uids=0,65532)`.
The previous full run passed that same check, so this was introduced by wiring
`benchmark_runner` into the sweep one commit ago.
`docker_exec` enters a container running as ROOT with the missions root
bind-mounted, and `cargo bench` writes `target/`. Run in the live tree it leaves
root-owned build output in a checkout owned by uid 65532 — the single-writer
invariant broken, and the next phase's cargo hitting permission-denied on a
directory it cannot write.
This is the SAME defect `evaluator_tools::Sandbox` was written for, found by the
same probe, and fixed the same way: benchmark a COPY. `BenchCopy` packs the
checkout through `mission_fs::pack_dir` (so it excludes exactly what the
delivered diff excludes — one exclusion list, now four consumers) into
`<missions_root>/_bench/<mission>`, a SIBLING of the per-mission dirs like
`_verify` and `_outputs`, so a mission reap cannot race a running bench. Removed
on drop, including on error paths.
The operator-triggered path (POST /api/missions/{id}/benchmark) had this bug
from the start and is fixed by the same change — it shares `run`.
Worth naming the pattern: measurement must not mutate what it measures. It
applies to the judge, to the `verifier` subagent that has no Edit or Write, and
now to the benchmark runner.
243 lib tests, zero warnings.
This commit is contained in:
@@ -159,8 +159,21 @@ pub async fn run(
|
|||||||
};
|
};
|
||||||
|
|
||||||
let (container, workdir) = exec_target(pool, mission_id).await?;
|
let (container, workdir) = exec_target(pool, mission_id).await?;
|
||||||
|
// Benchmark a COPY, never the mission's own checkout.
|
||||||
|
//
|
||||||
|
// `docker_exec` enters a container running as ROOT with the missions root
|
||||||
|
// bind-mounted, and `cargo bench` writes `target/`. Run in the live tree, it
|
||||||
|
// leaves root-owned build output in a checkout owned by uid 65532 — the
|
||||||
|
// single-writer invariant broken, and the next phase's cargo hitting
|
||||||
|
// permission-denied on a directory it cannot write.
|
||||||
|
//
|
||||||
|
// This is the SAME defect `evaluator_tools::Sandbox` exists for, found the
|
||||||
|
// same way: the harness's uid probe, reporting `uids=0,65532`. Measurement
|
||||||
|
// must not mutate what it measures — the rule this codebase already applies
|
||||||
|
// to the judge and to the `verifier` subagent.
|
||||||
|
let copy = BenchCopy::of(&workdir, &bench_copy_path(mission_id))?;
|
||||||
let cmd = harness.command();
|
let cmd = harness.command();
|
||||||
let raw = docker_exec(&container, &workdir, &cmd)
|
let raw = docker_exec(&container, copy.workdir(), &cmd)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("exec {cmd:?}: {e}"))?;
|
.map_err(|e| format!("exec {cmd:?}: {e}"))?;
|
||||||
let metrics = parse_output(&raw, &harness);
|
let metrics = parse_output(&raw, &harness);
|
||||||
@@ -236,6 +249,51 @@ async fn phase_config(pool: &PgPool, phase_id: Uuid) -> Result<Value, String> {
|
|||||||
|
|
||||||
/// Post-task-#23: shared runtime container + per-mission working dir.
|
/// Post-task-#23: shared runtime container + per-mission working dir.
|
||||||
/// See security_scan::exec_target for the same convention.
|
/// See security_scan::exec_target for the same convention.
|
||||||
|
/// `<missions_root>/_bench/<mission>` — a sibling of the per-mission dirs, like
|
||||||
|
/// `_verify` and `_outputs`, so reaping a mission never races a running bench.
|
||||||
|
fn bench_copy_path(mission_id: Uuid) -> std::path::PathBuf {
|
||||||
|
crate::mission_workspace::missions_root()
|
||||||
|
.join("_bench")
|
||||||
|
.join(mission_id.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A throwaway copy of the checkout for one benchmark run.
|
||||||
|
///
|
||||||
|
/// Removed on drop, including on the error paths — a `target/` left behind is
|
||||||
|
/// both disk and a stale tree a later run could measure by mistake.
|
||||||
|
struct BenchCopy {
|
||||||
|
root: std::path::PathBuf,
|
||||||
|
workdir: std::path::PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl BenchCopy {
|
||||||
|
fn of(source: &std::path::Path, root: &std::path::Path) -> Result<Self, String> {
|
||||||
|
// Through the transport packer, so the copy carries exactly what the
|
||||||
|
// delivered diff carries: no `target/`, no `node_modules/`. One
|
||||||
|
// exclusion list, now four consumers.
|
||||||
|
let _ = std::fs::remove_dir_all(root);
|
||||||
|
let archive = crate::mission_fs::pack_dir(source, "repo")
|
||||||
|
.map_err(|e| format!("pack checkout for benchmark: {e}"))?;
|
||||||
|
crate::mission_fs::unpack_into(&archive, root)
|
||||||
|
.map_err(|e| format!("unpack benchmark copy: {e}"))?;
|
||||||
|
let workdir = root.join("repo");
|
||||||
|
if !workdir.is_dir() {
|
||||||
|
return Err(format!("benchmark copy missing at {}", workdir.display()));
|
||||||
|
}
|
||||||
|
Ok(Self { root: root.to_path_buf(), workdir })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn workdir(&self) -> &std::path::Path {
|
||||||
|
&self.workdir
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for BenchCopy {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = std::fs::remove_dir_all(&self.root);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn exec_target(
|
async fn exec_target(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
mission_id: Uuid,
|
mission_id: Uuid,
|
||||||
@@ -401,3 +459,29 @@ fn compute_delta(before: &Value, after: &Value) -> Value {
|
|||||||
}
|
}
|
||||||
json!({ "kind": "opaque", "note": "before/after not structurally comparable" })
|
json!({ "kind": "opaque", "note": "before/after not structurally comparable" })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod bench_copy_tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// The benchmark copy must live OUTSIDE the mission directory, and must not
|
||||||
|
/// be the checkout itself.
|
||||||
|
///
|
||||||
|
/// Running `cargo bench` in the live tree left root-owned `target/` in a
|
||||||
|
/// checkout owned by uid 65532 — caught by the harness's uid probe
|
||||||
|
/// (`uids=0,65532`) after this runner was first wired into the sweep. The
|
||||||
|
/// same rule `evaluator_tools::Sandbox` follows: measurement must not mutate
|
||||||
|
/// what it measures.
|
||||||
|
#[test]
|
||||||
|
fn a_benchmark_runs_in_a_copy_outside_the_mission_directory() {
|
||||||
|
let mission = Uuid::now_v7();
|
||||||
|
let copy = bench_copy_path(mission);
|
||||||
|
let live = crate::mission_workspace::checkout_path(mission);
|
||||||
|
assert_ne!(copy, live, "the bench copy must not be the checkout");
|
||||||
|
assert!(
|
||||||
|
!copy.starts_with(crate::mission_workspace::missions_root().join(mission.to_string())),
|
||||||
|
"{copy:?} must be a SIBLING of the mission dir, or the reaper races it"
|
||||||
|
);
|
||||||
|
assert!(copy.starts_with(crate::mission_workspace::missions_root().join("_bench")), "{copy:?}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user