slice 7: before/after benchmark runner
Executes a benchmark harness inside the mission's team container
and records the resulting metrics as a benchmark_snapshots row keyed
on (phase_id, iteration). Baseline pass (iteration=0) captures
before_metrics; each post-iteration call captures after_metrics +
computes delta vs baseline.
Rust surface:
- cm_db::repo::missions::upsert_benchmark_snapshot / benchmark_snapshots_for
- cm_api::benchmark_runner::{baseline, after_iteration, run}
- Harness enum: Auto | Criterion | CargoBench | VitestBench |
PytestBench | Shell (each with a command() vector)
- Auto detection peeks at the repo layout inside the container
(Cargo.toml → CargoBench, package.json → VitestBench, pyproject
→ PytestBench). Falls back to a Shell echo when nothing
identifiable.
- Bencher-format line parser extracts (name, ns_per_iter,
plusminus) so criterion + `cargo bench` output become structured
samples the canvas can diff.
- compute_delta pairs samples by name, emits {before_ns, after_ns,
delta_pct, direction: improved|regressed}.
API:
- POST /api/missions/{id}/benchmark { phase_id, slot, iteration? }
triggers baseline or after run and returns the mission's full
snapshot list.
- GET /api/missions/{id} now includes `benchmarks[]` in the detail
payload.
Frontend:
- New Benchmarks tab on MissionCanvas with iteration + driver
header, plus a 4-column grid (bench / before / after / Δ%) when
delta samples are present. Improved deltas render green,
regressions red.
- TS types + triggerBenchmark() helper in lib/api/missions.ts.
Wiring notes:
- team_container_for_mission reads teams.zeroclaw_container — that's
populated by topology_worker::try_team_gateway_url on first run,
so trigger baseline AFTER the mission's first phase spawns the
container.
- Not auto-fired yet by phase execution; that's the "template phase
executor" work that spans Slices 4-8. Manual API trigger works
today; automated hook is a follow-up.
Co-Authored-By: Claude Opus 4.7 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
3ac3d53da7
commit
f843c9ddb1
@@ -529,3 +529,97 @@ pub async fn set_pdf_result(
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ── Benchmark snapshots (Slice 7) ────────────────────────────────
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BenchmarkSnapshot {
|
||||
pub id: Uuid,
|
||||
pub mission_id: Uuid,
|
||||
pub phase_id: Uuid,
|
||||
pub iteration: i32,
|
||||
pub before_metrics: Option<Value>,
|
||||
pub after_metrics: Option<Value>,
|
||||
pub delta: Option<Value>,
|
||||
pub driver: Option<String>,
|
||||
#[serde(with = "time::serde::rfc3339")]
|
||||
pub created_at: OffsetDateTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct UpsertBenchmarkSnapshot<'a> {
|
||||
pub mission_id: Uuid,
|
||||
pub phase_id: Uuid,
|
||||
pub iteration: i32,
|
||||
pub before_metrics: Option<&'a Value>,
|
||||
pub after_metrics: Option<&'a Value>,
|
||||
pub delta: Option<&'a Value>,
|
||||
pub driver: Option<&'a str>,
|
||||
}
|
||||
|
||||
/// UPSERT a snapshot keyed on (phase_id, iteration). Baseline pass
|
||||
/// uses iteration=0 with before_metrics only; each post-iteration
|
||||
/// call updates the same row with after_metrics + delta so the pair
|
||||
/// stays coherent for the canvas's side-by-side render.
|
||||
pub async fn upsert_benchmark_snapshot(
|
||||
pool: &PgPool,
|
||||
s: UpsertBenchmarkSnapshot<'_>,
|
||||
) -> Result<Uuid, DbError> {
|
||||
use sqlx::Row;
|
||||
let row = sqlx::query(
|
||||
"INSERT INTO benchmark_snapshots
|
||||
(id, mission_id, phase_id, iteration,
|
||||
before_metrics, after_metrics, delta, driver)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8)
|
||||
ON CONFLICT (phase_id, iteration) DO UPDATE SET
|
||||
-- Preserve non-NULL prior values so a baseline call
|
||||
-- doesn't wipe the after_metrics from a previous run.
|
||||
before_metrics = COALESCE(EXCLUDED.before_metrics, benchmark_snapshots.before_metrics),
|
||||
after_metrics = COALESCE(EXCLUDED.after_metrics, benchmark_snapshots.after_metrics),
|
||||
delta = COALESCE(EXCLUDED.delta, benchmark_snapshots.delta),
|
||||
driver = COALESCE(EXCLUDED.driver, benchmark_snapshots.driver)
|
||||
RETURNING id",
|
||||
)
|
||||
.bind(Uuid::now_v7())
|
||||
.bind(s.mission_id)
|
||||
.bind(s.phase_id)
|
||||
.bind(s.iteration)
|
||||
.bind(s.before_metrics)
|
||||
.bind(s.after_metrics)
|
||||
.bind(s.delta)
|
||||
.bind(s.driver)
|
||||
.fetch_one(pool)
|
||||
.await?;
|
||||
Ok(row.get("id"))
|
||||
}
|
||||
|
||||
pub async fn benchmark_snapshots_for(
|
||||
pool: &PgPool,
|
||||
mission_id: Uuid,
|
||||
) -> Result<Vec<BenchmarkSnapshot>, DbError> {
|
||||
use sqlx::Row;
|
||||
let rows = sqlx::query(
|
||||
"SELECT id, mission_id, phase_id, iteration,
|
||||
before_metrics, after_metrics, delta, driver, created_at
|
||||
FROM benchmark_snapshots
|
||||
WHERE mission_id = $1
|
||||
ORDER BY phase_id, iteration",
|
||||
)
|
||||
.bind(mission_id)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|r| BenchmarkSnapshot {
|
||||
id: r.get("id"),
|
||||
mission_id: r.get("mission_id"),
|
||||
phase_id: r.get("phase_id"),
|
||||
iteration: r.get("iteration"),
|
||||
before_metrics: r.get("before_metrics"),
|
||||
after_metrics: r.get("after_metrics"),
|
||||
delta: r.get("delta"),
|
||||
driver: r.get("driver"),
|
||||
created_at: r.get("created_at"),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user