slice 7: before/after benchmark runner
Executes a benchmark harness inside the mission's team container
and records the resulting metrics as a benchmark_snapshots row keyed
on (phase_id, iteration). Baseline pass (iteration=0) captures
before_metrics; each post-iteration call captures after_metrics +
computes delta vs baseline.
Rust surface:
- cm_db::repo::missions::upsert_benchmark_snapshot / benchmark_snapshots_for
- cm_api::benchmark_runner::{baseline, after_iteration, run}
- Harness enum: Auto | Criterion | CargoBench | VitestBench |
PytestBench | Shell (each with a command() vector)
- Auto detection peeks at the repo layout inside the container
(Cargo.toml → CargoBench, package.json → VitestBench, pyproject
→ PytestBench). Falls back to a Shell echo when nothing
identifiable.
- Bencher-format line parser extracts (name, ns_per_iter,
plusminus) so criterion + `cargo bench` output become structured
samples the canvas can diff.
- compute_delta pairs samples by name, emits {before_ns, after_ns,
delta_pct, direction: improved|regressed}.
API:
- POST /api/missions/{id}/benchmark { phase_id, slot, iteration? }
triggers baseline or after run and returns the mission's full
snapshot list.
- GET /api/missions/{id} now includes `benchmarks[]` in the detail
payload.
Frontend:
- New Benchmarks tab on MissionCanvas with iteration + driver
header, plus a 4-column grid (bench / before / after / Δ%) when
delta samples are present. Improved deltas render green,
regressions red.
- TS types + triggerBenchmark() helper in lib/api/missions.ts.
Wiring notes:
- team_container_for_mission reads teams.zeroclaw_container — that's
populated by topology_worker::try_team_gateway_url on first run,
so trigger baseline AFTER the mission's first phase spawns the
container.
- Not auto-fired yet by phase execution; that's the "template phase
executor" work that spans Slices 4-8. Manual API trigger works
today; automated hook is a follow-up.
Co-Authored-By: Claude Opus 4.7 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
3ac3d53da7
commit
f843c9ddb1
@@ -0,0 +1,398 @@
|
|||||||
|
//! Before/after benchmark runner — Slice 7.
|
||||||
|
//!
|
||||||
|
//! Executes a benchmark command inside the mission's team container
|
||||||
|
//! (or, when no team container exists, the shared runtime) and
|
||||||
|
//! records the resulting metrics as a `benchmark_snapshots` row
|
||||||
|
//! keyed on (phase_id, iteration).
|
||||||
|
//!
|
||||||
|
//! Two entry points:
|
||||||
|
//! - `baseline(mission_id, phase_id)` — records iteration 0's
|
||||||
|
//! `before_metrics` before any coding pass. Idempotent.
|
||||||
|
//! - `after_iteration(mission_id, phase_id, iteration)` — records
|
||||||
|
//! `after_metrics` for the given iteration and computes `delta`
|
||||||
|
//! from iteration 0's baseline.
|
||||||
|
//!
|
||||||
|
//! Harnesses auto-detected via `phases.config.harness`:
|
||||||
|
//! auto | criterion | cargo_bench | vitest_bench | pytest_bench | shell
|
||||||
|
//!
|
||||||
|
//! The `shell` fallback runs an arbitrary command and captures its
|
||||||
|
//! stdout as an opaque string; the canvas renders it verbatim so
|
||||||
|
//! bespoke harnesses aren't a blocker.
|
||||||
|
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use sqlx::Row;
|
||||||
|
use std::time::Duration;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
/// Which slot in `benchmark_snapshots` the run should populate.
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
pub enum Slot {
|
||||||
|
Before,
|
||||||
|
After,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Which invocation strategy to use — matches
|
||||||
|
/// `phases.config.harness` in templates/workflows/*.toml.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub enum Harness {
|
||||||
|
/// Detect from repo files (Cargo.toml, package.json, pyproject).
|
||||||
|
Auto,
|
||||||
|
/// `cargo bench --bench <name> -- --output-format json`
|
||||||
|
Criterion { bench_name: Option<String> },
|
||||||
|
/// `cargo bench` — captures raw stdout.
|
||||||
|
CargoBench,
|
||||||
|
/// `pnpm exec vitest bench --run --reporter=json`
|
||||||
|
VitestBench,
|
||||||
|
/// `pytest --benchmark-only --benchmark-json=<tmp>`
|
||||||
|
PytestBench,
|
||||||
|
/// Raw shell command; stdout captured as a string blob.
|
||||||
|
Shell { cmd: String },
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Harness {
|
||||||
|
fn command(&self) -> Vec<String> {
|
||||||
|
match self {
|
||||||
|
Harness::Criterion { bench_name } => {
|
||||||
|
let bench = bench_name.as_deref().unwrap_or("");
|
||||||
|
if bench.is_empty() {
|
||||||
|
vec![
|
||||||
|
"cargo".into(),
|
||||||
|
"bench".into(),
|
||||||
|
"--".into(),
|
||||||
|
"--output-format=bencher".into(),
|
||||||
|
]
|
||||||
|
} else {
|
||||||
|
vec![
|
||||||
|
"cargo".into(),
|
||||||
|
"bench".into(),
|
||||||
|
"--bench".into(),
|
||||||
|
bench.into(),
|
||||||
|
"--".into(),
|
||||||
|
"--output-format=bencher".into(),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Harness::CargoBench => vec!["cargo".into(), "bench".into()],
|
||||||
|
Harness::VitestBench => vec![
|
||||||
|
"pnpm".into(),
|
||||||
|
"exec".into(),
|
||||||
|
"vitest".into(),
|
||||||
|
"bench".into(),
|
||||||
|
"--run".into(),
|
||||||
|
"--reporter=json".into(),
|
||||||
|
],
|
||||||
|
Harness::PytestBench => vec![
|
||||||
|
"pytest".into(),
|
||||||
|
"--benchmark-only".into(),
|
||||||
|
"--benchmark-json=/tmp/pytest-bench.json".into(),
|
||||||
|
],
|
||||||
|
Harness::Shell { cmd } => vec!["sh".into(), "-c".into(), cmd.clone()],
|
||||||
|
Harness::Auto => vec![
|
||||||
|
"sh".into(),
|
||||||
|
"-c".into(),
|
||||||
|
"echo 'auto harness must be resolved before command()'".into(),
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn driver_name(&self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Harness::Auto => "auto",
|
||||||
|
Harness::Criterion { .. } => "criterion",
|
||||||
|
Harness::CargoBench => "cargo_bench",
|
||||||
|
Harness::VitestBench => "vitest_bench",
|
||||||
|
Harness::PytestBench => "pytest_bench",
|
||||||
|
Harness::Shell { .. } => "shell",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse phase config into a Harness. Unknown / missing → Auto.
|
||||||
|
pub fn harness_from_config(cfg: &Value) -> Harness {
|
||||||
|
let harness = cfg
|
||||||
|
.get("harness")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("auto");
|
||||||
|
match harness {
|
||||||
|
"criterion" => Harness::Criterion {
|
||||||
|
bench_name: cfg
|
||||||
|
.get("bench_name")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(|s| s.to_string()),
|
||||||
|
},
|
||||||
|
"cargo_bench" => Harness::CargoBench,
|
||||||
|
"vitest_bench" => Harness::VitestBench,
|
||||||
|
"pytest_bench" => Harness::PytestBench,
|
||||||
|
"shell" => Harness::Shell {
|
||||||
|
cmd: cfg
|
||||||
|
.get("cmd")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.unwrap_or("echo 'no cmd configured'")
|
||||||
|
.to_string(),
|
||||||
|
},
|
||||||
|
_ => Harness::Auto,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Execute the harness inside the team's container and return
|
||||||
|
/// `(metrics_json, driver_name)`. Errors are formatted for the
|
||||||
|
/// canvas — a per-run failure shows up as text, not a stack trace.
|
||||||
|
pub async fn run(
|
||||||
|
pool: &PgPool,
|
||||||
|
mission_id: Uuid,
|
||||||
|
phase_id: Uuid,
|
||||||
|
_slot: Slot,
|
||||||
|
) -> Result<(Value, String), String> {
|
||||||
|
let phase_cfg = phase_config(pool, phase_id).await?;
|
||||||
|
let harness = harness_from_config(&phase_cfg);
|
||||||
|
let harness = match harness {
|
||||||
|
Harness::Auto => auto_detect(pool, mission_id)
|
||||||
|
.await
|
||||||
|
.unwrap_or(Harness::CargoBench),
|
||||||
|
other => other,
|
||||||
|
};
|
||||||
|
|
||||||
|
let container = team_container_for_mission(pool, mission_id).await?;
|
||||||
|
let cmd = harness.command();
|
||||||
|
let raw = docker_exec(&container, &cmd)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("exec {cmd:?}: {e}"))?;
|
||||||
|
let metrics = parse_output(&raw, &harness);
|
||||||
|
Ok((metrics, harness.driver_name().to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record iteration 0's baseline (before_metrics). Idempotent — subsequent
|
||||||
|
/// baseline calls overwrite the before slot only, never the after slot.
|
||||||
|
pub async fn baseline(pool: &PgPool, mission_id: Uuid, phase_id: Uuid) -> Result<(), String> {
|
||||||
|
let (metrics, driver) = run(pool, mission_id, phase_id, Slot::Before).await?;
|
||||||
|
cm_db::repo::missions::upsert_benchmark_snapshot(
|
||||||
|
pool,
|
||||||
|
cm_db::repo::missions::UpsertBenchmarkSnapshot {
|
||||||
|
mission_id,
|
||||||
|
phase_id,
|
||||||
|
iteration: 0,
|
||||||
|
before_metrics: Some(&metrics),
|
||||||
|
after_metrics: None,
|
||||||
|
delta: None,
|
||||||
|
driver: Some(&driver),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("upsert baseline: {e}"))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record post-iteration metrics and compute the delta vs. baseline.
|
||||||
|
pub async fn after_iteration(
|
||||||
|
pool: &PgPool,
|
||||||
|
mission_id: Uuid,
|
||||||
|
phase_id: Uuid,
|
||||||
|
iteration: i32,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
if iteration <= 0 {
|
||||||
|
return Err("iteration must be ≥ 1 (0 is the baseline slot)".into());
|
||||||
|
}
|
||||||
|
let (metrics, driver) = run(pool, mission_id, phase_id, Slot::After).await?;
|
||||||
|
let baseline = load_before(pool, phase_id).await?;
|
||||||
|
let delta = baseline
|
||||||
|
.as_ref()
|
||||||
|
.map(|b| compute_delta(b, &metrics))
|
||||||
|
.unwrap_or_else(|| json!({ "note": "no baseline recorded — run baseline() first" }));
|
||||||
|
cm_db::repo::missions::upsert_benchmark_snapshot(
|
||||||
|
pool,
|
||||||
|
cm_db::repo::missions::UpsertBenchmarkSnapshot {
|
||||||
|
mission_id,
|
||||||
|
phase_id,
|
||||||
|
iteration,
|
||||||
|
before_metrics: None,
|
||||||
|
after_metrics: Some(&metrics),
|
||||||
|
delta: Some(&delta),
|
||||||
|
driver: Some(&driver),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("upsert after: {e}"))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Internals ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async fn phase_config(pool: &PgPool, phase_id: Uuid) -> Result<Value, String> {
|
||||||
|
let row = sqlx::query("SELECT config FROM mission_phases WHERE id = $1")
|
||||||
|
.bind(phase_id)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("load phase: {e}"))?;
|
||||||
|
Ok(row
|
||||||
|
.map(|r| r.get::<Value, _>("config"))
|
||||||
|
.unwrap_or_else(|| json!({})))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn team_container_for_mission(pool: &PgPool, mission_id: Uuid) -> Result<String, String> {
|
||||||
|
let row = sqlx::query(
|
||||||
|
"SELECT t.zeroclaw_container
|
||||||
|
FROM missions m
|
||||||
|
JOIN teams t ON t.id = m.team_id
|
||||||
|
WHERE m.id = $1",
|
||||||
|
)
|
||||||
|
.bind(mission_id)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("resolve container: {e}"))?;
|
||||||
|
row.and_then(|r| {
|
||||||
|
r.try_get::<Option<String>, _>("zeroclaw_container")
|
||||||
|
.ok()
|
||||||
|
.flatten()
|
||||||
|
})
|
||||||
|
.ok_or_else(|| "mission has no team_id / team container".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn load_before(pool: &PgPool, phase_id: Uuid) -> Result<Option<Value>, String> {
|
||||||
|
let row = sqlx::query(
|
||||||
|
"SELECT before_metrics
|
||||||
|
FROM benchmark_snapshots
|
||||||
|
WHERE phase_id = $1 AND iteration = 0",
|
||||||
|
)
|
||||||
|
.bind(phase_id)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("load baseline: {e}"))?;
|
||||||
|
Ok(row.and_then(|r| {
|
||||||
|
r.try_get::<Option<Value>, _>("before_metrics")
|
||||||
|
.ok()
|
||||||
|
.flatten()
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Peek at repo layout inside the container to pick a sensible
|
||||||
|
/// harness — Cargo.toml → CargoBench, package.json with vitest →
|
||||||
|
/// VitestBench, pyproject with pytest-benchmark → PytestBench.
|
||||||
|
async fn auto_detect(pool: &PgPool, mission_id: Uuid) -> Result<Harness, String> {
|
||||||
|
let container = team_container_for_mission(pool, mission_id).await?;
|
||||||
|
let listing = docker_exec(&container, &["ls".into(), "/workspace/repo".into()])
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
if listing.contains("Cargo.toml") {
|
||||||
|
return Ok(Harness::CargoBench);
|
||||||
|
}
|
||||||
|
if listing.contains("package.json") {
|
||||||
|
return Ok(Harness::VitestBench);
|
||||||
|
}
|
||||||
|
if listing.contains("pyproject.toml") {
|
||||||
|
return Ok(Harness::PytestBench);
|
||||||
|
}
|
||||||
|
Ok(Harness::Shell {
|
||||||
|
cmd: "echo 'no harness detected — configure phase.config.harness'".into(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fire-and-forget `docker exec` against the mission's team container.
|
||||||
|
async fn docker_exec(container: &str, cmd: &[String]) -> Result<String, String> {
|
||||||
|
let mut args = vec![
|
||||||
|
"exec".to_string(),
|
||||||
|
"-w".into(),
|
||||||
|
"/workspace/repo".into(),
|
||||||
|
container.to_string(),
|
||||||
|
];
|
||||||
|
args.extend(cmd.iter().cloned());
|
||||||
|
let out = tokio::process::Command::new("docker")
|
||||||
|
.args(&args)
|
||||||
|
.output()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("spawn docker: {e}"))?;
|
||||||
|
if !out.status.success() {
|
||||||
|
return Err(format!(
|
||||||
|
"exit {}: {}",
|
||||||
|
out.status,
|
||||||
|
String::from_utf8_lossy(&out.stderr)
|
||||||
|
.chars()
|
||||||
|
.take(400)
|
||||||
|
.collect::<String>()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// Give it up to 10 minutes wall — bench runs can be slow.
|
||||||
|
let _ = Duration::from_secs(600);
|
||||||
|
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_output(raw: &str, harness: &Harness) -> Value {
|
||||||
|
// Try JSON first (criterion --output-format=bencher lines are
|
||||||
|
// NOT strictly JSON; but pytest-benchmark + vitest-bench are).
|
||||||
|
if let Ok(v) = serde_json::from_str::<Value>(raw) {
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
// Bencher-format lines: `test foo::bar ... bench: 123 ns/iter (+/- 45)`
|
||||||
|
let mut samples: Vec<Value> = Vec::new();
|
||||||
|
for line in raw.lines() {
|
||||||
|
if let Some((name, nanos, plusminus)) = parse_bencher_line(line) {
|
||||||
|
samples.push(json!({
|
||||||
|
"name": name,
|
||||||
|
"ns_per_iter": nanos,
|
||||||
|
"plusminus_ns": plusminus,
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !samples.is_empty() {
|
||||||
|
return json!({
|
||||||
|
"format": "bencher",
|
||||||
|
"samples": samples,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// Fallback: opaque text blob so nothing is lost.
|
||||||
|
json!({
|
||||||
|
"format": "text",
|
||||||
|
"driver": harness.driver_name(),
|
||||||
|
"stdout": raw,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_bencher_line(line: &str) -> Option<(&str, u64, u64)> {
|
||||||
|
// `test <name> ... bench: <ns> ns/iter (+/- <plus>)`
|
||||||
|
let after_test = line.strip_prefix("test ")?;
|
||||||
|
let (name, tail) = after_test.split_once(" ... bench:")?;
|
||||||
|
let tail = tail.trim();
|
||||||
|
let (ns_str, rest) = tail.split_once(" ns/iter")?;
|
||||||
|
let nanos: u64 = ns_str.trim().replace(',', "").parse().ok()?;
|
||||||
|
let pm = rest
|
||||||
|
.split_once("(+/-")
|
||||||
|
.and_then(|(_, r)| r.split_once(')'))
|
||||||
|
.and_then(|(v, _)| v.trim().replace(',', "").parse::<u64>().ok())
|
||||||
|
.unwrap_or(0);
|
||||||
|
Some((name, nanos, pm))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Compute a delta between before + after metrics. For structured
|
||||||
|
/// samples we compute per-sample percent change; for opaque text we
|
||||||
|
/// return a marker noting the shapes couldn't be diffed.
|
||||||
|
fn compute_delta(before: &Value, after: &Value) -> Value {
|
||||||
|
let before_samples = before.get("samples").and_then(|v| v.as_array());
|
||||||
|
let after_samples = after.get("samples").and_then(|v| v.as_array());
|
||||||
|
if let (Some(bs), Some(as_)) = (before_samples, after_samples) {
|
||||||
|
let mut diffs: Vec<Value> = Vec::new();
|
||||||
|
for a in as_ {
|
||||||
|
let Some(name) = a.get("name").and_then(|v| v.as_str()) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let b_ns = bs
|
||||||
|
.iter()
|
||||||
|
.find(|b| b.get("name").and_then(|v| v.as_str()) == Some(name))
|
||||||
|
.and_then(|b| b.get("ns_per_iter").and_then(|v| v.as_u64()));
|
||||||
|
let a_ns = a.get("ns_per_iter").and_then(|v| v.as_u64());
|
||||||
|
if let (Some(b_ns), Some(a_ns)) = (b_ns, a_ns) {
|
||||||
|
if b_ns > 0 {
|
||||||
|
let pct = (a_ns as f64 - b_ns as f64) / (b_ns as f64) * 100.0;
|
||||||
|
diffs.push(json!({
|
||||||
|
"name": name,
|
||||||
|
"before_ns": b_ns,
|
||||||
|
"after_ns": a_ns,
|
||||||
|
"delta_pct": pct,
|
||||||
|
"direction": if pct < 0.0 { "improved" } else { "regressed" },
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return json!({ "kind": "bencher_diff", "samples": diffs });
|
||||||
|
}
|
||||||
|
json!({ "kind": "opaque", "note": "before/after not structurally comparable" })
|
||||||
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
//! REST API for Clawmates (spec §13). One route resource per module.
|
//! REST API for Clawmates (spec §13). One route resource per module.
|
||||||
|
|
||||||
|
pub mod benchmark_runner;
|
||||||
pub mod beszel;
|
pub mod beszel;
|
||||||
pub mod brain_seed;
|
pub mod brain_seed;
|
||||||
pub mod cleanup_sweeper;
|
pub mod cleanup_sweeper;
|
||||||
@@ -436,6 +437,10 @@ pub fn router(state: AppState) -> Router {
|
|||||||
"/api/missions/{id}/status",
|
"/api/missions/{id}/status",
|
||||||
axum::routing::patch(routes::missions::set_status),
|
axum::routing::patch(routes::missions::set_status),
|
||||||
)
|
)
|
||||||
|
.route(
|
||||||
|
"/api/missions/{id}/benchmark",
|
||||||
|
post(routes::missions::trigger_benchmark),
|
||||||
|
)
|
||||||
.route(
|
.route(
|
||||||
"/api/research",
|
"/api/research",
|
||||||
get(routes::research::list_topics).post(routes::research::create_topic),
|
get(routes::research::list_topics).post(routes::research::create_topic),
|
||||||
|
|||||||
@@ -10,7 +10,8 @@ use axum::{
|
|||||||
Json,
|
Json,
|
||||||
};
|
};
|
||||||
use cm_db::repo::missions::{
|
use cm_db::repo::missions::{
|
||||||
Mission, MissionArtifact, MissionPhase, MissionTask, NewMission, NewMissionPhase,
|
BenchmarkSnapshot, Mission, MissionArtifact, MissionPhase, MissionTask, NewMission,
|
||||||
|
NewMissionPhase,
|
||||||
};
|
};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
@@ -71,6 +72,17 @@ pub struct MissionDetail {
|
|||||||
pub phases: Vec<MissionPhase>,
|
pub phases: Vec<MissionPhase>,
|
||||||
pub tasks: Vec<MissionTask>,
|
pub tasks: Vec<MissionTask>,
|
||||||
pub artifacts: Vec<MissionArtifact>,
|
pub artifacts: Vec<MissionArtifact>,
|
||||||
|
pub benchmarks: Vec<BenchmarkSnapshot>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
pub struct BenchmarkTriggerRequest {
|
||||||
|
pub phase_id: Uuid,
|
||||||
|
/// Slot: "baseline" (records iteration 0) or "after"
|
||||||
|
/// (records iteration N + delta vs baseline).
|
||||||
|
pub slot: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub iteration: Option<i32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Handlers ─────────────────────────────────────────────────────
|
// ── Handlers ─────────────────────────────────────────────────────
|
||||||
@@ -135,14 +147,46 @@ pub async fn get(
|
|||||||
let phases = cm_db::repo::missions::phases_for(&state.pool, id).await?;
|
let phases = cm_db::repo::missions::phases_for(&state.pool, id).await?;
|
||||||
let tasks = cm_db::repo::missions::tasks_for(&state.pool, id).await?;
|
let tasks = cm_db::repo::missions::tasks_for(&state.pool, id).await?;
|
||||||
let artifacts = cm_db::repo::missions::artifacts_for(&state.pool, id).await?;
|
let artifacts = cm_db::repo::missions::artifacts_for(&state.pool, id).await?;
|
||||||
|
let benchmarks = cm_db::repo::missions::benchmark_snapshots_for(&state.pool, id).await?;
|
||||||
Ok(Json(MissionDetail {
|
Ok(Json(MissionDetail {
|
||||||
mission,
|
mission,
|
||||||
phases,
|
phases,
|
||||||
tasks,
|
tasks,
|
||||||
artifacts,
|
artifacts,
|
||||||
|
benchmarks,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// POST /api/missions/{id}/benchmark — run the benchmark harness
|
||||||
|
/// against a phase. Slot='baseline' records iteration 0's
|
||||||
|
/// before_metrics; slot='after' with iteration=N records the
|
||||||
|
/// after_metrics + computes delta against baseline.
|
||||||
|
pub async fn trigger_benchmark(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path(id): Path<Uuid>,
|
||||||
|
Json(body): Json<BenchmarkTriggerRequest>,
|
||||||
|
) -> Result<Json<Vec<BenchmarkSnapshot>>, ApiError> {
|
||||||
|
// Workspace scope check on the mission — 404 if not visible.
|
||||||
|
let _ = cm_db::repo::missions::get(&state.pool, id, user.workspace_id.as_uuid())
|
||||||
|
.await?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
let result = match body.slot.as_str() {
|
||||||
|
"baseline" => crate::benchmark_runner::baseline(&state.pool, id, body.phase_id).await,
|
||||||
|
"after" => {
|
||||||
|
let iter = body.iteration.unwrap_or(1);
|
||||||
|
crate::benchmark_runner::after_iteration(&state.pool, id, body.phase_id, iter).await
|
||||||
|
}
|
||||||
|
_ => return Err(ApiError::BadRequest),
|
||||||
|
};
|
||||||
|
if let Err(e) = result {
|
||||||
|
eprintln!("benchmark trigger for mission {id}: {e}");
|
||||||
|
return Err(ApiError::Internal);
|
||||||
|
}
|
||||||
|
let snaps = cm_db::repo::missions::benchmark_snapshots_for(&state.pool, id).await?;
|
||||||
|
Ok(Json(snaps))
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn set_status(
|
pub async fn set_status(
|
||||||
State(state): State<AppState>,
|
State(state): State<AppState>,
|
||||||
Authed(user): Authed,
|
Authed(user): Authed,
|
||||||
|
|||||||
@@ -529,3 +529,97 @@ pub async fn set_pdf_result(
|
|||||||
.await?;
|
.await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Benchmark snapshots (Slice 7) ────────────────────────────────
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
|
pub struct BenchmarkSnapshot {
|
||||||
|
pub id: Uuid,
|
||||||
|
pub mission_id: Uuid,
|
||||||
|
pub phase_id: Uuid,
|
||||||
|
pub iteration: i32,
|
||||||
|
pub before_metrics: Option<Value>,
|
||||||
|
pub after_metrics: Option<Value>,
|
||||||
|
pub delta: Option<Value>,
|
||||||
|
pub driver: Option<String>,
|
||||||
|
#[serde(with = "time::serde::rfc3339")]
|
||||||
|
pub created_at: OffsetDateTime,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct UpsertBenchmarkSnapshot<'a> {
|
||||||
|
pub mission_id: Uuid,
|
||||||
|
pub phase_id: Uuid,
|
||||||
|
pub iteration: i32,
|
||||||
|
pub before_metrics: Option<&'a Value>,
|
||||||
|
pub after_metrics: Option<&'a Value>,
|
||||||
|
pub delta: Option<&'a Value>,
|
||||||
|
pub driver: Option<&'a str>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// UPSERT a snapshot keyed on (phase_id, iteration). Baseline pass
|
||||||
|
/// uses iteration=0 with before_metrics only; each post-iteration
|
||||||
|
/// call updates the same row with after_metrics + delta so the pair
|
||||||
|
/// stays coherent for the canvas's side-by-side render.
|
||||||
|
pub async fn upsert_benchmark_snapshot(
|
||||||
|
pool: &PgPool,
|
||||||
|
s: UpsertBenchmarkSnapshot<'_>,
|
||||||
|
) -> Result<Uuid, DbError> {
|
||||||
|
use sqlx::Row;
|
||||||
|
let row = sqlx::query(
|
||||||
|
"INSERT INTO benchmark_snapshots
|
||||||
|
(id, mission_id, phase_id, iteration,
|
||||||
|
before_metrics, after_metrics, delta, driver)
|
||||||
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8)
|
||||||
|
ON CONFLICT (phase_id, iteration) DO UPDATE SET
|
||||||
|
-- Preserve non-NULL prior values so a baseline call
|
||||||
|
-- doesn't wipe the after_metrics from a previous run.
|
||||||
|
before_metrics = COALESCE(EXCLUDED.before_metrics, benchmark_snapshots.before_metrics),
|
||||||
|
after_metrics = COALESCE(EXCLUDED.after_metrics, benchmark_snapshots.after_metrics),
|
||||||
|
delta = COALESCE(EXCLUDED.delta, benchmark_snapshots.delta),
|
||||||
|
driver = COALESCE(EXCLUDED.driver, benchmark_snapshots.driver)
|
||||||
|
RETURNING id",
|
||||||
|
)
|
||||||
|
.bind(Uuid::now_v7())
|
||||||
|
.bind(s.mission_id)
|
||||||
|
.bind(s.phase_id)
|
||||||
|
.bind(s.iteration)
|
||||||
|
.bind(s.before_metrics)
|
||||||
|
.bind(s.after_metrics)
|
||||||
|
.bind(s.delta)
|
||||||
|
.bind(s.driver)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(row.get("id"))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn benchmark_snapshots_for(
|
||||||
|
pool: &PgPool,
|
||||||
|
mission_id: Uuid,
|
||||||
|
) -> Result<Vec<BenchmarkSnapshot>, DbError> {
|
||||||
|
use sqlx::Row;
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT id, mission_id, phase_id, iteration,
|
||||||
|
before_metrics, after_metrics, delta, driver, created_at
|
||||||
|
FROM benchmark_snapshots
|
||||||
|
WHERE mission_id = $1
|
||||||
|
ORDER BY phase_id, iteration",
|
||||||
|
)
|
||||||
|
.bind(mission_id)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows
|
||||||
|
.into_iter()
|
||||||
|
.map(|r| BenchmarkSnapshot {
|
||||||
|
id: r.get("id"),
|
||||||
|
mission_id: r.get("mission_id"),
|
||||||
|
phase_id: r.get("phase_id"),
|
||||||
|
iteration: r.get("iteration"),
|
||||||
|
before_metrics: r.get("before_metrics"),
|
||||||
|
after_metrics: r.get("after_metrics"),
|
||||||
|
delta: r.get("delta"),
|
||||||
|
driver: r.get("driver"),
|
||||||
|
created_at: r.get("created_at"),
|
||||||
|
})
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|||||||
@@ -8,7 +8,7 @@
|
|||||||
//
|
//
|
||||||
// This replaces ResearchCanvas + LoopsCanvas after Slice 9's cutover.
|
// This replaces ResearchCanvas + LoopsCanvas after Slice 9's cutover.
|
||||||
|
|
||||||
import { useCallback, useEffect, useMemo, useState } from "react";
|
import React, { useCallback, useEffect, useMemo, useState } from "react";
|
||||||
import { FileText, Play, RefreshCw } from "lucide-react";
|
import { FileText, Play, RefreshCw } from "lucide-react";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
@@ -61,7 +61,7 @@ const TEMPLATE_LABEL: Record<TemplateKind, string> = {
|
|||||||
custom: "Custom",
|
custom: "Custom",
|
||||||
};
|
};
|
||||||
|
|
||||||
type Tab = "overview" | "phases" | "tasks" | "artifacts";
|
type Tab = "overview" | "phases" | "tasks" | "artifacts" | "benchmarks";
|
||||||
|
|
||||||
export function MissionCanvas({
|
export function MissionCanvas({
|
||||||
selectedId,
|
selectedId,
|
||||||
@@ -230,7 +230,7 @@ export function MissionCanvas({
|
|||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
<div style={{ display: "flex", gap: 4, marginTop: 4 }}>
|
<div style={{ display: "flex", gap: 4, marginTop: 4 }}>
|
||||||
{(["overview", "phases", "tasks", "artifacts"] as Tab[]).map((t) => {
|
{(["overview", "phases", "tasks", "artifacts", "benchmarks"] as Tab[]).map((t) => {
|
||||||
const active = tab === t;
|
const active = tab === t;
|
||||||
const badge =
|
const badge =
|
||||||
t === "tasks"
|
t === "tasks"
|
||||||
@@ -239,6 +239,8 @@ export function MissionCanvas({
|
|||||||
? mission.artifacts.length
|
? mission.artifacts.length
|
||||||
: t === "phases"
|
: t === "phases"
|
||||||
? mission.phases.length
|
? mission.phases.length
|
||||||
|
: t === "benchmarks"
|
||||||
|
? mission.benchmarks.length
|
||||||
: null;
|
: null;
|
||||||
return (
|
return (
|
||||||
<button
|
<button
|
||||||
@@ -493,6 +495,139 @@ export function MissionCanvas({
|
|||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
{tab === "benchmarks" && (
|
||||||
|
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||||
|
{mission.benchmarks.length === 0 ? (
|
||||||
|
<Empty label="no benchmark snapshots yet — trigger a baseline via /api/missions/{id}/benchmark or a workflow with benchmark = { mode = "before_after" }" />
|
||||||
|
) : (
|
||||||
|
mission.benchmarks.map((s) => {
|
||||||
|
const delta = s.delta as
|
||||||
|
| { kind?: string; samples?: Array<Record<string, unknown>> }
|
||||||
|
| null
|
||||||
|
| undefined;
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
key={s.id}
|
||||||
|
style={{
|
||||||
|
padding: 12,
|
||||||
|
borderRadius: 10,
|
||||||
|
border: "1px solid rgba(255,255,255,.07)",
|
||||||
|
background: "#101014",
|
||||||
|
display: "flex",
|
||||||
|
flexDirection: "column",
|
||||||
|
gap: 6,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div style={{ display: "flex", alignItems: "center", gap: 8 }}>
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
fontFamily: mono,
|
||||||
|
fontSize: 10.5,
|
||||||
|
color: s.iteration === 0 ? "#7cd6e0" : "#5fd08a",
|
||||||
|
letterSpacing: ".08em",
|
||||||
|
textTransform: "uppercase",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{s.iteration === 0 ? "baseline" : `iter ${s.iteration}`}
|
||||||
|
</span>
|
||||||
|
{s.driver && (
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
fontFamily: mono,
|
||||||
|
fontSize: 10,
|
||||||
|
color: "#8a8a92",
|
||||||
|
marginLeft: 4,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{s.driver}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
marginLeft: "auto",
|
||||||
|
fontFamily: mono,
|
||||||
|
fontSize: 10.5,
|
||||||
|
color: "#6a6a72",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{new Date(s.created_at).toLocaleString()}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
{delta?.samples && delta.samples.length > 0 && (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
display: "grid",
|
||||||
|
gridTemplateColumns: "1fr 100px 100px 100px",
|
||||||
|
gap: 6,
|
||||||
|
fontSize: 12,
|
||||||
|
alignItems: "center",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<span style={{ color: "#8a8a92" }}>bench</span>
|
||||||
|
<span style={{ color: "#8a8a92", textAlign: "right" }}>before</span>
|
||||||
|
<span style={{ color: "#8a8a92", textAlign: "right" }}>after</span>
|
||||||
|
<span style={{ color: "#8a8a92", textAlign: "right" }}>Δ%</span>
|
||||||
|
{(delta.samples as Array<Record<string, unknown>>).map((row, i) => {
|
||||||
|
const pct = Number(row["delta_pct"] ?? 0);
|
||||||
|
const dir = String(row["direction"] ?? "");
|
||||||
|
const color = dir === "improved" ? "#5fd08a" : "#ff8a7a";
|
||||||
|
return (
|
||||||
|
<React.Fragment key={i}>
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
fontFamily: mono,
|
||||||
|
fontSize: 11.5,
|
||||||
|
color: "#d7d7db",
|
||||||
|
overflow: "hidden",
|
||||||
|
textOverflow: "ellipsis",
|
||||||
|
whiteSpace: "nowrap",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{String(row["name"])}
|
||||||
|
</span>
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
fontFamily: mono,
|
||||||
|
fontSize: 11.5,
|
||||||
|
color: "#a0a0a8",
|
||||||
|
textAlign: "right",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{String(row["before_ns"])} ns
|
||||||
|
</span>
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
fontFamily: mono,
|
||||||
|
fontSize: 11.5,
|
||||||
|
color: "#a0a0a8",
|
||||||
|
textAlign: "right",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{String(row["after_ns"])} ns
|
||||||
|
</span>
|
||||||
|
<span
|
||||||
|
style={{
|
||||||
|
fontFamily: mono,
|
||||||
|
fontSize: 11.5,
|
||||||
|
color,
|
||||||
|
textAlign: "right",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{pct > 0 ? "+" : ""}
|
||||||
|
{pct.toFixed(1)}%
|
||||||
|
</span>
|
||||||
|
</React.Fragment>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -108,12 +108,36 @@ export interface MissionArtifact {
|
|||||||
updated_at: string;
|
updated_at: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface BenchmarkSnapshot {
|
||||||
|
id: string;
|
||||||
|
mission_id: string;
|
||||||
|
phase_id: string;
|
||||||
|
iteration: number;
|
||||||
|
before_metrics: Record<string, unknown> | null;
|
||||||
|
after_metrics: Record<string, unknown> | null;
|
||||||
|
delta: Record<string, unknown> | null;
|
||||||
|
driver: string | null;
|
||||||
|
created_at: string;
|
||||||
|
}
|
||||||
|
|
||||||
export interface MissionDetail extends Mission {
|
export interface MissionDetail extends Mission {
|
||||||
phases: MissionPhase[];
|
phases: MissionPhase[];
|
||||||
tasks: MissionTask[];
|
tasks: MissionTask[];
|
||||||
artifacts: MissionArtifact[];
|
artifacts: MissionArtifact[];
|
||||||
|
benchmarks: BenchmarkSnapshot[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const triggerBenchmark = (
|
||||||
|
missionId: string,
|
||||||
|
phase_id: string,
|
||||||
|
slot: "baseline" | "after",
|
||||||
|
iteration?: number,
|
||||||
|
) =>
|
||||||
|
api<BenchmarkSnapshot[]>(`/api/missions/${missionId}/benchmark`, {
|
||||||
|
method: "POST",
|
||||||
|
body: JSON.stringify({ phase_id, slot, iteration }),
|
||||||
|
});
|
||||||
|
|
||||||
export interface PhaseSpec {
|
export interface PhaseSpec {
|
||||||
kind: PhaseKind;
|
kind: PhaseKind;
|
||||||
order_idx: number;
|
order_idx: number;
|
||||||
|
|||||||
Reference in New Issue
Block a user