feat(topology): runnable multi-topology benchmark example
`cargo run -p cm-orchestrator --example topology_bench --features provider` runs one task across hierarchical/pipeline/swarm topologies and prints a leaderboard + quality/cost Pareto front. Offline-deterministic via the scripted provider; set ANTHROPIC_API_KEY to run against a real model. Demonstrates milestone B end-to-end and seeds the reproducible paper harness. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
3725c96e4f
commit
c595cebab0
@@ -24,3 +24,7 @@ tokio = { workspace = true }
|
|||||||
|
|
||||||
[lints]
|
[lints]
|
||||||
workspace = true
|
workspace = true
|
||||||
|
|
||||||
|
[[example]]
|
||||||
|
name = "topology_bench"
|
||||||
|
required-features = ["provider"]
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
//! Runnable multi-topology benchmark — the reproducible "paper harness" seed.
|
||||||
|
//!
|
||||||
|
//! Runs one task across several topologies on the same executor and prints a
|
||||||
|
//! leaderboard + quality/cost Pareto front.
|
||||||
|
//!
|
||||||
|
//! ```text
|
||||||
|
//! cargo run -p cm-orchestrator --example topology_bench --features provider
|
||||||
|
//! ```
|
||||||
|
//!
|
||||||
|
//! Offline + deterministic by default (the scripted provider). Set
|
||||||
|
//! `ANTHROPIC_API_KEY` to run against a real model instead. The quality scorer
|
||||||
|
//! here is a deterministic length proxy; swap in `cm_orchestrator::JudgeScorer`
|
||||||
|
//! for real LLM-judged quality.
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use cm_llm::LlmProvider;
|
||||||
|
use cm_orchestrator::{compare, ProviderExecutor, RunRecord, Scorer};
|
||||||
|
use cm_topology::{Edge, EdgeKind, Node, TopologyGraph, TopologyKind};
|
||||||
|
|
||||||
|
/// Deterministic quality proxy: richer (longer) output scores higher.
|
||||||
|
struct LengthScorer;
|
||||||
|
impl Scorer for LengthScorer {
|
||||||
|
async fn score(&self, _task: &str, record: &RunRecord) -> f64 {
|
||||||
|
(record.final_output.len() as f64 / 400.0).min(1.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn node(id: &str, role: &str) -> Node {
|
||||||
|
Node::new(id, role)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hierarchical() -> TopologyGraph {
|
||||||
|
TopologyGraph::new(
|
||||||
|
TopologyKind::Hierarchical,
|
||||||
|
vec![node("lead", "coordinator"), node("w1", "researcher"), node("w2", "writer")],
|
||||||
|
vec![
|
||||||
|
Edge { from: "lead".into(), to: "w1".into(), kind: EdgeKind::DelegatesTo },
|
||||||
|
Edge { from: "lead".into(), to: "w2".into(), kind: EdgeKind::DelegatesTo },
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn pipeline() -> TopologyGraph {
|
||||||
|
TopologyGraph::new(
|
||||||
|
TopologyKind::Pipeline,
|
||||||
|
vec![node("a", "researcher"), node("b", "analyst"), node("c", "writer")],
|
||||||
|
vec![
|
||||||
|
Edge { from: "a".into(), to: "b".into(), kind: EdgeKind::PipesTo },
|
||||||
|
Edge { from: "b".into(), to: "c".into(), kind: EdgeKind::PipesTo },
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn swarm() -> TopologyGraph {
|
||||||
|
TopologyGraph::new(
|
||||||
|
TopologyKind::Swarm,
|
||||||
|
vec![node("a", "writer"), node("b", "writer"), node("lead", "coordinator")],
|
||||||
|
vec![],
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::main]
|
||||||
|
async fn main() {
|
||||||
|
let (provider, model): (Arc<dyn LlmProvider>, String) =
|
||||||
|
match std::env::var("ANTHROPIC_API_KEY") {
|
||||||
|
Ok(key) if !key.is_empty() => {
|
||||||
|
(Arc::new(cm_llm::AnthropicProvider::new(key)), "claude-sonnet-4-6".to_string())
|
||||||
|
}
|
||||||
|
_ => (
|
||||||
|
Arc::new(cm_llm::ScriptedProvider::from_toml("").unwrap()),
|
||||||
|
"scripted".to_string(),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
|
||||||
|
let executor = ProviderExecutor::new(provider, model.clone(), 512);
|
||||||
|
let task = "Draft a go-to-market launch plan for a new product.";
|
||||||
|
let graphs = vec![hierarchical(), pipeline(), swarm()];
|
||||||
|
|
||||||
|
let cmp = compare(&graphs, task, &executor, &LengthScorer)
|
||||||
|
.await
|
||||||
|
.expect("comparison failed");
|
||||||
|
|
||||||
|
println!("provider: {model}");
|
||||||
|
println!("task: {task}\n");
|
||||||
|
println!(
|
||||||
|
"{:<14} {:>8} {:>8} {:>6} {:>7}",
|
||||||
|
"topology", "quality", "tokens", "turns", "pareto"
|
||||||
|
);
|
||||||
|
println!("{}", "-".repeat(46));
|
||||||
|
for r in &cmp.results {
|
||||||
|
println!(
|
||||||
|
"{:<14} {:>8.2} {:>8} {:>6} {:>7}",
|
||||||
|
format!("{:?}", r.kind),
|
||||||
|
r.quality,
|
||||||
|
r.tokens,
|
||||||
|
r.turns,
|
||||||
|
if r.on_pareto { "*" } else { "" }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
println!();
|
||||||
|
if let Some(i) = cmp.best_quality {
|
||||||
|
println!("best quality: {:?}", cmp.results[i].kind);
|
||||||
|
}
|
||||||
|
if let Some(i) = cmp.best_value {
|
||||||
|
println!("best value: {:?} (quality per token)", cmp.results[i].kind);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user