//! Runnable multi-topology benchmark — the reproducible "paper harness" seed. //! //! Runs one task across several topologies on the same executor and prints a //! leaderboard + quality/cost Pareto front. //! //! ```text //! cargo run -p cm-orchestrator --example topology_bench --features provider //! ``` //! //! Offline + deterministic by default (the scripted provider). Set //! `ANTHROPIC_API_KEY` to run against a real model instead. The quality scorer //! here is a deterministic length proxy; swap in `cm_orchestrator::JudgeScorer` //! for real LLM-judged quality. use std::sync::Arc; use cm_llm::LlmProvider; use cm_orchestrator::{compare, evolve_all, run_workflow, ProviderExecutor, RunRecord, Scorer}; use cm_topology::{Edge, EdgeKind, Node, TopologyGraph, TopologyKind}; /// Deterministic quality proxy: richer (longer) output scores higher. struct LengthScorer; impl Scorer for LengthScorer { async fn score(&self, _task: &str, record: &RunRecord) -> f64 { (record.final_output.len() as f64 / 400.0).min(1.0) } } fn node(id: &str, role: &str) -> Node { Node::new(id, role) } fn hierarchical() -> TopologyGraph { TopologyGraph::new( TopologyKind::Hierarchical, vec![ node("lead", "coordinator"), node("w1", "researcher"), node("w2", "writer"), ], vec![ Edge { from: "lead".into(), to: "w1".into(), kind: EdgeKind::DelegatesTo, }, Edge { from: "lead".into(), to: "w2".into(), kind: EdgeKind::DelegatesTo, }, ], ) .unwrap() } fn pipeline() -> TopologyGraph { TopologyGraph::new( TopologyKind::Pipeline, vec![ node("a", "researcher"), node("b", "analyst"), node("c", "writer"), ], vec![ Edge { from: "a".into(), to: "b".into(), kind: EdgeKind::PipesTo, }, Edge { from: "b".into(), to: "c".into(), kind: EdgeKind::PipesTo, }, ], ) .unwrap() } fn swarm() -> TopologyGraph { TopologyGraph::new( TopologyKind::Swarm, vec![ node("a", "writer"), node("b", "writer"), node("lead", "coordinator"), ], vec![], ) .unwrap() } fn mesh() -> TopologyGraph { TopologyGraph::new( TopologyKind::Mesh, vec![ node("a", "analyst"), node("b", "analyst"), node("c", "analyst"), ], vec![ Edge { from: "a".into(), to: "b".into(), kind: EdgeKind::PeersWith, }, Edge { from: "b".into(), to: "c".into(), kind: EdgeKind::PeersWith, }, Edge { from: "a".into(), to: "c".into(), kind: EdgeKind::PeersWith, }, ], ) .unwrap() } fn debate() -> TopologyGraph { TopologyGraph::new( TopologyKind::Debate, vec![ node("proposer", "proposer"), node("critic", "critic"), node("judge", "judge"), ], vec![], ) .unwrap() } #[tokio::main] async fn main() { let (provider, model): (Arc, String) = match std::env::var("ANTHROPIC_API_KEY") { Ok(key) if !key.is_empty() => ( Arc::new(cm_llm::AnthropicProvider::new(key)), "claude-sonnet-4-6".to_string(), ), _ => ( Arc::new(cm_llm::ScriptedProvider::from_toml("").unwrap()), "scripted".to_string(), ), }; let executor = ProviderExecutor::new(provider, model.clone(), 512); let task = "Draft a go-to-market launch plan for a new product."; let graphs = vec![hierarchical(), pipeline(), swarm(), mesh(), debate()]; let cmp = compare(&graphs, task, &executor, &LengthScorer) .await .expect("comparison failed"); println!("provider: {model}"); println!("task: {task}\n"); println!( "{:<14} {:>8} {:>8} {:>6} {:>7}", "topology", "quality", "tokens", "turns", "pareto" ); println!("{}", "-".repeat(46)); for r in &cmp.results { println!( "{:<14} {:>8.2} {:>8} {:>6} {:>7}", format!("{:?}", r.kind), r.quality, r.tokens, r.turns, if r.on_pareto { "*" } else { "" } ); } println!(); if let Some(i) = cmp.best_quality { println!("best quality: {:?}", cmp.results[i].kind); } if let Some(i) = cmp.best_value { println!( "best value: {:?} (quality per token)", cmp.results[i].kind ); } // Workflow of topologies: brainstorm (swarm) → execute (hierarchical) → review (debate). let wf = run_workflow(&[swarm(), hierarchical(), debate()], task, &executor) .await .expect("workflow failed"); println!("\nworkflow: swarm -> hierarchical -> debate"); println!( " stages: {} turns: {} tokens: {}", wf.stages.len(), wf.totals.turns, wf.totals.tokens ); // Evolution: search every topology kind for the best fit for this task. let roles = ["coordinator", "researcher", "analyst", "writer"]; let ev = evolve_all(&roles, task, &executor, &LengthScorer) .await .expect("evolution failed"); println!("\nevolution: searched {} topologies", ev.evaluated); if let Some(i) = ev.best { let b = &ev.archive[i]; println!( " best: {:?} (size {}) quality {:.2} tokens {}", b.kind, b.size, b.quality, b.tokens ); } let pareto: Vec = ev .archive .iter() .filter(|c| c.on_pareto) .map(|c| format!("{:?}", c.kind)) .collect(); println!(" pareto-optimal: {}", pareto.join(", ")); }