feat(topology): runnable multi-topology benchmark example

`cargo run -p cm-orchestrator --example topology_bench --features provider` runs one task across hierarchical/pipeline/swarm topologies and prints a leaderboard + quality/cost Pareto front. Offline-deterministic via the scripted provider; set ANTHROPIC_API_KEY to run against a real model. Demonstrates milestone B end-to-end and seeds the reproducible paper harness. Co-Authored-By: Claude Opus 4.8 <[email protected]>
2026-06-15 20:45:26 -07:00
parent 3725c96e4f
commit c595cebab0
2 changed files with 115 additions and 0 deletions
@@ -0,0 +1,111 @@
+//! Runnable multi-topology benchmark — the reproducible "paper harness" seed.
+//!
+//! Runs one task across several topologies on the same executor and prints a
+//! leaderboard + quality/cost Pareto front.
+//!
+//! ```text
+//! cargo run -p cm-orchestrator --example topology_bench --features provider
+//! ```
+//!
+//! Offline + deterministic by default (the scripted provider). Set
+//! `ANTHROPIC_API_KEY` to run against a real model instead. The quality scorer
+//! here is a deterministic length proxy; swap in `cm_orchestrator::JudgeScorer`
+//! for real LLM-judged quality.
+
+use std::sync::Arc;
+
+use cm_llm::LlmProvider;
+use cm_orchestrator::{compare, ProviderExecutor, RunRecord, Scorer};
+use cm_topology::{Edge, EdgeKind, Node, TopologyGraph, TopologyKind};
+
+/// Deterministic quality proxy: richer (longer) output scores higher.
+struct LengthScorer;
+impl Scorer for LengthScorer {
+    async fn score(&self, _task: &str, record: &RunRecord) -> f64 {
+        (record.final_output.len() as f64 / 400.0).min(1.0)
+    }
+}
+
+fn node(id: &str, role: &str) -> Node {
+    Node::new(id, role)
+}
+
+fn hierarchical() -> TopologyGraph {
+    TopologyGraph::new(
+        TopologyKind::Hierarchical,
+        vec![node("lead", "coordinator"), node("w1", "researcher"), node("w2", "writer")],
+        vec![
+            Edge { from: "lead".into(), to: "w1".into(), kind: EdgeKind::DelegatesTo },
+            Edge { from: "lead".into(), to: "w2".into(), kind: EdgeKind::DelegatesTo },
+        ],
+    )
+    .unwrap()
+}
+
+fn pipeline() -> TopologyGraph {
+    TopologyGraph::new(
+        TopologyKind::Pipeline,
+        vec![node("a", "researcher"), node("b", "analyst"), node("c", "writer")],
+        vec![
+            Edge { from: "a".into(), to: "b".into(), kind: EdgeKind::PipesTo },
+            Edge { from: "b".into(), to: "c".into(), kind: EdgeKind::PipesTo },
+        ],
+    )
+    .unwrap()
+}
+
+fn swarm() -> TopologyGraph {
+    TopologyGraph::new(
+        TopologyKind::Swarm,
+        vec![node("a", "writer"), node("b", "writer"), node("lead", "coordinator")],
+        vec![],
+    )
+    .unwrap()
+}
+
+#[tokio::main]
+async fn main() {
+    let (provider, model): (Arc<dyn LlmProvider>, String) =
+        match std::env::var("ANTHROPIC_API_KEY") {
+            Ok(key) if !key.is_empty() => {
+                (Arc::new(cm_llm::AnthropicProvider::new(key)), "claude-sonnet-4-6".to_string())
+            }
+            _ => (
+                Arc::new(cm_llm::ScriptedProvider::from_toml("").unwrap()),
+                "scripted".to_string(),
+            ),
+        };
+
+    let executor = ProviderExecutor::new(provider, model.clone(), 512);
+    let task = "Draft a go-to-market launch plan for a new product.";
+    let graphs = vec![hierarchical(), pipeline(), swarm()];
+
+    let cmp = compare(&graphs, task, &executor, &LengthScorer)
+        .await
+        .expect("comparison failed");
+
+    println!("provider: {model}");
+    println!("task:     {task}\n");
+    println!(
+        "{:<14} {:>8} {:>8} {:>6} {:>7}",
+        "topology", "quality", "tokens", "turns", "pareto"
+    );
+    println!("{}", "-".repeat(46));
+    for r in &cmp.results {
+        println!(
+            "{:<14} {:>8.2} {:>8} {:>6} {:>7}",
+            format!("{:?}", r.kind),
+            r.quality,
+            r.tokens,
+            r.turns,
+            if r.on_pareto { "*" } else { "" }
+        );
+    }
+    println!();
+    if let Some(i) = cmp.best_quality {
+        println!("best quality: {:?}", cmp.results[i].kind);
+    }
+    if let Some(i) = cmp.best_value {
+        println!("best value:   {:?} (quality per token)", cmp.results[i].kind);
+    }
+}