clawmates/crates/cm-orchestrator/examples/topology_bench.rs

//! Runnable multi-topology benchmark — the reproducible "paper harness" seed.
//!
//! Runs one task across several topologies on the same executor and prints a
//! leaderboard + quality/cost Pareto front.
//!
//! ```text
//! cargo run -p cm-orchestrator --example topology_bench --features provider
//! ```
//!
//! Offline + deterministic by default (the scripted provider). Set
//! `ANTHROPIC_API_KEY` to run against a real model instead. The quality scorer
//! here is a deterministic length proxy; swap in `cm_orchestrator::JudgeScorer`
//! for real LLM-judged quality.

use std::sync::Arc;

use cm_llm::LlmProvider;
use cm_orchestrator::{compare, run_workflow, ProviderExecutor, RunRecord, Scorer};
use cm_topology::{Edge, EdgeKind, Node, TopologyGraph, TopologyKind};

/// Deterministic quality proxy: richer (longer) output scores higher.
struct LengthScorer;
impl Scorer for LengthScorer {
    async fn score(&self, _task: &str, record: &RunRecord) -> f64 {
        (record.final_output.len() as f64 / 400.0).min(1.0)
    }
}

fn node(id: &str, role: &str) -> Node {
    Node::new(id, role)
}

fn hierarchical() -> TopologyGraph {
    TopologyGraph::new(
        TopologyKind::Hierarchical,
        vec![node("lead", "coordinator"), node("w1", "researcher"), node("w2", "writer")],
        vec![
            Edge { from: "lead".into(), to: "w1".into(), kind: EdgeKind::DelegatesTo },
            Edge { from: "lead".into(), to: "w2".into(), kind: EdgeKind::DelegatesTo },
        ],
    )
    .unwrap()
}

fn pipeline() -> TopologyGraph {
    TopologyGraph::new(
        TopologyKind::Pipeline,
        vec![node("a", "researcher"), node("b", "analyst"), node("c", "writer")],
        vec![
            Edge { from: "a".into(), to: "b".into(), kind: EdgeKind::PipesTo },
            Edge { from: "b".into(), to: "c".into(), kind: EdgeKind::PipesTo },
        ],
    )
    .unwrap()
}

fn swarm() -> TopologyGraph {
    TopologyGraph::new(
        TopologyKind::Swarm,
        vec![node("a", "writer"), node("b", "writer"), node("lead", "coordinator")],
        vec![],
    )
    .unwrap()
}

fn mesh() -> TopologyGraph {
    TopologyGraph::new(
        TopologyKind::Mesh,
        vec![node("a", "analyst"), node("b", "analyst"), node("c", "analyst")],
        vec![
            Edge { from: "a".into(), to: "b".into(), kind: EdgeKind::PeersWith },
            Edge { from: "b".into(), to: "c".into(), kind: EdgeKind::PeersWith },
            Edge { from: "a".into(), to: "c".into(), kind: EdgeKind::PeersWith },
        ],
    )
    .unwrap()
}

fn debate() -> TopologyGraph {
    TopologyGraph::new(
        TopologyKind::Debate,
        vec![node("proposer", "proposer"), node("critic", "critic"), node("judge", "judge")],
        vec![],
    )
    .unwrap()
}

#[tokio::main]
async fn main() {
    let (provider, model): (Arc<dyn LlmProvider>, String) =
        match std::env::var("ANTHROPIC_API_KEY") {
            Ok(key) if !key.is_empty() => {
                (Arc::new(cm_llm::AnthropicProvider::new(key)), "claude-sonnet-4-6".to_string())
            }
            _ => (
                Arc::new(cm_llm::ScriptedProvider::from_toml("").unwrap()),
                "scripted".to_string(),
            ),
        };

    let executor = ProviderExecutor::new(provider, model.clone(), 512);
    let task = "Draft a go-to-market launch plan for a new product.";
    let graphs = vec![hierarchical(), pipeline(), swarm(), mesh(), debate()];

    let cmp = compare(&graphs, task, &executor, &LengthScorer)
        .await
        .expect("comparison failed");

    println!("provider: {model}");
    println!("task:     {task}\n");
    println!(
        "{:<14} {:>8} {:>8} {:>6} {:>7}",
        "topology", "quality", "tokens", "turns", "pareto"
    );
    println!("{}", "-".repeat(46));
    for r in &cmp.results {
        println!(
            "{:<14} {:>8.2} {:>8} {:>6} {:>7}",
            format!("{:?}", r.kind),
            r.quality,
            r.tokens,
            r.turns,
            if r.on_pareto { "*" } else { "" }
        );
    }
    println!();
    if let Some(i) = cmp.best_quality {
        println!("best quality: {:?}", cmp.results[i].kind);
    }
    if let Some(i) = cmp.best_value {
        println!("best value:   {:?} (quality per token)", cmp.results[i].kind);
    }

    // Workflow of topologies: brainstorm (swarm) → execute (hierarchical) → review (debate).
    let wf = run_workflow(&[swarm(), hierarchical(), debate()], task, &executor)
        .await
        .expect("workflow failed");
    println!("\nworkflow: swarm -> hierarchical -> debate");
    println!(
        "  stages: {}   turns: {}   tokens: {}",
        wf.stages.len(),
        wf.totals.turns,
        wf.totals.tokens
    );
}