clawmates/crates/cm-orchestrator/examples/topology_bench.rs

//! Runnable multi-topology benchmark — the reproducible "paper harness" seed.
//!
//! Runs one task across several topologies on the same executor and prints a
//! leaderboard + quality/cost Pareto front.
//!
//! ```text
//! cargo run -p cm-orchestrator --example topology_bench --features provider
//! ```
//!
//! Offline + deterministic by default (the scripted provider). Set
//! `ANTHROPIC_API_KEY` to run against a real model instead. The quality scorer
//! here is a deterministic length proxy; swap in `cm_orchestrator::JudgeScorer`
//! for real LLM-judged quality.

use std::sync::Arc;

use cm_llm::LlmProvider;
use cm_orchestrator::{compare, evolve_all, run_workflow, ProviderExecutor, RunRecord, Scorer};
use cm_topology::{Edge, EdgeKind, Node, TopologyGraph, TopologyKind};

/// Deterministic quality proxy: richer (longer) output scores higher.
struct LengthScorer;
impl Scorer for LengthScorer {
    async fn score(&self, _task: &str, record: &RunRecord) -> f64 {
        (record.final_output.len() as f64 / 400.0).min(1.0)
    }
}

fn node(id: &str, role: &str) -> Node {
    Node::new(id, role)
}

fn hierarchical() -> TopologyGraph {
    TopologyGraph::new(
        TopologyKind::Hierarchical,
        vec![
            node("lead", "coordinator"),
            node("w1", "researcher"),
            node("w2", "writer"),
        ],
        vec![
            Edge {
                from: "lead".into(),
                to: "w1".into(),
                kind: EdgeKind::DelegatesTo,
            },
            Edge {
                from: "lead".into(),
                to: "w2".into(),
                kind: EdgeKind::DelegatesTo,
            },
        ],
    )
    .unwrap()
}

fn pipeline() -> TopologyGraph {
    TopologyGraph::new(
        TopologyKind::Pipeline,
        vec![
            node("a", "researcher"),
            node("b", "analyst"),
            node("c", "writer"),
        ],
        vec![
            Edge {
                from: "a".into(),
                to: "b".into(),
                kind: EdgeKind::PipesTo,
            },
            Edge {
                from: "b".into(),
                to: "c".into(),
                kind: EdgeKind::PipesTo,
            },
        ],
    )
    .unwrap()
}

fn swarm() -> TopologyGraph {
    TopologyGraph::new(
        TopologyKind::Swarm,
        vec![
            node("a", "writer"),
            node("b", "writer"),
            node("lead", "coordinator"),
        ],
        vec![],
    )
    .unwrap()
}

fn mesh() -> TopologyGraph {
    TopologyGraph::new(
        TopologyKind::Mesh,
        vec![
            node("a", "analyst"),
            node("b", "analyst"),
            node("c", "analyst"),
        ],
        vec![
            Edge {
                from: "a".into(),
                to: "b".into(),
                kind: EdgeKind::PeersWith,
            },
            Edge {
                from: "b".into(),
                to: "c".into(),
                kind: EdgeKind::PeersWith,
            },
            Edge {
                from: "a".into(),
                to: "c".into(),
                kind: EdgeKind::PeersWith,
            },
        ],
    )
    .unwrap()
}

fn debate() -> TopologyGraph {
    TopologyGraph::new(
        TopologyKind::Debate,
        vec![
            node("proposer", "proposer"),
            node("critic", "critic"),
            node("judge", "judge"),
        ],
        vec![],
    )
    .unwrap()
}

#[tokio::main]
async fn main() {
    let (provider, model): (Arc<dyn LlmProvider>, String) = match std::env::var("ANTHROPIC_API_KEY")
    {
        Ok(key) if !key.is_empty() => (
            Arc::new(cm_llm::AnthropicProvider::new(key)),
            "claude-sonnet-4-6".to_string(),
        ),
        _ => (
            Arc::new(cm_llm::ScriptedProvider::from_toml("").unwrap()),
            "scripted".to_string(),
        ),
    };

    let executor = ProviderExecutor::new(provider, model.clone(), 512);
    let task = "Draft a go-to-market launch plan for a new product.";
    let graphs = vec![hierarchical(), pipeline(), swarm(), mesh(), debate()];

    let cmp = compare(&graphs, task, &executor, &LengthScorer)
        .await
        .expect("comparison failed");

    println!("provider: {model}");
    println!("task:     {task}\n");
    println!(
        "{:<14} {:>8} {:>8} {:>6} {:>7}",
        "topology", "quality", "tokens", "turns", "pareto"
    );
    println!("{}", "-".repeat(46));
    for r in &cmp.results {
        println!(
            "{:<14} {:>8.2} {:>8} {:>6} {:>7}",
            format!("{:?}", r.kind),
            r.quality,
            r.tokens,
            r.turns,
            if r.on_pareto { "*" } else { "" }
        );
    }
    println!();
    if let Some(i) = cmp.best_quality {
        println!("best quality: {:?}", cmp.results[i].kind);
    }
    if let Some(i) = cmp.best_value {
        println!(
            "best value:   {:?} (quality per token)",
            cmp.results[i].kind
        );
    }

    // Workflow of topologies: brainstorm (swarm) → execute (hierarchical) → review (debate).
    let wf = run_workflow(&[swarm(), hierarchical(), debate()], task, &executor)
        .await
        .expect("workflow failed");
    println!("\nworkflow: swarm -> hierarchical -> debate");
    println!(
        "  stages: {}   turns: {}   tokens: {}",
        wf.stages.len(),
        wf.totals.turns,
        wf.totals.tokens
    );

    // Evolution: search every topology kind for the best fit for this task.
    let roles = ["coordinator", "researcher", "analyst", "writer"];
    let ev = evolve_all(&roles, task, &executor, &LengthScorer)
        .await
        .expect("evolution failed");
    println!("\nevolution: searched {} topologies", ev.evaluated);
    if let Some(i) = ev.best {
        let b = &ev.archive[i];
        println!(
            "  best: {:?} (size {})  quality {:.2}  tokens {}",
            b.kind, b.size, b.quality, b.tokens
        );
    }
    let pareto: Vec<String> = ev
        .archive
        .iter()
        .filter(|c| c.on_pareto)
        .map(|c| format!("{:?}", c.kind))
        .collect();
    println!("  pareto-optimal: {}", pareto.join(", "));
}