clawmates/crates/cm-runtime/tests/live_anthropic.rs

//! The COMPLETE production path against the real Anthropic API
//! (CM_LIVE_LLM=1 + ANTHROPIC_API_KEY): a real model drives the runtime
//! loop — ungated tool execution with real usage metering, and the §15
//! chain: gated email intercepted, checkpointed, approved, and RESUMED
//! against the live API (checkpoint/resume fidelity, the plan's #1 risk).

use std::sync::Arc;
use std::time::Duration;

use cm_domain::{
    AccessPolicy, Agent, AgentId, AgentStatus, Role, RunState, User, UserId, Workspace, WorkspaceId,
};
use cm_llm::AnthropicProvider;
use cm_runtime::{RunEventBody, Runtime, RuntimeConfig};
use cm_safety::{approvals, Decision};

const MODEL: &str = "claude-haiku-4-5-20251001";

fn live() -> Option<String> {
    if std::env::var("CM_LIVE_LLM").as_deref() != Ok("1") {
        eprintln!("skipped: set CM_LIVE_LLM=1 to run");
        return None;
    }
    match std::env::var("ANTHROPIC_API_KEY") {
        Ok(key) => Some(key),
        Err(_) => {
            eprintln!("skipped: ANTHROPIC_API_KEY not set");
            None
        }
    }
}

async fn seeded(pool: &sqlx::PgPool) -> (Workspace, User, Agent) {
    let ws = Workspace {
        id: WorkspaceId::new(),
        name: "Acme".into(),
        plan: "team".into(),
    };
    cm_db::repo::workspaces::insert(pool, &ws).await.unwrap();
    let owner = User {
        id: UserId::new(),
        workspace_id: ws.id,
        email: format!("{}@acme.test", UserId::new()),
        role: Role::Owner,
        display_name: "Owner".into(),
        created_at: time::OffsetDateTime::UNIX_EPOCH,
    };
    cm_db::repo::users::insert(pool, &owner).await.unwrap();
    let agent = Agent {
        id: AgentId::new(),
        workspace_id: ws.id,
        name: "Scout".into(),
        job_title: "Analyst".into(),
        system_prompt: "Be terse. Use your tools when they apply.".into(),
        avatar: String::new(),
        accent: String::new(),
        wallpaper: String::new(),
        managed_by: owner.id,
        status: AgentStatus::Online,
    };
    cm_db::repo::agents::insert(pool, &agent, &AccessPolicy::default())
        .await
        .unwrap();
    cm_db::repo::credits::add_lot(pool, ws.id, 1000, "live")
        .await
        .unwrap();
    (ws, owner, agent)
}

#[tokio::test]
async fn a_real_model_runs_tools_and_gets_metered() {
    let Some(key) = live() else { return };
    let pool = cm_testkit::test_pool().await;
    let (ws, _, agent) = seeded(&pool).await;
    let rt = Runtime::new(
        pool.clone(),
        Arc::new(AnthropicProvider::new(key)),
        RuntimeConfig::basic(MODEL, 600),
    );
    let session = cm_db::repo::sessions::create(&pool, agent.id, ws.id, "Live")
        .await
        .unwrap();
    let started = rt
        .send_message(
            session.id,
            "What is the current UTC time? Use your clock tool, then state it.",
        )
        .await
        .unwrap();
    let mut rx = started.events;
    while let Ok(envelope) = rx.recv().await {
        if matches!(
            envelope.event,
            RunEventBody::RunCompleted { .. } | RunEventBody::Error { .. }
        ) {
            break;
        }
    }
    let run = cm_db::repo::runs::get(&pool, started.run_id).await.unwrap();
    assert_eq!(run.state, RunState::Completed);

    // A real clock.now step happened.
    let steps: i64 = sqlx::query_scalar(
        "SELECT count(*) FROM steps s JOIN messages m ON m.id = s.message_id
         WHERE m.session_id = $1 AND s.tool_name = 'clock.now' AND s.status = 'ok'",
    )
    .bind(session.id.as_uuid())
    .fetch_one(&pool)
    .await
    .unwrap();
    assert!(steps >= 1, "the model must have called the clock");

    // Real usage metered, real credits burned.
    let (tin, tout): (i64, i64) = sqlx::query_as(
        "SELECT COALESCE(SUM(tokens_in),0)::BIGINT, COALESCE(SUM(tokens_out),0)::BIGINT
         FROM usage_events WHERE workspace_id = $1",
    )
    .bind(ws.id.as_uuid())
    .fetch_one(&pool)
    .await
    .unwrap();
    assert!(tin > 0 && tout > 0, "real token usage: in={tin} out={tout}");
    assert!(
        cm_db::repo::credits::balance(&pool, ws.id).await.unwrap() < 1000,
        "credits decremented"
    );
}

#[tokio::test]
async fn the_gated_chain_suspends_and_resumes_against_the_live_api() {
    let Some(key) = live() else { return };
    let pool = cm_testkit::test_pool().await;
    let (ws, owner, agent) = seeded(&pool).await;
    let rt = Runtime::new(
        pool.clone(),
        Arc::new(AnthropicProvider::new(key)),
        RuntimeConfig::basic(MODEL, 600),
    );
    let session = cm_db::repo::sessions::create(&pool, agent.id, ws.id, "Live gated")
        .await
        .unwrap();
    let started = rt
        .send_message(
            session.id,
            "Use your email tool to send [email protected] an email with subject \
             'Q2' and body 'Numbers attached.' Do it now without asking.",
        )
        .await
        .unwrap();
    let mut rx = started.events;
    let mut suspended = false;
    while let Ok(envelope) = rx.recv().await {
        match envelope.event {
            RunEventBody::RunSuspended { .. } => {
                suspended = true;
                break;
            }
            RunEventBody::Error { .. } | RunEventBody::RunCompleted { .. } => break,
            _ => {}
        }
    }
    assert!(suspended, "a REAL model's email.send must be intercepted");

    let outbox: i64 = sqlx::query_scalar("SELECT count(*) FROM outbox")
        .fetch_one(&pool)
        .await
        .unwrap();
    assert_eq!(outbox, 0, "nothing sends while pending");

    // Approve → the checkpoint resumes against the LIVE API: the stored
    // assistant tool_use turn + our tool_result must reassemble into a
    // request Anthropic accepts (checkpoint fidelity, end to end).
    let pending = approvals::list_pending(&pool, ws.id).await.unwrap();
    assert_eq!(pending.len(), 1);
    approvals::decide(&pool, pending[0].id, owner.id, Decision::Approve)
        .await
        .unwrap();
    rt.resume_run(cm_safety::ResumeReady {
        run_id: started.run_id,
        approval_id: pending[0].id,
        approved: true,
    })
    .await
    .unwrap();

    let mut completed = false;
    for _ in 0..240 {
        let run = cm_db::repo::runs::get(&pool, started.run_id).await.unwrap();
        if run.state == RunState::Completed {
            completed = true;
            break;
        }
        tokio::time::sleep(Duration::from_millis(500)).await;
    }
    assert!(completed, "resume against the live API must complete");

    let outbox: i64 = sqlx::query_scalar("SELECT count(*) FROM outbox")
        .fetch_one(&pool)
        .await
        .unwrap();
    assert_eq!(outbox, 1, "approved email sent exactly once");
}