clawmates/crates/cm-runtime/tests/shell_exec.rs

//! Agents execute code in their REAL hardened sandbox: shell.exec runs in
//! a per-agent container (Docker driver), output returns to the model as a
//! step result, and /home/agent state persists across calls in a run.

use std::process::Command;
use std::sync::Arc;

use cm_domain::{
    AccessPolicy, Agent, AgentId, AgentStatus, Role, RunState, User, UserId, Workspace, WorkspaceId,
};
use cm_llm::ScriptedProvider;
use cm_runtime::{Runtime, RuntimeConfig, SandboxManager};
use cm_sandbox::DockerDriver;

const IMAGE: &str = "clawmates/agent-base:dev";

const SCENARIOS: &str = r#"
[[scenario]]
marker = "[[scenario:shell]]"

[[scenario.turns]]
events = [
    { type = "tool_use", name = "shell.exec", input = { command = "id -u && echo tick > /home/agent/state" } },
]

[[scenario.turns]]
events = [
    { type = "tool_use", name = "shell.exec", input = { command = "cat /home/agent/state" } },
]

[[scenario.turns]]
events = [
    { type = "text", text = "Ran both commands." },
]
"#;

fn ensure_image() {
    let exists = Command::new("docker")
        .args(["image", "inspect", IMAGE])
        .output()
        .expect("docker available")
        .status
        .success();
    if !exists {
        let root = env!("CARGO_MANIFEST_DIR");
        let status = Command::new("docker")
            .args([
                "build",
                "-t",
                IMAGE,
                "-f",
                &format!("{root}/../../images/agent-base/Dockerfile"),
                &format!("{root}/../../images/agent-base"),
            ])
            .status()
            .expect("docker build runs");
        assert!(status.success(), "agent-base image build failed");
    }
}

#[tokio::test]
async fn shell_exec_runs_in_the_agent_sandbox_with_persistent_home() {
    ensure_image();
    let pool = cm_testkit::test_pool().await;

    let ws = Workspace {
        id: WorkspaceId::new(),
        name: "Acme".into(),
        plan: "team".into(),
    };
    cm_db::repo::workspaces::insert(&pool, &ws).await.unwrap();
    let owner = User {
        id: UserId::new(),
        workspace_id: ws.id,
        email: format!("{}@acme.test", UserId::new()),
        role: Role::Owner,
        display_name: "Owner".into(),
        created_at: time::OffsetDateTime::UNIX_EPOCH,
    };
    cm_db::repo::users::insert(&pool, &owner).await.unwrap();
    let agent = Agent {
        id: AgentId::new(),
        workspace_id: ws.id,
        name: "Scout".into(),
        job_title: "Analyst".into(),
        system_prompt: String::new(),
        avatar: String::new(),
        accent: String::new(),
        wallpaper: String::new(),
        managed_by: owner.id,
        status: AgentStatus::Online,
    };
    cm_db::repo::agents::insert(&pool, &agent, &AccessPolicy::default())
        .await
        .unwrap();

    let driver = DockerDriver::connect().expect("docker reachable");
    let sandboxes = Arc::new(SandboxManager::new(
        Arc::new(driver),
        pool.clone(),
        "local",
        IMAGE,
    ));
    let rt = Runtime::new(
        pool.clone(),
        Arc::new(ScriptedProvider::from_toml(SCENARIOS).unwrap()),
        RuntimeConfig::basic("scripted", 1024).with_sandboxes(sandboxes.clone()),
    );

    let session = cm_db::repo::sessions::create(&pool, agent.id, ws.id, "Shell")
        .await
        .unwrap();
    let started = rt
        .send_message(session.id, "run it [[scenario:shell]]")
        .await
        .unwrap();
    let mut rx = started.events;
    while let Ok(envelope) = rx.recv().await {
        if matches!(
            envelope.event,
            cm_runtime::RunEventBody::RunCompleted { .. } | cm_runtime::RunEventBody::Error { .. }
        ) {
            break;
        }
    }
    let run = cm_db::repo::runs::get(&pool, started.run_id).await.unwrap();
    assert_eq!(run.state, RunState::Completed);

    // Step outputs prove kernel-level facts: uid 10001 inside, and the
    // second call saw the first call's file — same sandbox, same run.
    let outputs: Vec<serde_json::Value> = sqlx::query_scalar(
        "SELECT s.output FROM steps s
         JOIN messages m ON m.id = s.message_id
         WHERE m.session_id = $1 AND s.tool_name = 'shell.exec'
         ORDER BY s.seq",
    )
    .bind(session.id.as_uuid())
    .fetch_all(&pool)
    .await
    .unwrap();
    assert_eq!(outputs.len(), 2);
    assert_eq!(outputs[0]["exit_code"], 0, "first: {}", outputs[0]);
    assert!(
        outputs[0]["stdout"].as_str().unwrap().contains("10001"),
        "sandbox runs as uid 10001: {}",
        outputs[0]
    );
    assert_eq!(outputs[1]["exit_code"], 0, "second: {}", outputs[1]);
    assert!(
        outputs[1]["stdout"].as_str().unwrap().contains("tick"),
        "home persists across execs: {}",
        outputs[1]
    );

    // `shutdown` drains the warm pool; the container this exec ASSIGNED to the
    // agent survives it on purpose, so a redeploy can reuse it. A test has no
    // next deploy, so it must release its own or the container simply stays —
    // which is how 289 of them accumulated before anyone counted.
    sandboxes.release_agent(agent.id).await;
    sandboxes.shutdown().await;
}

#[tokio::test]
async fn shell_exec_without_a_sandbox_runtime_reports_a_tool_error() {
    let pool = cm_testkit::test_pool().await;
    let ws = Workspace {
        id: WorkspaceId::new(),
        name: "Acme".into(),
        plan: "team".into(),
    };
    cm_db::repo::workspaces::insert(&pool, &ws).await.unwrap();
    let owner = User {
        id: UserId::new(),
        workspace_id: ws.id,
        email: format!("{}@acme.test", UserId::new()),
        role: Role::Owner,
        display_name: "Owner".into(),
        created_at: time::OffsetDateTime::UNIX_EPOCH,
    };
    cm_db::repo::users::insert(&pool, &owner).await.unwrap();
    let agent = Agent {
        id: AgentId::new(),
        workspace_id: ws.id,
        name: "Scout".into(),
        job_title: "Analyst".into(),
        system_prompt: String::new(),
        avatar: String::new(),
        accent: String::new(),
        wallpaper: String::new(),
        managed_by: owner.id,
        status: AgentStatus::Online,
    };
    cm_db::repo::agents::insert(&pool, &agent, &AccessPolicy::default())
        .await
        .unwrap();

    let rt = Runtime::new(
        pool.clone(),
        Arc::new(ScriptedProvider::from_toml(SCENARIOS).unwrap()),
        RuntimeConfig::basic("scripted", 1024),
    );
    let session = cm_db::repo::sessions::create(&pool, agent.id, ws.id, "NoBox")
        .await
        .unwrap();
    let started = rt
        .send_message(session.id, "run it [[scenario:shell]]")
        .await
        .unwrap();
    let mut rx = started.events;
    while let Ok(envelope) = rx.recv().await {
        if matches!(
            envelope.event,
            cm_runtime::RunEventBody::RunCompleted { .. } | cm_runtime::RunEventBody::Error { .. }
        ) {
            break;
        }
    }
    // The run survives: the tool reports its error to the model, which
    // finishes the scenario.
    let run = cm_db::repo::runs::get(&pool, started.run_id).await.unwrap();
    assert_eq!(run.state, RunState::Completed);
    let statuses: Vec<String> = sqlx::query_scalar(
        "SELECT s.status FROM steps s
         JOIN messages m ON m.id = s.message_id
         WHERE m.session_id = $1 AND s.tool_name = 'shell.exec'
         ORDER BY s.seq",
    )
    .bind(session.id.as_uuid())
    .fetch_all(&pool)
    .await
    .unwrap();
    assert!(!statuses.is_empty());
    assert!(
        statuses.iter().all(|s| s == "error"),
        "steps must record the failure: {statuses:?}"
    );

}