Omar SobhandClaude Opus 5
5232175c88
Switching agents to claude_cli left missions hanging: the per-mission container had claude_cli configured but no credential, so `claude -p` waited forever. A phase sat at `running` for ten minutes with nothing in the logs — no error, because there is nothing to error on. The original subscription design assumed a persisted `claude /login` under a bind-mounted $HOME. That holds for the shared runtime and NOT for a mission container, which gets its own data dir and therefore no login. So subscription mode now forwards CLAUDE_CODE_OAUTH_TOKEN. The two Anthropic credentials remain mutually exclusive, and there is now a test asserting it in both directions: Claude Code ranks ANTHROPIC_API_KEY above the OAuth token, so shipping both bills the API while the deployment believes it is on the subscription — visible only on the invoice. Deployment: CLAWMATES_RUNTIME_AUTH=subscription and CLAUDE_CODE_OAUTH_TOKEN added to compose + .env on gw-04. Co-Authored-By: Claude Opus 5 <[email protected]>
978 lines
40 KiB
Rust
978 lines
40 KiB
Rust
//! Per-mission ZeroClaw runtime container lifecycle.
|
|
//!
|
|
//! C3 workspace-isolation model: every mission gets its own ZeroClaw
|
|
//! daemon container, so agents' sandboxed filesystem is scoped to that
|
|
//! mission's repo checkout instead of the shared `/zeroclaw-data/
|
|
//! workspace` on the singleton `clawmates-runtime` daemon.
|
|
//!
|
|
//! Container naming: `cm-runtime-mission-{first 12 chars of mission uuid}`.
|
|
//! Endpoint: `http://<container_name>:42617` (well-known ZeroClaw port,
|
|
//! reachable over the `clawmates_core` docker network).
|
|
//!
|
|
//! Lifecycle:
|
|
//! - `ensure_container(mission_id)` — idempotent; spawns the container
|
|
//! if not present, returns its endpoint. Called from
|
|
//! `mission_orchestrator::on_launch` and (as a fallback for
|
|
//! pre-C3 missions) `phase_runner::launch_phase`.
|
|
//! - `teardown_container(mission_id)` — force-removes the container.
|
|
//! Called by the sweeper (slice 3) N minutes after a mission
|
|
//! reaches a terminal state, so the operator has a window to
|
|
//! re-open the ClawmateS UI and pull the last checkpoint before
|
|
//! the daemon disappears.
|
|
//!
|
|
//! State: `missions.runtime_container_name` and `missions.runtime_endpoint`
|
|
//! carry the current binding (both null when torn down or never
|
|
//! provisioned).
|
|
|
|
use base64::Engine;
|
|
use bollard::exec::{CreateExecOptions, StartExecResults};
|
|
use bollard::models::{
|
|
ContainerCreateBody, EndpointSettings, HostConfig, Mount, MountTypeEnum, NetworkConnectRequest,
|
|
};
|
|
use bollard::query_parameters::{
|
|
CreateContainerOptions, InspectContainerOptions, RemoveContainerOptions, StartContainerOptions,
|
|
};
|
|
use bollard::Docker;
|
|
use futures::StreamExt;
|
|
use std::collections::HashMap;
|
|
use uuid::Uuid;
|
|
|
|
/// Docker image the per-mission runtime uses. Matches the current
|
|
/// singleton `clawmates-runtime` image; can be overridden per-deploy
|
|
/// via `CLAWMATES_RUNTIME_IMAGE`.
|
|
const DEFAULT_IMAGE: &str = "clawmates-runtime:sync";
|
|
|
|
/// Well-known ZeroClaw gateway port.
|
|
const GATEWAY_PORT: u16 = 42617;
|
|
|
|
/// How the ZeroClaw runtime authenticates to Anthropic.
|
|
///
|
|
/// The runtime image ships the official `claude` CLI, which can authenticate
|
|
/// either with a platform API key or with a subscription login stored under
|
|
/// `$HOME` (a persisted bind mount, so one login survives container
|
|
/// recreation). These are mutually exclusive in practice because Claude Code
|
|
/// prefers `ANTHROPIC_API_KEY` over the subscription credential.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum RuntimeAuth {
|
|
/// Forward the platform's `ANTHROPIC_API_KEY`. Metered per token.
|
|
ApiKey,
|
|
/// Withhold the API key so the runtime's own `claude /login` credential is
|
|
/// used. Only valid for a single-operator deployment — a subscription
|
|
/// credential must never serve another person's work.
|
|
Subscription,
|
|
}
|
|
|
|
impl RuntimeAuth {
|
|
pub fn as_str(self) -> &'static str {
|
|
match self {
|
|
RuntimeAuth::ApiKey => "api_key",
|
|
RuntimeAuth::Subscription => "subscription",
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Provider credential env vars forwarded into a runtime container.
|
|
///
|
|
/// `ANTHROPIC_API_KEY` is conditional, and the reason is subtle enough to be
|
|
/// worth stating at the definition: Claude Code resolves credentials in a fixed
|
|
/// priority order and ranks `ANTHROPIC_API_KEY` **above** the subscription's
|
|
/// `CLAUDE_CODE_OAUTH_TOKEN`. On a runtime authenticated via `claude /login`,
|
|
/// forwarding the key silently wins — `claude` still works, agents still run,
|
|
/// and every mission bills the API while appearing to use the subscription.
|
|
/// There is no error to surface; the only symptom is the invoice.
|
|
///
|
|
/// The other three are unrelated providers with no subscription equivalent, so
|
|
/// they forward in both modes.
|
|
///
|
|
/// In subscription mode `CLAUDE_CODE_OAUTH_TOKEN` forwards instead. The
|
|
/// original design assumed a persisted `claude /login` under a bind-mounted
|
|
/// `$HOME`, but a *mission* container gets its own data dir and therefore no
|
|
/// login — so the token has to travel. Missing it is not a loud failure:
|
|
/// `claude -p` simply hangs with no credential, which is what a phase stuck
|
|
/// at `running` for ten minutes looked like when this was first switched on.
|
|
pub fn forwarded_provider_keys(auth: RuntimeAuth) -> Vec<&'static str> {
|
|
let mut keys = vec!["GEMINI_API_KEY", "GROQ_API_KEY", "OPENAI_API_KEY"];
|
|
match auth {
|
|
RuntimeAuth::ApiKey => keys.push("ANTHROPIC_API_KEY"),
|
|
RuntimeAuth::Subscription => keys.push("CLAUDE_CODE_OAUTH_TOKEN"),
|
|
}
|
|
keys
|
|
}
|
|
|
|
/// Read `CLAWMATES_RUNTIME_AUTH`, defaulting to `api_key`.
|
|
///
|
|
/// Defaulting to the existing behaviour is deliberate: an unset or misspelled
|
|
/// value must not silently strip the API key and leave missions unable to
|
|
/// reach a model at all.
|
|
pub fn runtime_auth_mode() -> RuntimeAuth {
|
|
match std::env::var("CLAWMATES_RUNTIME_AUTH")
|
|
.unwrap_or_default()
|
|
.trim()
|
|
.to_ascii_lowercase()
|
|
.as_str()
|
|
{
|
|
"subscription" => RuntimeAuth::Subscription,
|
|
"" | "api_key" => RuntimeAuth::ApiKey,
|
|
other => {
|
|
eprintln!(
|
|
"mission_runtime: unknown CLAWMATES_RUNTIME_AUTH={other:?} — \
|
|
defaulting to api_key"
|
|
);
|
|
RuntimeAuth::ApiKey
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Docker networks the runtime container must be attached to.
|
|
/// - `clawmates_core`: talks to the server + database
|
|
/// - `clawmates_edge`: has egress for outbound provider calls
|
|
const CORE_NETWORK: &str = "clawmates_core";
|
|
const EDGE_NETWORK: &str = "clawmates_edge";
|
|
|
|
/// Host path (as seen by the docker engine, NOT the server container)
|
|
/// where the mission's checkouts live. Matches the mount source used
|
|
/// by `clawmates-runtime.service`.
|
|
const MISSIONS_HOST_ROOT: &str = "/var/lib/clawmates-missions";
|
|
|
|
/// Host path holding the shared ZeroClaw config + seeded agent library
|
|
/// (built up over time by the shared clawmates-runtime.service). Per-
|
|
/// mission runtime containers bind-mount this so their gateway
|
|
/// inherits all the `claw_*` agents that team_template_loader has
|
|
/// provisioned. Each per-mission container then MINTS ITS OWN pairing
|
|
/// code via /admin/paircode/new so the accepted-tokens list is
|
|
/// independent per mission. Overridable for dev via
|
|
/// `CLAWMATES_RUNTIME_SEED_DIR`.
|
|
///
|
|
/// Concurrency caveat: the sqlite files under .zeroclaw/data/ are
|
|
/// currently shared across all per-mission runtimes AND the shared
|
|
/// runtime. Concurrent daemons opening the same sessions.db can
|
|
/// interleave; in practice topology_worker sequentializes runs per
|
|
/// mission so this rarely bites. Long-term: copy-on-write per mission.
|
|
const DEFAULT_SEED_DIR: &str = "/root/clawmates-runtime/data";
|
|
|
|
/// Deterministic docker container name for a mission's runtime.
|
|
/// Uses the full UUID hex — UUIDv7 encodes time in the leading bytes,
|
|
/// so a short prefix isn't guaranteed unique across missions minted
|
|
/// in the same second. Docker permits up to 253 characters in a name,
|
|
/// so the extra length is free.
|
|
pub fn container_name(mission_id: Uuid) -> String {
|
|
format!("cm-runtime-mission-{}", mission_id.simple())
|
|
}
|
|
|
|
/// Endpoint URL the topology_worker's ZeroClawDriveExecutor will dial.
|
|
/// Uses the container name as hostname — resolves within the shared
|
|
/// `clawmates_core` docker network.
|
|
pub fn endpoint_url(container_name: &str) -> String {
|
|
format!("http://{}:{}", container_name, GATEWAY_PORT)
|
|
}
|
|
|
|
pub struct MissionRuntimeProvisioner {
|
|
docker: Docker,
|
|
image: String,
|
|
}
|
|
|
|
/// What `ensure_container` returns: everything the caller needs to
|
|
/// point a topology_worker at this mission's fresh gateway.
|
|
#[derive(Debug, Clone)]
|
|
pub struct EnsuredContainer {
|
|
pub endpoint: String,
|
|
/// One-time pairing code minted by the daemon at boot; may be
|
|
/// None on the reuse-existing path when we couldn't scrape it
|
|
/// back (log rotation). Callers keep the previously-persisted
|
|
/// value in that case.
|
|
pub pairing_code: Option<String>,
|
|
}
|
|
|
|
impl MissionRuntimeProvisioner {
|
|
/// Connect to the docker engine. Honors `DOCKER_HOST` (set in
|
|
/// compose to the socket proxy) and falls back to the local
|
|
/// socket. Returns None when docker is unreachable, so callers
|
|
/// can degrade gracefully (missions still launch, just against
|
|
/// the shared runtime).
|
|
pub fn from_env() -> Option<MissionRuntimeProvisioner> {
|
|
let docker = if let Ok(host) = std::env::var("DOCKER_HOST") {
|
|
Docker::connect_with_http(&host, 30, bollard::API_DEFAULT_VERSION).ok()?
|
|
} else {
|
|
Docker::connect_with_local_defaults().ok()?
|
|
};
|
|
let image =
|
|
std::env::var("CLAWMATES_RUNTIME_IMAGE").unwrap_or_else(|_| DEFAULT_IMAGE.to_string());
|
|
Some(MissionRuntimeProvisioner { docker, image })
|
|
}
|
|
|
|
/// Idempotent: returns the endpoint URL, creating the container
|
|
/// on first call. If the container exists but is stopped, starts
|
|
/// it. If it exists and is running, returns its endpoint.
|
|
pub async fn ensure_container(&self, mission_id: Uuid) -> Result<EnsuredContainer, String> {
|
|
let name = container_name(mission_id);
|
|
// Fast path: already running.
|
|
if let Ok(inspect) = self
|
|
.docker
|
|
.inspect_container(&name, None::<InspectContainerOptions>)
|
|
.await
|
|
{
|
|
let running = inspect
|
|
.state
|
|
.as_ref()
|
|
.and_then(|s| s.running)
|
|
.unwrap_or(false);
|
|
if running {
|
|
// Reuse; try to re-scrape the pairing code from logs,
|
|
// but it may have rotated out — caller falls back to
|
|
// the previously-persisted value in that case.
|
|
let pairing_code = self.mint_pairing_code(&name).await;
|
|
return Ok(EnsuredContainer {
|
|
endpoint: endpoint_url(&name),
|
|
pairing_code,
|
|
});
|
|
}
|
|
// Exists but not running — remove + recreate below rather
|
|
// than trying to restart a dirty-state container.
|
|
let _ = self
|
|
.docker
|
|
.remove_container(
|
|
&name,
|
|
Some(RemoveContainerOptions {
|
|
force: true,
|
|
..Default::default()
|
|
}),
|
|
)
|
|
.await;
|
|
}
|
|
|
|
// Create fresh. Ensure the bind source exists first — research-
|
|
// only missions (no repo checkout) still need the directory
|
|
// present or docker start fails with EACCES/ENOENT.
|
|
let mission_dir = format!("{MISSIONS_HOST_ROOT}/{mission_id}");
|
|
let _ = tokio::fs::create_dir_all(&mission_dir).await;
|
|
let seed_dir = std::env::var("CLAWMATES_RUNTIME_SEED_DIR")
|
|
.unwrap_or_else(|_| DEFAULT_SEED_DIR.to_string());
|
|
let mounts = vec![
|
|
// Mount just this mission's directory. Agents can navigate
|
|
// its `/repo` subdir but never see other missions'.
|
|
Mount {
|
|
target: Some("/mission".to_string()),
|
|
source: Some(mission_dir.clone()),
|
|
typ: Some(MountTypeEnum::BIND),
|
|
read_only: Some(false),
|
|
..Default::default()
|
|
},
|
|
// Share the shared-runtime data dir so this gateway inherits
|
|
// the seeded agent library (`claw_*` templates). We then
|
|
// mint a per-mission pairing code via /admin/paircode/new
|
|
// below — the mint writes into the shared devices.db but
|
|
// the resulting token is unique to this mission.
|
|
Mount {
|
|
target: Some("/zeroclaw-data".to_string()),
|
|
source: Some(seed_dir),
|
|
typ: Some(MountTypeEnum::BIND),
|
|
read_only: Some(false),
|
|
..Default::default()
|
|
},
|
|
];
|
|
|
|
let host_config = HostConfig {
|
|
mounts: Some(mounts),
|
|
restart_policy: Some(bollard::models::RestartPolicy {
|
|
name: Some(bollard::models::RestartPolicyNameEnum::UNLESS_STOPPED),
|
|
..Default::default()
|
|
}),
|
|
network_mode: Some(CORE_NETWORK.to_string()),
|
|
..Default::default()
|
|
};
|
|
|
|
// NOTE: do NOT set ZEROCLAW_WORKSPACE — despite the name, the
|
|
// daemon uses it (schema.rs:17467) as a legacy config-dir
|
|
// pointer that overrides ZEROCLAW_CONFIG_DIR/DATA_DIR. Setting
|
|
// it to /mission/repo makes the daemon compute its config dir
|
|
// as /mission/repo/.zeroclaw (empty!) and boot with a fresh
|
|
// defaults-only config — zero agents loaded.
|
|
//
|
|
// Per-agent workspace pinning belongs in the shared config
|
|
// under agents.<alias>.workspace = "/mission/repo", not env.
|
|
let mut env = vec![
|
|
format!("ZEROCLAW_GATEWAY_PORT={GATEWAY_PORT}"),
|
|
format!("CM_MISSION_ID={mission_id}"),
|
|
// Let the agents' git read the checkout.
|
|
//
|
|
// The server clones as uid 65532; this container runs as root, so
|
|
// every `git` an agent runs hits "detected dubious ownership" and
|
|
// refuses the repository. Agents do not report that as a failure —
|
|
// they improvise. On mission 019fc3ba one wrote a `.gitconfig_temp`
|
|
// containing `[safe] directory = /mission/repo` into the repository
|
|
// root, which then showed up in the captured diff and would have
|
|
// been committed and pushed to the user's repo alongside the real
|
|
// work.
|
|
//
|
|
// `GIT_CONFIG_*` is git's environment form of `-c` and is
|
|
// inherited by subprocesses, so it covers the agent's own git, any
|
|
// tool that shells out to git, and the `git_operations` tool alike.
|
|
// Scoped to the checkout; never `--global`.
|
|
"GIT_CONFIG_COUNT=1".to_string(),
|
|
"GIT_CONFIG_KEY_0=safe.directory".to_string(),
|
|
"GIT_CONFIG_VALUE_0=/mission/repo".to_string(),
|
|
];
|
|
// Provider credentials forwarded into the container.
|
|
//
|
|
// ANTHROPIC_API_KEY is conditional, and the reason is subtle enough to
|
|
// be worth stating: Claude Code resolves credentials in a fixed
|
|
// priority order, and ANTHROPIC_API_KEY ranks ABOVE the subscription's
|
|
// CLAUDE_CODE_OAUTH_TOKEN. So on a runtime authenticated via `claude
|
|
// /login`, forwarding the key here silently wins — `claude` still works,
|
|
// the agents still run, and every mission bills the API while appearing
|
|
// to use the subscription. Failing loudly is impossible; the only fix
|
|
// is not to send it.
|
|
//
|
|
// The other three are unrelated providers (Gemini/Groq/OpenAI) with no
|
|
// subscription equivalent, so they forward in both modes.
|
|
let auth_mode = runtime_auth_mode();
|
|
for key in forwarded_provider_keys(auth_mode) {
|
|
if let Ok(v) = std::env::var(key) {
|
|
env.push(format!("{key}={v}"));
|
|
}
|
|
}
|
|
eprintln!(
|
|
"mission_runtime: mission {mission_id} container auth mode = {} \
|
|
(ANTHROPIC_API_KEY {})",
|
|
auth_mode.as_str(),
|
|
if auth_mode == RuntimeAuth::ApiKey {
|
|
"forwarded"
|
|
} else {
|
|
"withheld so the runtime's subscription login is used"
|
|
}
|
|
);
|
|
|
|
let mut labels = HashMap::new();
|
|
labels.insert("clawmates.role".to_string(), "mission-runtime".to_string());
|
|
labels.insert("clawmates.mission_id".to_string(), mission_id.to_string());
|
|
|
|
let config = ContainerCreateBody {
|
|
image: Some(self.image.clone()),
|
|
cmd: Some(vec![
|
|
"daemon".to_string(),
|
|
"--host".to_string(),
|
|
"0.0.0.0".to_string(),
|
|
]),
|
|
env: Some(env),
|
|
host_config: Some(host_config),
|
|
labels: Some(labels),
|
|
..Default::default()
|
|
};
|
|
|
|
self.docker
|
|
.create_container(
|
|
Some(CreateContainerOptions {
|
|
name: Some(name.clone()),
|
|
..Default::default()
|
|
}),
|
|
config,
|
|
)
|
|
.await
|
|
.map_err(|e| format!("create mission runtime container: {e}"))?;
|
|
|
|
// Attach to the edge network for outbound provider egress.
|
|
let _ = self
|
|
.docker
|
|
.connect_network(
|
|
EDGE_NETWORK,
|
|
NetworkConnectRequest {
|
|
container: Some(name.clone()),
|
|
endpoint_config: Some(EndpointSettings::default()),
|
|
},
|
|
)
|
|
.await;
|
|
|
|
self.docker
|
|
.start_container(&name, None::<StartContainerOptions>)
|
|
.await
|
|
.map_err(|e| format!("start mission runtime container: {e}"))?;
|
|
|
|
// Give the daemon a moment to print its boot banner, then
|
|
// scrape the pairing code. Poll with a short deadline so a
|
|
// slow boot doesn't hang the launch — the topology_worker
|
|
// will retry pairing later if we came up empty.
|
|
let pairing_code = self.wait_for_pairing_code(&name).await;
|
|
Ok(EnsuredContainer {
|
|
endpoint: endpoint_url(&name),
|
|
pairing_code,
|
|
})
|
|
}
|
|
|
|
/// Poll the daemon's localhost admin endpoint until it responds
|
|
/// with a fresh pairing code. Boot takes ~1-2s; deadline is 15s.
|
|
/// Returns None on timeout so callers see a clear paired=false
|
|
/// signal in the mission binding log.
|
|
async fn wait_for_pairing_code(&self, name: &str) -> Option<String> {
|
|
let deadline = std::time::Duration::from_secs(15);
|
|
let start = std::time::Instant::now();
|
|
while start.elapsed() < deadline {
|
|
if let Some(code) = self.mint_pairing_code(name).await {
|
|
return Some(code);
|
|
}
|
|
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
|
}
|
|
None
|
|
}
|
|
|
|
/// `docker exec` into the container and hit the local admin
|
|
/// endpoint that mints a fresh pairing code. This works whether
|
|
/// the daemon booted "already paired" (no code in the log) or
|
|
/// "pairing required" (code in the log) — both surfaces mint on
|
|
/// demand.
|
|
async fn mint_pairing_code(&self, name: &str) -> Option<String> {
|
|
let exec = self
|
|
.docker
|
|
.create_exec(
|
|
name,
|
|
CreateExecOptions {
|
|
cmd: Some(
|
|
[
|
|
"curl",
|
|
"-fs",
|
|
"-X",
|
|
"POST",
|
|
"http://127.0.0.1:42617/admin/paircode/new",
|
|
]
|
|
.iter()
|
|
.map(|s| s.to_string())
|
|
.collect(),
|
|
),
|
|
attach_stdout: Some(true),
|
|
attach_stderr: Some(true),
|
|
..Default::default()
|
|
},
|
|
)
|
|
.await
|
|
.ok()?;
|
|
let started = self.docker.start_exec(&exec.id, None).await.ok()?;
|
|
let StartExecResults::Attached { mut output, .. } = started else {
|
|
return None;
|
|
};
|
|
let mut buf = String::new();
|
|
while let Some(chunk) = output.next().await {
|
|
if let Ok(c) = chunk {
|
|
buf.push_str(&c.to_string());
|
|
if buf.len() > 8_000 {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
extract_pairing_code_from_json(&buf)
|
|
}
|
|
|
|
/// Stamp `[agents.<alias>.workspace] path = "<workspace_path>"` into the
|
|
/// shared runtime config for each provisioned claw, by editing the
|
|
/// config file directly on the per-mission container. This is the
|
|
/// out-of-band path for workspace pinning: the ZeroClaw config prop API
|
|
/// cannot set `workspace.path` (an `Option<PathBuf>` the `Configurable`
|
|
/// macro skips from prop enumeration), so `provision_claw` leaves it
|
|
/// unset and we stamp it here. The daemon applies the change on the
|
|
/// same config reload that surfaces the freshly-provisioned claws for
|
|
/// the run.
|
|
///
|
|
/// Concurrency caveat (mirrors the seed-dir note above): the config
|
|
/// file is shared across the persistent runtime and every per-mission
|
|
/// daemon, so this read-modify-write can race a provision from another
|
|
/// mission launching at the same instant. Missions launch one at a
|
|
/// time in practice; the durable fix is per-mission config isolation.
|
|
pub async fn pin_agent_workspaces(
|
|
&self,
|
|
mission_id: Uuid,
|
|
claws: &[cm_domain::AgentId],
|
|
workspace_path: &str,
|
|
) -> Result<(), String> {
|
|
if claws.is_empty() {
|
|
return Ok(());
|
|
}
|
|
const CONFIG_PATH: &str = "/zeroclaw-data/.zeroclaw/config.toml";
|
|
let name = container_name(mission_id);
|
|
|
|
let raw = self
|
|
.exec_capture(&name, vec!["cat".into(), CONFIG_PATH.into()])
|
|
.await?;
|
|
let aliases: Vec<String> = claws
|
|
.iter()
|
|
.map(|c| crate::runtime_provision::claw_alias(c.as_uuid()))
|
|
.collect();
|
|
let (edited, pinned) = stamp_workspace_paths(&raw, &aliases, workspace_path)?;
|
|
if pinned == 0 {
|
|
return Ok(());
|
|
}
|
|
|
|
let b64 = base64::engine::general_purpose::STANDARD.encode(edited.as_bytes());
|
|
// Decode to a sibling temp then atomically move over the live file,
|
|
// so a partial write can never leave the daemon with truncated TOML.
|
|
let script = format!(
|
|
"printf %s '{b64}' | base64 -d > {CONFIG_PATH}.tmp && mv {CONFIG_PATH}.tmp {CONFIG_PATH}"
|
|
);
|
|
let out = self
|
|
.exec_capture(&name, vec!["sh".into(), "-c".into(), script])
|
|
.await?;
|
|
if !out.trim().is_empty() {
|
|
return Err(format!("write runtime config.toml: {out}"));
|
|
}
|
|
eprintln!(
|
|
"mission_runtime: pinned {pinned} workspace(s) → {workspace_path} for mission {mission_id}"
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
/// Run a command in the mission container and return its combined
|
|
/// stdout+stderr as a String. Used for small config round-trips.
|
|
async fn exec_capture(&self, name: &str, cmd: Vec<String>) -> Result<String, String> {
|
|
let exec = self
|
|
.docker
|
|
.create_exec(
|
|
name,
|
|
CreateExecOptions {
|
|
cmd: Some(cmd),
|
|
attach_stdout: Some(true),
|
|
attach_stderr: Some(true),
|
|
..Default::default()
|
|
},
|
|
)
|
|
.await
|
|
.map_err(|e| format!("create_exec on {name}: {e}"))?;
|
|
let started = self
|
|
.docker
|
|
.start_exec(&exec.id, None)
|
|
.await
|
|
.map_err(|e| format!("start_exec on {name}: {e}"))?;
|
|
let StartExecResults::Attached { mut output, .. } = started else {
|
|
return Err(format!("exec on {name} returned a detached result"));
|
|
};
|
|
let mut buf = String::new();
|
|
while let Some(chunk) = output.next().await {
|
|
match chunk {
|
|
Ok(c) => buf.push_str(&c.to_string()),
|
|
Err(e) => return Err(format!("exec output stream on {name}: {e}")),
|
|
}
|
|
}
|
|
Ok(buf)
|
|
}
|
|
}
|
|
|
|
/// Format-preserving stamp of `[agents.<alias>.workspace] path = "<path>"`
|
|
/// for each alias present in `raw`. Keeps the operator's comments, ordering,
|
|
/// and every untouched byte intact; only the `path` keys change. Aliases not
|
|
/// already present are skipped (never fabricated — a bare agent table would
|
|
/// drop that agent's model/risk_profile/bundles). Returns the edited document
|
|
/// and how many agents were pinned.
|
|
fn stamp_workspace_paths(
|
|
raw: &str,
|
|
aliases: &[String],
|
|
workspace_path: &str,
|
|
) -> Result<(String, usize), String> {
|
|
let mut doc = raw
|
|
.parse::<toml_edit::DocumentMut>()
|
|
.map_err(|e| format!("parse runtime config.toml: {e}"))?;
|
|
let Some(agents) = doc.get_mut("agents").and_then(|i| i.as_table_like_mut()) else {
|
|
return Err("runtime config has no [agents] table".to_string());
|
|
};
|
|
let mut pinned = 0usize;
|
|
for alias in aliases {
|
|
let Some(agent) = agents.get_mut(alias).and_then(|i| i.as_table_like_mut()) else {
|
|
eprintln!("mission_runtime: pin skip — {alias} absent from config");
|
|
continue;
|
|
};
|
|
if agent.get("workspace").is_none() {
|
|
agent.insert("workspace", toml_edit::Item::Table(toml_edit::Table::new()));
|
|
}
|
|
if let Some(ws) = agent
|
|
.get_mut("workspace")
|
|
.and_then(|i| i.as_table_like_mut())
|
|
{
|
|
ws.insert("path", toml_edit::value(workspace_path));
|
|
pinned += 1;
|
|
}
|
|
}
|
|
Ok((doc.to_string(), pinned))
|
|
}
|
|
|
|
/// Parse the JSON `{ "pairing_code": "NNNNNN", ... }` body from the
|
|
/// admin/paircode/new endpoint.
|
|
fn extract_pairing_code_from_json(body: &str) -> Option<String> {
|
|
let v: serde_json::Value = serde_json::from_str(body.trim()).ok()?;
|
|
v.get("pairing_code")
|
|
.and_then(|x| x.as_str())
|
|
.filter(|s| !s.is_empty())
|
|
.map(String::from)
|
|
}
|
|
|
|
impl MissionRuntimeProvisioner {
|
|
/// Restart the mission's runtime container and wait for its gateway to
|
|
/// answer again. Needed after `pin_agent_workspaces`: the daemon reads
|
|
/// config once at boot and never re-reads the file, so a file-only setting
|
|
/// (`workspace.path`, which the config prop API cannot set) only takes
|
|
/// effect across a restart. Agents provisioned through the daemon's own
|
|
/// config API are already persisted to that file, so they survive.
|
|
pub async fn restart_container(&self, mission_id: Uuid) -> Result<(), String> {
|
|
let name = container_name(mission_id);
|
|
self.docker
|
|
.restart_container(
|
|
&name,
|
|
None::<bollard::query_parameters::RestartContainerOptions>,
|
|
)
|
|
.await
|
|
.map_err(|e| format!("restart mission runtime container: {e}"))?;
|
|
// Wait for the gateway to serve again so the caller can launch a run
|
|
// immediately after. ~20s ceiling; the daemon normally boots in ~2s.
|
|
for _ in 0..40 {
|
|
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
|
|
// NOTE: exec_capture only fails on docker errors — a curl that
|
|
// can't connect still "succeeds" (it prints `curl: (7) …`), so we
|
|
// must inspect the BODY. /health answers `{"paired":…}`.
|
|
let body = self
|
|
.exec_capture(
|
|
&name,
|
|
vec![
|
|
"curl".into(),
|
|
"-fsS".into(),
|
|
"-m".into(),
|
|
"2".into(),
|
|
format!("http://127.0.0.1:{GATEWAY_PORT}/health"),
|
|
],
|
|
)
|
|
.await
|
|
.unwrap_or_default();
|
|
if body.contains("\"paired\"") {
|
|
eprintln!("mission_runtime: restarted {name}, gateway healthy");
|
|
return Ok(());
|
|
}
|
|
}
|
|
Err(format!("{name} gateway did not come back after restart"))
|
|
}
|
|
|
|
/// Force-remove the mission's runtime container AND its host workspace
|
|
/// dir (the `/mission/repo` checkout bind source). Idempotent: a missing
|
|
/// container or dir is not an error — this is called both by the terminal
|
|
/// sweeper and by mission delete, where the container may already be gone.
|
|
pub async fn teardown_container(&self, mission_id: Uuid) -> Result<(), String> {
|
|
let name = container_name(mission_id);
|
|
if let Err(e) = self
|
|
.docker
|
|
.remove_container(
|
|
&name,
|
|
Some(RemoveContainerOptions {
|
|
force: true,
|
|
..Default::default()
|
|
}),
|
|
)
|
|
.await
|
|
{
|
|
// 404 (already gone) is fine; anything else is worth surfacing.
|
|
let msg = e.to_string();
|
|
if !msg.contains("No such container") && !msg.contains("404") {
|
|
return Err(format!("remove mission runtime container: {e}"));
|
|
}
|
|
}
|
|
// Remove the per-mission workspace dir (repo checkout + scratch). This
|
|
// path is bind-mounted into cm-api, so we can reap it directly.
|
|
let mission_dir = format!("{MISSIONS_HOST_ROOT}/{mission_id}");
|
|
if let Err(e) = tokio::fs::remove_dir_all(&mission_dir).await {
|
|
if e.kind() != std::io::ErrorKind::NotFound {
|
|
eprintln!("mission_runtime: rm workspace dir {mission_dir}: {e}");
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// Background sweeper: force-remove runtime containers for missions
|
|
/// that reached a terminal state ≥ `grace` ago. Keeps the container
|
|
/// around briefly after `completed`/`failed`/`cancelled` so the
|
|
/// operator can re-open the UI and pull the last checkpoint before
|
|
/// the daemon disappears. Runs on the same cadence as the phase
|
|
/// runner (10s) with a much longer per-mission grace.
|
|
pub fn spawn_sweeper(pool: sqlx::PgPool, grace: std::time::Duration) {
|
|
tokio::spawn(async move {
|
|
// Wait past the phase_runner boot so we don't fight over
|
|
// just-launched missions.
|
|
tokio::time::sleep(std::time::Duration::from_secs(30)).await;
|
|
let mut ticker = tokio::time::interval(std::time::Duration::from_secs(60));
|
|
ticker.tick().await;
|
|
loop {
|
|
ticker.tick().await;
|
|
if let Err(e) = sweep_once(&pool, grace).await {
|
|
eprintln!("mission_runtime::sweeper: sweep failed: {e}");
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
async fn sweep_once(pool: &sqlx::PgPool, grace: std::time::Duration) -> Result<(), String> {
|
|
use sqlx::Row;
|
|
let grace_secs = grace.as_secs() as f64;
|
|
let rows = sqlx::query(
|
|
"SELECT id, workspace_id, runtime_container_name
|
|
FROM missions
|
|
WHERE status IN ('completed', 'failed', 'cancelled')
|
|
AND runtime_endpoint IS NOT NULL
|
|
AND completed_at IS NOT NULL
|
|
AND completed_at < now() - make_interval(secs => $1::float)
|
|
LIMIT 20",
|
|
)
|
|
.bind(grace_secs)
|
|
.fetch_all(pool)
|
|
.await
|
|
.map_err(|e| format!("query terminal missions: {e}"))?;
|
|
if rows.is_empty() {
|
|
return Ok(());
|
|
}
|
|
let Some(prov) = MissionRuntimeProvisioner::from_env() else {
|
|
return Ok(());
|
|
};
|
|
for row in rows {
|
|
let id: Uuid = row.get("id");
|
|
let workspace_id: Uuid = row.get("workspace_id");
|
|
// Last chance. `teardown_container` deletes the checkout, so anything
|
|
// not captured by now is gone for good. The phase sweep should have
|
|
// handled this minutes ago; this covers the cases it cannot — a phase
|
|
// that ended `failed` rather than `completed`, or a capture that kept
|
|
// erroring until the grace window ran out.
|
|
if let Err(e) = capture_outstanding_phases(pool, id).await {
|
|
eprintln!("mission_runtime::sweeper: last-chance capture for {id}: {e}");
|
|
}
|
|
if let Err(e) = prov.teardown_container(id).await {
|
|
// A not-found is expected when the container was already
|
|
// reaped by a docker restart or a manual op; log at info
|
|
// level (via eprintln) and clear the binding anyway so the
|
|
// sweeper doesn't retry forever.
|
|
eprintln!("mission_runtime::sweeper: teardown mission {id}: {e}");
|
|
}
|
|
if let Err(e) =
|
|
cm_db::repo::missions::set_runtime_binding(pool, id, workspace_id, None, None, None)
|
|
.await
|
|
{
|
|
eprintln!("mission_runtime::sweeper: clear binding for {id}: {e}");
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Capture any phase of `mission_id` that has a repo and no `code_diff` yet,
|
|
/// regardless of how the phase ended.
|
|
///
|
|
/// The phase sweep only captures `completed` phases. A mission that failed
|
|
/// mid-coding still has real work in its checkout, and deleting it
|
|
/// unexamined is how a debugging session loses the only evidence of what the
|
|
/// agents actually did.
|
|
async fn capture_outstanding_phases(pool: &sqlx::PgPool, mission_id: Uuid) -> Result<(), String> {
|
|
use sqlx::Row;
|
|
let rows = sqlx::query(
|
|
"SELECT mp.id
|
|
FROM mission_phases mp
|
|
JOIN missions m ON m.id = mp.mission_id
|
|
WHERE mp.mission_id = $1
|
|
AND m.repo_id IS NOT NULL
|
|
AND NOT EXISTS (
|
|
SELECT 1 FROM mission_artifacts a
|
|
WHERE a.mission_id = mp.mission_id
|
|
AND a.phase_id = mp.id
|
|
AND a.kind = 'code_diff'
|
|
)",
|
|
)
|
|
.bind(mission_id)
|
|
.fetch_all(pool)
|
|
.await
|
|
.map_err(|e| format!("select uncaptured phases: {e}"))?;
|
|
|
|
for row in rows {
|
|
let phase_id: Uuid = row.get("id");
|
|
if let Err(e) =
|
|
crate::mission_delivery::capture_phase_diff(pool, mission_id, phase_id).await
|
|
{
|
|
eprintln!(
|
|
"mission_runtime::sweeper: capture mission {mission_id} phase {phase_id}: {e}"
|
|
);
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
/// The regression guard for the whole subscription feature.
|
|
///
|
|
/// Claude Code ranks `ANTHROPIC_API_KEY` above the subscription's OAuth
|
|
/// credential, so forwarding it into a container whose runtime is logged in
|
|
/// means every mission silently bills the API while looking correct. There
|
|
/// is no error to observe — only the invoice. If this test ever goes red,
|
|
/// the subscription path is off even though nothing appears broken.
|
|
#[test]
|
|
fn subscription_mode_withholds_the_anthropic_api_key() {
|
|
let keys = forwarded_provider_keys(RuntimeAuth::Subscription);
|
|
assert!(
|
|
!keys.contains(&"ANTHROPIC_API_KEY"),
|
|
"ANTHROPIC_API_KEY outranks the subscription credential; forwarding \
|
|
it silently bills the API. Forwarded: {keys:?}"
|
|
);
|
|
// Unrelated providers have no subscription equivalent and must survive.
|
|
for k in ["GEMINI_API_KEY", "GROQ_API_KEY", "OPENAI_API_KEY"] {
|
|
assert!(keys.contains(&k), "{k} should still be forwarded");
|
|
}
|
|
// And the subscription credential MUST travel. A mission container
|
|
// has its own data dir, so unlike the shared runtime it has no
|
|
// persisted `claude /login` to fall back on. Without this the CLI
|
|
// has no credential and simply hangs — a phase stuck at `running`
|
|
// with nothing in the logs, which is exactly how this was found.
|
|
assert!(
|
|
keys.contains(&"CLAUDE_CODE_OAUTH_TOKEN"),
|
|
"subscription mode must forward the token; without it `claude -p` \
|
|
hangs with no credential. Forwarded: {keys:?}"
|
|
);
|
|
}
|
|
|
|
/// The two credentials must never travel together: Claude Code would pick
|
|
/// the API key and bill it while the deployment believes it is on the
|
|
/// subscription.
|
|
#[test]
|
|
fn the_two_anthropic_credentials_are_mutually_exclusive() {
|
|
for mode in [RuntimeAuth::ApiKey, RuntimeAuth::Subscription] {
|
|
let keys = forwarded_provider_keys(mode);
|
|
let both = keys.contains(&"ANTHROPIC_API_KEY")
|
|
&& keys.contains(&"CLAUDE_CODE_OAUTH_TOKEN");
|
|
assert!(!both, "{mode:?} forwards both credentials: {keys:?}");
|
|
}
|
|
}
|
|
|
|
/// Default behaviour is unchanged, so a deployment that never opts in keeps
|
|
/// working exactly as before.
|
|
#[test]
|
|
fn api_key_mode_forwards_everything() {
|
|
let keys = forwarded_provider_keys(RuntimeAuth::ApiKey);
|
|
for k in [
|
|
"ANTHROPIC_API_KEY",
|
|
"GEMINI_API_KEY",
|
|
"GROQ_API_KEY",
|
|
"OPENAI_API_KEY",
|
|
] {
|
|
assert!(keys.contains(&k), "{k} should be forwarded in api_key mode");
|
|
}
|
|
assert!(
|
|
!keys.contains(&"CLAUDE_CODE_OAUTH_TOKEN"),
|
|
"api_key mode must not also ship the subscription token"
|
|
);
|
|
}
|
|
|
|
/// An unset or misspelled value must fall back to the *existing* behaviour.
|
|
/// Defaulting to subscription on a typo would leave missions with no
|
|
/// credential at all.
|
|
#[test]
|
|
fn auth_mode_defaults_to_api_key() {
|
|
// Can't safely mutate process env in a parallel test binary, so assert
|
|
// the mapping the parser implements rather than the env read itself.
|
|
for (input, expected) in [
|
|
("subscription", RuntimeAuth::Subscription),
|
|
("SUBSCRIPTION", RuntimeAuth::Subscription),
|
|
("api_key", RuntimeAuth::ApiKey),
|
|
("", RuntimeAuth::ApiKey),
|
|
("nonsense", RuntimeAuth::ApiKey),
|
|
] {
|
|
let got = match input.trim().to_ascii_lowercase().as_str() {
|
|
"subscription" => RuntimeAuth::Subscription,
|
|
"" | "api_key" => RuntimeAuth::ApiKey,
|
|
_ => RuntimeAuth::ApiKey,
|
|
};
|
|
assert_eq!(got, expected, "input {input:?}");
|
|
}
|
|
}
|
|
|
|
const SAMPLE_CONFIG: &str = r#"# top comment
|
|
[agents.claw_a]
|
|
model_provider = "anthropic.default"
|
|
risk_profile = "coding_readwrite"
|
|
mcp_bundles = ["clawmates_door"]
|
|
|
|
[agents.claw_a.workspace]
|
|
unrestricted_filesystem = false
|
|
|
|
[agents.claw_b]
|
|
risk_profile = "research_readonly"
|
|
|
|
[risk_profiles.coding_readwrite]
|
|
# keep this comment
|
|
allowed_tools = ["file_read", "file_edit"]
|
|
"#;
|
|
|
|
#[test]
|
|
fn stamp_pins_path_and_preserves_existing_workspace_fields() {
|
|
let (out, n) =
|
|
stamp_workspace_paths(SAMPLE_CONFIG, &["claw_a".to_string()], "/mission/repo").unwrap();
|
|
assert_eq!(n, 1);
|
|
assert!(out.contains(r#"path = "/mission/repo""#));
|
|
// The sibling field in the same table is untouched.
|
|
assert!(out.contains("unrestricted_filesystem = false"));
|
|
// Comments and unrelated sections survive the round-trip.
|
|
assert!(out.contains("# top comment"));
|
|
assert!(out.contains("# keep this comment"));
|
|
assert!(out.contains("[risk_profiles.coding_readwrite]"));
|
|
}
|
|
|
|
#[test]
|
|
fn stamp_creates_workspace_table_when_absent() {
|
|
let (out, n) =
|
|
stamp_workspace_paths(SAMPLE_CONFIG, &["claw_b".to_string()], "/mission/repo").unwrap();
|
|
assert_eq!(n, 1);
|
|
// claw_b had no [workspace] table; it now has one with the path.
|
|
let doc = out.parse::<toml_edit::DocumentMut>().unwrap();
|
|
assert_eq!(
|
|
doc["agents"]["claw_b"]["workspace"]["path"].as_str(),
|
|
Some("/mission/repo")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn stamp_skips_absent_aliases_without_fabricating_them() {
|
|
let (out, n) = stamp_workspace_paths(
|
|
SAMPLE_CONFIG,
|
|
&["claw_missing".to_string()],
|
|
"/mission/repo",
|
|
)
|
|
.unwrap();
|
|
assert_eq!(n, 0);
|
|
assert!(!out.contains("claw_missing"));
|
|
}
|
|
|
|
#[test]
|
|
fn stamp_is_idempotent_overwriting_a_prior_path() {
|
|
let once = stamp_workspace_paths(SAMPLE_CONFIG, &["claw_a".to_string()], "/old/path")
|
|
.unwrap()
|
|
.0;
|
|
let (twice, n) =
|
|
stamp_workspace_paths(&once, &["claw_a".to_string()], "/mission/repo").unwrap();
|
|
assert_eq!(n, 1);
|
|
assert!(twice.contains(r#"path = "/mission/repo""#));
|
|
assert!(!twice.contains("/old/path"));
|
|
// Exactly one path key for claw_a (no duplication).
|
|
assert_eq!(twice.matches("path = ").count(), 1);
|
|
}
|
|
|
|
#[test]
|
|
fn container_name_is_stable_and_prefixed() {
|
|
let id = Uuid::parse_str("019f84a0-f2a2-7bd0-be9b-86713ec73693").unwrap();
|
|
let name = container_name(id);
|
|
assert_eq!(name, "cm-runtime-mission-019f84a0f2a27bd0be9b86713ec73693");
|
|
// Determinism: same input → same output.
|
|
assert_eq!(name, container_name(id));
|
|
}
|
|
|
|
#[test]
|
|
fn container_names_differ_across_missions() {
|
|
// Two UUIDs differing only in the trailing hex char — full-uuid
|
|
// naming must distinguish them (UUIDv7's timestamp shares
|
|
// leading bytes for missions minted in the same second).
|
|
let a = container_name(Uuid::parse_str("019f84a0-f2a2-7bd0-be9b-86713ec73693").unwrap());
|
|
let b = container_name(Uuid::parse_str("019f84a0-f2a2-7bd0-be9b-86713ec7369f").unwrap());
|
|
assert_ne!(a, b);
|
|
}
|
|
|
|
#[test]
|
|
fn endpoint_url_uses_gateway_port() {
|
|
let url = endpoint_url("cm-runtime-mission-abc");
|
|
assert_eq!(url, "http://cm-runtime-mission-abc:42617");
|
|
}
|
|
}
|