missions: schema + provisioner skeleton for per-mission runtime containers (C3 slice 1)
- migration 0058: adds missions.runtime_container_name + runtime_endpoint
- new mission_runtime module (bollard): ensure_container /
teardown_container. Container is spawned on clawmates_core +
clawmates_edge networks with just /var/lib/clawmates-missions/{id}
bind-mounted so agents scoped to /mission/repo can only see this
missions repo.
- provider API keys forwarded from the server envs so per-mission
runtimes inherit them.
- Mission struct + repo helpers updated for the two new columns +
set_runtime_binding().
- Unit tests cover container naming determinism + entropy.
Not wired to the orchestrator yet — that lands in slice 2.
This commit is contained in:
@@ -13,6 +13,7 @@ mod mcp_door;
|
|||||||
mod mcp_skills;
|
mod mcp_skills;
|
||||||
pub mod mission_orchestrator;
|
pub mod mission_orchestrator;
|
||||||
pub mod mission_refiner;
|
pub mod mission_refiner;
|
||||||
|
pub mod mission_runtime;
|
||||||
pub mod mission_workspace;
|
pub mod mission_workspace;
|
||||||
pub mod node_rules;
|
pub mod node_rules;
|
||||||
pub mod pdf_renderer;
|
pub mod pdf_renderer;
|
||||||
|
|||||||
@@ -0,0 +1,259 @@
|
|||||||
|
//! Per-mission ZeroClaw runtime container lifecycle.
|
||||||
|
//!
|
||||||
|
//! C3 workspace-isolation model: every mission gets its own ZeroClaw
|
||||||
|
//! daemon container, so agents' sandboxed filesystem is scoped to that
|
||||||
|
//! mission's repo checkout instead of the shared `/zeroclaw-data/
|
||||||
|
//! workspace` on the singleton `clawmates-runtime` daemon.
|
||||||
|
//!
|
||||||
|
//! Container naming: `cm-runtime-mission-{first 12 chars of mission uuid}`.
|
||||||
|
//! Endpoint: `http://<container_name>:42617` (well-known ZeroClaw port,
|
||||||
|
//! reachable over the `clawmates_core` docker network).
|
||||||
|
//!
|
||||||
|
//! Lifecycle:
|
||||||
|
//! - `ensure_container(mission_id)` — idempotent; spawns the container
|
||||||
|
//! if not present, returns its endpoint. Called from
|
||||||
|
//! `mission_orchestrator::on_launch` and (as a fallback for
|
||||||
|
//! pre-C3 missions) `phase_runner::launch_phase`.
|
||||||
|
//! - `teardown_container(mission_id)` — force-removes the container.
|
||||||
|
//! Called by the sweeper (slice 3) N minutes after a mission
|
||||||
|
//! reaches a terminal state, so the operator has a window to
|
||||||
|
//! re-open the ClawmateS UI and pull the last checkpoint before
|
||||||
|
//! the daemon disappears.
|
||||||
|
//!
|
||||||
|
//! State: `missions.runtime_container_name` and `missions.runtime_endpoint`
|
||||||
|
//! carry the current binding (both null when torn down or never
|
||||||
|
//! provisioned).
|
||||||
|
|
||||||
|
use bollard::models::{ContainerCreateBody, EndpointSettings, HostConfig, Mount, MountTypeEnum};
|
||||||
|
use bollard::query_parameters::{
|
||||||
|
ConnectNetworkOptions, CreateContainerOptions, InspectContainerOptions,
|
||||||
|
RemoveContainerOptions, StartContainerOptions,
|
||||||
|
};
|
||||||
|
use bollard::Docker;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
/// Docker image the per-mission runtime uses. Matches the current
|
||||||
|
/// singleton `clawmates-runtime` image; can be overridden per-deploy
|
||||||
|
/// via `CLAWMATES_RUNTIME_IMAGE`.
|
||||||
|
const DEFAULT_IMAGE: &str = "clawmates-runtime:sync";
|
||||||
|
|
||||||
|
/// Well-known ZeroClaw gateway port.
|
||||||
|
const GATEWAY_PORT: u16 = 42617;
|
||||||
|
|
||||||
|
/// Docker networks the runtime container must be attached to.
|
||||||
|
/// - `clawmates_core`: talks to the server + database
|
||||||
|
/// - `clawmates_edge`: has egress for outbound provider calls
|
||||||
|
const CORE_NETWORK: &str = "clawmates_core";
|
||||||
|
const EDGE_NETWORK: &str = "clawmates_edge";
|
||||||
|
|
||||||
|
/// Host path (as seen by the docker engine, NOT the server container)
|
||||||
|
/// where the mission's checkouts live. Matches the mount source used
|
||||||
|
/// by `clawmates-runtime.service`.
|
||||||
|
const MISSIONS_HOST_ROOT: &str = "/var/lib/clawmates-missions";
|
||||||
|
|
||||||
|
/// Deterministic docker container name for a mission's runtime. Short
|
||||||
|
/// enough to fit alongside project prefixes, unique per mission.
|
||||||
|
pub fn container_name(mission_id: Uuid) -> String {
|
||||||
|
let hex = mission_id.simple().to_string();
|
||||||
|
// 12 hex chars = 48 bits of the uuid — enough to avoid collision
|
||||||
|
// among the missions any single deployment will ever see, and
|
||||||
|
// short enough to keep container names glanceable in `docker ps`.
|
||||||
|
format!("cm-runtime-mission-{}", &hex[..12])
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Endpoint URL the topology_worker's ZeroClawDriveExecutor will dial.
|
||||||
|
/// Uses the container name as hostname — resolves within the shared
|
||||||
|
/// `clawmates_core` docker network.
|
||||||
|
pub fn endpoint_url(container_name: &str) -> String {
|
||||||
|
format!("http://{}:{}", container_name, GATEWAY_PORT)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct MissionRuntimeProvisioner {
|
||||||
|
docker: Docker,
|
||||||
|
image: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MissionRuntimeProvisioner {
|
||||||
|
/// Connect to the docker engine. Honors `DOCKER_HOST` (set in
|
||||||
|
/// compose to the socket proxy) and falls back to the local
|
||||||
|
/// socket. Returns None when docker is unreachable, so callers
|
||||||
|
/// can degrade gracefully (missions still launch, just against
|
||||||
|
/// the shared runtime).
|
||||||
|
pub fn from_env() -> Option<MissionRuntimeProvisioner> {
|
||||||
|
let docker = if let Ok(host) = std::env::var("DOCKER_HOST") {
|
||||||
|
Docker::connect_with_http(&host, 30, bollard::API_DEFAULT_VERSION).ok()?
|
||||||
|
} else {
|
||||||
|
Docker::connect_with_local_defaults().ok()?
|
||||||
|
};
|
||||||
|
let image =
|
||||||
|
std::env::var("CLAWMATES_RUNTIME_IMAGE").unwrap_or_else(|_| DEFAULT_IMAGE.to_string());
|
||||||
|
Some(MissionRuntimeProvisioner { docker, image })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Idempotent: returns the endpoint URL, creating the container
|
||||||
|
/// on first call. If the container exists but is stopped, starts
|
||||||
|
/// it. If it exists and is running, returns its endpoint.
|
||||||
|
pub async fn ensure_container(&self, mission_id: Uuid) -> Result<String, String> {
|
||||||
|
let name = container_name(mission_id);
|
||||||
|
// Fast path: already running.
|
||||||
|
if let Ok(inspect) = self
|
||||||
|
.docker
|
||||||
|
.inspect_container(&name, None::<InspectContainerOptions>)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
let running = inspect
|
||||||
|
.state
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|s| s.running)
|
||||||
|
.unwrap_or(false);
|
||||||
|
if running {
|
||||||
|
return Ok(endpoint_url(&name));
|
||||||
|
}
|
||||||
|
// Exists but not running — remove + recreate below rather
|
||||||
|
// than trying to restart a dirty-state container.
|
||||||
|
let _ = self
|
||||||
|
.docker
|
||||||
|
.remove_container(
|
||||||
|
&name,
|
||||||
|
Some(RemoveContainerOptions {
|
||||||
|
force: true,
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create fresh.
|
||||||
|
let mission_dir = format!("{MISSIONS_HOST_ROOT}/{mission_id}");
|
||||||
|
let mounts = vec![
|
||||||
|
// Mount just this mission's directory. Agents can navigate
|
||||||
|
// its `/repo` subdir but never see other missions'.
|
||||||
|
Mount {
|
||||||
|
target: Some("/mission".to_string()),
|
||||||
|
source: Some(mission_dir.clone()),
|
||||||
|
typ: Some(MountTypeEnum::BIND),
|
||||||
|
read_only: Some(false),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
let host_config = HostConfig {
|
||||||
|
mounts: Some(mounts),
|
||||||
|
restart_policy: Some(bollard::models::RestartPolicy {
|
||||||
|
name: Some(bollard::models::RestartPolicyNameEnum::UNLESS_STOPPED),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
network_mode: Some(CORE_NETWORK.to_string()),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut env = vec![
|
||||||
|
format!("ZEROCLAW_GATEWAY_PORT={GATEWAY_PORT}"),
|
||||||
|
"ZEROCLAW_WORKSPACE=/mission/repo".to_string(),
|
||||||
|
format!("CM_MISSION_ID={mission_id}"),
|
||||||
|
];
|
||||||
|
for key in [
|
||||||
|
"ANTHROPIC_API_KEY",
|
||||||
|
"GEMINI_API_KEY",
|
||||||
|
"GROQ_API_KEY",
|
||||||
|
"OPENAI_API_KEY",
|
||||||
|
] {
|
||||||
|
if let Ok(v) = std::env::var(key) {
|
||||||
|
env.push(format!("{key}={v}"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut labels = HashMap::new();
|
||||||
|
labels.insert("clawmates.role".to_string(), "mission-runtime".to_string());
|
||||||
|
labels.insert("clawmates.mission_id".to_string(), mission_id.to_string());
|
||||||
|
|
||||||
|
let config = ContainerCreateBody {
|
||||||
|
image: Some(self.image.clone()),
|
||||||
|
cmd: Some(vec![
|
||||||
|
"daemon".to_string(),
|
||||||
|
"--host".to_string(),
|
||||||
|
"0.0.0.0".to_string(),
|
||||||
|
]),
|
||||||
|
env: Some(env),
|
||||||
|
host_config: Some(host_config),
|
||||||
|
labels: Some(labels),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
self.docker
|
||||||
|
.create_container(
|
||||||
|
Some(CreateContainerOptions {
|
||||||
|
name: Some(name.clone()),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
config,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("create mission runtime container: {e}"))?;
|
||||||
|
|
||||||
|
// Attach to the edge network for outbound provider egress.
|
||||||
|
let _ = self
|
||||||
|
.docker
|
||||||
|
.connect_network(
|
||||||
|
EDGE_NETWORK,
|
||||||
|
ConnectNetworkOptions {
|
||||||
|
container: name.clone(),
|
||||||
|
endpoint_config: EndpointSettings::default(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
self.docker
|
||||||
|
.start_container(&name, None::<StartContainerOptions>)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("start mission runtime container: {e}"))?;
|
||||||
|
|
||||||
|
Ok(endpoint_url(&name))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Force-remove the mission's runtime container. Idempotent.
|
||||||
|
pub async fn teardown_container(&self, mission_id: Uuid) -> Result<(), String> {
|
||||||
|
let name = container_name(mission_id);
|
||||||
|
self.docker
|
||||||
|
.remove_container(
|
||||||
|
&name,
|
||||||
|
Some(RemoveContainerOptions {
|
||||||
|
force: true,
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("remove mission runtime container: {e}"))?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn container_name_is_stable_and_prefixed() {
|
||||||
|
let id = Uuid::parse_str("019f84a0-f2a2-7bd0-be9b-86713ec73693").unwrap();
|
||||||
|
let name = container_name(id);
|
||||||
|
assert_eq!(name, "cm-runtime-mission-019f84a0f2a2");
|
||||||
|
// Determinism: same input → same output.
|
||||||
|
assert_eq!(name, container_name(id));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn container_names_differ_across_missions() {
|
||||||
|
let a = container_name(Uuid::parse_str("019f84a0-f2a2-7bd0-be9b-86713ec73693").unwrap());
|
||||||
|
let b = container_name(Uuid::parse_str("019f84a0-f2a2-7bd0-be9b-86713ec7369f").unwrap());
|
||||||
|
// 12-char slug covers the leading bits, so different tails still
|
||||||
|
// share prefix — this test guards the *choice* of slug length.
|
||||||
|
assert_ne!(a, b, "container name slug must include enough entropy");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn endpoint_url_uses_gateway_port() {
|
||||||
|
let url = endpoint_url("cm-runtime-mission-abc");
|
||||||
|
assert_eq!(url, "http://cm-runtime-mission-abc:42617");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -34,6 +34,11 @@ pub struct Mission {
|
|||||||
pub runtime_kind: String,
|
pub runtime_kind: String,
|
||||||
/// FK → nodes(id); only relevant when runtime_kind = 'local_herdr'
|
/// FK → nodes(id); only relevant when runtime_kind = 'local_herdr'
|
||||||
pub target_node_id: Option<Uuid>,
|
pub target_node_id: Option<Uuid>,
|
||||||
|
/// Per-mission ZeroClaw runtime container name (C3 workspace isolation).
|
||||||
|
/// Null until `mission_runtime::ensure_container` provisions it.
|
||||||
|
pub runtime_container_name: Option<String>,
|
||||||
|
/// Gateway URL the topology_worker dials for this mission's runs.
|
||||||
|
pub runtime_endpoint: Option<String>,
|
||||||
#[serde(with = "time::serde::rfc3339")]
|
#[serde(with = "time::serde::rfc3339")]
|
||||||
pub created_at: OffsetDateTime,
|
pub created_at: OffsetDateTime,
|
||||||
#[serde(with = "time::serde::rfc3339")]
|
#[serde(with = "time::serde::rfc3339")]
|
||||||
@@ -181,6 +186,7 @@ pub async fn get(pool: &PgPool, id: Uuid, workspace_id: Uuid) -> Result<Option<M
|
|||||||
"SELECT id, workspace_id, title, template_kind, team_id,
|
"SELECT id, workspace_id, title, template_kind, team_id,
|
||||||
team_template_id, repo_id, schedule, status,
|
team_template_id, repo_id, schedule, status,
|
||||||
description, config, runtime_kind, target_node_id,
|
description, config, runtime_kind, target_node_id,
|
||||||
|
runtime_container_name, runtime_endpoint,
|
||||||
created_at, updated_at, completed_at
|
created_at, updated_at, completed_at
|
||||||
FROM missions WHERE id = $1 AND workspace_id = $2",
|
FROM missions WHERE id = $1 AND workspace_id = $2",
|
||||||
)
|
)
|
||||||
@@ -202,6 +208,8 @@ pub async fn get(pool: &PgPool, id: Uuid, workspace_id: Uuid) -> Result<Option<M
|
|||||||
config: r.get("config"),
|
config: r.get("config"),
|
||||||
runtime_kind: r.get("runtime_kind"),
|
runtime_kind: r.get("runtime_kind"),
|
||||||
target_node_id: r.get("target_node_id"),
|
target_node_id: r.get("target_node_id"),
|
||||||
|
runtime_container_name: r.get("runtime_container_name"),
|
||||||
|
runtime_endpoint: r.get("runtime_endpoint"),
|
||||||
created_at: r.get("created_at"),
|
created_at: r.get("created_at"),
|
||||||
updated_at: r.get("updated_at"),
|
updated_at: r.get("updated_at"),
|
||||||
completed_at: r.get("completed_at"),
|
completed_at: r.get("completed_at"),
|
||||||
@@ -219,6 +227,7 @@ pub async fn list_by_workspace(
|
|||||||
"SELECT id, workspace_id, title, template_kind, team_id,
|
"SELECT id, workspace_id, title, template_kind, team_id,
|
||||||
team_template_id, repo_id, schedule, status,
|
team_template_id, repo_id, schedule, status,
|
||||||
description, config, runtime_kind, target_node_id,
|
description, config, runtime_kind, target_node_id,
|
||||||
|
runtime_container_name, runtime_endpoint,
|
||||||
created_at, updated_at, completed_at
|
created_at, updated_at, completed_at
|
||||||
FROM missions WHERE workspace_id = $1
|
FROM missions WHERE workspace_id = $1
|
||||||
ORDER BY created_at DESC LIMIT $2",
|
ORDER BY created_at DESC LIMIT $2",
|
||||||
@@ -243,6 +252,8 @@ pub async fn list_by_workspace(
|
|||||||
config: r.get("config"),
|
config: r.get("config"),
|
||||||
runtime_kind: r.get("runtime_kind"),
|
runtime_kind: r.get("runtime_kind"),
|
||||||
target_node_id: r.get("target_node_id"),
|
target_node_id: r.get("target_node_id"),
|
||||||
|
runtime_container_name: r.get("runtime_container_name"),
|
||||||
|
runtime_endpoint: r.get("runtime_endpoint"),
|
||||||
created_at: r.get("created_at"),
|
created_at: r.get("created_at"),
|
||||||
updated_at: r.get("updated_at"),
|
updated_at: r.get("updated_at"),
|
||||||
completed_at: r.get("completed_at"),
|
completed_at: r.get("completed_at"),
|
||||||
@@ -294,6 +305,33 @@ pub async fn update_meta(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Bind a mission to its per-mission runtime container + endpoint.
|
||||||
|
/// Called by `mission_runtime::ensure_container` after the docker
|
||||||
|
/// container is running. Null endpoint clears the binding (used by
|
||||||
|
/// the teardown sweeper).
|
||||||
|
pub async fn set_runtime_binding(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
container_name: Option<&str>,
|
||||||
|
endpoint: Option<&str>,
|
||||||
|
) -> Result<(), DbError> {
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE missions
|
||||||
|
SET runtime_container_name = $3,
|
||||||
|
runtime_endpoint = $4,
|
||||||
|
updated_at = now()
|
||||||
|
WHERE id = $1 AND workspace_id = $2",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(container_name)
|
||||||
|
.bind(endpoint)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// Hard-delete a mission. Cascades via FKs on mission_phases /
|
/// Hard-delete a mission. Cascades via FKs on mission_phases /
|
||||||
/// mission_tasks / mission_artifacts / benchmark_snapshots (all
|
/// mission_tasks / mission_artifacts / benchmark_snapshots (all
|
||||||
/// declared ON DELETE CASCADE in 0047).
|
/// declared ON DELETE CASCADE in 0047).
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
-- Per-mission ZeroClaw runtime container.
|
||||||
|
--
|
||||||
|
-- C3 workspace-isolation model: each mission gets its own runtime
|
||||||
|
-- container so agents' sandboxed filesystem is scoped to that mission's
|
||||||
|
-- repo checkout. Replaces the shared clawmates-runtime container that
|
||||||
|
-- previously served all missions from one /zeroclaw-data/workspace.
|
||||||
|
--
|
||||||
|
-- runtime_container_name: docker container name owned by the mission.
|
||||||
|
-- Convention: cm-runtime-mission-{first 12 chars of mission uuid}.
|
||||||
|
-- runtime_endpoint: fully-qualified gateway URL that the
|
||||||
|
-- topology_worker's ZeroClawDriveExecutor connects to (e.g.
|
||||||
|
-- http://cm-runtime-mission-019f84a0f2a2:42617). Null when the
|
||||||
|
-- container hasn't been provisioned yet (or was torn down after
|
||||||
|
-- completion).
|
||||||
|
--
|
||||||
|
-- Both columns are set during on_launch (or lazily by phase_runner on
|
||||||
|
-- retries against pre-C3 missions) and cleared when the mission
|
||||||
|
-- terminates and the sweeper reaps its container.
|
||||||
|
|
||||||
|
ALTER TABLE missions
|
||||||
|
ADD COLUMN runtime_container_name TEXT,
|
||||||
|
ADD COLUMN runtime_endpoint TEXT;
|
||||||
Reference in New Issue
Block a user