Scaling Phase 1: multi-tenant onboarding + replica-safe coordination

Decouples "many users" + "many server replicas" from "many machines" so the
platform is tenant-isolated and horizontally safe on the current single node.

- Per-signup workspaces (cm-auth): a new hosted-identity sign-in provisions and
  owns its own workspace instead of joining the first. Config-gated by
  auth.per_signup_workspace (default off); concurrent first-logins serialized by
  a per-subject advisory lock so no duplicate workspaces.
- Terminal tickets in Postgres (migration 0016, hashed, single-use): any replica
  can redeem a ticket minted by another. Drops the in-process ticket map.
- Container registry in Postgres (migration 0017, agent_containers): Terminal
  and Sandbox managers resolve an agent's container through a shared registry,
  so a 2nd replica reuses it instead of spawning a duplicate. node_id recorded
  as 'local' (Phase 2 hook). Boot reconcile removes only true orphans, so
  terminals now survive a redeploy (tmux sessions resume).
- Per-workspace quotas (cm-api/quota.rs): plan-tier caps on agents + live
  containers, enforced at agent create + terminal spin-up (reconnects allowed),
  returned as HTTP 402. New GET /api/quota surfaces usage vs limits.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Omar Sobh
2026-06-23 18:24:51 -07:00
co-authored by Claude Opus 4.8
parent f8f2b65e1f
commit e9ce368ec1
19 changed files with 669 additions and 205 deletions
+11 -1
View File
@@ -153,10 +153,17 @@ async fn run() -> Result<(), String> {
std::sync::Arc::new(driver); std::sync::Arc::new(driver);
let agents = std::sync::Arc::new(cm_runtime::SandboxManager::new( let agents = std::sync::Arc::new(cm_runtime::SandboxManager::new(
driver.clone(), driver.clone(),
pool.clone(),
"local",
&config.sandbox.image, &config.sandbox.image,
)); ));
let browser = std::sync::Arc::new( let browser = std::sync::Arc::new(
cm_runtime::SandboxManager::new(driver.clone(), &config.sandbox.browser_image) cm_runtime::SandboxManager::new(
driver.clone(),
pool.clone(),
"local",
&config.sandbox.browser_image,
)
.with_egress(), .with_egress(),
); );
// Themed interactive terminal containers (zsh + oh-my-zsh + p10k) // Themed interactive terminal containers (zsh + oh-my-zsh + p10k)
@@ -172,6 +179,8 @@ async fn run() -> Result<(), String> {
}; };
let terminals = std::sync::Arc::new(cm_runtime::TerminalManager::new( let terminals = std::sync::Arc::new(cm_runtime::TerminalManager::new(
driver, driver,
pool.clone(),
"local",
&config.sandbox.terminal_image, &config.sandbox.terminal_image,
config.sandbox.terminal_egress, config.sandbox.terminal_egress,
drives, drives,
@@ -276,6 +285,7 @@ async fn run() -> Result<(), String> {
(config.storage.backend == cm_config::StorageBackend::Local) (config.storage.backend == cm_config::StorageBackend::Local)
.then(|| PathBuf::from(&config.storage.data_dir)), .then(|| PathBuf::from(&config.storage.data_dir)),
) )
.with_per_signup_workspace(config.auth.per_signup_workspace)
.pipe_auth_verifier(auth_verifier), .pipe_auth_verifier(auth_verifier),
); );
if e2e::enabled() { if e2e::enabled() {
+3
View File
@@ -17,6 +17,8 @@ pub enum ApiError {
NotFound, NotFound,
#[error("conflict")] #[error("conflict")]
Conflict, Conflict,
#[error("{0}")]
Quota(String),
#[error("internal error")] #[error("internal error")]
Internal, Internal,
} }
@@ -55,6 +57,7 @@ impl IntoResponse for ApiError {
ApiError::Forbidden => StatusCode::FORBIDDEN, ApiError::Forbidden => StatusCode::FORBIDDEN,
ApiError::NotFound => StatusCode::NOT_FOUND, ApiError::NotFound => StatusCode::NOT_FOUND,
ApiError::Conflict => StatusCode::CONFLICT, ApiError::Conflict => StatusCode::CONFLICT,
ApiError::Quota(_) => StatusCode::PAYMENT_REQUIRED,
ApiError::Internal => StatusCode::INTERNAL_SERVER_ERROR, ApiError::Internal => StatusCode::INTERNAL_SERVER_ERROR,
}; };
(status, Json(json!({ "error": self.to_string() }))).into_response() (status, Json(json!({ "error": self.to_string() }))).into_response()
+8 -2
View File
@@ -4,6 +4,7 @@ pub mod cleanup_sweeper;
mod error; mod error;
mod extract; mod extract;
mod mcp_door; mod mcp_door;
pub mod quota;
mod recursive_exec; mod recursive_exec;
mod routes; mod routes;
mod runtime_provision; mod runtime_provision;
@@ -30,7 +31,6 @@ pub struct AppState {
pub oauth: cm_config::OAuthConfig, pub oauth: cm_config::OAuthConfig,
pub billing: cm_config::BillingConfig, pub billing: cm_config::BillingConfig,
/// Short-lived single-use tickets for the Terminal WebSocket. /// Short-lived single-use tickets for the Terminal WebSocket.
pub terminal_tickets: routes::terminal::TerminalTickets,
/// Local blob-store root (Some on the Local backend) so the Files app can /// Local blob-store root (Some on the Local backend) so the Files app can
/// reconcile its index with files the Terminal wrote into the drives. /// reconcile its index with files the Terminal wrote into the drives.
pub file_root: Option<std::path::PathBuf>, pub file_root: Option<std::path::PathBuf>,
@@ -46,7 +46,6 @@ impl AppState {
broker_socket: None, broker_socket: None,
oauth: cm_config::OAuthConfig::default(), oauth: cm_config::OAuthConfig::default(),
billing: cm_config::BillingConfig::default(), billing: cm_config::BillingConfig::default(),
terminal_tickets: routes::terminal::TerminalTickets::default(),
file_root: None, file_root: None,
} }
} }
@@ -61,6 +60,12 @@ impl AppState {
self self
} }
/// SaaS: give each new hosted-identity sign-in its own workspace.
pub fn with_per_signup_workspace(mut self, enabled: bool) -> AppState {
self.auth = self.auth.with_per_signup_workspace(enabled);
self
}
pub fn with_billing(mut self, billing: cm_config::BillingConfig) -> AppState { pub fn with_billing(mut self, billing: cm_config::BillingConfig) -> AppState {
self.billing = billing; self.billing = billing;
self self
@@ -96,6 +101,7 @@ impl AppState {
pub fn router(state: AppState) -> Router { pub fn router(state: AppState) -> Router {
Router::new() Router::new()
.route("/healthz", get(routes::health::healthz)) .route("/healthz", get(routes::health::healthz))
.route("/api/quota", get(quota::get_quota))
.route("/mcp", post(mcp_door::mcp)) .route("/mcp", post(mcp_door::mcp))
.route("/api/auth/login", post(routes::auth::login)) .route("/api/auth/login", post(routes::auth::login))
.route("/api/auth/logout", post(routes::auth::logout)) .route("/api/auth/logout", post(routes::auth::logout))
+104
View File
@@ -0,0 +1,104 @@
//! Per-workspace resource quotas keyed off the workspace plan. Phase 1 of the
//! multi-tenant scaling work: keep one tenant from exhausting the shared pool.
use axum::extract::State;
use axum::Json;
use cm_domain::WorkspaceId;
use serde::Serialize;
use sqlx::Row;
use crate::{ApiError, AppState, Authed};
/// Caps for a plan tier.
pub struct Quota {
pub max_agents: i64,
pub max_live_containers: i64,
}
/// Per-plan limits. Unknown plans fall back to the free tier.
pub fn plan_quota(plan: &str) -> Quota {
match plan {
"team" => Quota {
max_agents: 50,
max_live_containers: 50,
},
"pro" => Quota {
max_agents: 20,
max_live_containers: 20,
},
_ => Quota {
max_agents: 3,
max_live_containers: 3,
},
}
}
/// The workspace's plan name (defaults to "free").
async fn plan_of(state: &AppState, workspace_id: WorkspaceId) -> Result<String, ApiError> {
let row = sqlx::query("SELECT plan FROM workspaces WHERE id = $1")
.bind(workspace_id.as_uuid())
.fetch_optional(&state.pool)
.await?;
Ok(row
.map(|r| r.get::<String, _>("plan"))
.unwrap_or_else(|| "free".to_string()))
}
/// Reject creating another agent if the workspace is at its plan cap.
pub async fn enforce_new_agent(state: &AppState, workspace_id: WorkspaceId) -> Result<(), ApiError> {
let plan = plan_of(state, workspace_id).await?;
let quota = plan_quota(&plan);
let used = cm_db::repo::agents::count_active(&state.pool, workspace_id).await?;
if used >= quota.max_agents {
return Err(ApiError::Quota(format!(
"agent limit reached ({} on the {plan} plan) — upgrade your plan or remove an agent",
quota.max_agents
)));
}
Ok(())
}
/// Reject spinning up another container if the workspace is at its plan cap.
pub async fn enforce_new_container(
state: &AppState,
workspace_id: WorkspaceId,
) -> Result<(), ApiError> {
let plan = plan_of(state, workspace_id).await?;
let quota = plan_quota(&plan);
let used = cm_db::repo::agent_containers::count_for_workspace(&state.pool, workspace_id).await?;
if used >= quota.max_live_containers {
return Err(ApiError::Quota(format!(
"live container limit reached ({} on the {plan} plan) — close a terminal/agent or upgrade",
quota.max_live_containers
)));
}
Ok(())
}
#[derive(Serialize)]
pub struct QuotaUsage {
plan: String,
agents_used: i64,
max_agents: i64,
containers_used: i64,
max_live_containers: i64,
}
/// `GET /api/quota` — the caller's workspace usage + limits (for the UI).
pub async fn get_quota(
State(state): State<AppState>,
Authed(user): Authed,
) -> Result<Json<QuotaUsage>, ApiError> {
let plan = plan_of(&state, user.workspace_id).await?;
let quota = plan_quota(&plan);
let agents_used = cm_db::repo::agents::count_active(&state.pool, user.workspace_id).await?;
let containers_used =
cm_db::repo::agent_containers::count_for_workspace(&state.pool, user.workspace_id).await?;
Ok(Json(QuotaUsage {
plan,
agents_used,
max_agents: quota.max_agents,
containers_used,
max_live_containers: quota.max_live_containers,
}))
}
+1
View File
@@ -742,6 +742,7 @@ pub async fn create(
Authed(user): Authed, Authed(user): Authed,
Json(body): Json<CreateClawRequest>, Json(body): Json<CreateClawRequest>,
) -> Result<(StatusCode, Json<Agent>), ApiError> { ) -> Result<(StatusCode, Json<Agent>), ApiError> {
crate::quota::enforce_new_agent(&state, user.workspace_id).await?;
let agent = Agent { let agent = Agent {
id: AgentId::new(), id: AgentId::new(),
workspace_id: user.workspace_id, workspace_id: user.workspace_id,
+63 -47
View File
@@ -5,9 +5,7 @@
//! ticket, then opens the WS with `?ticket=`. The WS itself is routed straight //! ticket, then opens the WS with `?ticket=`. The WS itself is routed straight
//! to this server by the edge (Traefik), bypassing the HTTP-only Next proxy. //! to this server by the edge (Traefik), bypassing the HTTP-only Next proxy.
use std::collections::HashMap; use std::sync::Arc;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade};
use axum::extract::{Path, Query, State}; use axum::extract::{Path, Query, State};
@@ -19,56 +17,67 @@ use cm_domain::{AgentId, WorkspaceId};
use futures::{SinkExt, StreamExt}; use futures::{SinkExt, StreamExt};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::json; use serde_json::json;
use sqlx::{PgPool, Row};
use tokio::io::AsyncWriteExt; use tokio::io::AsyncWriteExt;
use crate::routes::claws::workspace_agent; use crate::routes::claws::workspace_agent;
use crate::{ApiError, AppState, Authed}; use crate::{ApiError, AppState, Authed};
const TICKET_TTL: Duration = Duration::from_secs(30); /// Single-use WS tickets live in Postgres (hashed), so any server replica can
/// redeem one — not just the instance that minted it. A ticket is deleted on
/// redeem; expired rows are swept opportunistically. TTL is 30s (in the SQL).
fn hash_ticket(token: &str) -> String {
use sha2::{Digest, Sha256};
let mut h = Sha256::new();
h.update(token.as_bytes());
format!("{:x}", h.finalize())
}
/// In-memory single-use WS tickets (token → agent + greeting label + expiry). /// Mint a short-lived single-use ticket; returns the raw token.
/// Tiny + ephemeral; a ticket is consumed on redeem, expired ones swept lazily. async fn issue_ticket(
struct Ticket { pool: &PgPool,
agent_id: AgentId, agent_id: AgentId,
workspace_id: WorkspaceId, workspace_id: WorkspaceId,
/// Display name to greet the user with in the terminal MOTD. label: &str,
label: String, ) -> Result<String, sqlx::Error> {
expires: Instant,
}
#[derive(Clone, Default)]
pub struct TerminalTickets {
inner: Arc<Mutex<HashMap<String, Ticket>>>,
}
impl TerminalTickets {
pub fn issue(&self, agent_id: AgentId, workspace_id: WorkspaceId, label: String) -> String {
let token = uuid::Uuid::new_v4().simple().to_string(); let token = uuid::Uuid::new_v4().simple().to_string();
let mut g = self.inner.lock().expect("tickets lock"); let _ = sqlx::query("DELETE FROM terminal_ws_tickets WHERE expires_at < now()")
let now = Instant::now(); .execute(pool)
g.retain(|_, t| t.expires > now); .await;
g.insert( sqlx::query(
token.clone(), "INSERT INTO terminal_ws_tickets (token_hash, agent_id, workspace_id, label, expires_at)
Ticket { VALUES ($1, $2, $3, $4, now() + interval '30 seconds')",
agent_id, )
workspace_id, .bind(hash_ticket(&token))
label, .bind(agent_id.as_uuid())
expires: now + TICKET_TTL, .bind(workspace_id.as_uuid())
}, .bind(label)
); .execute(pool)
token .await?;
} Ok(token)
}
/// Consume a ticket; returns its (workspace, greeting label) iff valid for this agent. /// Consume a ticket; returns (workspace, greeting label) iff valid for this agent.
pub fn redeem(&self, token: &str, agent_id: AgentId) -> Option<(WorkspaceId, String)> { async fn redeem_ticket(
let now = Instant::now(); pool: &PgPool,
let mut g = self.inner.lock().expect("tickets lock"); token: &str,
g.retain(|_, t| t.expires > now); agent_id: AgentId,
match g.remove(token) { ) -> Option<(WorkspaceId, String)> {
Some(t) if t.agent_id == agent_id && t.expires > now => Some((t.workspace_id, t.label)), let row = sqlx::query(
_ => None, "DELETE FROM terminal_ws_tickets
} WHERE token_hash = $1 AND agent_id = $2 AND expires_at > now()
} RETURNING workspace_id, label",
)
.bind(hash_ticket(token))
.bind(agent_id.as_uuid())
.fetch_optional(pool)
.await
.ok()
.flatten()?;
Some((
WorkspaceId::from(row.get::<uuid::Uuid, _>("workspace_id")),
row.get::<String, _>("label"),
))
} }
#[derive(Serialize)] #[derive(Serialize)]
@@ -84,6 +93,15 @@ pub async fn ticket(
) -> Result<Json<TicketResponse>, ApiError> { ) -> Result<Json<TicketResponse>, ApiError> {
// Tenant isolation: a foreign agent looks non-existent. // Tenant isolation: a foreign agent looks non-existent.
workspace_agent(&state, &user, agent_id).await?; workspace_agent(&state, &user, agent_id).await?;
// Quota: gate only when this would spin up a NEW container (a reconnect to an
// already-running terminal is always allowed).
let existing = cm_db::repo::agent_containers::get(&state.pool, agent_id, "terminal")
.await
.ok()
.flatten();
if existing.is_none() {
crate::quota::enforce_new_container(&state, user.workspace_id).await?;
}
// Greet by display name, falling back to the email local-part. // Greet by display name, falling back to the email local-part.
let label = match cm_db::repo::users::get(&state.pool, user.user_id).await { let label = match cm_db::repo::users::get(&state.pool, user.user_id).await {
Ok(u) if !u.display_name.trim().is_empty() => u.display_name.trim().to_string(), Ok(u) if !u.display_name.trim().is_empty() => u.display_name.trim().to_string(),
@@ -96,9 +114,7 @@ pub async fn ticket(
.to_string(), .to_string(),
Err(_) => "there".to_string(), Err(_) => "there".to_string(),
}; };
let ticket = state let ticket = issue_ticket(&state.pool, agent_id, user.workspace_id, &label).await?;
.terminal_tickets
.issue(agent_id, user.workspace_id, label);
cm_db::repo::audit::append( cm_db::repo::audit::append(
&state.pool, &state.pool,
user.workspace_id, user.workspace_id,
@@ -200,7 +216,7 @@ pub async fn ws(
Query(q): Query<WsQuery>, Query(q): Query<WsQuery>,
upgrade: WebSocketUpgrade, upgrade: WebSocketUpgrade,
) -> Response { ) -> Response {
let (workspace_id, label) = match state.terminal_tickets.redeem(&q.ticket, agent_id) { let (workspace_id, label) = match redeem_ticket(&state.pool, &q.ticket, agent_id).await {
Some(v) => v, Some(v) => v,
None => return (StatusCode::UNAUTHORIZED, "invalid or expired ticket").into_response(), None => return (StatusCode::UNAUTHORIZED, "invalid or expired ticket").into_response(),
}; };
+1
View File
@@ -16,6 +16,7 @@ serde = { workspace = true }
tokio = { workspace = true } tokio = { workspace = true }
sha2 = "0.10" sha2 = "0.10"
sqlx = { workspace = true } sqlx = { workspace = true }
uuid = { workspace = true }
cm-db = { path = "../cm-db" } cm-db = { path = "../cm-db" }
cm-domain = { path = "../cm-domain" } cm-domain = { path = "../cm-domain" }
thiserror = { workspace = true } thiserror = { workspace = true }
+76 -25
View File
@@ -2,7 +2,7 @@ use argon2::password_hash::rand_core::OsRng;
use argon2::password_hash::{PasswordHash, PasswordHasher, PasswordVerifier, SaltString}; use argon2::password_hash::{PasswordHash, PasswordHasher, PasswordVerifier, SaltString};
use argon2::Argon2; use argon2::Argon2;
use cm_domain::{Role, UserId, WorkspaceId}; use cm_domain::{Role, UserId, WorkspaceId};
use sqlx::PgPool; use sqlx::{PgPool, Row};
use time::{Duration, OffsetDateTime}; use time::{Duration, OffsetDateTime};
use crate::token::{hash_token, SessionToken}; use crate::token::{hash_token, SessionToken};
@@ -37,6 +37,9 @@ pub enum AuthError {
pub struct AuthService { pub struct AuthService {
pool: PgPool, pool: PgPool,
verifier: Option<std::sync::Arc<crate::JwtVerifier>>, verifier: Option<std::sync::Arc<crate::JwtVerifier>>,
/// SaaS mode: a brand-new external identity provisions its own workspace
/// (owner) rather than joining the instance's single workspace.
per_signup_workspace: bool,
} }
impl AuthService { impl AuthService {
@@ -44,6 +47,7 @@ impl AuthService {
AuthService { AuthService {
pool, pool,
verifier: None, verifier: None,
per_signup_workspace: false,
} }
} }
@@ -54,6 +58,13 @@ impl AuthService {
self self
} }
/// In SaaS deployments, give each new hosted-identity sign-in its own
/// workspace (they become its owner) instead of joining the first one.
pub fn with_per_signup_workspace(mut self, enabled: bool) -> AuthService {
self.per_signup_workspace = enabled;
self
}
/// Verifies an issuer session token and JIT-provisions the user on /// Verifies an issuer session token and JIT-provisions the user on
/// first sight. Role tracks the issuer claim on every login (Clerk's /// first sight. Role tracks the issuer claim on every login (Clerk's
/// org role is the source of truth for SSO users). /// org role is the source of truth for SSO users).
@@ -112,42 +123,82 @@ impl AuthService {
} }
} }
// First sight: provision into the instance's workspace. // First sight. Serialize concurrent first-logins for the same subject
let workspace = // (the authed shell fires several API calls at once) with a per-subject
sqlx::query_scalar!("SELECT id FROM workspaces ORDER BY created_at, id LIMIT 1",) // advisory lock, so SaaS mode can't create duplicate workspaces and the
.fetch_optional(&self.pool) // appliance path can't double-insert the user.
.await?
.ok_or(AuthError::Unauthenticated)?;
let email = claims let email = claims
.email .email
.clone() .clone()
.unwrap_or_else(|| format!("{}@sso.local", claims.sub)); .unwrap_or_else(|| format!("{}@sso.local", claims.sub));
let display_name = email.split('@').next().unwrap_or("teammate").to_owned(); let display_name = email.split('@').next().unwrap_or("teammate").to_owned();
let mut tx = self.pool.begin().await?;
sqlx::query("SELECT pg_advisory_xact_lock(hashtext($1))")
.bind(&claims.sub)
.execute(&mut *tx)
.await?;
// A concurrent winner may have already provisioned this identity.
if let Some(row) = sqlx::query("SELECT id, workspace_id, role FROM users WHERE auth_subject = $1")
.bind(&claims.sub)
.fetch_optional(&mut *tx)
.await?
{
tx.commit().await?;
let role_db: String = row.get("role");
return Ok(AuthedUser {
user_id: UserId::from(row.get::<uuid::Uuid, _>("id")),
workspace_id: WorkspaceId::from(row.get::<uuid::Uuid, _>("workspace_id")),
role: if role_db == "owner" {
Role::Owner
} else {
Role::Member
},
});
}
// Pick the workspace: SaaS provisions a fresh one this user owns;
// appliance joins the instance's single workspace.
let (workspace, effective_role): (uuid::Uuid, &str) = if self.per_signup_workspace {
let ws = WorkspaceId::new();
sqlx::query("INSERT INTO workspaces (id, name, plan) VALUES ($1, $2, 'free')")
.bind(ws.as_uuid())
.bind(format!("{display_name}'s workspace"))
.execute(&mut *tx)
.await?;
(ws.as_uuid(), "owner")
} else {
let ws: uuid::Uuid =
sqlx::query_scalar("SELECT id FROM workspaces ORDER BY created_at, id LIMIT 1")
.fetch_optional(&mut *tx)
.await?
.ok_or(AuthError::Unauthenticated)?;
(ws, role_str)
};
let user_id = UserId::new(); let user_id = UserId::new();
// Idempotent under concurrent first-login: the authed shell fires let row = sqlx::query(
// several API calls at once, and each would otherwise race to INSERT
// this row and trip the partial unique index on auth_subject (the
// losers 500'd). ON CONFLICT resolves to the row the winning request
// created, so every concurrent caller returns the same identity.
let row = sqlx::query!(
"INSERT INTO users (id, workspace_id, email, role, display_name, auth_subject) "INSERT INTO users (id, workspace_id, email, role, display_name, auth_subject)
VALUES ($1, $2, $3, $4, $5, $6) VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (auth_subject) WHERE auth_subject IS NOT NULL
DO UPDATE SET role = EXCLUDED.role
RETURNING id, workspace_id", RETURNING id, workspace_id",
user_id.as_uuid(),
workspace,
email,
role_str,
display_name,
claims.sub,
) )
.fetch_one(&self.pool) .bind(user_id.as_uuid())
.bind(workspace)
.bind(&email)
.bind(effective_role)
.bind(&display_name)
.bind(&claims.sub)
.fetch_one(&mut *tx)
.await?; .await?;
tx.commit().await?;
Ok(AuthedUser { Ok(AuthedUser {
user_id: UserId::from(row.id), user_id: UserId::from(row.get::<uuid::Uuid, _>("id")),
workspace_id: WorkspaceId::from(row.workspace_id), workspace_id: WorkspaceId::from(row.get::<uuid::Uuid, _>("workspace_id")),
role, role: if effective_role == "owner" {
Role::Owner
} else {
Role::Member
},
}) })
} }
+5
View File
@@ -97,6 +97,11 @@ pub struct AuthConfig {
pub mode: AuthMode, pub mode: AuthMode,
pub issuer_url: Option<String>, pub issuer_url: Option<String>,
pub client_id: Option<String>, pub client_id: Option<String>,
/// When true (SaaS), a brand-new hosted-identity (Clerk/OIDC) sign-in
/// provisions its OWN workspace and owns it. When false (appliance), the
/// first sign-in joins the instance's single workspace as a member.
#[serde(default)]
pub per_signup_workspace: bool,
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
+159
View File
@@ -0,0 +1,159 @@
//! Durable registry of the live container backing each agent, so any server
//! replica can find + reuse it (replacing the former in-process handle map).
use cm_domain::{AgentId, WorkspaceId};
use sqlx::{PgPool, Row};
use crate::DbError;
/// A registry row for one agent's container of a given kind.
#[derive(Debug, Clone)]
pub struct ContainerRow {
pub node_id: String,
pub container_id: String,
pub name: String,
pub session_count: i32,
}
/// One container to act on (idle reap / reconcile / shutdown).
#[derive(Debug, Clone)]
pub struct ManagedRow {
pub agent_id: AgentId,
pub node_id: String,
pub container_id: String,
pub name: String,
}
pub async fn get(
pool: &PgPool,
agent_id: AgentId,
kind: &str,
) -> Result<Option<ContainerRow>, DbError> {
let row = sqlx::query(
"SELECT node_id, container_id, name, session_count
FROM agent_containers WHERE agent_id = $1 AND kind = $2",
)
.bind(agent_id.as_uuid())
.bind(kind)
.fetch_optional(pool)
.await?;
Ok(row.map(|r| ContainerRow {
node_id: r.get("node_id"),
container_id: r.get("container_id"),
name: r.get("name"),
session_count: r.get("session_count"),
}))
}
/// Record the (new) container for `(agent, kind)`; resets the session count.
/// The workspace is derived from the agent row (single source of truth).
pub async fn upsert(
pool: &PgPool,
agent_id: AgentId,
kind: &str,
node_id: &str,
container_id: &str,
name: &str,
) -> Result<(), DbError> {
sqlx::query(
"INSERT INTO agent_containers
(agent_id, kind, node_id, container_id, name, workspace_id, session_count, last_seen)
SELECT $1, $2, $3, $4, $5, a.workspace_id, 0, now() FROM agents a WHERE a.id = $1
ON CONFLICT (agent_id, kind) DO UPDATE SET
node_id = excluded.node_id, container_id = excluded.container_id,
name = excluded.name, workspace_id = excluded.workspace_id,
session_count = 0, last_seen = now()",
)
.bind(agent_id.as_uuid())
.bind(kind)
.bind(node_id)
.bind(container_id)
.bind(name)
.execute(pool)
.await?;
Ok(())
}
pub async fn delete(pool: &PgPool, agent_id: AgentId, kind: &str) -> Result<(), DbError> {
sqlx::query("DELETE FROM agent_containers WHERE agent_id = $1 AND kind = $2")
.bind(agent_id.as_uuid())
.bind(kind)
.execute(pool)
.await?;
Ok(())
}
/// Adjust the live-session count (clamped at 0) and bump last_seen.
pub async fn add_session(
pool: &PgPool,
agent_id: AgentId,
kind: &str,
delta: i32,
) -> Result<(), DbError> {
sqlx::query(
"UPDATE agent_containers
SET session_count = GREATEST(0, session_count + $3), last_seen = now()
WHERE agent_id = $1 AND kind = $2",
)
.bind(agent_id.as_uuid())
.bind(kind)
.bind(delta)
.execute(pool)
.await?;
Ok(())
}
/// Zero the live-session counts for a kind (graceful shutdown: the WS sessions
/// are ending, so the containers become idle-reapable but are not destroyed).
pub async fn reset_sessions(pool: &PgPool, kind: &str) -> Result<(), DbError> {
sqlx::query("UPDATE agent_containers SET session_count = 0 WHERE kind = $1")
.bind(kind)
.execute(pool)
.await?;
Ok(())
}
/// Session-less containers untouched for longer than `idle_secs` (0 = all).
pub async fn idle(
pool: &PgPool,
kind: &str,
idle_secs: i64,
) -> Result<Vec<ManagedRow>, DbError> {
let rows = sqlx::query(
"SELECT agent_id, node_id, container_id, name FROM agent_containers
WHERE kind = $1 AND session_count <= 0
AND last_seen < now() - ($2 * interval '1 second')",
)
.bind(kind)
.bind(idle_secs)
.fetch_all(pool)
.await?;
Ok(rows.into_iter().map(map_managed).collect())
}
/// Every recorded container of a kind (for reconcile + shutdown).
pub async fn all(pool: &PgPool, kind: &str) -> Result<Vec<ManagedRow>, DbError> {
let rows = sqlx::query("SELECT agent_id, node_id, container_id, name FROM agent_containers WHERE kind = $1")
.bind(kind)
.fetch_all(pool)
.await?;
Ok(rows.into_iter().map(map_managed).collect())
}
/// Count live containers in a workspace (any kind) — for quota checks.
pub async fn count_for_workspace(pool: &PgPool, workspace_id: WorkspaceId) -> Result<i64, DbError> {
let row = sqlx::query("SELECT count(*) AS n FROM agent_containers WHERE workspace_id = $1")
.bind(workspace_id.as_uuid())
.fetch_one(pool)
.await?;
Ok(row.get::<i64, _>("n"))
}
fn map_managed(r: sqlx::postgres::PgRow) -> ManagedRow {
ManagedRow {
agent_id: AgentId::from(r.get::<uuid::Uuid, _>("agent_id")),
node_id: r.get("node_id"),
container_id: r.get("container_id"),
name: r.get("name"),
}
}
+11
View File
@@ -6,6 +6,17 @@ use uuid::Uuid;
use crate::DbError; use crate::DbError;
/// Count of non-deleted agents in a workspace (for per-workspace quotas).
pub async fn count_active(pool: &PgPool, workspace_id: WorkspaceId) -> Result<i64, DbError> {
let n: i64 = sqlx::query_scalar(
"SELECT count(*) FROM agents WHERE workspace_id = $1 AND deleted_at IS NULL",
)
.bind(workspace_id.as_uuid())
.fetch_one(pool)
.await?;
Ok(n)
}
/// Inserts an agent together with its access policy in one transaction — /// Inserts an agent together with its access policy in one transaction —
/// an agent without a policy must never be observable (§7.7). /// an agent without a policy must never be observable (§7.7).
pub async fn insert(pool: &PgPool, agent: &Agent, policy: &AccessPolicy) -> Result<(), DbError> { pub async fn insert(pool: &PgPool, agent: &Agent, policy: &AccessPolicy) -> Result<(), DbError> {
+1
View File
@@ -1,3 +1,4 @@
pub mod agent_containers;
pub mod agents; pub mod agents;
pub mod audit; pub mod audit;
pub mod cleanup; pub mod cleanup;
+74 -39
View File
@@ -2,45 +2,59 @@
//! container per agent, provisioned lazily on first use and reused for //! container per agent, provisioned lazily on first use and reused for
//! the manager's lifetime; a dead sandbox is replaced transparently. //! the manager's lifetime; a dead sandbox is replaced transparently.
use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
use cm_domain::AgentId; use cm_domain::AgentId;
use cm_sandbox::{ExecResult, SandboxDriver, SandboxHandle, SandboxSpec}; use cm_sandbox::{ExecResult, SandboxDriver, SandboxHandle, SandboxSpec};
use sqlx::PgPool;
use tokio::sync::Mutex; use tokio::sync::Mutex;
impl std::fmt::Debug for SandboxManager { impl std::fmt::Debug for SandboxManager {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SandboxManager") f.debug_struct("SandboxManager")
.field("image", &self.image) .field("image", &self.image)
.field("node_id", &self.node_id)
.finish_non_exhaustive() .finish_non_exhaustive()
} }
} }
pub struct SandboxManager { pub struct SandboxManager {
driver: Arc<dyn SandboxDriver>, driver: Arc<dyn SandboxDriver>,
db: PgPool,
/// Placement node this manager's driver provisions onto ("local" today).
node_id: String,
image: String, image: String,
egress: bool, egress: bool,
handles: Mutex<HashMap<AgentId, SandboxHandle>>, /// Pre-provisioned, unassigned sandboxes (the warm pool, per-replica).
/// Pre-provisioned, unassigned sandboxes (the warm pool).
pool: Mutex<Vec<SandboxHandle>>, pool: Mutex<Vec<SandboxHandle>>,
/// Warm-pool target; the background warmer keeps `pool` at this size. /// Warm-pool target; the background warmer keeps `pool` at this size.
warm_target: std::sync::atomic::AtomicUsize, warm_target: std::sync::atomic::AtomicUsize,
} }
impl SandboxManager { impl SandboxManager {
pub fn new(driver: Arc<dyn SandboxDriver>, image: &str) -> SandboxManager { pub fn new(
driver: Arc<dyn SandboxDriver>,
db: PgPool,
node_id: &str,
image: &str,
) -> SandboxManager {
SandboxManager { SandboxManager {
driver, driver,
db,
node_id: node_id.to_owned(),
image: image.to_owned(), image: image.to_owned(),
egress: false, egress: false,
handles: Mutex::new(HashMap::new()),
pool: Mutex::new(Vec::new()), pool: Mutex::new(Vec::new()),
warm_target: std::sync::atomic::AtomicUsize::new(0), warm_target: std::sync::atomic::AtomicUsize::new(0),
} }
} }
/// The `agent_containers.kind` discriminator for this manager.
fn kind(&self) -> &'static str {
cm_sandbox::sandbox_kind(self.egress)
}
/// Starts the background warmer: keeps `target` sandboxes /// Starts the background warmer: keeps `target` sandboxes
/// pre-provisioned so an agent's first exec skips container startup. /// pre-provisioned so an agent's first exec skips container startup.
pub fn warm(self: Arc<Self>, target: usize) -> Arc<Self> { pub fn warm(self: Arc<Self>, target: usize) -> Arc<Self> {
@@ -106,14 +120,26 @@ impl SandboxManager {
/// first use. The sandbox has no egress and no credentials — running /// first use. The sandbox has no egress and no credentials — running
/// agent-authored code here is the point of the architecture. /// agent-authored code here is the point of the architecture.
pub async fn exec(&self, agent_id: AgentId, command: &str) -> Result<ExecResult, String> { pub async fn exec(&self, agent_id: AgentId, command: &str) -> Result<ExecResult, String> {
let mut handles = self.handles.lock().await; let kind = self.kind();
let alive = match handles.get(&agent_id) { // Reuse the registry-recorded sandbox for this agent if it's alive — so a
Some(handle) => self.driver.health(handle).await.unwrap_or(false), // different replica finds the same container instead of making another.
None => false, if let Ok(Some(row)) = cm_db::repo::agent_containers::get(&self.db, agent_id, kind).await {
let handle = SandboxHandle {
id: row.container_id,
name: row.name,
}; };
if !alive { if self.driver.health(&handle).await.unwrap_or(false) {
// A warm sandbox if one is ready (and still healthy); return self
// otherwise provision inline. .driver
.exec(&handle, &["sh", "-lc", command])
.await
.map_err(|e| format!("sandbox exec failed: {e}"));
}
// Recorded but dead: clean both the container and the stale row.
let _ = self.driver.destroy(&handle).await;
let _ = cm_db::repo::agent_containers::delete(&self.db, agent_id, kind).await;
}
// Take a warm sandbox if one is ready (and still healthy), else provision.
let mut assigned = None; let mut assigned = None;
while let Some(candidate) = self.pool.lock().await.pop() { while let Some(candidate) = self.pool.lock().await.pop() {
if self.driver.health(&candidate).await.unwrap_or(false) { if self.driver.health(&candidate).await.unwrap_or(false) {
@@ -126,11 +152,18 @@ impl SandboxManager {
Some(handle) => handle, Some(handle) => handle,
None => self.provision_one().await?, None => self.provision_one().await?,
}; };
handles.insert(agent_id, handle); cm_db::repo::agent_containers::upsert(
} &self.db,
let handle = handles.get(&agent_id).expect("just ensured"); agent_id,
kind,
&self.node_id,
&handle.id,
&handle.name,
)
.await
.map_err(|e| format!("registry upsert failed: {e}"))?;
self.driver self.driver
.exec(handle, &["sh", "-lc", command]) .exec(&handle, &["sh", "-lc", command])
.await .await
.map_err(|e| format!("sandbox exec failed: {e}")) .map_err(|e| format!("sandbox exec failed: {e}"))
} }
@@ -138,30 +171,28 @@ impl SandboxManager {
/// Tear down a single agent's sandbox if it has one (on agent deletion). /// Tear down a single agent's sandbox if it has one (on agent deletion).
/// Returns whether a container existed and was destroyed. /// Returns whether a container existed and was destroyed.
pub async fn release_agent(&self, agent_id: AgentId) -> bool { pub async fn release_agent(&self, agent_id: AgentId) -> bool {
let handle = { self.handles.lock().await.remove(&agent_id) }; match cm_db::repo::agent_containers::get(&self.db, agent_id, self.kind()).await {
match handle { Ok(Some(row)) => {
Some(h) => { let handle = SandboxHandle {
if let Err(e) = self.driver.destroy(&h).await { id: row.container_id,
eprintln!("sandbox release: failed to remove {}: {e}", h.id); name: row.name,
};
if let Err(e) = self.driver.destroy(&handle).await {
eprintln!("sandbox release: failed to remove {}: {e}", handle.id);
} }
let _ = cm_db::repo::agent_containers::delete(&self.db, agent_id, self.kind()).await;
true true
} }
None => false, _ => false,
} }
} }
/// Destroys every sandbox this manager provisioned — assigned and /// Graceful shutdown (SIGTERM): assigned sandboxes persist in the registry
/// pooled — and stops the warmer. Called on graceful shutdown (SIGTERM). /// (reused after a redeploy); only the per-replica warm pool is torn down,
/// and the warmer is stopped.
pub async fn shutdown(&self) { pub async fn shutdown(&self) {
self.warm_target self.warm_target
.store(0, std::sync::atomic::Ordering::Relaxed); .store(0, std::sync::atomic::Ordering::Relaxed);
let mut handles = self.handles.lock().await;
for (_, handle) in handles.drain() {
if let Err(e) = self.driver.destroy(&handle).await {
eprintln!("sandbox shutdown: failed to remove {}: {e}", handle.id);
}
}
drop(handles);
let mut pool = self.pool.lock().await; let mut pool = self.pool.lock().await;
for handle in pool.drain(..) { for handle in pool.drain(..) {
if let Err(e) = self.driver.destroy(&handle).await { if let Err(e) = self.driver.destroy(&handle).await {
@@ -184,15 +215,19 @@ impl SandboxManager {
return 0; return 0;
} }
}; };
let live: std::collections::HashSet<String> = { let mut live: std::collections::HashSet<String> =
let handles = self.handles.lock().await; cm_db::repo::agent_containers::all(&self.db, kind)
.await
.unwrap_or_default()
.into_iter()
.map(|r| r.container_id)
.collect();
{
let pool = self.pool.lock().await; let pool = self.pool.lock().await;
handles for h in pool.iter() {
.values() live.insert(h.id.clone());
.map(|h| h.id.clone()) }
.chain(pool.iter().map(|h| h.id.clone())) }
.collect()
};
let now = std::time::SystemTime::now() let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH) .duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs() as i64) .map(|d| d.as_secs() as i64)
+103 -73
View File
@@ -3,15 +3,23 @@
//! WebSocket connect and reused across reconnects. Unlike the agent tool //! WebSocket connect and reused across reconnects. Unlike the agent tool
//! sandboxes, these are interactive PTYs (`exec -it zsh`), so a small idle //! sandboxes, these are interactive PTYs (`exec -it zsh`), so a small idle
//! sweeper reaps containers that no live session has touched for a while. //! sweeper reaps containers that no live session has touched for a while.
//!
//! The agent→container mapping lives in Postgres (`agent_containers`) rather
//! than process memory, so ANY server replica reuses the same container instead
//! of spawning a duplicate — and registry-tracked terminals survive a redeploy
//! (their tmux sessions resume), since boot reconciliation removes only true
//! orphans (engine containers with no registry row).
use std::collections::HashMap;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::Arc; use std::sync::Arc;
use std::time::{Duration, Instant}; use std::time::Duration;
use cm_domain::{AgentId, WorkspaceId}; use cm_domain::{AgentId, WorkspaceId};
use cm_sandbox::{DriveMount, PtySession, SandboxDriver, SandboxHandle, SandboxKind, SandboxSpec}; use cm_sandbox::{DriveMount, PtySession, SandboxDriver, SandboxHandle, SandboxKind, SandboxSpec};
use tokio::sync::Mutex; use sqlx::PgPool;
/// The `agent_containers.kind` discriminator for terminal containers.
const KIND: &str = "terminal";
/// Where the agent's Files drives live, so the Terminal can mount them. /// Where the agent's Files drives live, so the Terminal can mount them.
#[derive(Clone)] #[derive(Clone)]
@@ -25,21 +33,20 @@ pub struct DriveConfig {
pub struct TerminalManager { pub struct TerminalManager {
driver: Arc<dyn SandboxDriver>, driver: Arc<dyn SandboxDriver>,
pool: PgPool,
/// Placement node this manager's driver provisions onto ("local" today).
node_id: String,
image: String, image: String,
egress: bool, egress: bool,
/// Drive mounts; None disables the ~/drives mapping. /// Drive mounts; None disables the ~/drives mapping.
drives: Option<DriveConfig>, drives: Option<DriveConfig>,
handles: Mutex<HashMap<AgentId, SandboxHandle>>,
/// Live WebSocket sessions per agent + when each agent was last touched —
/// an agent with zero sessions, idle past the TTL, gets its container reaped.
active: Mutex<HashMap<AgentId, usize>>,
last_seen: Mutex<HashMap<AgentId, Instant>>,
} }
impl std::fmt::Debug for TerminalManager { impl std::fmt::Debug for TerminalManager {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TerminalManager") f.debug_struct("TerminalManager")
.field("image", &self.image) .field("image", &self.image)
.field("node_id", &self.node_id)
.finish_non_exhaustive() .finish_non_exhaustive()
} }
} }
@@ -47,18 +54,19 @@ impl std::fmt::Debug for TerminalManager {
impl TerminalManager { impl TerminalManager {
pub fn new( pub fn new(
driver: Arc<dyn SandboxDriver>, driver: Arc<dyn SandboxDriver>,
pool: PgPool,
node_id: &str,
image: &str, image: &str,
egress: bool, egress: bool,
drives: Option<DriveConfig>, drives: Option<DriveConfig>,
) -> TerminalManager { ) -> TerminalManager {
TerminalManager { TerminalManager {
driver, driver,
pool,
node_id: node_id.to_owned(),
image: image.to_owned(), image: image.to_owned(),
egress, egress,
drives, drives,
handles: Mutex::new(HashMap::new()),
active: Mutex::new(HashMap::new()),
last_seen: Mutex::new(HashMap::new()),
} }
} }
@@ -118,25 +126,39 @@ impl TerminalManager {
} }
/// The agent's terminal container, provisioned on first use; a dead one is /// The agent's terminal container, provisioned on first use; a dead one is
/// replaced transparently. /// replaced transparently. The mapping is read from / written to the shared
/// registry, so a different replica finds the same container.
async fn ensure( async fn ensure(
&self, &self,
workspace_id: WorkspaceId, workspace_id: WorkspaceId,
agent_id: AgentId, agent_id: AgentId,
) -> Result<SandboxHandle, String> { ) -> Result<SandboxHandle, String> {
let mut handles = self.handles.lock().await; if let Ok(Some(row)) =
let alive = match handles.get(&agent_id) { cm_db::repo::agent_containers::get(&self.pool, agent_id, KIND).await
Some(handle) => self.driver.health(handle).await.unwrap_or(false), {
None => false, let handle = SandboxHandle {
id: row.container_id.clone(),
name: row.name.clone(),
}; };
if !alive { if self.driver.health(&handle).await.unwrap_or(false) {
if let Some(stale) = handles.remove(&agent_id) { return Ok(handle);
let _ = self.driver.destroy(&stale).await; }
// Recorded but dead: clean both the container and the stale row.
let _ = self.driver.destroy(&handle).await;
let _ = cm_db::repo::agent_containers::delete(&self.pool, agent_id, KIND).await;
} }
let handle = self.provision_one(workspace_id, agent_id).await?; let handle = self.provision_one(workspace_id, agent_id).await?;
handles.insert(agent_id, handle); cm_db::repo::agent_containers::upsert(
} &self.pool,
Ok(handles.get(&agent_id).expect("just ensured").clone()) agent_id,
KIND,
&self.node_id,
&handle.id,
&handle.name,
)
.await
.map_err(|e| format!("registry upsert failed: {e}"))?;
Ok(handle)
} }
/// Open an interactive login zsh in the agent's terminal container. `env` /// Open an interactive login zsh in the agent's terminal container. `env`
@@ -168,8 +190,7 @@ impl TerminalManager {
) )
.await .await
.map_err(|e| format!("terminal attach failed: {e}"))?; .map_err(|e| format!("terminal attach failed: {e}"))?;
*self.active.lock().await.entry(agent_id).or_insert(0) += 1; let _ = cm_db::repo::agent_containers::add_session(&self.pool, agent_id, KIND, 1).await;
self.last_seen.lock().await.insert(agent_id, Instant::now());
Ok(session) Ok(session)
} }
@@ -183,83 +204,92 @@ impl TerminalManager {
/// A session ended; the container stays warm for reconnects but becomes a /// A session ended; the container stays warm for reconnects but becomes a
/// candidate for the idle sweeper once no session remains. /// candidate for the idle sweeper once no session remains.
pub async fn detach(&self, agent_id: AgentId) { pub async fn detach(&self, agent_id: AgentId) {
if let Some(n) = self.active.lock().await.get_mut(&agent_id) { let _ = cm_db::repo::agent_containers::add_session(&self.pool, agent_id, KIND, -1).await;
*n = n.saturating_sub(1);
}
self.last_seen.lock().await.insert(agent_id, Instant::now());
} }
/// Tear down a single agent's terminal container (on agent deletion / idle). /// Tear down a single agent's terminal container (on agent deletion / idle).
/// Returns whether one existed. /// Returns whether one existed.
pub async fn release_agent(&self, agent_id: AgentId) -> bool { pub async fn release_agent(&self, agent_id: AgentId) -> bool {
self.active.lock().await.remove(&agent_id); match cm_db::repo::agent_containers::get(&self.pool, agent_id, KIND).await {
self.last_seen.lock().await.remove(&agent_id); Ok(Some(row)) => {
let handle = { self.handles.lock().await.remove(&agent_id) }; let handle = SandboxHandle {
match handle { id: row.container_id,
Some(h) => { name: row.name,
if let Err(e) = self.driver.destroy(&h).await { };
eprintln!("terminal release: failed to remove {}: {e}", h.id); if let Err(e) = self.driver.destroy(&handle).await {
eprintln!("terminal release: failed to remove {}: {e}", handle.id);
} }
let _ = cm_db::repo::agent_containers::delete(&self.pool, agent_id, KIND).await;
true true
} }
None => false, _ => false,
} }
} }
/// Destroy every terminal container on graceful shutdown (SIGTERM). /// Graceful shutdown: terminals are NOT destroyed — they persist (tmux
/// sessions survive the redeploy) and become idle-reapable. We only zero the
/// session counts since the WS connections are dropping.
pub async fn shutdown(&self) { pub async fn shutdown(&self) {
let mut handles = self.handles.lock().await; let _ = cm_db::repo::agent_containers::reset_sessions(&self.pool, KIND).await;
for (_, handle) in handles.drain() {
if let Err(e) = self.driver.destroy(&handle).await {
eprintln!("terminal shutdown: failed to remove {}: {e}", handle.id);
}
}
} }
/// Reap idle terminals (no live session, untouched past `idle_ttl`) and /// Reap idle terminals (no live session, untouched past `idle_ttl`).
/// orphan containers the engine still holds that no live handle owns. async fn reap_idle(&self, idle_ttl: Duration) -> usize {
async fn sweep(&self, idle_ttl: Duration) -> usize { let rows = match cm_db::repo::agent_containers::idle(
// 1. Idle, session-less terminals we still track. &self.pool,
let now = Instant::now(); KIND,
let idle: Vec<AgentId> = { idle_ttl.as_secs() as i64,
let active = self.active.lock().await; )
let last = self.last_seen.lock().await; .await
last.iter() {
.filter(|(id, seen)| { Ok(r) => r,
active.get(*id).copied().unwrap_or(0) == 0 Err(e) => {
&& now.duration_since(**seen) > idle_ttl eprintln!("terminal reaper: idle query failed: {e}");
}) return 0;
.map(|(id, _)| *id) }
.collect()
}; };
let mut reaped = 0; let mut reaped = 0;
for id in idle { for r in rows {
if self.release_agent(id).await { let handle = SandboxHandle {
id: r.container_id.clone(),
name: r.name.clone(),
};
let _ = self.driver.destroy(&handle).await;
let _ = cm_db::repo::agent_containers::delete(&self.pool, r.agent_id, KIND).await;
reaped += 1; reaped += 1;
} }
reaped
} }
// 2. Orphans from a crashed process (label-filtered, no live handle).
/// Reap orphan containers the engine holds that no registry row owns (and,
/// when `min_age` > 0, that are older than it). Registry-tracked terminals
/// are left alone.
async fn reap_orphans(&self, min_age: Duration) -> usize {
let managed = match self.driver.list_managed(SandboxKind::Terminal.label()).await { let managed = match self.driver.list_managed(SandboxKind::Terminal.label()).await {
Ok(m) => m, Ok(m) => m,
Err(e) => { Err(e) => {
eprintln!("terminal reaper: list failed: {e}"); eprintln!("terminal reaper: list failed: {e}");
return reaped; return 0;
} }
}; };
let live: std::collections::HashSet<String> = { let tracked: std::collections::HashSet<String> =
let handles = self.handles.lock().await; cm_db::repo::agent_containers::all(&self.pool, KIND)
handles.values().map(|h| h.id.clone()).collect() .await
}; .unwrap_or_default()
.into_iter()
.map(|r| r.container_id)
.collect();
let now_unix = std::time::SystemTime::now() let now_unix = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH) .duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs() as i64) .map(|d| d.as_secs() as i64)
.unwrap_or(0); .unwrap_or(0);
let min_age = idle_ttl.as_secs() as i64; let min = min_age.as_secs() as i64;
let mut reaped = 0;
for m in managed { for m in managed {
if live.contains(&m.id) { if tracked.contains(&m.id) {
continue; continue;
} }
if min_age > 0 && now_unix - m.created_unix < min_age { if min > 0 && now_unix - m.created_unix < min {
continue; continue;
} }
let handle = SandboxHandle { let handle = SandboxHandle {
@@ -274,10 +304,10 @@ impl TerminalManager {
reaped reaped
} }
/// Boot reconciliation: remove every terminal container the engine still /// Boot reconciliation: remove only true orphans (engine containers with no
/// holds (all are orphans from a dead process — we track none yet). /// registry row). Registry-tracked terminals are preserved across a redeploy.
pub async fn reconcile_orphans(&self) -> usize { pub async fn reconcile_orphans(&self) -> usize {
self.sweep(Duration::ZERO).await self.reap_orphans(Duration::ZERO).await
} }
/// Background reaper: every `interval`, reap idle (`idle_ttl`) + orphan terminals. /// Background reaper: every `interval`, reap idle (`idle_ttl`) + orphan terminals.
@@ -286,7 +316,7 @@ impl TerminalManager {
let mut tick = tokio::time::interval(interval); let mut tick = tokio::time::interval(interval);
loop { loop {
tick.tick().await; tick.tick().await;
let n = self.sweep(idle_ttl).await; let n = self.reap_idle(idle_ttl).await + self.reap_orphans(idle_ttl).await;
if n > 0 { if n > 0 {
eprintln!("terminal reaper: removed {n} terminal container(s)"); eprintln!("terminal reaper: removed {n} terminal container(s)");
} }
+2 -1
View File
@@ -119,7 +119,8 @@ async fn browsing_returns_web_tainted_content_and_taints_later_gated_actions() {
let driver: Arc<dyn cm_sandbox::SandboxDriver> = let driver: Arc<dyn cm_sandbox::SandboxDriver> =
Arc::new(DockerDriver::connect().expect("docker reachable")); Arc::new(DockerDriver::connect().expect("docker reachable"));
let browser = Arc::new(SandboxManager::new(driver, BROWSER_IMAGE).with_egress()); let browser =
Arc::new(SandboxManager::new(driver, pool.clone(), "local", BROWSER_IMAGE).with_egress());
let blob = Arc::new(cm_files::LocalBlobStore::new( let blob = Arc::new(cm_files::LocalBlobStore::new(
std::env::temp_dir().join(format!("tc-brw-{}", uuid::Uuid::now_v7())), std::env::temp_dir().join(format!("tc-brw-{}", uuid::Uuid::now_v7())),
)); ));
+1 -1
View File
@@ -95,7 +95,7 @@ async fn shell_exec_runs_in_the_agent_sandbox_with_persistent_home() {
.unwrap(); .unwrap();
let driver = DockerDriver::connect().expect("docker reachable"); let driver = DockerDriver::connect().expect("docker reachable");
let sandboxes = Arc::new(SandboxManager::new(Arc::new(driver), IMAGE)); let sandboxes = Arc::new(SandboxManager::new(Arc::new(driver), pool.clone(), "local", IMAGE));
let rt = Runtime::new( let rt = Runtime::new(
pool.clone(), pool.clone(),
Arc::new(ScriptedProvider::from_toml(SCENARIOS).unwrap()), Arc::new(ScriptedProvider::from_toml(SCENARIOS).unwrap()),
+2 -1
View File
@@ -54,7 +54,8 @@ async fn the_pool_prefills_assigns_and_refills() {
ensure_image(); ensure_image();
let driver: Arc<dyn cm_sandbox::SandboxDriver> = let driver: Arc<dyn cm_sandbox::SandboxDriver> =
Arc::new(DockerDriver::connect().expect("docker reachable")); Arc::new(DockerDriver::connect().expect("docker reachable"));
let manager = Arc::new(SandboxManager::new(driver, IMAGE)).warm(2); let pool = cm_testkit::test_pool().await;
let manager = Arc::new(SandboxManager::new(driver, pool, "local", IMAGE)).warm(2);
// The warmer fills the pool without any exec happening. // The warmer fills the pool without any exec happening.
pool_reaches(&manager, 2).await; pool_reaches(&manager, 2).await;
+12
View File
@@ -0,0 +1,12 @@
-- Short-lived single-use terminal-WS tickets, stored hashed in Postgres so any
-- server replica (not just the one that minted it) can redeem the handshake.
-- Replaces the former in-process ticket map. Rows are deleted on redeem and
-- swept on expiry.
CREATE TABLE terminal_ws_tickets (
token_hash TEXT PRIMARY KEY,
agent_id UUID NOT NULL REFERENCES agents (id) ON DELETE CASCADE,
workspace_id UUID NOT NULL REFERENCES workspaces (id) ON DELETE CASCADE,
label TEXT NOT NULL,
expires_at TIMESTAMPTZ NOT NULL
);
CREATE INDEX terminal_ws_tickets_expires_idx ON terminal_ws_tickets (expires_at);
+17
View File
@@ -0,0 +1,17 @@
-- Durable registry of the live container backing each agent (terminal / agent /
-- browser), so ANY server replica can find + reuse it instead of the former
-- in-process map (which made a 2nd replica spawn duplicate containers). node_id
-- records placement for the multi-node node-pool (Phase 2); it is 'local' today.
CREATE TABLE agent_containers (
agent_id UUID NOT NULL REFERENCES agents (id) ON DELETE CASCADE,
kind TEXT NOT NULL, -- 'terminal' | 'agent' | 'browser'
node_id TEXT NOT NULL DEFAULT 'local',
container_id TEXT NOT NULL,
name TEXT NOT NULL,
workspace_id UUID NOT NULL REFERENCES workspaces (id) ON DELETE CASCADE,
session_count INTEGER NOT NULL DEFAULT 0, -- live attached sessions (terminals)
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
last_seen TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (agent_id, kind)
);
CREATE INDEX agent_containers_idle_idx ON agent_containers (kind, last_seen);