Scaling Phase 1: multi-tenant onboarding + replica-safe coordination
Decouples "many users" + "many server replicas" from "many machines" so the platform is tenant-isolated and horizontally safe on the current single node. - Per-signup workspaces (cm-auth): a new hosted-identity sign-in provisions and owns its own workspace instead of joining the first. Config-gated by auth.per_signup_workspace (default off); concurrent first-logins serialized by a per-subject advisory lock so no duplicate workspaces. - Terminal tickets in Postgres (migration 0016, hashed, single-use): any replica can redeem a ticket minted by another. Drops the in-process ticket map. - Container registry in Postgres (migration 0017, agent_containers): Terminal and Sandbox managers resolve an agent's container through a shared registry, so a 2nd replica reuses it instead of spawning a duplicate. node_id recorded as 'local' (Phase 2 hook). Boot reconcile removes only true orphans, so terminals now survive a redeploy (tmux sessions resume). - Per-workspace quotas (cm-api/quota.rs): plan-tier caps on agents + live containers, enforced at agent create + terminal spin-up (reconnects allowed), returned as HTTP 402. New GET /api/quota surfaces usage vs limits. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
f8f2b65e1f
commit
e9ce368ec1
@@ -153,10 +153,17 @@ async fn run() -> Result<(), String> {
|
|||||||
std::sync::Arc::new(driver);
|
std::sync::Arc::new(driver);
|
||||||
let agents = std::sync::Arc::new(cm_runtime::SandboxManager::new(
|
let agents = std::sync::Arc::new(cm_runtime::SandboxManager::new(
|
||||||
driver.clone(),
|
driver.clone(),
|
||||||
|
pool.clone(),
|
||||||
|
"local",
|
||||||
&config.sandbox.image,
|
&config.sandbox.image,
|
||||||
));
|
));
|
||||||
let browser = std::sync::Arc::new(
|
let browser = std::sync::Arc::new(
|
||||||
cm_runtime::SandboxManager::new(driver.clone(), &config.sandbox.browser_image)
|
cm_runtime::SandboxManager::new(
|
||||||
|
driver.clone(),
|
||||||
|
pool.clone(),
|
||||||
|
"local",
|
||||||
|
&config.sandbox.browser_image,
|
||||||
|
)
|
||||||
.with_egress(),
|
.with_egress(),
|
||||||
);
|
);
|
||||||
// Themed interactive terminal containers (zsh + oh-my-zsh + p10k)
|
// Themed interactive terminal containers (zsh + oh-my-zsh + p10k)
|
||||||
@@ -172,6 +179,8 @@ async fn run() -> Result<(), String> {
|
|||||||
};
|
};
|
||||||
let terminals = std::sync::Arc::new(cm_runtime::TerminalManager::new(
|
let terminals = std::sync::Arc::new(cm_runtime::TerminalManager::new(
|
||||||
driver,
|
driver,
|
||||||
|
pool.clone(),
|
||||||
|
"local",
|
||||||
&config.sandbox.terminal_image,
|
&config.sandbox.terminal_image,
|
||||||
config.sandbox.terminal_egress,
|
config.sandbox.terminal_egress,
|
||||||
drives,
|
drives,
|
||||||
@@ -276,6 +285,7 @@ async fn run() -> Result<(), String> {
|
|||||||
(config.storage.backend == cm_config::StorageBackend::Local)
|
(config.storage.backend == cm_config::StorageBackend::Local)
|
||||||
.then(|| PathBuf::from(&config.storage.data_dir)),
|
.then(|| PathBuf::from(&config.storage.data_dir)),
|
||||||
)
|
)
|
||||||
|
.with_per_signup_workspace(config.auth.per_signup_workspace)
|
||||||
.pipe_auth_verifier(auth_verifier),
|
.pipe_auth_verifier(auth_verifier),
|
||||||
);
|
);
|
||||||
if e2e::enabled() {
|
if e2e::enabled() {
|
||||||
|
|||||||
@@ -17,6 +17,8 @@ pub enum ApiError {
|
|||||||
NotFound,
|
NotFound,
|
||||||
#[error("conflict")]
|
#[error("conflict")]
|
||||||
Conflict,
|
Conflict,
|
||||||
|
#[error("{0}")]
|
||||||
|
Quota(String),
|
||||||
#[error("internal error")]
|
#[error("internal error")]
|
||||||
Internal,
|
Internal,
|
||||||
}
|
}
|
||||||
@@ -55,6 +57,7 @@ impl IntoResponse for ApiError {
|
|||||||
ApiError::Forbidden => StatusCode::FORBIDDEN,
|
ApiError::Forbidden => StatusCode::FORBIDDEN,
|
||||||
ApiError::NotFound => StatusCode::NOT_FOUND,
|
ApiError::NotFound => StatusCode::NOT_FOUND,
|
||||||
ApiError::Conflict => StatusCode::CONFLICT,
|
ApiError::Conflict => StatusCode::CONFLICT,
|
||||||
|
ApiError::Quota(_) => StatusCode::PAYMENT_REQUIRED,
|
||||||
ApiError::Internal => StatusCode::INTERNAL_SERVER_ERROR,
|
ApiError::Internal => StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
};
|
};
|
||||||
(status, Json(json!({ "error": self.to_string() }))).into_response()
|
(status, Json(json!({ "error": self.to_string() }))).into_response()
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ pub mod cleanup_sweeper;
|
|||||||
mod error;
|
mod error;
|
||||||
mod extract;
|
mod extract;
|
||||||
mod mcp_door;
|
mod mcp_door;
|
||||||
|
pub mod quota;
|
||||||
mod recursive_exec;
|
mod recursive_exec;
|
||||||
mod routes;
|
mod routes;
|
||||||
mod runtime_provision;
|
mod runtime_provision;
|
||||||
@@ -30,7 +31,6 @@ pub struct AppState {
|
|||||||
pub oauth: cm_config::OAuthConfig,
|
pub oauth: cm_config::OAuthConfig,
|
||||||
pub billing: cm_config::BillingConfig,
|
pub billing: cm_config::BillingConfig,
|
||||||
/// Short-lived single-use tickets for the Terminal WebSocket.
|
/// Short-lived single-use tickets for the Terminal WebSocket.
|
||||||
pub terminal_tickets: routes::terminal::TerminalTickets,
|
|
||||||
/// Local blob-store root (Some on the Local backend) so the Files app can
|
/// Local blob-store root (Some on the Local backend) so the Files app can
|
||||||
/// reconcile its index with files the Terminal wrote into the drives.
|
/// reconcile its index with files the Terminal wrote into the drives.
|
||||||
pub file_root: Option<std::path::PathBuf>,
|
pub file_root: Option<std::path::PathBuf>,
|
||||||
@@ -46,7 +46,6 @@ impl AppState {
|
|||||||
broker_socket: None,
|
broker_socket: None,
|
||||||
oauth: cm_config::OAuthConfig::default(),
|
oauth: cm_config::OAuthConfig::default(),
|
||||||
billing: cm_config::BillingConfig::default(),
|
billing: cm_config::BillingConfig::default(),
|
||||||
terminal_tickets: routes::terminal::TerminalTickets::default(),
|
|
||||||
file_root: None,
|
file_root: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -61,6 +60,12 @@ impl AppState {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// SaaS: give each new hosted-identity sign-in its own workspace.
|
||||||
|
pub fn with_per_signup_workspace(mut self, enabled: bool) -> AppState {
|
||||||
|
self.auth = self.auth.with_per_signup_workspace(enabled);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
pub fn with_billing(mut self, billing: cm_config::BillingConfig) -> AppState {
|
pub fn with_billing(mut self, billing: cm_config::BillingConfig) -> AppState {
|
||||||
self.billing = billing;
|
self.billing = billing;
|
||||||
self
|
self
|
||||||
@@ -96,6 +101,7 @@ impl AppState {
|
|||||||
pub fn router(state: AppState) -> Router {
|
pub fn router(state: AppState) -> Router {
|
||||||
Router::new()
|
Router::new()
|
||||||
.route("/healthz", get(routes::health::healthz))
|
.route("/healthz", get(routes::health::healthz))
|
||||||
|
.route("/api/quota", get(quota::get_quota))
|
||||||
.route("/mcp", post(mcp_door::mcp))
|
.route("/mcp", post(mcp_door::mcp))
|
||||||
.route("/api/auth/login", post(routes::auth::login))
|
.route("/api/auth/login", post(routes::auth::login))
|
||||||
.route("/api/auth/logout", post(routes::auth::logout))
|
.route("/api/auth/logout", post(routes::auth::logout))
|
||||||
|
|||||||
@@ -0,0 +1,104 @@
|
|||||||
|
//! Per-workspace resource quotas keyed off the workspace plan. Phase 1 of the
|
||||||
|
//! multi-tenant scaling work: keep one tenant from exhausting the shared pool.
|
||||||
|
|
||||||
|
use axum::extract::State;
|
||||||
|
use axum::Json;
|
||||||
|
use cm_domain::WorkspaceId;
|
||||||
|
use serde::Serialize;
|
||||||
|
use sqlx::Row;
|
||||||
|
|
||||||
|
use crate::{ApiError, AppState, Authed};
|
||||||
|
|
||||||
|
/// Caps for a plan tier.
|
||||||
|
pub struct Quota {
|
||||||
|
pub max_agents: i64,
|
||||||
|
pub max_live_containers: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Per-plan limits. Unknown plans fall back to the free tier.
|
||||||
|
pub fn plan_quota(plan: &str) -> Quota {
|
||||||
|
match plan {
|
||||||
|
"team" => Quota {
|
||||||
|
max_agents: 50,
|
||||||
|
max_live_containers: 50,
|
||||||
|
},
|
||||||
|
"pro" => Quota {
|
||||||
|
max_agents: 20,
|
||||||
|
max_live_containers: 20,
|
||||||
|
},
|
||||||
|
_ => Quota {
|
||||||
|
max_agents: 3,
|
||||||
|
max_live_containers: 3,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The workspace's plan name (defaults to "free").
|
||||||
|
async fn plan_of(state: &AppState, workspace_id: WorkspaceId) -> Result<String, ApiError> {
|
||||||
|
let row = sqlx::query("SELECT plan FROM workspaces WHERE id = $1")
|
||||||
|
.bind(workspace_id.as_uuid())
|
||||||
|
.fetch_optional(&state.pool)
|
||||||
|
.await?;
|
||||||
|
Ok(row
|
||||||
|
.map(|r| r.get::<String, _>("plan"))
|
||||||
|
.unwrap_or_else(|| "free".to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reject creating another agent if the workspace is at its plan cap.
|
||||||
|
pub async fn enforce_new_agent(state: &AppState, workspace_id: WorkspaceId) -> Result<(), ApiError> {
|
||||||
|
let plan = plan_of(state, workspace_id).await?;
|
||||||
|
let quota = plan_quota(&plan);
|
||||||
|
let used = cm_db::repo::agents::count_active(&state.pool, workspace_id).await?;
|
||||||
|
if used >= quota.max_agents {
|
||||||
|
return Err(ApiError::Quota(format!(
|
||||||
|
"agent limit reached ({} on the {plan} plan) — upgrade your plan or remove an agent",
|
||||||
|
quota.max_agents
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reject spinning up another container if the workspace is at its plan cap.
|
||||||
|
pub async fn enforce_new_container(
|
||||||
|
state: &AppState,
|
||||||
|
workspace_id: WorkspaceId,
|
||||||
|
) -> Result<(), ApiError> {
|
||||||
|
let plan = plan_of(state, workspace_id).await?;
|
||||||
|
let quota = plan_quota(&plan);
|
||||||
|
let used = cm_db::repo::agent_containers::count_for_workspace(&state.pool, workspace_id).await?;
|
||||||
|
if used >= quota.max_live_containers {
|
||||||
|
return Err(ApiError::Quota(format!(
|
||||||
|
"live container limit reached ({} on the {plan} plan) — close a terminal/agent or upgrade",
|
||||||
|
quota.max_live_containers
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub struct QuotaUsage {
|
||||||
|
plan: String,
|
||||||
|
agents_used: i64,
|
||||||
|
max_agents: i64,
|
||||||
|
containers_used: i64,
|
||||||
|
max_live_containers: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `GET /api/quota` — the caller's workspace usage + limits (for the UI).
|
||||||
|
pub async fn get_quota(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
) -> Result<Json<QuotaUsage>, ApiError> {
|
||||||
|
let plan = plan_of(&state, user.workspace_id).await?;
|
||||||
|
let quota = plan_quota(&plan);
|
||||||
|
let agents_used = cm_db::repo::agents::count_active(&state.pool, user.workspace_id).await?;
|
||||||
|
let containers_used =
|
||||||
|
cm_db::repo::agent_containers::count_for_workspace(&state.pool, user.workspace_id).await?;
|
||||||
|
Ok(Json(QuotaUsage {
|
||||||
|
plan,
|
||||||
|
agents_used,
|
||||||
|
max_agents: quota.max_agents,
|
||||||
|
containers_used,
|
||||||
|
max_live_containers: quota.max_live_containers,
|
||||||
|
}))
|
||||||
|
}
|
||||||
@@ -742,6 +742,7 @@ pub async fn create(
|
|||||||
Authed(user): Authed,
|
Authed(user): Authed,
|
||||||
Json(body): Json<CreateClawRequest>,
|
Json(body): Json<CreateClawRequest>,
|
||||||
) -> Result<(StatusCode, Json<Agent>), ApiError> {
|
) -> Result<(StatusCode, Json<Agent>), ApiError> {
|
||||||
|
crate::quota::enforce_new_agent(&state, user.workspace_id).await?;
|
||||||
let agent = Agent {
|
let agent = Agent {
|
||||||
id: AgentId::new(),
|
id: AgentId::new(),
|
||||||
workspace_id: user.workspace_id,
|
workspace_id: user.workspace_id,
|
||||||
|
|||||||
@@ -5,9 +5,7 @@
|
|||||||
//! ticket, then opens the WS with `?ticket=`. The WS itself is routed straight
|
//! ticket, then opens the WS with `?ticket=`. The WS itself is routed straight
|
||||||
//! to this server by the edge (Traefik), bypassing the HTTP-only Next proxy.
|
//! to this server by the edge (Traefik), bypassing the HTTP-only Next proxy.
|
||||||
|
|
||||||
use std::collections::HashMap;
|
use std::sync::Arc;
|
||||||
use std::sync::{Arc, Mutex};
|
|
||||||
use std::time::{Duration, Instant};
|
|
||||||
|
|
||||||
use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade};
|
use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade};
|
||||||
use axum::extract::{Path, Query, State};
|
use axum::extract::{Path, Query, State};
|
||||||
@@ -19,56 +17,67 @@ use cm_domain::{AgentId, WorkspaceId};
|
|||||||
use futures::{SinkExt, StreamExt};
|
use futures::{SinkExt, StreamExt};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
use sqlx::{PgPool, Row};
|
||||||
use tokio::io::AsyncWriteExt;
|
use tokio::io::AsyncWriteExt;
|
||||||
|
|
||||||
use crate::routes::claws::workspace_agent;
|
use crate::routes::claws::workspace_agent;
|
||||||
use crate::{ApiError, AppState, Authed};
|
use crate::{ApiError, AppState, Authed};
|
||||||
|
|
||||||
const TICKET_TTL: Duration = Duration::from_secs(30);
|
/// Single-use WS tickets live in Postgres (hashed), so any server replica can
|
||||||
|
/// redeem one — not just the instance that minted it. A ticket is deleted on
|
||||||
|
/// redeem; expired rows are swept opportunistically. TTL is 30s (in the SQL).
|
||||||
|
fn hash_ticket(token: &str) -> String {
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
let mut h = Sha256::new();
|
||||||
|
h.update(token.as_bytes());
|
||||||
|
format!("{:x}", h.finalize())
|
||||||
|
}
|
||||||
|
|
||||||
/// In-memory single-use WS tickets (token → agent + greeting label + expiry).
|
/// Mint a short-lived single-use ticket; returns the raw token.
|
||||||
/// Tiny + ephemeral; a ticket is consumed on redeem, expired ones swept lazily.
|
async fn issue_ticket(
|
||||||
struct Ticket {
|
pool: &PgPool,
|
||||||
agent_id: AgentId,
|
agent_id: AgentId,
|
||||||
workspace_id: WorkspaceId,
|
workspace_id: WorkspaceId,
|
||||||
/// Display name to greet the user with in the terminal MOTD.
|
label: &str,
|
||||||
label: String,
|
) -> Result<String, sqlx::Error> {
|
||||||
expires: Instant,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Clone, Default)]
|
|
||||||
pub struct TerminalTickets {
|
|
||||||
inner: Arc<Mutex<HashMap<String, Ticket>>>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl TerminalTickets {
|
|
||||||
pub fn issue(&self, agent_id: AgentId, workspace_id: WorkspaceId, label: String) -> String {
|
|
||||||
let token = uuid::Uuid::new_v4().simple().to_string();
|
let token = uuid::Uuid::new_v4().simple().to_string();
|
||||||
let mut g = self.inner.lock().expect("tickets lock");
|
let _ = sqlx::query("DELETE FROM terminal_ws_tickets WHERE expires_at < now()")
|
||||||
let now = Instant::now();
|
.execute(pool)
|
||||||
g.retain(|_, t| t.expires > now);
|
.await;
|
||||||
g.insert(
|
sqlx::query(
|
||||||
token.clone(),
|
"INSERT INTO terminal_ws_tickets (token_hash, agent_id, workspace_id, label, expires_at)
|
||||||
Ticket {
|
VALUES ($1, $2, $3, $4, now() + interval '30 seconds')",
|
||||||
agent_id,
|
)
|
||||||
workspace_id,
|
.bind(hash_ticket(&token))
|
||||||
label,
|
.bind(agent_id.as_uuid())
|
||||||
expires: now + TICKET_TTL,
|
.bind(workspace_id.as_uuid())
|
||||||
},
|
.bind(label)
|
||||||
);
|
.execute(pool)
|
||||||
token
|
.await?;
|
||||||
}
|
Ok(token)
|
||||||
|
}
|
||||||
|
|
||||||
/// Consume a ticket; returns its (workspace, greeting label) iff valid for this agent.
|
/// Consume a ticket; returns (workspace, greeting label) iff valid for this agent.
|
||||||
pub fn redeem(&self, token: &str, agent_id: AgentId) -> Option<(WorkspaceId, String)> {
|
async fn redeem_ticket(
|
||||||
let now = Instant::now();
|
pool: &PgPool,
|
||||||
let mut g = self.inner.lock().expect("tickets lock");
|
token: &str,
|
||||||
g.retain(|_, t| t.expires > now);
|
agent_id: AgentId,
|
||||||
match g.remove(token) {
|
) -> Option<(WorkspaceId, String)> {
|
||||||
Some(t) if t.agent_id == agent_id && t.expires > now => Some((t.workspace_id, t.label)),
|
let row = sqlx::query(
|
||||||
_ => None,
|
"DELETE FROM terminal_ws_tickets
|
||||||
}
|
WHERE token_hash = $1 AND agent_id = $2 AND expires_at > now()
|
||||||
}
|
RETURNING workspace_id, label",
|
||||||
|
)
|
||||||
|
.bind(hash_ticket(token))
|
||||||
|
.bind(agent_id.as_uuid())
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.flatten()?;
|
||||||
|
Some((
|
||||||
|
WorkspaceId::from(row.get::<uuid::Uuid, _>("workspace_id")),
|
||||||
|
row.get::<String, _>("label"),
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
@@ -84,6 +93,15 @@ pub async fn ticket(
|
|||||||
) -> Result<Json<TicketResponse>, ApiError> {
|
) -> Result<Json<TicketResponse>, ApiError> {
|
||||||
// Tenant isolation: a foreign agent looks non-existent.
|
// Tenant isolation: a foreign agent looks non-existent.
|
||||||
workspace_agent(&state, &user, agent_id).await?;
|
workspace_agent(&state, &user, agent_id).await?;
|
||||||
|
// Quota: gate only when this would spin up a NEW container (a reconnect to an
|
||||||
|
// already-running terminal is always allowed).
|
||||||
|
let existing = cm_db::repo::agent_containers::get(&state.pool, agent_id, "terminal")
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.flatten();
|
||||||
|
if existing.is_none() {
|
||||||
|
crate::quota::enforce_new_container(&state, user.workspace_id).await?;
|
||||||
|
}
|
||||||
// Greet by display name, falling back to the email local-part.
|
// Greet by display name, falling back to the email local-part.
|
||||||
let label = match cm_db::repo::users::get(&state.pool, user.user_id).await {
|
let label = match cm_db::repo::users::get(&state.pool, user.user_id).await {
|
||||||
Ok(u) if !u.display_name.trim().is_empty() => u.display_name.trim().to_string(),
|
Ok(u) if !u.display_name.trim().is_empty() => u.display_name.trim().to_string(),
|
||||||
@@ -96,9 +114,7 @@ pub async fn ticket(
|
|||||||
.to_string(),
|
.to_string(),
|
||||||
Err(_) => "there".to_string(),
|
Err(_) => "there".to_string(),
|
||||||
};
|
};
|
||||||
let ticket = state
|
let ticket = issue_ticket(&state.pool, agent_id, user.workspace_id, &label).await?;
|
||||||
.terminal_tickets
|
|
||||||
.issue(agent_id, user.workspace_id, label);
|
|
||||||
cm_db::repo::audit::append(
|
cm_db::repo::audit::append(
|
||||||
&state.pool,
|
&state.pool,
|
||||||
user.workspace_id,
|
user.workspace_id,
|
||||||
@@ -200,7 +216,7 @@ pub async fn ws(
|
|||||||
Query(q): Query<WsQuery>,
|
Query(q): Query<WsQuery>,
|
||||||
upgrade: WebSocketUpgrade,
|
upgrade: WebSocketUpgrade,
|
||||||
) -> Response {
|
) -> Response {
|
||||||
let (workspace_id, label) = match state.terminal_tickets.redeem(&q.ticket, agent_id) {
|
let (workspace_id, label) = match redeem_ticket(&state.pool, &q.ticket, agent_id).await {
|
||||||
Some(v) => v,
|
Some(v) => v,
|
||||||
None => return (StatusCode::UNAUTHORIZED, "invalid or expired ticket").into_response(),
|
None => return (StatusCode::UNAUTHORIZED, "invalid or expired ticket").into_response(),
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ serde = { workspace = true }
|
|||||||
tokio = { workspace = true }
|
tokio = { workspace = true }
|
||||||
sha2 = "0.10"
|
sha2 = "0.10"
|
||||||
sqlx = { workspace = true }
|
sqlx = { workspace = true }
|
||||||
|
uuid = { workspace = true }
|
||||||
cm-db = { path = "../cm-db" }
|
cm-db = { path = "../cm-db" }
|
||||||
cm-domain = { path = "../cm-domain" }
|
cm-domain = { path = "../cm-domain" }
|
||||||
thiserror = { workspace = true }
|
thiserror = { workspace = true }
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ use argon2::password_hash::rand_core::OsRng;
|
|||||||
use argon2::password_hash::{PasswordHash, PasswordHasher, PasswordVerifier, SaltString};
|
use argon2::password_hash::{PasswordHash, PasswordHasher, PasswordVerifier, SaltString};
|
||||||
use argon2::Argon2;
|
use argon2::Argon2;
|
||||||
use cm_domain::{Role, UserId, WorkspaceId};
|
use cm_domain::{Role, UserId, WorkspaceId};
|
||||||
use sqlx::PgPool;
|
use sqlx::{PgPool, Row};
|
||||||
use time::{Duration, OffsetDateTime};
|
use time::{Duration, OffsetDateTime};
|
||||||
|
|
||||||
use crate::token::{hash_token, SessionToken};
|
use crate::token::{hash_token, SessionToken};
|
||||||
@@ -37,6 +37,9 @@ pub enum AuthError {
|
|||||||
pub struct AuthService {
|
pub struct AuthService {
|
||||||
pool: PgPool,
|
pool: PgPool,
|
||||||
verifier: Option<std::sync::Arc<crate::JwtVerifier>>,
|
verifier: Option<std::sync::Arc<crate::JwtVerifier>>,
|
||||||
|
/// SaaS mode: a brand-new external identity provisions its own workspace
|
||||||
|
/// (owner) rather than joining the instance's single workspace.
|
||||||
|
per_signup_workspace: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl AuthService {
|
impl AuthService {
|
||||||
@@ -44,6 +47,7 @@ impl AuthService {
|
|||||||
AuthService {
|
AuthService {
|
||||||
pool,
|
pool,
|
||||||
verifier: None,
|
verifier: None,
|
||||||
|
per_signup_workspace: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,6 +58,13 @@ impl AuthService {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// In SaaS deployments, give each new hosted-identity sign-in its own
|
||||||
|
/// workspace (they become its owner) instead of joining the first one.
|
||||||
|
pub fn with_per_signup_workspace(mut self, enabled: bool) -> AuthService {
|
||||||
|
self.per_signup_workspace = enabled;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
/// Verifies an issuer session token and JIT-provisions the user on
|
/// Verifies an issuer session token and JIT-provisions the user on
|
||||||
/// first sight. Role tracks the issuer claim on every login (Clerk's
|
/// first sight. Role tracks the issuer claim on every login (Clerk's
|
||||||
/// org role is the source of truth for SSO users).
|
/// org role is the source of truth for SSO users).
|
||||||
@@ -112,42 +123,82 @@ impl AuthService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// First sight: provision into the instance's workspace.
|
// First sight. Serialize concurrent first-logins for the same subject
|
||||||
let workspace =
|
// (the authed shell fires several API calls at once) with a per-subject
|
||||||
sqlx::query_scalar!("SELECT id FROM workspaces ORDER BY created_at, id LIMIT 1",)
|
// advisory lock, so SaaS mode can't create duplicate workspaces and the
|
||||||
.fetch_optional(&self.pool)
|
// appliance path can't double-insert the user.
|
||||||
.await?
|
|
||||||
.ok_or(AuthError::Unauthenticated)?;
|
|
||||||
let email = claims
|
let email = claims
|
||||||
.email
|
.email
|
||||||
.clone()
|
.clone()
|
||||||
.unwrap_or_else(|| format!("{}@sso.local", claims.sub));
|
.unwrap_or_else(|| format!("{}@sso.local", claims.sub));
|
||||||
let display_name = email.split('@').next().unwrap_or("teammate").to_owned();
|
let display_name = email.split('@').next().unwrap_or("teammate").to_owned();
|
||||||
|
|
||||||
|
let mut tx = self.pool.begin().await?;
|
||||||
|
sqlx::query("SELECT pg_advisory_xact_lock(hashtext($1))")
|
||||||
|
.bind(&claims.sub)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
// A concurrent winner may have already provisioned this identity.
|
||||||
|
if let Some(row) = sqlx::query("SELECT id, workspace_id, role FROM users WHERE auth_subject = $1")
|
||||||
|
.bind(&claims.sub)
|
||||||
|
.fetch_optional(&mut *tx)
|
||||||
|
.await?
|
||||||
|
{
|
||||||
|
tx.commit().await?;
|
||||||
|
let role_db: String = row.get("role");
|
||||||
|
return Ok(AuthedUser {
|
||||||
|
user_id: UserId::from(row.get::<uuid::Uuid, _>("id")),
|
||||||
|
workspace_id: WorkspaceId::from(row.get::<uuid::Uuid, _>("workspace_id")),
|
||||||
|
role: if role_db == "owner" {
|
||||||
|
Role::Owner
|
||||||
|
} else {
|
||||||
|
Role::Member
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pick the workspace: SaaS provisions a fresh one this user owns;
|
||||||
|
// appliance joins the instance's single workspace.
|
||||||
|
let (workspace, effective_role): (uuid::Uuid, &str) = if self.per_signup_workspace {
|
||||||
|
let ws = WorkspaceId::new();
|
||||||
|
sqlx::query("INSERT INTO workspaces (id, name, plan) VALUES ($1, $2, 'free')")
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.bind(format!("{display_name}'s workspace"))
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?;
|
||||||
|
(ws.as_uuid(), "owner")
|
||||||
|
} else {
|
||||||
|
let ws: uuid::Uuid =
|
||||||
|
sqlx::query_scalar("SELECT id FROM workspaces ORDER BY created_at, id LIMIT 1")
|
||||||
|
.fetch_optional(&mut *tx)
|
||||||
|
.await?
|
||||||
|
.ok_or(AuthError::Unauthenticated)?;
|
||||||
|
(ws, role_str)
|
||||||
|
};
|
||||||
let user_id = UserId::new();
|
let user_id = UserId::new();
|
||||||
// Idempotent under concurrent first-login: the authed shell fires
|
let row = sqlx::query(
|
||||||
// several API calls at once, and each would otherwise race to INSERT
|
|
||||||
// this row and trip the partial unique index on auth_subject (the
|
|
||||||
// losers 500'd). ON CONFLICT resolves to the row the winning request
|
|
||||||
// created, so every concurrent caller returns the same identity.
|
|
||||||
let row = sqlx::query!(
|
|
||||||
"INSERT INTO users (id, workspace_id, email, role, display_name, auth_subject)
|
"INSERT INTO users (id, workspace_id, email, role, display_name, auth_subject)
|
||||||
VALUES ($1, $2, $3, $4, $5, $6)
|
VALUES ($1, $2, $3, $4, $5, $6)
|
||||||
ON CONFLICT (auth_subject) WHERE auth_subject IS NOT NULL
|
|
||||||
DO UPDATE SET role = EXCLUDED.role
|
|
||||||
RETURNING id, workspace_id",
|
RETURNING id, workspace_id",
|
||||||
user_id.as_uuid(),
|
|
||||||
workspace,
|
|
||||||
email,
|
|
||||||
role_str,
|
|
||||||
display_name,
|
|
||||||
claims.sub,
|
|
||||||
)
|
)
|
||||||
.fetch_one(&self.pool)
|
.bind(user_id.as_uuid())
|
||||||
|
.bind(workspace)
|
||||||
|
.bind(&email)
|
||||||
|
.bind(effective_role)
|
||||||
|
.bind(&display_name)
|
||||||
|
.bind(&claims.sub)
|
||||||
|
.fetch_one(&mut *tx)
|
||||||
.await?;
|
.await?;
|
||||||
|
tx.commit().await?;
|
||||||
Ok(AuthedUser {
|
Ok(AuthedUser {
|
||||||
user_id: UserId::from(row.id),
|
user_id: UserId::from(row.get::<uuid::Uuid, _>("id")),
|
||||||
workspace_id: WorkspaceId::from(row.workspace_id),
|
workspace_id: WorkspaceId::from(row.get::<uuid::Uuid, _>("workspace_id")),
|
||||||
role,
|
role: if effective_role == "owner" {
|
||||||
|
Role::Owner
|
||||||
|
} else {
|
||||||
|
Role::Member
|
||||||
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -97,6 +97,11 @@ pub struct AuthConfig {
|
|||||||
pub mode: AuthMode,
|
pub mode: AuthMode,
|
||||||
pub issuer_url: Option<String>,
|
pub issuer_url: Option<String>,
|
||||||
pub client_id: Option<String>,
|
pub client_id: Option<String>,
|
||||||
|
/// When true (SaaS), a brand-new hosted-identity (Clerk/OIDC) sign-in
|
||||||
|
/// provisions its OWN workspace and owns it. When false (appliance), the
|
||||||
|
/// first sign-in joins the instance's single workspace as a member.
|
||||||
|
#[serde(default)]
|
||||||
|
pub per_signup_workspace: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
|
||||||
|
|||||||
@@ -0,0 +1,159 @@
|
|||||||
|
//! Durable registry of the live container backing each agent, so any server
|
||||||
|
//! replica can find + reuse it (replacing the former in-process handle map).
|
||||||
|
|
||||||
|
use cm_domain::{AgentId, WorkspaceId};
|
||||||
|
use sqlx::{PgPool, Row};
|
||||||
|
|
||||||
|
use crate::DbError;
|
||||||
|
|
||||||
|
/// A registry row for one agent's container of a given kind.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ContainerRow {
|
||||||
|
pub node_id: String,
|
||||||
|
pub container_id: String,
|
||||||
|
pub name: String,
|
||||||
|
pub session_count: i32,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One container to act on (idle reap / reconcile / shutdown).
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ManagedRow {
|
||||||
|
pub agent_id: AgentId,
|
||||||
|
pub node_id: String,
|
||||||
|
pub container_id: String,
|
||||||
|
pub name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get(
|
||||||
|
pool: &PgPool,
|
||||||
|
agent_id: AgentId,
|
||||||
|
kind: &str,
|
||||||
|
) -> Result<Option<ContainerRow>, DbError> {
|
||||||
|
let row = sqlx::query(
|
||||||
|
"SELECT node_id, container_id, name, session_count
|
||||||
|
FROM agent_containers WHERE agent_id = $1 AND kind = $2",
|
||||||
|
)
|
||||||
|
.bind(agent_id.as_uuid())
|
||||||
|
.bind(kind)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(row.map(|r| ContainerRow {
|
||||||
|
node_id: r.get("node_id"),
|
||||||
|
container_id: r.get("container_id"),
|
||||||
|
name: r.get("name"),
|
||||||
|
session_count: r.get("session_count"),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record the (new) container for `(agent, kind)`; resets the session count.
|
||||||
|
/// The workspace is derived from the agent row (single source of truth).
|
||||||
|
pub async fn upsert(
|
||||||
|
pool: &PgPool,
|
||||||
|
agent_id: AgentId,
|
||||||
|
kind: &str,
|
||||||
|
node_id: &str,
|
||||||
|
container_id: &str,
|
||||||
|
name: &str,
|
||||||
|
) -> Result<(), DbError> {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO agent_containers
|
||||||
|
(agent_id, kind, node_id, container_id, name, workspace_id, session_count, last_seen)
|
||||||
|
SELECT $1, $2, $3, $4, $5, a.workspace_id, 0, now() FROM agents a WHERE a.id = $1
|
||||||
|
ON CONFLICT (agent_id, kind) DO UPDATE SET
|
||||||
|
node_id = excluded.node_id, container_id = excluded.container_id,
|
||||||
|
name = excluded.name, workspace_id = excluded.workspace_id,
|
||||||
|
session_count = 0, last_seen = now()",
|
||||||
|
)
|
||||||
|
.bind(agent_id.as_uuid())
|
||||||
|
.bind(kind)
|
||||||
|
.bind(node_id)
|
||||||
|
.bind(container_id)
|
||||||
|
.bind(name)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn delete(pool: &PgPool, agent_id: AgentId, kind: &str) -> Result<(), DbError> {
|
||||||
|
sqlx::query("DELETE FROM agent_containers WHERE agent_id = $1 AND kind = $2")
|
||||||
|
.bind(agent_id.as_uuid())
|
||||||
|
.bind(kind)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Adjust the live-session count (clamped at 0) and bump last_seen.
|
||||||
|
pub async fn add_session(
|
||||||
|
pool: &PgPool,
|
||||||
|
agent_id: AgentId,
|
||||||
|
kind: &str,
|
||||||
|
delta: i32,
|
||||||
|
) -> Result<(), DbError> {
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE agent_containers
|
||||||
|
SET session_count = GREATEST(0, session_count + $3), last_seen = now()
|
||||||
|
WHERE agent_id = $1 AND kind = $2",
|
||||||
|
)
|
||||||
|
.bind(agent_id.as_uuid())
|
||||||
|
.bind(kind)
|
||||||
|
.bind(delta)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Zero the live-session counts for a kind (graceful shutdown: the WS sessions
|
||||||
|
/// are ending, so the containers become idle-reapable but are not destroyed).
|
||||||
|
pub async fn reset_sessions(pool: &PgPool, kind: &str) -> Result<(), DbError> {
|
||||||
|
sqlx::query("UPDATE agent_containers SET session_count = 0 WHERE kind = $1")
|
||||||
|
.bind(kind)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Session-less containers untouched for longer than `idle_secs` (0 = all).
|
||||||
|
pub async fn idle(
|
||||||
|
pool: &PgPool,
|
||||||
|
kind: &str,
|
||||||
|
idle_secs: i64,
|
||||||
|
) -> Result<Vec<ManagedRow>, DbError> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT agent_id, node_id, container_id, name FROM agent_containers
|
||||||
|
WHERE kind = $1 AND session_count <= 0
|
||||||
|
AND last_seen < now() - ($2 * interval '1 second')",
|
||||||
|
)
|
||||||
|
.bind(kind)
|
||||||
|
.bind(idle_secs)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows.into_iter().map(map_managed).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every recorded container of a kind (for reconcile + shutdown).
|
||||||
|
pub async fn all(pool: &PgPool, kind: &str) -> Result<Vec<ManagedRow>, DbError> {
|
||||||
|
let rows = sqlx::query("SELECT agent_id, node_id, container_id, name FROM agent_containers WHERE kind = $1")
|
||||||
|
.bind(kind)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows.into_iter().map(map_managed).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Count live containers in a workspace (any kind) — for quota checks.
|
||||||
|
pub async fn count_for_workspace(pool: &PgPool, workspace_id: WorkspaceId) -> Result<i64, DbError> {
|
||||||
|
let row = sqlx::query("SELECT count(*) AS n FROM agent_containers WHERE workspace_id = $1")
|
||||||
|
.bind(workspace_id.as_uuid())
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(row.get::<i64, _>("n"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn map_managed(r: sqlx::postgres::PgRow) -> ManagedRow {
|
||||||
|
ManagedRow {
|
||||||
|
agent_id: AgentId::from(r.get::<uuid::Uuid, _>("agent_id")),
|
||||||
|
node_id: r.get("node_id"),
|
||||||
|
container_id: r.get("container_id"),
|
||||||
|
name: r.get("name"),
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,6 +6,17 @@ use uuid::Uuid;
|
|||||||
|
|
||||||
use crate::DbError;
|
use crate::DbError;
|
||||||
|
|
||||||
|
/// Count of non-deleted agents in a workspace (for per-workspace quotas).
|
||||||
|
pub async fn count_active(pool: &PgPool, workspace_id: WorkspaceId) -> Result<i64, DbError> {
|
||||||
|
let n: i64 = sqlx::query_scalar(
|
||||||
|
"SELECT count(*) FROM agents WHERE workspace_id = $1 AND deleted_at IS NULL",
|
||||||
|
)
|
||||||
|
.bind(workspace_id.as_uuid())
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(n)
|
||||||
|
}
|
||||||
|
|
||||||
/// Inserts an agent together with its access policy in one transaction —
|
/// Inserts an agent together with its access policy in one transaction —
|
||||||
/// an agent without a policy must never be observable (§7.7).
|
/// an agent without a policy must never be observable (§7.7).
|
||||||
pub async fn insert(pool: &PgPool, agent: &Agent, policy: &AccessPolicy) -> Result<(), DbError> {
|
pub async fn insert(pool: &PgPool, agent: &Agent, policy: &AccessPolicy) -> Result<(), DbError> {
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
pub mod agent_containers;
|
||||||
pub mod agents;
|
pub mod agents;
|
||||||
pub mod audit;
|
pub mod audit;
|
||||||
pub mod cleanup;
|
pub mod cleanup;
|
||||||
|
|||||||
@@ -2,45 +2,59 @@
|
|||||||
//! container per agent, provisioned lazily on first use and reused for
|
//! container per agent, provisioned lazily on first use and reused for
|
||||||
//! the manager's lifetime; a dead sandbox is replaced transparently.
|
//! the manager's lifetime; a dead sandbox is replaced transparently.
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use cm_domain::AgentId;
|
use cm_domain::AgentId;
|
||||||
use cm_sandbox::{ExecResult, SandboxDriver, SandboxHandle, SandboxSpec};
|
use cm_sandbox::{ExecResult, SandboxDriver, SandboxHandle, SandboxSpec};
|
||||||
|
use sqlx::PgPool;
|
||||||
use tokio::sync::Mutex;
|
use tokio::sync::Mutex;
|
||||||
|
|
||||||
impl std::fmt::Debug for SandboxManager {
|
impl std::fmt::Debug for SandboxManager {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
f.debug_struct("SandboxManager")
|
f.debug_struct("SandboxManager")
|
||||||
.field("image", &self.image)
|
.field("image", &self.image)
|
||||||
|
.field("node_id", &self.node_id)
|
||||||
.finish_non_exhaustive()
|
.finish_non_exhaustive()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub struct SandboxManager {
|
pub struct SandboxManager {
|
||||||
driver: Arc<dyn SandboxDriver>,
|
driver: Arc<dyn SandboxDriver>,
|
||||||
|
db: PgPool,
|
||||||
|
/// Placement node this manager's driver provisions onto ("local" today).
|
||||||
|
node_id: String,
|
||||||
image: String,
|
image: String,
|
||||||
egress: bool,
|
egress: bool,
|
||||||
handles: Mutex<HashMap<AgentId, SandboxHandle>>,
|
/// Pre-provisioned, unassigned sandboxes (the warm pool, per-replica).
|
||||||
/// Pre-provisioned, unassigned sandboxes (the warm pool).
|
|
||||||
pool: Mutex<Vec<SandboxHandle>>,
|
pool: Mutex<Vec<SandboxHandle>>,
|
||||||
/// Warm-pool target; the background warmer keeps `pool` at this size.
|
/// Warm-pool target; the background warmer keeps `pool` at this size.
|
||||||
warm_target: std::sync::atomic::AtomicUsize,
|
warm_target: std::sync::atomic::AtomicUsize,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SandboxManager {
|
impl SandboxManager {
|
||||||
pub fn new(driver: Arc<dyn SandboxDriver>, image: &str) -> SandboxManager {
|
pub fn new(
|
||||||
|
driver: Arc<dyn SandboxDriver>,
|
||||||
|
db: PgPool,
|
||||||
|
node_id: &str,
|
||||||
|
image: &str,
|
||||||
|
) -> SandboxManager {
|
||||||
SandboxManager {
|
SandboxManager {
|
||||||
driver,
|
driver,
|
||||||
|
db,
|
||||||
|
node_id: node_id.to_owned(),
|
||||||
image: image.to_owned(),
|
image: image.to_owned(),
|
||||||
egress: false,
|
egress: false,
|
||||||
handles: Mutex::new(HashMap::new()),
|
|
||||||
pool: Mutex::new(Vec::new()),
|
pool: Mutex::new(Vec::new()),
|
||||||
warm_target: std::sync::atomic::AtomicUsize::new(0),
|
warm_target: std::sync::atomic::AtomicUsize::new(0),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The `agent_containers.kind` discriminator for this manager.
|
||||||
|
fn kind(&self) -> &'static str {
|
||||||
|
cm_sandbox::sandbox_kind(self.egress)
|
||||||
|
}
|
||||||
|
|
||||||
/// Starts the background warmer: keeps `target` sandboxes
|
/// Starts the background warmer: keeps `target` sandboxes
|
||||||
/// pre-provisioned so an agent's first exec skips container startup.
|
/// pre-provisioned so an agent's first exec skips container startup.
|
||||||
pub fn warm(self: Arc<Self>, target: usize) -> Arc<Self> {
|
pub fn warm(self: Arc<Self>, target: usize) -> Arc<Self> {
|
||||||
@@ -106,14 +120,26 @@ impl SandboxManager {
|
|||||||
/// first use. The sandbox has no egress and no credentials — running
|
/// first use. The sandbox has no egress and no credentials — running
|
||||||
/// agent-authored code here is the point of the architecture.
|
/// agent-authored code here is the point of the architecture.
|
||||||
pub async fn exec(&self, agent_id: AgentId, command: &str) -> Result<ExecResult, String> {
|
pub async fn exec(&self, agent_id: AgentId, command: &str) -> Result<ExecResult, String> {
|
||||||
let mut handles = self.handles.lock().await;
|
let kind = self.kind();
|
||||||
let alive = match handles.get(&agent_id) {
|
// Reuse the registry-recorded sandbox for this agent if it's alive — so a
|
||||||
Some(handle) => self.driver.health(handle).await.unwrap_or(false),
|
// different replica finds the same container instead of making another.
|
||||||
None => false,
|
if let Ok(Some(row)) = cm_db::repo::agent_containers::get(&self.db, agent_id, kind).await {
|
||||||
|
let handle = SandboxHandle {
|
||||||
|
id: row.container_id,
|
||||||
|
name: row.name,
|
||||||
};
|
};
|
||||||
if !alive {
|
if self.driver.health(&handle).await.unwrap_or(false) {
|
||||||
// A warm sandbox if one is ready (and still healthy);
|
return self
|
||||||
// otherwise provision inline.
|
.driver
|
||||||
|
.exec(&handle, &["sh", "-lc", command])
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("sandbox exec failed: {e}"));
|
||||||
|
}
|
||||||
|
// Recorded but dead: clean both the container and the stale row.
|
||||||
|
let _ = self.driver.destroy(&handle).await;
|
||||||
|
let _ = cm_db::repo::agent_containers::delete(&self.db, agent_id, kind).await;
|
||||||
|
}
|
||||||
|
// Take a warm sandbox if one is ready (and still healthy), else provision.
|
||||||
let mut assigned = None;
|
let mut assigned = None;
|
||||||
while let Some(candidate) = self.pool.lock().await.pop() {
|
while let Some(candidate) = self.pool.lock().await.pop() {
|
||||||
if self.driver.health(&candidate).await.unwrap_or(false) {
|
if self.driver.health(&candidate).await.unwrap_or(false) {
|
||||||
@@ -126,11 +152,18 @@ impl SandboxManager {
|
|||||||
Some(handle) => handle,
|
Some(handle) => handle,
|
||||||
None => self.provision_one().await?,
|
None => self.provision_one().await?,
|
||||||
};
|
};
|
||||||
handles.insert(agent_id, handle);
|
cm_db::repo::agent_containers::upsert(
|
||||||
}
|
&self.db,
|
||||||
let handle = handles.get(&agent_id).expect("just ensured");
|
agent_id,
|
||||||
|
kind,
|
||||||
|
&self.node_id,
|
||||||
|
&handle.id,
|
||||||
|
&handle.name,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("registry upsert failed: {e}"))?;
|
||||||
self.driver
|
self.driver
|
||||||
.exec(handle, &["sh", "-lc", command])
|
.exec(&handle, &["sh", "-lc", command])
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("sandbox exec failed: {e}"))
|
.map_err(|e| format!("sandbox exec failed: {e}"))
|
||||||
}
|
}
|
||||||
@@ -138,30 +171,28 @@ impl SandboxManager {
|
|||||||
/// Tear down a single agent's sandbox if it has one (on agent deletion).
|
/// Tear down a single agent's sandbox if it has one (on agent deletion).
|
||||||
/// Returns whether a container existed and was destroyed.
|
/// Returns whether a container existed and was destroyed.
|
||||||
pub async fn release_agent(&self, agent_id: AgentId) -> bool {
|
pub async fn release_agent(&self, agent_id: AgentId) -> bool {
|
||||||
let handle = { self.handles.lock().await.remove(&agent_id) };
|
match cm_db::repo::agent_containers::get(&self.db, agent_id, self.kind()).await {
|
||||||
match handle {
|
Ok(Some(row)) => {
|
||||||
Some(h) => {
|
let handle = SandboxHandle {
|
||||||
if let Err(e) = self.driver.destroy(&h).await {
|
id: row.container_id,
|
||||||
eprintln!("sandbox release: failed to remove {}: {e}", h.id);
|
name: row.name,
|
||||||
|
};
|
||||||
|
if let Err(e) = self.driver.destroy(&handle).await {
|
||||||
|
eprintln!("sandbox release: failed to remove {}: {e}", handle.id);
|
||||||
}
|
}
|
||||||
|
let _ = cm_db::repo::agent_containers::delete(&self.db, agent_id, self.kind()).await;
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
None => false,
|
_ => false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Destroys every sandbox this manager provisioned — assigned and
|
/// Graceful shutdown (SIGTERM): assigned sandboxes persist in the registry
|
||||||
/// pooled — and stops the warmer. Called on graceful shutdown (SIGTERM).
|
/// (reused after a redeploy); only the per-replica warm pool is torn down,
|
||||||
|
/// and the warmer is stopped.
|
||||||
pub async fn shutdown(&self) {
|
pub async fn shutdown(&self) {
|
||||||
self.warm_target
|
self.warm_target
|
||||||
.store(0, std::sync::atomic::Ordering::Relaxed);
|
.store(0, std::sync::atomic::Ordering::Relaxed);
|
||||||
let mut handles = self.handles.lock().await;
|
|
||||||
for (_, handle) in handles.drain() {
|
|
||||||
if let Err(e) = self.driver.destroy(&handle).await {
|
|
||||||
eprintln!("sandbox shutdown: failed to remove {}: {e}", handle.id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
drop(handles);
|
|
||||||
let mut pool = self.pool.lock().await;
|
let mut pool = self.pool.lock().await;
|
||||||
for handle in pool.drain(..) {
|
for handle in pool.drain(..) {
|
||||||
if let Err(e) = self.driver.destroy(&handle).await {
|
if let Err(e) = self.driver.destroy(&handle).await {
|
||||||
@@ -184,15 +215,19 @@ impl SandboxManager {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let live: std::collections::HashSet<String> = {
|
let mut live: std::collections::HashSet<String> =
|
||||||
let handles = self.handles.lock().await;
|
cm_db::repo::agent_containers::all(&self.db, kind)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default()
|
||||||
|
.into_iter()
|
||||||
|
.map(|r| r.container_id)
|
||||||
|
.collect();
|
||||||
|
{
|
||||||
let pool = self.pool.lock().await;
|
let pool = self.pool.lock().await;
|
||||||
handles
|
for h in pool.iter() {
|
||||||
.values()
|
live.insert(h.id.clone());
|
||||||
.map(|h| h.id.clone())
|
}
|
||||||
.chain(pool.iter().map(|h| h.id.clone()))
|
}
|
||||||
.collect()
|
|
||||||
};
|
|
||||||
let now = std::time::SystemTime::now()
|
let now = std::time::SystemTime::now()
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
.map(|d| d.as_secs() as i64)
|
.map(|d| d.as_secs() as i64)
|
||||||
|
|||||||
@@ -3,15 +3,23 @@
|
|||||||
//! WebSocket connect and reused across reconnects. Unlike the agent tool
|
//! WebSocket connect and reused across reconnects. Unlike the agent tool
|
||||||
//! sandboxes, these are interactive PTYs (`exec -it zsh`), so a small idle
|
//! sandboxes, these are interactive PTYs (`exec -it zsh`), so a small idle
|
||||||
//! sweeper reaps containers that no live session has touched for a while.
|
//! sweeper reaps containers that no live session has touched for a while.
|
||||||
|
//!
|
||||||
|
//! The agent→container mapping lives in Postgres (`agent_containers`) rather
|
||||||
|
//! than process memory, so ANY server replica reuses the same container instead
|
||||||
|
//! of spawning a duplicate — and registry-tracked terminals survive a redeploy
|
||||||
|
//! (their tmux sessions resume), since boot reconciliation removes only true
|
||||||
|
//! orphans (engine containers with no registry row).
|
||||||
|
|
||||||
use std::collections::HashMap;
|
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::Duration;
|
||||||
|
|
||||||
use cm_domain::{AgentId, WorkspaceId};
|
use cm_domain::{AgentId, WorkspaceId};
|
||||||
use cm_sandbox::{DriveMount, PtySession, SandboxDriver, SandboxHandle, SandboxKind, SandboxSpec};
|
use cm_sandbox::{DriveMount, PtySession, SandboxDriver, SandboxHandle, SandboxKind, SandboxSpec};
|
||||||
use tokio::sync::Mutex;
|
use sqlx::PgPool;
|
||||||
|
|
||||||
|
/// The `agent_containers.kind` discriminator for terminal containers.
|
||||||
|
const KIND: &str = "terminal";
|
||||||
|
|
||||||
/// Where the agent's Files drives live, so the Terminal can mount them.
|
/// Where the agent's Files drives live, so the Terminal can mount them.
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
@@ -25,21 +33,20 @@ pub struct DriveConfig {
|
|||||||
|
|
||||||
pub struct TerminalManager {
|
pub struct TerminalManager {
|
||||||
driver: Arc<dyn SandboxDriver>,
|
driver: Arc<dyn SandboxDriver>,
|
||||||
|
pool: PgPool,
|
||||||
|
/// Placement node this manager's driver provisions onto ("local" today).
|
||||||
|
node_id: String,
|
||||||
image: String,
|
image: String,
|
||||||
egress: bool,
|
egress: bool,
|
||||||
/// Drive mounts; None disables the ~/drives mapping.
|
/// Drive mounts; None disables the ~/drives mapping.
|
||||||
drives: Option<DriveConfig>,
|
drives: Option<DriveConfig>,
|
||||||
handles: Mutex<HashMap<AgentId, SandboxHandle>>,
|
|
||||||
/// Live WebSocket sessions per agent + when each agent was last touched —
|
|
||||||
/// an agent with zero sessions, idle past the TTL, gets its container reaped.
|
|
||||||
active: Mutex<HashMap<AgentId, usize>>,
|
|
||||||
last_seen: Mutex<HashMap<AgentId, Instant>>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl std::fmt::Debug for TerminalManager {
|
impl std::fmt::Debug for TerminalManager {
|
||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
f.debug_struct("TerminalManager")
|
f.debug_struct("TerminalManager")
|
||||||
.field("image", &self.image)
|
.field("image", &self.image)
|
||||||
|
.field("node_id", &self.node_id)
|
||||||
.finish_non_exhaustive()
|
.finish_non_exhaustive()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -47,18 +54,19 @@ impl std::fmt::Debug for TerminalManager {
|
|||||||
impl TerminalManager {
|
impl TerminalManager {
|
||||||
pub fn new(
|
pub fn new(
|
||||||
driver: Arc<dyn SandboxDriver>,
|
driver: Arc<dyn SandboxDriver>,
|
||||||
|
pool: PgPool,
|
||||||
|
node_id: &str,
|
||||||
image: &str,
|
image: &str,
|
||||||
egress: bool,
|
egress: bool,
|
||||||
drives: Option<DriveConfig>,
|
drives: Option<DriveConfig>,
|
||||||
) -> TerminalManager {
|
) -> TerminalManager {
|
||||||
TerminalManager {
|
TerminalManager {
|
||||||
driver,
|
driver,
|
||||||
|
pool,
|
||||||
|
node_id: node_id.to_owned(),
|
||||||
image: image.to_owned(),
|
image: image.to_owned(),
|
||||||
egress,
|
egress,
|
||||||
drives,
|
drives,
|
||||||
handles: Mutex::new(HashMap::new()),
|
|
||||||
active: Mutex::new(HashMap::new()),
|
|
||||||
last_seen: Mutex::new(HashMap::new()),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -118,25 +126,39 @@ impl TerminalManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// The agent's terminal container, provisioned on first use; a dead one is
|
/// The agent's terminal container, provisioned on first use; a dead one is
|
||||||
/// replaced transparently.
|
/// replaced transparently. The mapping is read from / written to the shared
|
||||||
|
/// registry, so a different replica finds the same container.
|
||||||
async fn ensure(
|
async fn ensure(
|
||||||
&self,
|
&self,
|
||||||
workspace_id: WorkspaceId,
|
workspace_id: WorkspaceId,
|
||||||
agent_id: AgentId,
|
agent_id: AgentId,
|
||||||
) -> Result<SandboxHandle, String> {
|
) -> Result<SandboxHandle, String> {
|
||||||
let mut handles = self.handles.lock().await;
|
if let Ok(Some(row)) =
|
||||||
let alive = match handles.get(&agent_id) {
|
cm_db::repo::agent_containers::get(&self.pool, agent_id, KIND).await
|
||||||
Some(handle) => self.driver.health(handle).await.unwrap_or(false),
|
{
|
||||||
None => false,
|
let handle = SandboxHandle {
|
||||||
|
id: row.container_id.clone(),
|
||||||
|
name: row.name.clone(),
|
||||||
};
|
};
|
||||||
if !alive {
|
if self.driver.health(&handle).await.unwrap_or(false) {
|
||||||
if let Some(stale) = handles.remove(&agent_id) {
|
return Ok(handle);
|
||||||
let _ = self.driver.destroy(&stale).await;
|
}
|
||||||
|
// Recorded but dead: clean both the container and the stale row.
|
||||||
|
let _ = self.driver.destroy(&handle).await;
|
||||||
|
let _ = cm_db::repo::agent_containers::delete(&self.pool, agent_id, KIND).await;
|
||||||
}
|
}
|
||||||
let handle = self.provision_one(workspace_id, agent_id).await?;
|
let handle = self.provision_one(workspace_id, agent_id).await?;
|
||||||
handles.insert(agent_id, handle);
|
cm_db::repo::agent_containers::upsert(
|
||||||
}
|
&self.pool,
|
||||||
Ok(handles.get(&agent_id).expect("just ensured").clone())
|
agent_id,
|
||||||
|
KIND,
|
||||||
|
&self.node_id,
|
||||||
|
&handle.id,
|
||||||
|
&handle.name,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("registry upsert failed: {e}"))?;
|
||||||
|
Ok(handle)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Open an interactive login zsh in the agent's terminal container. `env`
|
/// Open an interactive login zsh in the agent's terminal container. `env`
|
||||||
@@ -168,8 +190,7 @@ impl TerminalManager {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("terminal attach failed: {e}"))?;
|
.map_err(|e| format!("terminal attach failed: {e}"))?;
|
||||||
*self.active.lock().await.entry(agent_id).or_insert(0) += 1;
|
let _ = cm_db::repo::agent_containers::add_session(&self.pool, agent_id, KIND, 1).await;
|
||||||
self.last_seen.lock().await.insert(agent_id, Instant::now());
|
|
||||||
Ok(session)
|
Ok(session)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -183,83 +204,92 @@ impl TerminalManager {
|
|||||||
/// A session ended; the container stays warm for reconnects but becomes a
|
/// A session ended; the container stays warm for reconnects but becomes a
|
||||||
/// candidate for the idle sweeper once no session remains.
|
/// candidate for the idle sweeper once no session remains.
|
||||||
pub async fn detach(&self, agent_id: AgentId) {
|
pub async fn detach(&self, agent_id: AgentId) {
|
||||||
if let Some(n) = self.active.lock().await.get_mut(&agent_id) {
|
let _ = cm_db::repo::agent_containers::add_session(&self.pool, agent_id, KIND, -1).await;
|
||||||
*n = n.saturating_sub(1);
|
|
||||||
}
|
|
||||||
self.last_seen.lock().await.insert(agent_id, Instant::now());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Tear down a single agent's terminal container (on agent deletion / idle).
|
/// Tear down a single agent's terminal container (on agent deletion / idle).
|
||||||
/// Returns whether one existed.
|
/// Returns whether one existed.
|
||||||
pub async fn release_agent(&self, agent_id: AgentId) -> bool {
|
pub async fn release_agent(&self, agent_id: AgentId) -> bool {
|
||||||
self.active.lock().await.remove(&agent_id);
|
match cm_db::repo::agent_containers::get(&self.pool, agent_id, KIND).await {
|
||||||
self.last_seen.lock().await.remove(&agent_id);
|
Ok(Some(row)) => {
|
||||||
let handle = { self.handles.lock().await.remove(&agent_id) };
|
let handle = SandboxHandle {
|
||||||
match handle {
|
id: row.container_id,
|
||||||
Some(h) => {
|
name: row.name,
|
||||||
if let Err(e) = self.driver.destroy(&h).await {
|
};
|
||||||
eprintln!("terminal release: failed to remove {}: {e}", h.id);
|
if let Err(e) = self.driver.destroy(&handle).await {
|
||||||
|
eprintln!("terminal release: failed to remove {}: {e}", handle.id);
|
||||||
}
|
}
|
||||||
|
let _ = cm_db::repo::agent_containers::delete(&self.pool, agent_id, KIND).await;
|
||||||
true
|
true
|
||||||
}
|
}
|
||||||
None => false,
|
_ => false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Destroy every terminal container on graceful shutdown (SIGTERM).
|
/// Graceful shutdown: terminals are NOT destroyed — they persist (tmux
|
||||||
|
/// sessions survive the redeploy) and become idle-reapable. We only zero the
|
||||||
|
/// session counts since the WS connections are dropping.
|
||||||
pub async fn shutdown(&self) {
|
pub async fn shutdown(&self) {
|
||||||
let mut handles = self.handles.lock().await;
|
let _ = cm_db::repo::agent_containers::reset_sessions(&self.pool, KIND).await;
|
||||||
for (_, handle) in handles.drain() {
|
|
||||||
if let Err(e) = self.driver.destroy(&handle).await {
|
|
||||||
eprintln!("terminal shutdown: failed to remove {}: {e}", handle.id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Reap idle terminals (no live session, untouched past `idle_ttl`) and
|
/// Reap idle terminals (no live session, untouched past `idle_ttl`).
|
||||||
/// orphan containers the engine still holds that no live handle owns.
|
async fn reap_idle(&self, idle_ttl: Duration) -> usize {
|
||||||
async fn sweep(&self, idle_ttl: Duration) -> usize {
|
let rows = match cm_db::repo::agent_containers::idle(
|
||||||
// 1. Idle, session-less terminals we still track.
|
&self.pool,
|
||||||
let now = Instant::now();
|
KIND,
|
||||||
let idle: Vec<AgentId> = {
|
idle_ttl.as_secs() as i64,
|
||||||
let active = self.active.lock().await;
|
)
|
||||||
let last = self.last_seen.lock().await;
|
.await
|
||||||
last.iter()
|
{
|
||||||
.filter(|(id, seen)| {
|
Ok(r) => r,
|
||||||
active.get(*id).copied().unwrap_or(0) == 0
|
Err(e) => {
|
||||||
&& now.duration_since(**seen) > idle_ttl
|
eprintln!("terminal reaper: idle query failed: {e}");
|
||||||
})
|
return 0;
|
||||||
.map(|(id, _)| *id)
|
}
|
||||||
.collect()
|
|
||||||
};
|
};
|
||||||
let mut reaped = 0;
|
let mut reaped = 0;
|
||||||
for id in idle {
|
for r in rows {
|
||||||
if self.release_agent(id).await {
|
let handle = SandboxHandle {
|
||||||
|
id: r.container_id.clone(),
|
||||||
|
name: r.name.clone(),
|
||||||
|
};
|
||||||
|
let _ = self.driver.destroy(&handle).await;
|
||||||
|
let _ = cm_db::repo::agent_containers::delete(&self.pool, r.agent_id, KIND).await;
|
||||||
reaped += 1;
|
reaped += 1;
|
||||||
}
|
}
|
||||||
|
reaped
|
||||||
}
|
}
|
||||||
// 2. Orphans from a crashed process (label-filtered, no live handle).
|
|
||||||
|
/// Reap orphan containers the engine holds that no registry row owns (and,
|
||||||
|
/// when `min_age` > 0, that are older than it). Registry-tracked terminals
|
||||||
|
/// are left alone.
|
||||||
|
async fn reap_orphans(&self, min_age: Duration) -> usize {
|
||||||
let managed = match self.driver.list_managed(SandboxKind::Terminal.label()).await {
|
let managed = match self.driver.list_managed(SandboxKind::Terminal.label()).await {
|
||||||
Ok(m) => m,
|
Ok(m) => m,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("terminal reaper: list failed: {e}");
|
eprintln!("terminal reaper: list failed: {e}");
|
||||||
return reaped;
|
return 0;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let live: std::collections::HashSet<String> = {
|
let tracked: std::collections::HashSet<String> =
|
||||||
let handles = self.handles.lock().await;
|
cm_db::repo::agent_containers::all(&self.pool, KIND)
|
||||||
handles.values().map(|h| h.id.clone()).collect()
|
.await
|
||||||
};
|
.unwrap_or_default()
|
||||||
|
.into_iter()
|
||||||
|
.map(|r| r.container_id)
|
||||||
|
.collect();
|
||||||
let now_unix = std::time::SystemTime::now()
|
let now_unix = std::time::SystemTime::now()
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
.map(|d| d.as_secs() as i64)
|
.map(|d| d.as_secs() as i64)
|
||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
let min_age = idle_ttl.as_secs() as i64;
|
let min = min_age.as_secs() as i64;
|
||||||
|
let mut reaped = 0;
|
||||||
for m in managed {
|
for m in managed {
|
||||||
if live.contains(&m.id) {
|
if tracked.contains(&m.id) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if min_age > 0 && now_unix - m.created_unix < min_age {
|
if min > 0 && now_unix - m.created_unix < min {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let handle = SandboxHandle {
|
let handle = SandboxHandle {
|
||||||
@@ -274,10 +304,10 @@ impl TerminalManager {
|
|||||||
reaped
|
reaped
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Boot reconciliation: remove every terminal container the engine still
|
/// Boot reconciliation: remove only true orphans (engine containers with no
|
||||||
/// holds (all are orphans from a dead process — we track none yet).
|
/// registry row). Registry-tracked terminals are preserved across a redeploy.
|
||||||
pub async fn reconcile_orphans(&self) -> usize {
|
pub async fn reconcile_orphans(&self) -> usize {
|
||||||
self.sweep(Duration::ZERO).await
|
self.reap_orphans(Duration::ZERO).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Background reaper: every `interval`, reap idle (`idle_ttl`) + orphan terminals.
|
/// Background reaper: every `interval`, reap idle (`idle_ttl`) + orphan terminals.
|
||||||
@@ -286,7 +316,7 @@ impl TerminalManager {
|
|||||||
let mut tick = tokio::time::interval(interval);
|
let mut tick = tokio::time::interval(interval);
|
||||||
loop {
|
loop {
|
||||||
tick.tick().await;
|
tick.tick().await;
|
||||||
let n = self.sweep(idle_ttl).await;
|
let n = self.reap_idle(idle_ttl).await + self.reap_orphans(idle_ttl).await;
|
||||||
if n > 0 {
|
if n > 0 {
|
||||||
eprintln!("terminal reaper: removed {n} terminal container(s)");
|
eprintln!("terminal reaper: removed {n} terminal container(s)");
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -119,7 +119,8 @@ async fn browsing_returns_web_tainted_content_and_taints_later_gated_actions() {
|
|||||||
|
|
||||||
let driver: Arc<dyn cm_sandbox::SandboxDriver> =
|
let driver: Arc<dyn cm_sandbox::SandboxDriver> =
|
||||||
Arc::new(DockerDriver::connect().expect("docker reachable"));
|
Arc::new(DockerDriver::connect().expect("docker reachable"));
|
||||||
let browser = Arc::new(SandboxManager::new(driver, BROWSER_IMAGE).with_egress());
|
let browser =
|
||||||
|
Arc::new(SandboxManager::new(driver, pool.clone(), "local", BROWSER_IMAGE).with_egress());
|
||||||
let blob = Arc::new(cm_files::LocalBlobStore::new(
|
let blob = Arc::new(cm_files::LocalBlobStore::new(
|
||||||
std::env::temp_dir().join(format!("tc-brw-{}", uuid::Uuid::now_v7())),
|
std::env::temp_dir().join(format!("tc-brw-{}", uuid::Uuid::now_v7())),
|
||||||
));
|
));
|
||||||
|
|||||||
@@ -95,7 +95,7 @@ async fn shell_exec_runs_in_the_agent_sandbox_with_persistent_home() {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let driver = DockerDriver::connect().expect("docker reachable");
|
let driver = DockerDriver::connect().expect("docker reachable");
|
||||||
let sandboxes = Arc::new(SandboxManager::new(Arc::new(driver), IMAGE));
|
let sandboxes = Arc::new(SandboxManager::new(Arc::new(driver), pool.clone(), "local", IMAGE));
|
||||||
let rt = Runtime::new(
|
let rt = Runtime::new(
|
||||||
pool.clone(),
|
pool.clone(),
|
||||||
Arc::new(ScriptedProvider::from_toml(SCENARIOS).unwrap()),
|
Arc::new(ScriptedProvider::from_toml(SCENARIOS).unwrap()),
|
||||||
|
|||||||
@@ -54,7 +54,8 @@ async fn the_pool_prefills_assigns_and_refills() {
|
|||||||
ensure_image();
|
ensure_image();
|
||||||
let driver: Arc<dyn cm_sandbox::SandboxDriver> =
|
let driver: Arc<dyn cm_sandbox::SandboxDriver> =
|
||||||
Arc::new(DockerDriver::connect().expect("docker reachable"));
|
Arc::new(DockerDriver::connect().expect("docker reachable"));
|
||||||
let manager = Arc::new(SandboxManager::new(driver, IMAGE)).warm(2);
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let manager = Arc::new(SandboxManager::new(driver, pool, "local", IMAGE)).warm(2);
|
||||||
|
|
||||||
// The warmer fills the pool without any exec happening.
|
// The warmer fills the pool without any exec happening.
|
||||||
pool_reaches(&manager, 2).await;
|
pool_reaches(&manager, 2).await;
|
||||||
|
|||||||
@@ -0,0 +1,12 @@
|
|||||||
|
-- Short-lived single-use terminal-WS tickets, stored hashed in Postgres so any
|
||||||
|
-- server replica (not just the one that minted it) can redeem the handshake.
|
||||||
|
-- Replaces the former in-process ticket map. Rows are deleted on redeem and
|
||||||
|
-- swept on expiry.
|
||||||
|
CREATE TABLE terminal_ws_tickets (
|
||||||
|
token_hash TEXT PRIMARY KEY,
|
||||||
|
agent_id UUID NOT NULL REFERENCES agents (id) ON DELETE CASCADE,
|
||||||
|
workspace_id UUID NOT NULL REFERENCES workspaces (id) ON DELETE CASCADE,
|
||||||
|
label TEXT NOT NULL,
|
||||||
|
expires_at TIMESTAMPTZ NOT NULL
|
||||||
|
);
|
||||||
|
CREATE INDEX terminal_ws_tickets_expires_idx ON terminal_ws_tickets (expires_at);
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
-- Durable registry of the live container backing each agent (terminal / agent /
|
||||||
|
-- browser), so ANY server replica can find + reuse it instead of the former
|
||||||
|
-- in-process map (which made a 2nd replica spawn duplicate containers). node_id
|
||||||
|
-- records placement for the multi-node node-pool (Phase 2); it is 'local' today.
|
||||||
|
CREATE TABLE agent_containers (
|
||||||
|
agent_id UUID NOT NULL REFERENCES agents (id) ON DELETE CASCADE,
|
||||||
|
kind TEXT NOT NULL, -- 'terminal' | 'agent' | 'browser'
|
||||||
|
node_id TEXT NOT NULL DEFAULT 'local',
|
||||||
|
container_id TEXT NOT NULL,
|
||||||
|
name TEXT NOT NULL,
|
||||||
|
workspace_id UUID NOT NULL REFERENCES workspaces (id) ON DELETE CASCADE,
|
||||||
|
session_count INTEGER NOT NULL DEFAULT 0, -- live attached sessions (terminals)
|
||||||
|
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
last_seen TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
PRIMARY KEY (agent_id, kind)
|
||||||
|
);
|
||||||
|
CREATE INDEX agent_containers_idle_idx ON agent_containers (kind, last_seen);
|
||||||
Reference in New Issue
Block a user