//! Recursive sub-topology executor — the engine behind the upper deploy rungs. //! //! A team run drives **claws** directly (the leaf [`ZeroClawDriveExecutor`]). A //! *company* run is a topology whose nodes are **teams**; an *org* run is a //! topology whose nodes are **companies**. This executor makes a parent "turn" //! mean *run the child's whole sub-topology to completion and return its final //! output* — so the entire `org → company → team → claw` tree collapses into //! nested [`execute_resumable`] calls, reusing every planner unchanged. Only the //! leaf ever touches the runtime. //! //! **Durability.** A parent run can spend minutes inside one node executing a //! child sub-topology. Two guards keep that safe on the durable worker: //! - every leaf turn **touches the parent run's `updated_at`** (via the //! `keepalive` callback) so the 180s stale sweep never requeues the parent //! mid-subtree; //! - the same callback observes **parent cancellation** and halts the whole //! subtree at the next leaf boundary. //! //! Resume is *coarse* in v1: the outer worker checkpoints parent-node-level //! progress, so a crash re-runs only the in-flight child subtree (completed //! sibling nodes are skipped). Per-leaf nested checkpointing is a future slice. use std::future::Future; use std::pin::Pin; use std::sync::Arc; use cm_db::repo; use cm_domain::WorkspaceId; use cm_orchestrator::{ execute_resumable, GatedAction, OrchestratorError, RunProgress, RunRecord, TurnExecutor, TurnOutcome, TurnRequest, }; use cm_topology::TopologyGraph; use sqlx::PgPool; use uuid::Uuid; use crate::topology_exec::ZeroClawDriveExecutor; /// Backstop against a malformed binding cycle infinitely recursing /// (org → company → team is depth 2; this caps well above any real nesting). const MAX_DEPTH: u32 = 4; /// Which deploy tier this executor drives. The leaf (`team`/claw) tier is the /// plain [`ZeroClawDriveExecutor`], not represented here. #[derive(Clone, Copy, PartialEq, Eq)] pub enum Tier { /// Nodes bind companies (`attrs["company_id"]`); each runs a company tier. Org, /// Nodes bind teams (`attrs["team_id"]`); each runs the leaf claw tier. Company, } /// A [`TurnExecutor`] whose "turn" runs a sub-topology one tier down. pub struct SubTopologyExecutor { pool: PgPool, workspace_id: WorkspaceId, tier: Tier, /// The parent durable run id — every leaf turn touches its `updated_at`. parent_run_id: Uuid, /// Shared leaf executor that drives real claws over the gateway. leaf: Arc, depth: u32, } impl SubTopologyExecutor { /// Build the top of a recursive run (depth 0) for a durable parent run. pub fn new( pool: PgPool, workspace_id: WorkspaceId, tier: Tier, parent_run_id: Uuid, leaf: Arc, ) -> Self { SubTopologyExecutor { pool, workspace_id, tier, parent_run_id, leaf, depth: 0, } } /// A child executor one tier down, sharing the same run id + leaf. fn child(&self, tier: Tier) -> Self { SubTopologyExecutor { pool: self.pool.clone(), workspace_id: self.workspace_id, tier, parent_run_id: self.parent_run_id, leaf: Arc::clone(&self.leaf), depth: self.depth + 1, } } /// Run a child graph on a nested `SubTopologyExecutor`. Boxed so the /// org→company recursion has a finite future size (the trait method would /// otherwise contain itself). fn run_nested<'a>( &'a self, graph: TopologyGraph, task: String, child: SubTopologyExecutor, ) -> Pin> + Send + 'a>> { Box::pin(async move { let pool = self.pool.clone(); let run_id = self.parent_run_id; execute_resumable( &graph, &task, &child, RunProgress::default(), move |_snap| { let pool = pool.clone(); async move { keepalive(&pool, run_id).await } }, ) .await }) } } impl TurnExecutor for SubTopologyExecutor { async fn run_turn(&self, req: TurnRequest) -> Result { if self.depth >= MAX_DEPTH { return Err(OrchestratorError::Executor(format!( "topology nesting exceeds max depth {MAX_DEPTH}" ))); } let record = match self.tier { Tier::Company => { // Child = a team; run its graph directly on the claws (leaf). let team_id = child_id(&req, "team_id")?; let team = repo::teams::get_team(&self.pool, team_id, self.workspace_id) .await .map_err(|e| { OrchestratorError::Executor(format!("load team {team_id}: {e}")) })?; let graph = parse_graph(&team.graph)?; let pool = self.pool.clone(); let run_id = self.parent_run_id; execute_resumable( &graph, &req.task, &*self.leaf, RunProgress::default(), move |_snap| { let pool = pool.clone(); async move { keepalive(&pool, run_id).await } }, ) .await? } Tier::Org => { // Child = a company; recurse with a company-tier executor. let company_id = child_id(&req, "company_id")?; let company = repo::companies::get(&self.pool, company_id, self.workspace_id) .await .map_err(|e| { OrchestratorError::Executor(format!("load company {company_id}: {e}")) })?; let graph = parse_graph(&company.graph)?; let child = self.child(Tier::Company); self.run_nested(graph, req.task.clone(), child).await? } }; // Bubble the child journal's gated actions + tokens up to the parent // node, so §15 audit and the run record see the whole subtree. let gated: Vec = record.steps.iter().flat_map(|s| s.gated.clone()).collect(); Ok(TurnOutcome { output: record.final_output, tokens: record.totals.tokens, gated, spend: Default::default(), }) } } /// Keep the parent run alive and honor its cancellation from a leaf step. async fn keepalive(pool: &PgPool, run_id: Uuid) -> Result<(), OrchestratorError> { // Touch updated_at so the stale-run sweep treats this long run as alive. let _ = repo::topology_runs::touch(pool, run_id).await; // Stop the whole subtree if the parent run was cancelled. if matches!( repo::topology_runs::current_status(pool, run_id).await, Ok(Some(ref s)) if s == "cancelled" ) { return Err(OrchestratorError::Executor("run cancelled".into())); } Ok(()) } /// Resolve a node's child binding (`team_id` / `company_id`) from its attrs, /// falling back to the `agent` slot (set to the same id at build time). fn child_id(req: &TurnRequest, key: &str) -> Result { let raw = req .attrs .get(key) .cloned() .or_else(|| req.agent.clone()) .ok_or_else(|| { OrchestratorError::Executor(format!("node {} missing {key} binding", req.node_id)) })?; Uuid::parse_str(raw.trim()).map_err(|_| { OrchestratorError::Executor(format!("node {} has invalid {key}: {raw}", req.node_id)) }) } fn parse_graph(v: &serde_json::Value) -> Result { serde_json::from_value(v.clone()) .map_err(|e| OrchestratorError::Executor(format!("invalid child graph: {e}"))) }