clawmates/crates/cm-api/src/recursive_exec.rs

//! Recursive sub-topology executor — the engine behind the upper deploy rungs.
//!
//! A team run drives **claws** directly (the leaf [`ZeroClawDriveExecutor`]). A
//! *company* run is a topology whose nodes are **teams**; an *org* run is a
//! topology whose nodes are **companies**. This executor makes a parent "turn"
//! mean *run the child's whole sub-topology to completion and return its final
//! output* — so the entire `org → company → team → claw` tree collapses into
//! nested [`execute_resumable`] calls, reusing every planner unchanged. Only the
//! leaf ever touches the runtime.
//!
//! **Durability.** A parent run can spend minutes inside one node executing a
//! child sub-topology. Two guards keep that safe on the durable worker:
//! - every leaf turn **touches the parent run's `updated_at`** (via the
//!   `keepalive` callback) so the 180s stale sweep never requeues the parent
//!   mid-subtree;
//! - the same callback observes **parent cancellation** and halts the whole
//!   subtree at the next leaf boundary.
//!
//! Resume is *coarse* in v1: the outer worker checkpoints parent-node-level
//! progress, so a crash re-runs only the in-flight child subtree (completed
//! sibling nodes are skipped). Per-leaf nested checkpointing is a future slice.

use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;

use cm_db::repo;
use cm_domain::WorkspaceId;
use cm_orchestrator::{
    execute_resumable, GatedAction, OrchestratorError, RunProgress, RunRecord, TurnExecutor,
    TurnOutcome, TurnRequest,
};
use cm_topology::TopologyGraph;
use sqlx::PgPool;
use uuid::Uuid;

use crate::topology_exec::ZeroClawDriveExecutor;

/// Backstop against a malformed binding cycle infinitely recursing
/// (org → company → team is depth 2; this caps well above any real nesting).
const MAX_DEPTH: u32 = 4;

/// Which deploy tier this executor drives. The leaf (`team`/claw) tier is the
/// plain [`ZeroClawDriveExecutor`], not represented here.
#[derive(Clone, Copy, PartialEq, Eq)]
pub enum Tier {
    /// Nodes bind companies (`attrs["company_id"]`); each runs a company tier.
    Org,
    /// Nodes bind teams (`attrs["team_id"]`); each runs the leaf claw tier.
    Company,
}

/// A [`TurnExecutor`] whose "turn" runs a sub-topology one tier down.
pub struct SubTopologyExecutor {
    pool: PgPool,
    workspace_id: WorkspaceId,
    tier: Tier,
    /// The parent durable run id — every leaf turn touches its `updated_at`.
    parent_run_id: Uuid,
    /// Shared leaf executor that drives real claws over the gateway.
    leaf: Arc<ZeroClawDriveExecutor>,
    depth: u32,
}

impl SubTopologyExecutor {
    /// Build the top of a recursive run (depth 0) for a durable parent run.
    pub fn new(
        pool: PgPool,
        workspace_id: WorkspaceId,
        tier: Tier,
        parent_run_id: Uuid,
        leaf: Arc<ZeroClawDriveExecutor>,
    ) -> Self {
        SubTopologyExecutor {
            pool,
            workspace_id,
            tier,
            parent_run_id,
            leaf,
            depth: 0,
        }
    }

    /// A child executor one tier down, sharing the same run id + leaf.
    fn child(&self, tier: Tier) -> Self {
        SubTopologyExecutor {
            pool: self.pool.clone(),
            workspace_id: self.workspace_id,
            tier,
            parent_run_id: self.parent_run_id,
            leaf: Arc::clone(&self.leaf),
            depth: self.depth + 1,
        }
    }

    /// Run a child graph on a nested `SubTopologyExecutor`. Boxed so the
    /// org→company recursion has a finite future size (the trait method would
    /// otherwise contain itself).
    fn run_nested<'a>(
        &'a self,
        graph: TopologyGraph,
        task: String,
        child: SubTopologyExecutor,
    ) -> Pin<Box<dyn Future<Output = Result<RunRecord, OrchestratorError>> + Send + 'a>> {
        Box::pin(async move {
            let pool = self.pool.clone();
            let run_id = self.parent_run_id;
            execute_resumable(
                &graph,
                &task,
                &child,
                RunProgress::default(),
                move |_snap| {
                    let pool = pool.clone();
                    async move { keepalive(&pool, run_id).await }
                },
            )
            .await
        })
    }
}

impl TurnExecutor for SubTopologyExecutor {
    async fn run_turn(&self, req: TurnRequest) -> Result<TurnOutcome, OrchestratorError> {
        if self.depth >= MAX_DEPTH {
            return Err(OrchestratorError::Executor(format!(
                "topology nesting exceeds max depth {MAX_DEPTH}"
            )));
        }

        let record = match self.tier {
            Tier::Company => {
                // Child = a team; run its graph directly on the claws (leaf).
                let team_id = child_id(&req, "team_id")?;
                let team = repo::teams::get_team(&self.pool, team_id, self.workspace_id)
                    .await
                    .map_err(|e| {
                        OrchestratorError::Executor(format!("load team {team_id}: {e}"))
                    })?;
                let graph = parse_graph(&team.graph)?;
                let pool = self.pool.clone();
                let run_id = self.parent_run_id;
                execute_resumable(
                    &graph,
                    &req.task,
                    &*self.leaf,
                    RunProgress::default(),
                    move |_snap| {
                        let pool = pool.clone();
                        async move { keepalive(&pool, run_id).await }
                    },
                )
                .await?
            }
            Tier::Org => {
                // Child = a company; recurse with a company-tier executor.
                let company_id = child_id(&req, "company_id")?;
                let company = repo::companies::get(&self.pool, company_id, self.workspace_id)
                    .await
                    .map_err(|e| {
                        OrchestratorError::Executor(format!("load company {company_id}: {e}"))
                    })?;
                let graph = parse_graph(&company.graph)?;
                let child = self.child(Tier::Company);
                self.run_nested(graph, req.task.clone(), child).await?
            }
        };

        // Bubble the child journal's gated actions + tokens up to the parent
        // node, so §15 audit and the run record see the whole subtree.
        let gated: Vec<GatedAction> = record.steps.iter().flat_map(|s| s.gated.clone()).collect();
        Ok(TurnOutcome {
            output: record.final_output,
            tokens: record.totals.tokens,
            gated,
            spend: Default::default(),
        })
    }
}

/// Keep the parent run alive and honor its cancellation from a leaf step.
async fn keepalive(pool: &PgPool, run_id: Uuid) -> Result<(), OrchestratorError> {
    // Touch updated_at so the stale-run sweep treats this long run as alive.
    let _ = repo::topology_runs::touch(pool, run_id).await;
    // Stop the whole subtree if the parent run was cancelled.
    if matches!(
        repo::topology_runs::current_status(pool, run_id).await,
        Ok(Some(ref s)) if s == "cancelled"
    ) {
        return Err(OrchestratorError::Executor("run cancelled".into()));
    }
    Ok(())
}

/// Resolve a node's child binding (`team_id` / `company_id`) from its attrs,
/// falling back to the `agent` slot (set to the same id at build time).
fn child_id(req: &TurnRequest, key: &str) -> Result<Uuid, OrchestratorError> {
    let raw = req
        .attrs
        .get(key)
        .cloned()
        .or_else(|| req.agent.clone())
        .ok_or_else(|| {
            OrchestratorError::Executor(format!("node {} missing {key} binding", req.node_id))
        })?;
    Uuid::parse_str(raw.trim()).map_err(|_| {
        OrchestratorError::Executor(format!("node {} has invalid {key}: {raw}", req.node_id))
    })
}

fn parse_graph(v: &serde_json::Value) -> Result<TopologyGraph, OrchestratorError> {
    serde_json::from_value(v.clone())
        .map_err(|e| OrchestratorError::Executor(format!("invalid child graph: {e}")))
}