clawmates/crates/cm-api/src/team_template_loader.rs

//! Load builtin team template TOML recipes from disk into
//! `team_templates` + `template_roles` at server boot.
//!
//! Templates ship under `templates/teams/*.toml` (repo-root path,
//! bundled into the server container image). The loader is idempotent:
//! re-upserts every boot so template edits go live on the next
//! deploy without needing a manual migration.

use serde::Deserialize;
use sha2::{Digest, Sha256};
use sqlx::PgPool;
use std::path::PathBuf;

use cm_db::repo::team_templates::{upsert_builtin, UpsertBuiltin, UpsertBuiltinRole};

/// Deterministic id per builtin template key. Rolling to sha256 of a
/// stable namespace + the key gives us a v4-shaped id that never
/// changes across boots (the uuid crate's v5 feature isn't enabled in
/// the workspace and we didn't want to bump it just for this).
fn builtin_id(key: &str) -> uuid::Uuid {
    let mut h = Sha256::new();
    h.update(b"clawmates.builtin.team_template\x00");
    h.update(key.as_bytes());
    let digest = h.finalize();
    let mut bytes = [0u8; 16];
    bytes.copy_from_slice(&digest[..16]);
    // Force the v4 layout so the id passes any downstream v4-shaped
    // checks (version nibble = 4, variant bits = 10xx).
    bytes[6] = (bytes[6] & 0x0f) | 0x40;
    bytes[8] = (bytes[8] & 0x3f) | 0x80;
    uuid::Uuid::from_bytes(bytes)
}

#[derive(Debug, Deserialize)]
struct TemplateFile {
    key: String,
    name: String,
    #[serde(default)]
    stack: Vec<String>,
    #[serde(default = "default_category")]
    category: String,
    default_topology: String,
    risk_profile: String,
    #[serde(default)]
    mcp_bundles: Vec<String>,
    #[serde(default = "default_version")]
    version: i32,
    #[serde(default)]
    description: Option<String>,
    #[serde(default)]
    config: serde_json::Value,
    #[serde(default)]
    roles: Vec<TemplateRoleFile>,
}
fn default_version() -> i32 {
    1
}

fn default_category() -> String {
    "development".to_string()
}

#[derive(Debug, Deserialize)]
struct TemplateRoleFile {
    slot: String,
    order_idx: i32,
    system_prompt: String,
    #[serde(default)]
    skills: Vec<String>,
    #[serde(default)]
    brain_seed: Option<String>,
    /// Which model this role's claw runs on. Omitted means the mint's default,
    /// which is what every authored template does today — so adding the field
    /// changes nothing until a template uses it.
    #[serde(default)]
    model: Option<String>,
}

fn templates_dir() -> PathBuf {
    if let Ok(d) = std::env::var("CLAWMATES_TEAM_TEMPLATES_DIR") {
        return PathBuf::from(d);
    }
    // Container default — Dockerfile copies templates/ to /etc/clawmates/templates.
    let container = PathBuf::from("/etc/clawmates/templates/teams");
    if container.exists() {
        return container;
    }
    // Dev fallback — repo-relative.
    PathBuf::from("templates/teams")
}

/// Read every `*.toml` under the templates dir and upsert. Skips files
/// that fail to parse but logs the reason so a broken template can't
/// block server boot entirely.
pub async fn load_builtins(pool: &PgPool) -> usize {
    let dir = templates_dir();
    let entries = match std::fs::read_dir(&dir) {
        Ok(r) => r,
        Err(e) => {
            eprintln!(
                "team_template_loader: templates dir {} not readable: {e} — skipping builtin seed",
                dir.display()
            );
            return 0;
        }
    };
    let mut loaded = 0usize;
    for entry in entries.flatten() {
        let path = entry.path();
        if path.extension().and_then(|s| s.to_str()) != Some("toml") {
            continue;
        }
        match load_one(pool, &path).await {
            Ok(key) => {
                loaded += 1;
                eprintln!("team_template_loader: upserted builtin {key}");
            }
            Err(e) => {
                eprintln!(
                    "team_template_loader: failed to load {}: {e}",
                    path.display()
                );
            }
        }
    }
    loaded
}

async fn load_one(pool: &PgPool, path: &std::path::Path) -> Result<String, String> {
    let text =
        std::fs::read_to_string(path).map_err(|e| format!("read {}: {e}", path.display()))?;
    let file: TemplateFile =
        toml::from_str(&text).map_err(|e| format!("parse {}: {e}", path.display()))?;
    let key = file.key.clone();

    let roles: Vec<UpsertBuiltinRole<'_>> = file
        .roles
        .iter()
        .map(|r| UpsertBuiltinRole {
            slot: &r.slot,
            order_idx: r.order_idx,
            system_prompt: &r.system_prompt,
            skills: r.skills.clone(),
            brain_seed: r.brain_seed.as_deref(),
            model: r.model.as_deref(),
        })
        .collect();

    let builtin = UpsertBuiltin {
        id: builtin_id(&file.key),
        key: &file.key,
        name: &file.name,
        stack: file.stack.clone(),
        default_topology: &file.default_topology,
        risk_profile: &file.risk_profile,
        mcp_bundles: file.mcp_bundles.clone(),
        version: file.version,
        description: file.description.as_deref(),
        config: file.config.clone(),
        category: &file.category,
        roles,
    };
    let template_id = upsert_builtin(pool, builtin)
        .await
        .map_err(|e| format!("upsert {key}: {e}"))?;

    // Bind skills to template roles (Slice 3.5d): replace the entire
    // set so a TOML edit that removes a skill from a role's list also
    // removes the DB binding. Skills the loader can't find by name
    // are skipped with a log — usually means the skill file hasn't
    // been authored yet (or the name is a typo).
    if let Err(e) = cm_db::repo::skills_catalog::clear_template_role_skills(pool, template_id).await
    {
        eprintln!("team_template_loader: clear_template_role_skills({key}) failed: {e}");
    }
    // Unresolved names are aggregated into one line per template rather than
    // logged individually: the per-name spam (128 lines at last count) scrolled
    // past unread for long enough that every template's skill bindings were
    // silently empty, because the TOMLs used snake_case slugs while the authored
    // skills in `skills/**/*.md` use kebab-case names. A count is noticeable.
    let mut unresolved: Vec<String> = Vec::new();
    let mut bound = 0usize;
    for role in &file.roles {
        for (idx, skill_name) in role.skills.iter().enumerate() {
            match cm_db::repo::skills_catalog::get_by_name(pool, None, skill_name).await {
                Ok(Some(skill)) => {
                    // Pin the foundation + first-N-per-role skills for
                    // now; the finer policy (per-skill pin flag) lives
                    // in the skill's own metadata in a later slice.
                    let pin = idx < 2 || skill.tags.iter().any(|t| t == "foundation");
                    let bind = cm_db::repo::skills_catalog::AttachRoleSkill {
                        template_id,
                        slot: &role.slot,
                        skill_id: skill.id,
                        pin_in_context: pin,
                        order_idx: idx as i32,
                    };
                    if let Err(e) = cm_db::repo::skills_catalog::attach_role_skill(pool, bind).await
                    {
                        eprintln!(
                            "team_template_loader: attach skill {skill_name} → {key}.{}: {e}",
                            role.slot
                        );
                    } else {
                        bound += 1;
                    }
                }
                Ok(None) => unresolved.push(format!("{}.{skill_name}", role.slot)),
                Err(e) => {
                    eprintln!("team_template_loader: lookup skill '{skill_name}' failed: {e}");
                }
            }
        }
    }
    if unresolved.is_empty() {
        eprintln!("team_template_loader: {key} — {bound} role skills bound");
    } else {
        eprintln!(
            "team_template_loader: {key} — {bound} role skills bound, {} unresolved (no such skill authored under skills/): {}",
            unresolved.len(),
            unresolved.join(", "),
        );
    }

    Ok(key)
}

#[cfg(test)]
mod tests {
    use std::collections::HashSet;
    use std::path::{Path, PathBuf};

    fn repo_root() -> PathBuf {
        // crates/cm-api → repo root
        Path::new(env!("CARGO_MANIFEST_DIR"))
            .ancestors()
            .nth(2)
            .expect("repo root above crates/cm-api")
            .to_path_buf()
    }

    fn authored_skill_names(dir: &Path, out: &mut HashSet<String>) {
        let Ok(entries) = std::fs::read_dir(dir) else {
            return;
        };
        for e in entries.flatten() {
            let p = e.path();
            if p.is_dir() {
                authored_skill_names(&p, out);
            } else if p.extension().is_some_and(|x| x == "md") {
                let body = std::fs::read_to_string(&p).unwrap_or_default();
                if let Some(name) = body
                    .lines()
                    .find_map(|l| l.strip_prefix("name:").map(str::trim))
                {
                    out.insert(name.to_string());
                }
            }
        }
    }

    fn referenced_skill_names() -> HashSet<String> {
        let mut refs = HashSet::new();
        let dir = repo_root().join("templates/teams");
        for e in std::fs::read_dir(&dir)
            .expect("templates/teams readable")
            .flatten()
        {
            let body = std::fs::read_to_string(e.path()).unwrap_or_default();
            let parsed: toml::Value = match body.parse() {
                Ok(v) => v,
                Err(e) => panic!("{:?} is not valid TOML: {e}", e),
            };
            if let Some(roles) = parsed.get("roles").and_then(|r| r.as_array()) {
                for role in roles {
                    if let Some(skills) = role.get("skills").and_then(|s| s.as_array()) {
                        refs.extend(skills.iter().filter_map(|s| s.as_str()).map(str::to_string));
                    }
                }
            }
        }
        refs
    }

    /// Every skill authored under `skills/**/*.md` must be reachable by at
    /// least one team role.
    ///
    /// This is the half of the naming drift that was invisible: the TOMLs used
    /// snake_case slugs (`write_rust`) while the authored skills use kebab-case
    /// names (`write-rust-current-edition`), so `get_by_name` missed on every
    /// lookup — no role got any skill, and ten authored skills were reachable
    /// by nobody. Both halves are silent at runtime; only a test catches them.
    #[test]
    fn every_authored_skill_is_referenced_by_some_role() {
        let mut authored = HashSet::new();
        authored_skill_names(&repo_root().join("skills"), &mut authored);
        assert!(
            !authored.is_empty(),
            "no authored skills found — check the skills/ path"
        );
        let referenced = referenced_skill_names();
        let orphans: Vec<_> = authored.difference(&referenced).cloned().collect();
        assert!(
            orphans.is_empty(),
            "authored skills no team role references (they can never reach an agent): {orphans:?}"
        );
    }

    /// EVERY referenced name must resolve to an authored skill.
    ///
    /// The other direction, and the one that was missing. Both existing tests
    /// assert `authored ⊆ referenced` — true of all 30 authored skills, so both
    /// passed while 55 of 85 bindings resolved to nothing and ten roles ran with
    /// an empty context bundle.
    ///
    /// The old comment on the test below called the gap "deliberately
    /// aspirational". An aspirational binding is indistinguishable at runtime
    /// from a typo: `get_by_name` returns Ok(None), the loader logs a line
    /// nobody reads, and the role ships without the instructions its prompt
    /// assumes it has. If a skill is worth naming it is worth authoring, and if
    /// it is not, the name should not be in the template.
    #[test]
    fn every_referenced_skill_resolves_to_an_authored_one() {
        let mut authored = HashSet::new();
        authored_skill_names(&repo_root().join("skills"), &mut authored);
        let referenced = referenced_skill_names();
        let mut missing: Vec<_> = referenced.difference(&authored).cloned().collect();
        missing.sort();
        assert!(
            missing.is_empty(),
            "{} referenced skill(s) bind to nothing — the role gets no instructions \
             and nothing errors: {missing:#?}",
            missing.len()
        );
    }

    /// A referenced name that matches no authored skill binds to nothing. Some
    /// are deliberately aspirational, so this asserts the *resolvable* ones
    /// stay resolvable rather than demanding every name exist.
    #[test]
    fn referenced_skills_that_exist_use_the_authored_spelling() {
        let mut authored = HashSet::new();
        authored_skill_names(&repo_root().join("skills"), &mut authored);
        let referenced = referenced_skill_names();
        let resolvable = referenced.intersection(&authored).count();
        assert_eq!(
            resolvable,
            authored.len(),
            "every authored skill should be referenced by its exact name",
        );
    }
}

#[cfg(test)]
mod bundle_tests {
    use std::collections::HashSet;

    fn repo() -> std::path::PathBuf {
        std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
            .join("../..")
            .canonicalize()
            .expect("repo root")
    }

    /// Every `mcp_bundles` name a template asks for must be one the runtime
    /// config actually defines.
    ///
    /// This was harmless while `provision_claw` wrote a constant bundle list
    /// and ignored the templates. It is not harmless now that the list is
    /// honoured: an undefined name is a capability the agent is told it has and
    /// does not, which is the same failure as an unresolved skill binding one
    /// layer down. `gitea_forge` was named by seven team templates, one
    /// workflow recipe, the auto-provision path and a user-selectable dropdown,
    /// and defined nowhere.
    #[test]
    fn every_named_mcp_bundle_is_defined_by_the_runtime_config() {
        let cfg = std::fs::read_to_string(
            repo().join("deploy/clawmates-runtime/agent.config.example.toml"),
        )
        .expect("runtime config");
        let defined: HashSet<String> = cfg
            .lines()
            .filter_map(|l| l.trim().strip_prefix("[mcp_bundles."))
            .filter_map(|r| r.strip_suffix(']'))
            .map(|s| s.to_string())
            .collect();
        assert!(
            defined.contains("clawmates_door"),
            "parsed no bundles from the runtime config — the parser, not the \
             templates, is what broke"
        );

        let mut missing: Vec<String> = Vec::new();
        for dir in ["templates/teams", "templates/workflows"] {
            for entry in std::fs::read_dir(repo().join(dir)).expect("template dir") {
                let path = entry.expect("entry").path();
                if path.extension().and_then(|e| e.to_str()) != Some("toml") {
                    continue;
                }
                let body = std::fs::read_to_string(&path).expect("read template");
                for line in body.lines() {
                    let t = line.trim();
                    // Skip comments: several deliberately NAME a bundle while
                    // explaining that it is not delivered.
                    if t.starts_with('#') || !t.starts_with("mcp_bundles") {
                        continue;
                    }
                    let Some(inner) = t.split_once('[').and_then(|(_, r)| r.rsplit_once(']'))
                    else {
                        continue;
                    };
                    for name in inner.0.split(',') {
                        let name = name.trim().trim_matches('"');
                        if !name.is_empty() && !defined.contains(name) {
                            missing.push(format!(
                                "{}: {name}",
                                path.file_name().unwrap().to_string_lossy()
                            ));
                        }
                    }
                }
            }
        }
        missing.sort();
        missing.dedup();
        assert!(
            missing.is_empty(),
            "{} template(s) name an MCP bundle the runtime does not define, so \
             the agent is provisioned with a capability that resolves to \
             nothing:\n  {}",
            missing.len(),
            missing.join("\n  ")
        );
    }
}