clawmates/crates/cm-brain/src/hub.rs

//! ClawBrainHub pull/push, over `claw-brain`'s production registry client.
//!
//! Pull works anonymously for public brains (uses `BRAINHUB_API_KEY` when set);
//! push requires the key. Brains arrive either as the canonical HDF5 brain-pack
//! or as a JSON export (the public reference brains are unsigned JSON) — pull
//! detects which and normalizes JSON into a brain-pack `.brain` on disk.

use std::path::Path;

use claw_brain::{parse_brain_ref, BrainRegistryClient, BrainRegistryConfig};
use claw_core::prelude::BrainRef;
use serde::Deserialize;

use crate::{be, BrainError, ClawBrain};

const DEFAULT_BASE: &str = "https://clawbrainhub.com/api/v1";

fn config() -> BrainRegistryConfig {
    BrainRegistryConfig {
        base_url: std::env::var("BRAINHUB_BASE_URL").unwrap_or_else(|_| DEFAULT_BASE.to_string()),
        api_key: std::env::var("BRAINHUB_API_KEY").unwrap_or_default(),
        timeout_secs: 30,
        // ClawMates runs pulled claws sandboxed and re-scans itself, so it does
        // not gate pulls on the hub's trust score or signature presence.
        min_trust_score: 0.0,
        require_signature: false,
        trusted_publishers: vec![],
    }
}

#[derive(Debug, Clone, serde::Serialize)]
pub struct PulledMeta {
    pub reference: String,
    pub size_bytes: u64,
    pub trust_score: f32,
    pub description: String,
}

#[derive(Debug, Clone)]
pub struct PulledBrain {
    pub meta: PulledMeta,
    /// Suggested claw name (from the brain meta or the ref).
    pub name: String,
    pub system_prompt: String,
}

/// Pull `owner/name[:version]` and normalize it into a brain-pack `.brain` at
/// `dest`. Returns metadata + the extracted identity for claw creation.
pub async fn pull(reference: &str, dest: &Path) -> Result<PulledBrain, BrainError> {
    if let Some(d) = dest.parent() {
        std::fs::create_dir_all(d).map_err(be)?;
    }
    let tmp = dest.with_extension("pull-tmp");
    let client = BrainRegistryClient::new(config());
    let entry = client
        .fetch_metadata(&BrainRef::from(reference))
        .await
        .map_err(be)?;
    client.download(&entry, &tmp).await.map_err(be)?;
    let bytes = std::fs::read(&tmp).map_err(be)?;
    let parts = parse_brain_ref(&entry.brain_ref).map_err(be)?;
    let meta = PulledMeta {
        reference: format!("{}/{}:{}", parts.owner, parts.name, parts.tag),
        size_bytes: entry.size_bytes,
        trust_score: entry.trust_score,
        description: entry.description.clone(),
    };

    // HDF5 brain-pack → use as-is; JSON export → convert to a brain-pack.
    let (name, system_prompt) = if bytes.starts_with(b"\x89HDF") {
        let _ = std::fs::remove_file(dest);
        std::fs::rename(&tmp, dest).map_err(be)?;
        let b = ClawBrain::open_or_create(dest, reference)?;
        (parts.name.clone(), b.system_prompt().unwrap_or_default())
    } else {
        let r = build_from_json(&bytes, dest, reference, &parts.name)?;
        let _ = std::fs::remove_file(&tmp);
        r
    };
    Ok(PulledBrain {
        meta,
        name,
        system_prompt,
    })
}

#[derive(Debug, Clone, serde::Serialize)]
pub struct BrainListing {
    /// `owner/name` (no version) — apply/pull resolve `:latest` themselves.
    pub reference: String,
    pub owner: String,
    pub name: String,
    pub version: String,
    pub description: String,
    pub trust_score: f32,
    pub size_bytes: u64,
}

/// List/search registry brains (anonymous OK). The hub's `/search` is
/// keyword‑only with no list‑all, so an **empty query** approximates "browse" by
/// merging a handful of seed‑keyword searches (deduped by reference).
pub async fn list(query: &str) -> Result<Vec<BrainListing>, BrainError> {
    let client = BrainRegistryClient::new(config());
    let q = query.trim();
    let raw = if !q.is_empty() {
        search_one(&client, q).await?
    } else {
        const SEEDS: &[&str] = &[
            "assistant",
            "agent",
            "code",
            "react",
            "data",
            "research",
            "write",
            "general",
            "support",
        ];
        let mut seen = std::collections::HashSet::new();
        let mut out = Vec::new();
        for s in SEEDS {
            if let Ok(items) = search_one(&client, s).await {
                for it in items {
                    if seen.insert(it.reference.clone()) {
                        out.push(it);
                    }
                }
            }
        }
        out
    };
    Ok(filter_loadable(raw).await)
}

/// Drop brains that don't actually load with usable data — corrupt downloads
/// (registry 500) or empty brains — so the registry widget only shows brains a
/// claw can really use. Checks run concurrently.
async fn filter_loadable(raw: Vec<BrainListing>) -> Vec<BrainListing> {
    let checks = raw.into_iter().map(|b| async move {
        match preview(&b.reference).await {
            Ok(pv) if !is_empty_preview(&pv) => Some(b),
            _ => None,
        }
    });
    futures::future::join_all(checks)
        .await
        .into_iter()
        .flatten()
        .collect()
}

async fn search_one(
    client: &BrainRegistryClient,
    q: &str,
) -> Result<Vec<BrainListing>, BrainError> {
    let entries = client.list(Some(q), None, None).await.map_err(be)?;
    let mut out = Vec::new();
    for e in entries {
        if let Ok(p) = parse_brain_ref(&e.brain_ref) {
            out.push(BrainListing {
                reference: format!("{}/{}", p.owner, p.name),
                owner: p.owner,
                name: p.name,
                version: p.tag,
                description: e.description,
                trust_score: e.trust_score,
                size_bytes: e.size_bytes,
            });
        }
    }
    Ok(out)
}

#[derive(Debug, Clone, serde::Serialize)]
pub struct SectionText {
    pub populated: bool,
    pub chars: usize,
    pub preview: String,
}

/// A read-only overview of a brain's contents (which sections are populated vs
/// empty), for the registry detail slide-out. Pulls + introspects, then discards.
#[derive(Debug, Clone, serde::Serialize)]
pub struct BrainPreview {
    pub reference: String,
    pub size_bytes: u64,
    pub system_prompt: SectionText,
    pub agent_md: SectionText,
    pub personality: SectionText,
    pub skills: Vec<String>,
    pub tools: Vec<(String, String)>,
    pub memory_count: usize,
    pub memory_recent: Vec<String>,
    pub runtime: bool,
    pub provenance: bool,
}

fn section_text(s: Option<String>) -> SectionText {
    let txt = s.unwrap_or_default();
    SectionText {
        populated: !txt.trim().is_empty(),
        chars: txt.chars().count(),
        preview: txt.chars().take(240).collect(),
    }
}

/// Pull a brain and summarize its sections without applying it to any agent.
pub async fn preview(reference: &str) -> Result<BrainPreview, BrainError> {
    let safe: String = reference
        .chars()
        .map(|c| if c == '/' || c == ':' { '_' } else { c })
        .collect();
    let tmp = std::env::temp_dir().join(format!(
        "cm-brain-preview-{}-{}.brain",
        std::process::id(),
        safe
    ));
    let _ = std::fs::remove_file(&tmp);
    let pulled = pull(reference, &tmp).await?;
    let b = ClawBrain::open_or_create(&tmp, reference)?;
    let pv = BrainPreview {
        reference: pulled.meta.reference,
        size_bytes: pulled.meta.size_bytes,
        system_prompt: section_text(b.system_prompt()),
        agent_md: section_text(b.agent_md()),
        personality: section_text(b.personality()),
        skills: b.skills().into_iter().map(|(n, _)| n).collect(),
        tools: b.tools(),
        memory_count: b.memory_count(),
        memory_recent: b.recent_memory(4).into_iter().map(|(_, t)| t).collect(),
        runtime: b.runtime().map(|s| !s.trim().is_empty()).unwrap_or(false),
        provenance: b
            .provenance()
            .map(|s| !s.trim().is_empty())
            .unwrap_or(false),
    };
    let _ = std::fs::remove_file(&tmp);
    Ok(pv)
}

/// Pull `reference` and **merge** its contents into the existing brain at
/// `dest_brain` (apply‑to‑existing‑agent): overwrite identity (soul/agent_md/
/// persona), upsert skills + tools, append memory. The returned
/// `PulledBrain.system_prompt` is the merged brain's `assembled_identity()` —
/// the authoritative system prompt the caller should persist.
pub async fn pull_merge(reference: &str, dest_brain: &Path) -> Result<PulledBrain, BrainError> {
    let tmp = dest_brain.with_extension("incoming");
    let _ = std::fs::remove_file(&tmp);
    let mut pulled = pull(reference, &tmp).await?;
    let assembled = {
        let src = ClawBrain::open_or_create(&tmp, reference)?;
        let mut dst = ClawBrain::open_or_create(dest_brain, reference)?;
        if let Some(s) = src.system_prompt() {
            dst.set_system_prompt(&s)?;
        }
        if let Some(a) = src.agent_md() {
            dst.set_agent_md(&a)?;
        }
        if let Some(p) = src.personality() {
            dst.set_personality(&p)?;
        }
        for (n, b) in src.skills() {
            dst.set_skill(&n, &b)?;
        }
        for (n, st) in src.tools() {
            dst.set_tool(&n, &st)?;
        }
        if let Some(rt) = src.runtime() {
            let _ = dst.set_runtime(&rt);
        }
        for (_, text) in src.recent_memory(200) {
            let _ = dst.remember("memory", &text, "brain-apply");
        }
        dst.assembled_identity()
    };
    let _ = std::fs::remove_file(&tmp);
    if !assembled.trim().is_empty() {
        pulled.system_prompt = assembled;
    }
    Ok(pulled)
}

/// Push a local `.brain` (brain-pack) as `owner/name:version`. Requires
/// `BRAINHUB_API_KEY`.
pub async fn push(
    reference: &str,
    path: &Path,
    description: &str,
    tags: &[String],
) -> Result<(), BrainError> {
    if std::env::var("BRAINHUB_API_KEY")
        .unwrap_or_default()
        .trim()
        .is_empty()
    {
        return Err(BrainError::Backend(
            "BRAINHUB_API_KEY is not set — cannot push to ClawBrainHub".into(),
        ));
    }
    let client = BrainRegistryClient::new(config());
    client
        .push(&BrainRef::from(reference), path, description, tags)
        .await
        .map_err(be)
}

/// The owner namespace the configured `BRAINHUB_API_KEY` authenticates as
/// (where pushes/enhanced versions land).
pub async fn whoami() -> Result<String, BrainError> {
    let client = BrainRegistryClient::new(config());
    Ok(client.whoami().await.map_err(be)?.owner)
}

/// Delete a brain (all versions) from the registry. Requires `BRAINHUB_API_KEY`
/// and ownership of the namespace.
pub async fn delete(reference: &str) -> Result<(), BrainError> {
    if std::env::var("BRAINHUB_API_KEY")
        .unwrap_or_default()
        .trim()
        .is_empty()
    {
        return Err(BrainError::Backend("BRAINHUB_API_KEY not set".into()));
    }
    let client = BrainRegistryClient::new(config());
    client.delete(&BrainRef::from(reference)).await.map_err(be)
}

/// True when a brain carries no usable content (all sections empty).
pub fn is_empty_preview(pv: &BrainPreview) -> bool {
    !pv.system_prompt.populated
        && !pv.agent_md.populated
        && !pv.personality.populated
        && pv.skills.is_empty()
        && pv.tools.is_empty()
        && pv.memory_count == 0
}

// ── JSON export → brain-pack conversion ─────────────────────────────────────
#[derive(Deserialize, Default)]
struct HubExport {
    meta: Option<HubMeta>,
    identity: Option<HubIdentity>,
    skills: Option<HubSkills>,
    memory: Option<HubMemory>,
    runtime: Option<serde_json::Value>,
}
#[derive(Deserialize, Default)]
struct HubMeta {
    brain_name: Option<String>,
}
#[derive(Deserialize, Default)]
struct HubIdentity {
    soul_md: Option<String>,
    agent_md: Option<String>,
    system_prompt: Option<String>,
    persona: Option<String>,
}
#[derive(Deserialize, Default)]
struct HubSkills {
    skills_md: Option<String>,
}
#[derive(Deserialize, Default)]
struct HubMemory {
    entries: Option<Vec<HubMemEntry>>,
}
#[derive(Deserialize, Default)]
struct HubMemEntry {
    chunk: Option<String>,
}

fn nonempty(s: Option<String>) -> Option<String> {
    s.filter(|t| !t.trim().is_empty())
}

fn build_from_json(
    bytes: &[u8],
    dest: &Path,
    reference: &str,
    ref_name: &str,
) -> Result<(String, String), BrainError> {
    let exp: HubExport = serde_json::from_slice(bytes).map_err(be)?;
    let _ = std::fs::remove_file(dest); // start clean
    let mut brain = ClawBrain::open_or_create(dest, reference)?;

    let id = exp.identity.unwrap_or_default();
    // soul_md is the rich identity; the public brains leave system_prompt empty.
    let system_prompt = nonempty(id.system_prompt)
        .or_else(|| nonempty(id.soul_md))
        .unwrap_or_default();
    if !system_prompt.is_empty() {
        brain.set_system_prompt(&system_prompt)?;
    }
    if let Some(a) = nonempty(id.agent_md) {
        brain.set_agent_md(&a)?;
    }
    if let Some(p) = nonempty(id.persona) {
        brain.set_personality(&p)?;
    }
    if let Some(md) = exp.skills.and_then(|s| nonempty(s.skills_md)) {
        brain.set_skills_md(&md)?;
    }
    if let Some(entries) = exp.memory.and_then(|m| m.entries) {
        for e in entries {
            if let Some(chunk) = nonempty(e.chunk) {
                let _ = brain.remember("import", &chunk, "pack-import");
            }
        }
    }
    if let Some(rt) = exp.runtime {
        let _ = brain.set_runtime(&rt.to_string());
    }

    let name = exp
        .meta
        .and_then(|m| nonempty(m.brain_name))
        .unwrap_or_else(|| ref_name.to_string());
    Ok((name, system_prompt))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    #[ignore = "hits live clawbrainhub.com"]
    async fn live_pull_general_assistant() {
        let dest = std::env::temp_dir().join(format!("cm-brain-pull-{}.brain", std::process::id()));
        let _ = std::fs::remove_file(&dest);
        let pulled = pull("redclawsystems/general-assistant", &dest)
            .await
            .expect("pull should succeed anonymously");
        assert!(dest.exists(), "brain file written");
        assert!(!pulled.system_prompt.is_empty(), "identity extracted");
        let b = ClawBrain::open_or_create(&dest, "x").unwrap();
        assert!(!b.skills().is_empty(), "skills parsed from skills_md");
        eprintln!(
            "pulled {} ({} bytes, {} skills, {} memories)",
            pulled.meta.reference,
            pulled.meta.size_bytes,
            b.skills().len(),
            b.memory_count()
        );
        let _ = std::fs::remove_file(&dest);
    }

    #[tokio::test]
    #[ignore = "hits live clawbrainhub.com"]
    async fn live_pull_merge_into_existing() {
        let dest =
            std::env::temp_dir().join(format!("cm-brain-merge-{}.brain", std::process::id()));
        let _ = std::fs::remove_file(&dest);
        {
            let mut b = ClawBrain::open_or_create(&dest, "x").unwrap();
            b.set_skill("existing", "keep me").unwrap();
        }
        let pulled = pull_merge("redclawsystems/general-assistant", &dest)
            .await
            .expect("merge");
        assert!(
            !pulled.system_prompt.is_empty(),
            "assembled identity returned"
        );
        let b = ClawBrain::open_or_create(&dest, "x").unwrap();
        let names: Vec<String> = b.skills().into_iter().map(|(n, _)| n).collect();
        assert!(
            names.iter().any(|n| n == "existing"),
            "existing skill preserved; got {names:?}"
        );
        assert!(names.len() > 1, "merged skills added; got {names:?}");
        eprintln!(
            "merged → {} skills, {} memories",
            names.len(),
            b.memory_count()
        );
        let _ = std::fs::remove_file(&dest);
    }

    #[tokio::test]
    #[ignore = "hits live clawbrainhub.com"]
    async fn live_list() {
        let items = list("").await.expect("list");
        eprintln!(
            "registry browse returned {} brains: {:?}",
            items.len(),
            items.iter().map(|b| &b.reference).collect::<Vec<_>>()
        );
        assert!(!items.is_empty(), "seed-browse should surface brains");
        assert!(
            items.iter().any(|b| b.reference == "omar/react-native"),
            "react-native should appear"
        );
    }

    #[tokio::test]
    #[ignore = "dumps a brain's sections to /tmp/brain-dump.txt"]
    async fn live_dump_brain() {
        let r = std::env::var("DUMP_REF").unwrap_or_else(|_| "omar/rust-2024".to_string());
        let dest = std::env::temp_dir().join("dump.brain");
        let _ = std::fs::remove_file(&dest);
        pull(&r, &dest).await.expect("pull");
        let b = ClawBrain::open_or_create(&dest, &r).unwrap();
        let sp = b.system_prompt().unwrap_or_default();
        let am = b.agent_md().unwrap_or_default();
        let pe = b.personality().unwrap_or_default();
        let sk = b
            .skills()
            .into_iter()
            .map(|(n, bd)| format!("## {n}\n{bd}"))
            .collect::<Vec<_>>()
            .join("\n\n");
        eprintln!(
            "{r}: system_prompt={} agent_md={} persona={} skills_md={} chars (skills={})",
            sp.len(),
            am.len(),
            pe.len(),
            sk.len(),
            b.skills().len()
        );
        let payload = format!("BRAIN: {r}\n\n=== SYSTEM PROMPT ===\n{sp}\n\n=== AGENTS.md ===\n{am}\n\n=== PERSONA ===\n{pe}\n\n=== SKILLS ===\n{sk}");
        std::fs::write("/tmp/brain-dump.txt", &payload).unwrap();
        eprintln!("wrote /tmp/brain-dump.txt ({} chars)", payload.len());
        let _ = std::fs::remove_file(&dest);
    }

    #[tokio::test]
    #[ignore = "hits live clawbrainhub.com — read-only scan"]
    async fn live_scan_empty_brains() {
        let queries = [
            "",
            "omar",
            "assistant",
            "default",
            "test",
            "claw",
            "agent",
            "brain",
            "react",
            "general",
            "data",
            "code",
        ];
        let mut seen = std::collections::HashSet::new();
        for q in queries {
            for it in list(q).await.unwrap_or_default() {
                if !seen.insert(it.reference.clone()) {
                    continue;
                }
                match preview(&it.reference).await {
                    Ok(pv) => eprintln!(
                        "{:42} owner={:34} EMPTY={} (sp={} agent={} persona={} skills={} tools={} mem={})",
                        it.reference, it.owner, is_empty_preview(&pv),
                        pv.system_prompt.populated, pv.agent_md.populated, pv.personality.populated,
                        pv.skills.len(), pv.tools.len(), pv.memory_count,
                    ),
                    Err(e) => eprintln!("{:42} owner={:34} preview-error: {e}", it.reference, it.owner),
                }
            }
        }
    }

    #[tokio::test]
    #[ignore = "hits live clawbrainhub.com"]
    async fn live_apply_react_native_hdf5() {
        let dest = std::env::temp_dir().join(format!("cm-brain-rn-{}.brain", std::process::id()));
        let _ = std::fs::remove_file(&dest);
        let pulled = pull_merge("omar/react-native", &dest)
            .await
            .expect("merge HDF5 brain");
        let b = ClawBrain::open_or_create(&dest, "x").unwrap();
        eprintln!(
            "react-native → system_prompt {} chars, {} skills",
            pulled.system_prompt.len(),
            b.skills().len()
        );
        let _ = std::fs::remove_file(&dest);
    }
}