Large World graph, agent platform, brain stack & dashboard rebuild

Frontend
- Large World: collapse org/company/team tiers into one expandable React Flow
  hierarchy (WorldFlow) with per-click expand, persisted node positions, a
  compact tree sidebar, wrench multi-select delete across levels, and a sized
  right slide-out (phone/tablet/full) showing an agent summary + drill button.
- Agent page: GitHub-style animated contribution grid (VitalsCard), collapsible
  System Prompt + Personality cards, restructured anatomy cards, bigger avatar
  with name/title header row, Markdown/JSON-aware rendering, brain registry +
  history, avatar generate/upload.
- User-icon menu (Infrastructure/Brains/Tools/Profile/Credits) + ToolPanel;
  Master Planner deploy wizard (Specialists/Swarm/Scheduled/Triggered);
  Team Runs view; reap-progress modal; dashboard is the single live interface.

Backend
- cm-brain crate (.brain as the agent definition) + brain apply/history.
- Hard-purge reap (FK-ordered) + sandbox release + SSE batch-delete.
- Swarm self-verifying loop, mode-aware planner, web.search tool, webhooks
  (migration 0013), org/company/team delete endpoints, scheduler sweeps.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Omar Sobh
2026-06-22 23:21:54 -07:00
co-authored by Claude Opus 4.8
parent 9f266d5806
commit 34f744734b
123 changed files with 9591 additions and 1098 deletions
+480
View File
@@ -0,0 +1,480 @@
//! ClawBrainHub pull/push, over `claw-brain`'s production registry client.
//!
//! Pull works anonymously for public brains (uses `BRAINHUB_API_KEY` when set);
//! push requires the key. Brains arrive either as the canonical HDF5 brain-pack
//! or as a JSON export (the public reference brains are unsigned JSON) — pull
//! detects which and normalizes JSON into a brain-pack `.brain` on disk.
use std::path::Path;
use claw_brain::{parse_brain_ref, BrainRegistryClient, BrainRegistryConfig};
use claw_core::prelude::BrainRef;
use serde::Deserialize;
use crate::{be, BrainError, ClawBrain};
const DEFAULT_BASE: &str = "https://clawbrainhub.com/api/v1";
fn config() -> BrainRegistryConfig {
BrainRegistryConfig {
base_url: std::env::var("BRAINHUB_BASE_URL").unwrap_or_else(|_| DEFAULT_BASE.to_string()),
api_key: std::env::var("BRAINHUB_API_KEY").unwrap_or_default(),
timeout_secs: 30,
// ClawMates runs pulled claws sandboxed and re-scans itself, so it does
// not gate pulls on the hub's trust score or signature presence.
min_trust_score: 0.0,
require_signature: false,
trusted_publishers: vec![],
}
}
#[derive(Debug, Clone, serde::Serialize)]
pub struct PulledMeta {
pub reference: String,
pub size_bytes: u64,
pub trust_score: f32,
pub description: String,
}
#[derive(Debug, Clone)]
pub struct PulledBrain {
pub meta: PulledMeta,
/// Suggested claw name (from the brain meta or the ref).
pub name: String,
pub system_prompt: String,
}
/// Pull `owner/name[:version]` and normalize it into a brain-pack `.brain` at
/// `dest`. Returns metadata + the extracted identity for claw creation.
pub async fn pull(reference: &str, dest: &Path) -> Result<PulledBrain, BrainError> {
if let Some(d) = dest.parent() {
std::fs::create_dir_all(d).map_err(be)?;
}
let tmp = dest.with_extension("pull-tmp");
let client = BrainRegistryClient::new(config());
let entry = client
.fetch_metadata(&BrainRef::from(reference))
.await
.map_err(be)?;
client.download(&entry, &tmp).await.map_err(be)?;
let bytes = std::fs::read(&tmp).map_err(be)?;
let parts = parse_brain_ref(&entry.brain_ref).map_err(be)?;
let meta = PulledMeta {
reference: format!("{}/{}:{}", parts.owner, parts.name, parts.tag),
size_bytes: entry.size_bytes,
trust_score: entry.trust_score,
description: entry.description.clone(),
};
// HDF5 brain-pack → use as-is; JSON export → convert to a brain-pack.
let (name, system_prompt) = if bytes.starts_with(b"\x89HDF") {
let _ = std::fs::remove_file(dest);
std::fs::rename(&tmp, dest).map_err(be)?;
let b = ClawBrain::open_or_create(dest, reference)?;
(parts.name.clone(), b.system_prompt().unwrap_or_default())
} else {
let r = build_from_json(&bytes, dest, reference, &parts.name)?;
let _ = std::fs::remove_file(&tmp);
r
};
Ok(PulledBrain { meta, name, system_prompt })
}
#[derive(Debug, Clone, serde::Serialize)]
pub struct BrainListing {
/// `owner/name` (no version) — apply/pull resolve `:latest` themselves.
pub reference: String,
pub owner: String,
pub name: String,
pub version: String,
pub description: String,
pub trust_score: f32,
pub size_bytes: u64,
}
/// List/search registry brains (anonymous OK). The hub's `/search` is
/// keywordonly with no listall, so an **empty query** approximates "browse" by
/// merging a handful of seedkeyword searches (deduped by reference).
pub async fn list(query: &str) -> Result<Vec<BrainListing>, BrainError> {
let client = BrainRegistryClient::new(config());
let q = query.trim();
let raw = if !q.is_empty() {
search_one(&client, q).await?
} else {
const SEEDS: &[&str] = &[
"assistant", "agent", "code", "react", "data", "research", "write", "general", "support",
];
let mut seen = std::collections::HashSet::new();
let mut out = Vec::new();
for s in SEEDS {
if let Ok(items) = search_one(&client, s).await {
for it in items {
if seen.insert(it.reference.clone()) {
out.push(it);
}
}
}
}
out
};
Ok(filter_loadable(raw).await)
}
/// Drop brains that don't actually load with usable data — corrupt downloads
/// (registry 500) or empty brains — so the registry widget only shows brains a
/// claw can really use. Checks run concurrently.
async fn filter_loadable(raw: Vec<BrainListing>) -> Vec<BrainListing> {
let checks = raw.into_iter().map(|b| async move {
match preview(&b.reference).await {
Ok(pv) if !is_empty_preview(&pv) => Some(b),
_ => None,
}
});
futures::future::join_all(checks)
.await
.into_iter()
.flatten()
.collect()
}
async fn search_one(client: &BrainRegistryClient, q: &str) -> Result<Vec<BrainListing>, BrainError> {
let entries = client.list(Some(q), None, None).await.map_err(be)?;
let mut out = Vec::new();
for e in entries {
if let Ok(p) = parse_brain_ref(&e.brain_ref) {
out.push(BrainListing {
reference: format!("{}/{}", p.owner, p.name),
owner: p.owner,
name: p.name,
version: p.tag,
description: e.description,
trust_score: e.trust_score,
size_bytes: e.size_bytes,
});
}
}
Ok(out)
}
#[derive(Debug, Clone, serde::Serialize)]
pub struct SectionText {
pub populated: bool,
pub chars: usize,
pub preview: String,
}
/// A read-only overview of a brain's contents (which sections are populated vs
/// empty), for the registry detail slide-out. Pulls + introspects, then discards.
#[derive(Debug, Clone, serde::Serialize)]
pub struct BrainPreview {
pub reference: String,
pub size_bytes: u64,
pub system_prompt: SectionText,
pub agent_md: SectionText,
pub personality: SectionText,
pub skills: Vec<String>,
pub tools: Vec<(String, String)>,
pub memory_count: usize,
pub memory_recent: Vec<String>,
pub runtime: bool,
pub provenance: bool,
}
fn section_text(s: Option<String>) -> SectionText {
let txt = s.unwrap_or_default();
SectionText {
populated: !txt.trim().is_empty(),
chars: txt.chars().count(),
preview: txt.chars().take(240).collect(),
}
}
/// Pull a brain and summarize its sections without applying it to any agent.
pub async fn preview(reference: &str) -> Result<BrainPreview, BrainError> {
let safe: String = reference.chars().map(|c| if c == '/' || c == ':' { '_' } else { c }).collect();
let tmp = std::env::temp_dir().join(format!("cm-brain-preview-{}-{}.brain", std::process::id(), safe));
let _ = std::fs::remove_file(&tmp);
let pulled = pull(reference, &tmp).await?;
let b = ClawBrain::open_or_create(&tmp, reference)?;
let pv = BrainPreview {
reference: pulled.meta.reference,
size_bytes: pulled.meta.size_bytes,
system_prompt: section_text(b.system_prompt()),
agent_md: section_text(b.agent_md()),
personality: section_text(b.personality()),
skills: b.skills().into_iter().map(|(n, _)| n).collect(),
tools: b.tools(),
memory_count: b.memory_count(),
memory_recent: b.recent_memory(4).into_iter().map(|(_, t)| t).collect(),
runtime: b.runtime().map(|s| !s.trim().is_empty()).unwrap_or(false),
provenance: b.provenance().map(|s| !s.trim().is_empty()).unwrap_or(false),
};
let _ = std::fs::remove_file(&tmp);
Ok(pv)
}
/// Pull `reference` and **merge** its contents into the existing brain at
/// `dest_brain` (applytoexistingagent): overwrite identity (soul/agent_md/
/// persona), upsert skills + tools, append memory. The returned
/// `PulledBrain.system_prompt` is the merged brain's `assembled_identity()` —
/// the authoritative system prompt the caller should persist.
pub async fn pull_merge(reference: &str, dest_brain: &Path) -> Result<PulledBrain, BrainError> {
let tmp = dest_brain.with_extension("incoming");
let _ = std::fs::remove_file(&tmp);
let mut pulled = pull(reference, &tmp).await?;
let assembled = {
let src = ClawBrain::open_or_create(&tmp, reference)?;
let mut dst = ClawBrain::open_or_create(dest_brain, reference)?;
if let Some(s) = src.system_prompt() { dst.set_system_prompt(&s)?; }
if let Some(a) = src.agent_md() { dst.set_agent_md(&a)?; }
if let Some(p) = src.personality() { dst.set_personality(&p)?; }
for (n, b) in src.skills() { dst.set_skill(&n, &b)?; }
for (n, st) in src.tools() { dst.set_tool(&n, &st)?; }
if let Some(rt) = src.runtime() { let _ = dst.set_runtime(&rt); }
for (_, text) in src.recent_memory(200) { let _ = dst.remember("memory", &text, "brain-apply"); }
dst.assembled_identity()
};
let _ = std::fs::remove_file(&tmp);
if !assembled.trim().is_empty() {
pulled.system_prompt = assembled;
}
Ok(pulled)
}
/// Push a local `.brain` (brain-pack) as `owner/name:version`. Requires
/// `BRAINHUB_API_KEY`.
pub async fn push(
reference: &str,
path: &Path,
description: &str,
tags: &[String],
) -> Result<(), BrainError> {
if std::env::var("BRAINHUB_API_KEY").unwrap_or_default().trim().is_empty() {
return Err(BrainError::Backend(
"BRAINHUB_API_KEY is not set — cannot push to ClawBrainHub".into(),
));
}
let client = BrainRegistryClient::new(config());
client
.push(&BrainRef::from(reference), path, description, tags)
.await
.map_err(be)
}
/// The owner namespace the configured `BRAINHUB_API_KEY` authenticates as
/// (where pushes/enhanced versions land).
pub async fn whoami() -> Result<String, BrainError> {
let client = BrainRegistryClient::new(config());
Ok(client.whoami().await.map_err(be)?.owner)
}
/// Delete a brain (all versions) from the registry. Requires `BRAINHUB_API_KEY`
/// and ownership of the namespace.
pub async fn delete(reference: &str) -> Result<(), BrainError> {
if std::env::var("BRAINHUB_API_KEY").unwrap_or_default().trim().is_empty() {
return Err(BrainError::Backend("BRAINHUB_API_KEY not set".into()));
}
let client = BrainRegistryClient::new(config());
client.delete(&BrainRef::from(reference)).await.map_err(be)
}
/// True when a brain carries no usable content (all sections empty).
pub fn is_empty_preview(pv: &BrainPreview) -> bool {
!pv.system_prompt.populated
&& !pv.agent_md.populated
&& !pv.personality.populated
&& pv.skills.is_empty()
&& pv.tools.is_empty()
&& pv.memory_count == 0
}
// ── JSON export → brain-pack conversion ─────────────────────────────────────
#[derive(Deserialize, Default)]
struct HubExport {
meta: Option<HubMeta>,
identity: Option<HubIdentity>,
skills: Option<HubSkills>,
memory: Option<HubMemory>,
runtime: Option<serde_json::Value>,
}
#[derive(Deserialize, Default)]
struct HubMeta {
brain_name: Option<String>,
}
#[derive(Deserialize, Default)]
struct HubIdentity {
soul_md: Option<String>,
agent_md: Option<String>,
system_prompt: Option<String>,
persona: Option<String>,
}
#[derive(Deserialize, Default)]
struct HubSkills {
skills_md: Option<String>,
}
#[derive(Deserialize, Default)]
struct HubMemory {
entries: Option<Vec<HubMemEntry>>,
}
#[derive(Deserialize, Default)]
struct HubMemEntry {
chunk: Option<String>,
}
fn nonempty(s: Option<String>) -> Option<String> {
s.filter(|t| !t.trim().is_empty())
}
fn build_from_json(
bytes: &[u8],
dest: &Path,
reference: &str,
ref_name: &str,
) -> Result<(String, String), BrainError> {
let exp: HubExport = serde_json::from_slice(bytes).map_err(be)?;
let _ = std::fs::remove_file(dest); // start clean
let mut brain = ClawBrain::open_or_create(dest, reference)?;
let id = exp.identity.unwrap_or_default();
// soul_md is the rich identity; the public brains leave system_prompt empty.
let system_prompt = nonempty(id.system_prompt)
.or_else(|| nonempty(id.soul_md))
.unwrap_or_default();
if !system_prompt.is_empty() {
brain.set_system_prompt(&system_prompt)?;
}
if let Some(a) = nonempty(id.agent_md) {
brain.set_agent_md(&a)?;
}
if let Some(p) = nonempty(id.persona) {
brain.set_personality(&p)?;
}
if let Some(md) = exp.skills.and_then(|s| nonempty(s.skills_md)) {
brain.set_skills_md(&md)?;
}
if let Some(entries) = exp.memory.and_then(|m| m.entries) {
for e in entries {
if let Some(chunk) = nonempty(e.chunk) {
let _ = brain.remember("import", &chunk, "pack-import");
}
}
}
if let Some(rt) = exp.runtime {
let _ = brain.set_runtime(&rt.to_string());
}
let name = exp
.meta
.and_then(|m| nonempty(m.brain_name))
.unwrap_or_else(|| ref_name.to_string());
Ok((name, system_prompt))
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
#[ignore = "hits live clawbrainhub.com"]
async fn live_pull_general_assistant() {
let dest =
std::env::temp_dir().join(format!("cm-brain-pull-{}.brain", std::process::id()));
let _ = std::fs::remove_file(&dest);
let pulled = pull("redclawsystems/general-assistant", &dest)
.await
.expect("pull should succeed anonymously");
assert!(dest.exists(), "brain file written");
assert!(!pulled.system_prompt.is_empty(), "identity extracted");
let b = ClawBrain::open_or_create(&dest, "x").unwrap();
assert!(!b.skills().is_empty(), "skills parsed from skills_md");
eprintln!(
"pulled {} ({} bytes, {} skills, {} memories)",
pulled.meta.reference,
pulled.meta.size_bytes,
b.skills().len(),
b.memory_count()
);
let _ = std::fs::remove_file(&dest);
}
#[tokio::test]
#[ignore = "hits live clawbrainhub.com"]
async fn live_pull_merge_into_existing() {
let dest = std::env::temp_dir().join(format!("cm-brain-merge-{}.brain", std::process::id()));
let _ = std::fs::remove_file(&dest);
{
let mut b = ClawBrain::open_or_create(&dest, "x").unwrap();
b.set_skill("existing", "keep me").unwrap();
}
let pulled = pull_merge("redclawsystems/general-assistant", &dest).await.expect("merge");
assert!(!pulled.system_prompt.is_empty(), "assembled identity returned");
let b = ClawBrain::open_or_create(&dest, "x").unwrap();
let names: Vec<String> = b.skills().into_iter().map(|(n, _)| n).collect();
assert!(names.iter().any(|n| n == "existing"), "existing skill preserved; got {names:?}");
assert!(names.len() > 1, "merged skills added; got {names:?}");
eprintln!("merged → {} skills, {} memories", names.len(), b.memory_count());
let _ = std::fs::remove_file(&dest);
}
#[tokio::test]
#[ignore = "hits live clawbrainhub.com"]
async fn live_list() {
let items = list("").await.expect("list");
eprintln!("registry browse returned {} brains: {:?}", items.len(), items.iter().map(|b| &b.reference).collect::<Vec<_>>());
assert!(!items.is_empty(), "seed-browse should surface brains");
assert!(items.iter().any(|b| b.reference == "omar/react-native"), "react-native should appear");
}
#[tokio::test]
#[ignore = "dumps a brain's sections to /tmp/brain-dump.txt"]
async fn live_dump_brain() {
let r = std::env::var("DUMP_REF").unwrap_or_else(|_| "omar/rust-2024".to_string());
let dest = std::env::temp_dir().join("dump.brain");
let _ = std::fs::remove_file(&dest);
pull(&r, &dest).await.expect("pull");
let b = ClawBrain::open_or_create(&dest, &r).unwrap();
let sp = b.system_prompt().unwrap_or_default();
let am = b.agent_md().unwrap_or_default();
let pe = b.personality().unwrap_or_default();
let sk = b.skills().into_iter().map(|(n, bd)| format!("## {n}\n{bd}")).collect::<Vec<_>>().join("\n\n");
eprintln!("{r}: system_prompt={} agent_md={} persona={} skills_md={} chars (skills={})", sp.len(), am.len(), pe.len(), sk.len(), b.skills().len());
let payload = format!("BRAIN: {r}\n\n=== SYSTEM PROMPT ===\n{sp}\n\n=== AGENTS.md ===\n{am}\n\n=== PERSONA ===\n{pe}\n\n=== SKILLS ===\n{sk}");
std::fs::write("/tmp/brain-dump.txt", &payload).unwrap();
eprintln!("wrote /tmp/brain-dump.txt ({} chars)", payload.len());
let _ = std::fs::remove_file(&dest);
}
#[tokio::test]
#[ignore = "hits live clawbrainhub.com — read-only scan"]
async fn live_scan_empty_brains() {
let queries = ["", "omar", "assistant", "default", "test", "claw", "agent", "brain", "react", "general", "data", "code"];
let mut seen = std::collections::HashSet::new();
for q in queries {
for it in list(q).await.unwrap_or_default() {
if !seen.insert(it.reference.clone()) {
continue;
}
match preview(&it.reference).await {
Ok(pv) => eprintln!(
"{:42} owner={:34} EMPTY={} (sp={} agent={} persona={} skills={} tools={} mem={})",
it.reference, it.owner, is_empty_preview(&pv),
pv.system_prompt.populated, pv.agent_md.populated, pv.personality.populated,
pv.skills.len(), pv.tools.len(), pv.memory_count,
),
Err(e) => eprintln!("{:42} owner={:34} preview-error: {e}", it.reference, it.owner),
}
}
}
}
#[tokio::test]
#[ignore = "hits live clawbrainhub.com"]
async fn live_apply_react_native_hdf5() {
let dest = std::env::temp_dir().join(format!("cm-brain-rn-{}.brain", std::process::id()));
let _ = std::fs::remove_file(&dest);
let pulled = pull_merge("omar/react-native", &dest).await.expect("merge HDF5 brain");
let b = ClawBrain::open_or_create(&dest, "x").unwrap();
eprintln!("react-native → system_prompt {} chars, {} skills", pulled.system_prompt.len(), b.skills().len());
let _ = std::fs::remove_file(&dest);
}
}