ClawBrainHub: read clawhdf5 (HDF5) .brains, not just JSON

Add the pure-Rust clawhdf5 crate (git dep, no C deps) and parse HDF5 .brain
blobs: pull fetches raw bytes, brain_from_blob branches on the magic byte
(JSON '{' vs HDF5 \x89HDF) and reads the same datasets (identity/agent_md,
soul_md, skills/skills_md, tool_defs_json) into the shared card normalizer.
All 19 of omar's HDF5 brains now pull + appear in the picker (was JSON-only).
Verified live: omar/rust-2024 yields real skills; list now 22 brains.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>

cardclaws-backend/Cargo.lock

@@ -264,6 +264,7 @@ dependencies = [
  "cardclaws-types",
  "cardclaws-wallet",
  "chrono",
+ "clawhdf5",
  "http-body-util",
  "maxminddb",
  "rand 0.8.6",
@@ -394,6 +395,44 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "clawhdf5"
+version = "2.0.0"
+source = "git+https://github.com/quantumclaws/clawhdf5#4b467199270b6e1d6a8b4ea56c7c1984b1794195"
+dependencies = [
+ "clawhdf5-format",
+ "clawhdf5-io",
+]
+
+[[package]]
+name = "clawhdf5-format"
+version = "2.0.0"
+source = "git+https://github.com/quantumclaws/clawhdf5#4b467199270b6e1d6a8b4ea56c7c1984b1794195"
+dependencies = [
+ "byteorder",
+ "flate2",
+ "sha2",
+]
+
+[[package]]
+name = "clawhdf5-io"
+version = "2.0.0"
+source = "git+https://github.com/quantumclaws/clawhdf5#4b467199270b6e1d6a8b4ea56c7c1984b1794195"
+dependencies = [
+ "clawhdf5-format",
+ "libc",
+ "memmap2",
+]
+
+[[package]]
+name = "cmake"
+version = "0.1.58"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "combine"
 version = "4.6.7"
@@ -657,6 +696,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
 dependencies = [
  "crc32fast",
+ "libz-ng-sys",
  "miniz_oxide",
 ]
 
@@ -1296,6 +1336,16 @@ dependencies = [
  "vcpkg",
 ]
 
+[[package]]
+name = "libz-ng-sys"
+version = "1.1.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "879917b256f6317769b9f374b435805a8697013098aacce5a38ac106cd6a9469"
+dependencies = [
+ "cmake",
+ "libc",
+]
+
 [[package]]
 name = "litemap"
 version = "0.8.2"
@@ -1366,6 +1416,15 @@ version = "2.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"
 
+[[package]]
+name = "memmap2"
+version = "0.9.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "mime"
 version = "0.3.17"

cardclaws-backend/crates/cardclaws-api/Cargo.toml

@@ -44,6 +44,8 @@ rand = { workspace = true }
 base64 = { workspace = true }
 validator = { workspace = true }
 async-trait = { workspace = true }
+# Pure-Rust HDF5 reader: parse clawhdf5-format .brain files pulled from ClawBrainHub.
+clawhdf5 = { git = "https://github.com/quantumclaws/clawhdf5", package = "clawhdf5" }
 maxminddb = { version = "0.24", optional = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }

cardclaws-backend/crates/cardclaws-api/src/brainhub.rs

@@ -126,12 +126,16 @@ impl BrainHubClient for HttpBrainHubClient {
             );
             let (owner, name, version) = (s.owner.clone(), s.name.clone(), s.version.clone());
             set.spawn(async move {
-                let usable = match fetch_json(&http, &token, &url).await {
+                let usable = match fetch_bytes(&http, &token, &url).await {
-                    Ok(json) => {
+                    Ok(bytes) => match brain_from_blob(&owner, &name, &version, bytes) {
-                        let b = parse_brain(&owner, &name, &version, &json);
+                        Ok(b) => {
-                        !b.skills.is_empty() || !b.tagline.is_empty() || !b.capabilities.is_empty()
+                            !b.skills.is_empty()
+                                || !b.tagline.is_empty()
+                                || !b.capabilities.is_empty()
                         }
                         Err(_) => false,
+                    },
+                    Err(_) => false,
                 };
                 (i, usable)
             });
@@ -157,8 +161,8 @@ impl BrainHubClient for HttpBrainHubClient {
         version: &str,
     ) -> Result<AgentBrain, BrainHubError> {
         let url = format!("{}/brains/{}/{}/{}/pull", self.base, owner, name, version);
-        let json = self.get_json(&url).await?;
+        let bytes = fetch_bytes(&self.http, &self.token, &url).await?;
-        Ok(parse_brain(owner, name, version, &json))
+        brain_from_blob(owner, name, version, bytes)
     }
 }
 
@@ -185,6 +189,30 @@ async fn fetch_json(
         .map_err(|e| BrainHubError::Parse(e.to_string()))
 }
 
+/// GET a URL as raw bytes (a .brain blob may be JSON or HDF5).
+async fn fetch_bytes(
+    http: &reqwest::Client,
+    token: &Option<String>,
+    url: &str,
+) -> Result<Vec<u8>, BrainHubError> {
+    let mut req = http.get(url);
+    if let Some(t) = token {
+        req = req.bearer_auth(t);
+    }
+    let resp = req
+        .send()
+        .await
+        .map_err(|e| BrainHubError::Http(e.to_string()))?;
+    if !resp.status().is_success() {
+        return Err(BrainHubError::Http(format!("status {}", resp.status())));
+    }
+    Ok(resp
+        .bytes()
+        .await
+        .map_err(|e| BrainHubError::Http(e.to_string()))?
+        .to_vec())
+}
+
 /// Parse a `/brains/{owner}` list response into summaries (latest version each).
 fn parse_summaries(json: &serde_json::Value) -> Vec<BrainSummary> {
     let arr = match json.as_array() {
@@ -225,29 +253,47 @@ fn parse_summaries(json: &serde_json::Value) -> Vec<BrainSummary> {
         .collect()
 }
 
-/// Normalize a pulled `.brain` JSON into card fields.
+/// Normalize a pulled `.brain` blob (JSON or clawhdf5 HDF5) into card fields.
-fn parse_brain(owner: &str, name: &str, version: &str, json: &serde_json::Value) -> AgentBrain {
+/// `name` is the registry name (used for the resource URL).
-    let meta_name = json
+fn brain_from_blob(
+    owner: &str,
+    name: &str,
+    version: &str,
+    bytes: Vec<u8>,
+) -> Result<AgentBrain, BrainHubError> {
+    match bytes.first() {
+        // JSON .brain ("{ ...").
+        Some(b'{') => {
+            let json: serde_json::Value =
+                serde_json::from_slice(&bytes).map_err(|e| BrainHubError::Parse(e.to_string()))?;
+            Ok(parse_brain_json(owner, name, version, &json))
+        }
+        // HDF5 .brain (magic "\x89HDF\r\n\x1a\n").
+        _ if bytes.starts_with(b"\x89HDF\r\n\x1a\n") => {
+            parse_brain_hdf5(owner, name, version, bytes)
+        }
+        _ => Err(BrainHubError::Parse("unrecognized .brain format".into())),
+    }
+}
+
+/// Normalize a JSON `.brain` into card fields.
+fn parse_brain_json(
+    owner: &str,
+    name: &str,
+    version: &str,
+    json: &serde_json::Value,
+) -> AgentBrain {
+    let str_at = |p: &str| {
+        json.pointer(p)
+            .and_then(|v| v.as_str())
+            .unwrap_or("")
+            .to_string()
+    };
+    let display = json
         .pointer("/meta/brain_name")
         .and_then(|v| v.as_str())
         .unwrap_or(name);
-    let agent_md = json
+    let tools = json
-        .pointer("/identity/agent_md")
-        .and_then(|v| v.as_str())
-        .unwrap_or("");
-    let soul_md = json
-        .pointer("/identity/soul_md")
-        .and_then(|v| v.as_str())
-        .unwrap_or("");
-    let skills_md = json
-        .pointer("/skills/skills_md")
-        .and_then(|v| v.as_str())
-        .unwrap_or("");
-
-    let mut skills = md_headings(skills_md);
-    skills.truncate(8);
-
-    let mut tools: Vec<String> = json
         .pointer("/skills/tool_defs")
         .and_then(|v| v.as_array())
         .map(|a| {
@@ -256,30 +302,107 @@ fn parse_brain(owner: &str, name: &str, version: &str, json: &serde_json::Value)
                 .collect()
         })
         .unwrap_or_default();
+    let (agent_md, soul_md, skills_md) = (
+        str_at("/identity/agent_md"),
+        str_at("/identity/soul_md"),
+        str_at("/skills/skills_md"),
+    );
+    build_agent_brain(BrainParts {
+        owner,
+        reg_name: name,
+        display_name: display,
+        version,
+        agent_md: &agent_md,
+        soul_md: &soul_md,
+        skills_md: &skills_md,
+        tools,
+    })
+}
+
+/// Normalize a clawhdf5 HDF5 `.brain` into card fields (same datasets as JSON).
+fn parse_brain_hdf5(
+    owner: &str,
+    name: &str,
+    version: &str,
+    bytes: Vec<u8>,
+) -> Result<AgentBrain, BrainHubError> {
+    let file = clawhdf5::File::from_bytes(bytes)
+        .map_err(|e| BrainHubError::Parse(format!("hdf5: {e}")))?;
+    let read = |path: &str| -> String {
+        file.dataset(path)
+            .ok()
+            .and_then(|ds| ds.read_selection(&clawhdf5::Selection::All).ok())
+            .map(|b| String::from_utf8_lossy(&b).into_owned())
+            .unwrap_or_default()
+    };
+    let tools = serde_json::from_str::<Vec<serde_json::Value>>(&read("skills/tool_defs_json"))
+        .ok()
+        .map(|a| {
+            a.iter()
+                .filter_map(|t| t.get("name").and_then(|n| n.as_str()).map(str::to_string))
+                .collect()
+        })
+        .unwrap_or_default();
+    let (agent_md, soul_md, skills_md) = (
+        read("identity/agent_md"),
+        read("identity/soul_md"),
+        read("skills/skills_md"),
+    );
+    Ok(build_agent_brain(BrainParts {
+        owner,
+        reg_name: name,
+        display_name: name,
+        version,
+        agent_md: &agent_md,
+        soul_md: &soul_md,
+        skills_md: &skills_md,
+        tools,
+    }))
+}
+
+/// Inputs to `build_agent_brain` (grouped to keep the arg count sane).
+struct BrainParts<'a> {
+    owner: &'a str,
+    /// Registry name (used for the shareable URL).
+    reg_name: &'a str,
+    /// Display name shown on the card (meta brain_name, falls back to reg_name).
+    display_name: &'a str,
+    version: &'a str,
+    agent_md: &'a str,
+    soul_md: &'a str,
+    skills_md: &'a str,
+    tools: Vec<String>,
+}
+
+/// Shared normalization from the brain's markdown fields → card fields.
+fn build_agent_brain(p: BrainParts) -> AgentBrain {
+    let mut skills = md_headings(p.skills_md);
+    skills.truncate(8);
+    let mut tools = p.tools;
     tools.truncate(8);
 
-    let mut capabilities = md_bullets(agent_md);
+    let mut capabilities = md_bullets(p.agent_md);
     if capabilities.is_empty() {
-        capabilities = md_headings(agent_md)
+        capabilities = md_headings(p.agent_md)
             .into_iter()
             .filter(|h| h != "Purpose")
             .collect();
     }
     capabilities.truncate(6);
 
-    let tagline = first_paragraph(soul_md)
+    let tagline = first_paragraph(p.soul_md)
-        .or_else(|| first_paragraph(agent_md))
+        .or_else(|| first_paragraph(p.agent_md))
         .unwrap_or_default();
 
     AgentBrain {
-        owner: owner.to_string(),
+        owner: p.owner.to_string(),
-        name: meta_name.to_string(),
+        name: p.display_name.to_string(),
-        version: version.to_string(),
+        version: p.version.to_string(),
         tagline,
         skills,
         tools,
         capabilities,
-        resource_url: format!("{WEB_BASE}/{owner}/{name}"),
+        resource_url: format!("{WEB_BASE}/{}/{}", p.owner, p.reg_name),
     }
 }
 
@@ -384,7 +507,7 @@ mod tests {
             },
             "skills": { "skills_md": "# Skills\n\n## web_search\nSearch the web.\n## code_review\nReview code." }
         });
-        let b = parse_brain("redclawsystems", "general-assistant", "1.0.0", &json);
+        let b = parse_brain_json("redclawsystems", "general-assistant", "1.0.0", &json);
         assert_eq!(b.name, "general-assistant");
         assert_eq!(b.skills, vec!["Web Search", "Code Review"]);
         assert_eq!(b.capabilities, vec!["Writing", "Planning"]);