Omar SobhandClaude Opus 5
76ac3714f1
Three fail-open paths on the §15 door: no env at all meant allow-all; a
governor that could not be reached approved with a WARNING; and a reply
that never said DENY — empty, truncated, a refusal — approved, because the
rule was !contains("DENY"). On the two days the judge plan emptied every
outbound action was approved by nobody.
Now: governor_allows() needs an explicit ALLOW and no DENY; both judge()
implementations return false when unreachable; with no governor the door
opens only on CLAWMATES_DOOR_POLICY=allow. Open Agent Passport (arXiv
2603.20953): 74.6% social-engineering success under a permissive policy,
0 of 879 under a restrictive one. Local override gains the governor prod
already runs.
skill_self_authoring: default flipped to OFF. No agent-authored skill has
ever been delivered to a mission or scored; prod held zero proposals.
Enable with CLAWMATES_SKILL_SELF_AUTHORING=1 once promoted skills go
through the files arm and get a Skill-Use score.
Co-Authored-By: Claude Opus 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01WZb5A2kfVfjpdwSochkuHz
834 lines
30 KiB
Rust
834 lines
30 KiB
Rust
//! Level-up proposer + applier — Slice 8.5.
|
|
//!
|
|
//! Two entry points:
|
|
//! propose_agent(agent_id) — reads agent's brain + last N runs,
|
|
//! asks the LLM to propose brain consolidation / identity
|
|
//! refinement / skill add / skill candidate items. Persists as
|
|
//! a pending `level_up_proposals` row.
|
|
//! propose_team(team_id) — aggregates each team agent's context +
|
|
//! recent mission outcomes; LLM proposes roster changes + MCP
|
|
//! bundle changes on top of per-role items.
|
|
//!
|
|
//! apply(proposal_id, approved_item_ids) commits only the ids the
|
|
//! reviewer picked. Rejected proposals move to status='rejected';
|
|
//! partial approvals move to status='partial'.
|
|
//!
|
|
//! The proposer model resolves through the provider REGISTRY
|
|
//! (`Runtime::resolve_provider`), the same path the evaluator uses, and defaults
|
|
//! to `glm:glm-4.7`. Configurable via `CLAWMATES_LEVEL_UP_MODEL` as a registry
|
|
//! spec (`glm:glm-4.7`, `kimi:k2`, `claude-sonnet-5`, …).
|
|
//!
|
|
//! It used to call Gemini directly over bespoke HTTP with `GEMINI_API_KEY`. Two
|
|
//! problems with that, one fatal: it was the only thing standing between this
|
|
//! feature and a dead prepayment balance, and it duplicated a provider client
|
|
//! the codebase already has. Going through the registry means every provider the
|
|
//! platform can already reach works here, and no single vendor's billing can
|
|
//! take the feature down.
|
|
|
|
use serde_json::{json, Value};
|
|
use sqlx::PgPool;
|
|
use sqlx::Row;
|
|
use uuid::Uuid;
|
|
|
|
/// Registry spec, not a bare model name — the registry needs the provider.
|
|
///
|
|
/// GLM: cheap, reliable at structured output, and already the validator this
|
|
/// project measured and chose (see `scripts/judge-eval.sh`).
|
|
const DEFAULT_MODEL: &str = "glm:glm-4.7";
|
|
|
|
fn model_name() -> String {
|
|
std::env::var("CLAWMATES_LEVEL_UP_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
|
}
|
|
|
|
/// Analyze an agent + insert a pending proposal. Returns the proposal id.
|
|
pub async fn propose_agent(
|
|
pool: &PgPool,
|
|
runtime: &cm_runtime::Runtime,
|
|
workspace_id: cm_domain::WorkspaceId,
|
|
created_by: cm_domain::UserId,
|
|
agent_id: Uuid,
|
|
) -> Result<Uuid, String> {
|
|
let agent = cm_db::repo::agents::get(pool, cm_domain::AgentId::from(agent_id))
|
|
.await
|
|
.map_err(|e| format!("load agent: {e}"))?;
|
|
if agent.workspace_id != workspace_id {
|
|
return Err("agent not in workspace".into());
|
|
}
|
|
|
|
let brain_summary = load_brain_summary(agent_id).await;
|
|
let recent_runs = recent_run_summary(pool, agent_id, 10).await?;
|
|
let link = cm_db::repo::agent_template_link::get(pool, agent_id)
|
|
.await
|
|
.ok()
|
|
.flatten();
|
|
|
|
let payload = call_llm_for_agent(
|
|
runtime,
|
|
&agent.name,
|
|
&agent.job_title,
|
|
&agent.system_prompt,
|
|
&brain_summary,
|
|
&recent_runs,
|
|
link.as_ref(),
|
|
)
|
|
.await?;
|
|
|
|
let model = model_name();
|
|
let id = cm_db::repo::level_up::insert(
|
|
pool,
|
|
cm_db::repo::level_up::NewProposal {
|
|
workspace_id: workspace_id.as_uuid(),
|
|
agent_id: Some(agent_id),
|
|
team_id: None,
|
|
payload: &payload,
|
|
model: Some(&model),
|
|
created_by: Some(created_by.as_uuid()),
|
|
},
|
|
)
|
|
.await
|
|
.map_err(|e| format!("insert proposal: {e}"))?;
|
|
Ok(id)
|
|
}
|
|
|
|
/// Analyze a team + insert a pending proposal. Returns the proposal id.
|
|
pub async fn propose_team(
|
|
pool: &PgPool,
|
|
runtime: &cm_runtime::Runtime,
|
|
workspace_id: cm_domain::WorkspaceId,
|
|
created_by: cm_domain::UserId,
|
|
team_id: Uuid,
|
|
) -> Result<Uuid, String> {
|
|
let members = sqlx::query(
|
|
"SELECT a.id, a.name, a.job_title, a.system_prompt, m.role_slot
|
|
FROM team_members m
|
|
JOIN agents a ON a.id = m.claw_id
|
|
WHERE m.team_id = $1
|
|
ORDER BY m.role_slot",
|
|
)
|
|
.bind(team_id)
|
|
.fetch_all(pool)
|
|
.await
|
|
.map_err(|e| format!("load members: {e}"))?;
|
|
|
|
if members.is_empty() {
|
|
return Err("team has no members".into());
|
|
}
|
|
|
|
let mut member_summaries: Vec<Value> = Vec::new();
|
|
for r in &members {
|
|
let id: Uuid = r.get("id");
|
|
let name: String = r.get("name");
|
|
let role: String = r.get("role_slot");
|
|
let brain = load_brain_summary(id).await;
|
|
let runs = recent_run_summary(pool, id, 3).await.unwrap_or_default();
|
|
member_summaries.push(json!({
|
|
"agent_id": id,
|
|
"name": name,
|
|
"role_slot": role,
|
|
"brain": brain,
|
|
"recent_runs": runs,
|
|
}));
|
|
}
|
|
|
|
let payload = call_llm_for_team(runtime, &member_summaries).await?;
|
|
|
|
let model = model_name();
|
|
let id = cm_db::repo::level_up::insert(
|
|
pool,
|
|
cm_db::repo::level_up::NewProposal {
|
|
workspace_id: workspace_id.as_uuid(),
|
|
agent_id: None,
|
|
team_id: Some(team_id),
|
|
payload: &payload,
|
|
model: Some(&model),
|
|
created_by: Some(created_by.as_uuid()),
|
|
},
|
|
)
|
|
.await
|
|
.map_err(|e| format!("insert team proposal: {e}"))?;
|
|
Ok(id)
|
|
}
|
|
|
|
/// Apply the reviewer-approved subset of a proposal. Item kinds
|
|
/// (from payload.suggested_items[].kind) each map to a small applier:
|
|
/// identity_refinement → agents.set_system_prompt (via patch)
|
|
/// skill_add → agent_skills_ext INSERT
|
|
/// skill_candidate → skills_catalog::upsert workspace-scoped
|
|
/// brain_consolidation → set_agent_md on the brain
|
|
/// roster_change → not automated in this slice (logs a
|
|
/// reminder — human runs the team-wizard)
|
|
/// mcp_bundle_change → not automated in this slice (same)
|
|
pub async fn apply(
|
|
pool: &PgPool,
|
|
workspace_id: cm_domain::WorkspaceId,
|
|
approved_by: cm_domain::UserId,
|
|
proposal_id: Uuid,
|
|
approved_item_ids: &[String],
|
|
) -> Result<(), String> {
|
|
let proposal = cm_db::repo::level_up::get(pool, proposal_id, workspace_id.as_uuid())
|
|
.await
|
|
.map_err(|e| format!("load proposal: {e}"))?
|
|
.ok_or_else(|| "proposal not found".to_string())?;
|
|
if proposal.status != "pending" {
|
|
return Err(format!("proposal already {}", proposal.status));
|
|
}
|
|
|
|
let items = proposal
|
|
.payload
|
|
.get("suggested_items")
|
|
.and_then(|v| v.as_array())
|
|
.cloned()
|
|
.unwrap_or_default();
|
|
|
|
let mut actually_applied: Vec<String> = Vec::new();
|
|
for item in items {
|
|
let Some(item_id) = item.get("id").and_then(|v| v.as_str()) else {
|
|
continue;
|
|
};
|
|
if !approved_item_ids.iter().any(|s| s == item_id) {
|
|
continue;
|
|
}
|
|
let kind = item.get("kind").and_then(|v| v.as_str()).unwrap_or("");
|
|
let applied = match kind {
|
|
"identity_refinement" => apply_identity(pool, &proposal, &item).await,
|
|
"skill_add" => apply_skill_add(pool, &proposal, &item).await,
|
|
"skill_candidate" => apply_skill_candidate(pool, &proposal, &item).await,
|
|
"brain_consolidation" => apply_brain_consolidation(&proposal, &item).await,
|
|
"roster_change" | "mcp_bundle_change" => {
|
|
eprintln!(
|
|
"level_up: {kind} item {item_id} — not auto-applied, human runs team-wizard"
|
|
);
|
|
Ok(())
|
|
}
|
|
other => {
|
|
eprintln!("level_up: unknown item kind `{other}` — skipping {item_id}");
|
|
Ok(())
|
|
}
|
|
};
|
|
match applied {
|
|
Ok(()) => actually_applied.push(item_id.to_string()),
|
|
Err(e) => eprintln!("level_up: apply {item_id} failed: {e}"),
|
|
}
|
|
}
|
|
|
|
let partial = actually_applied.len() != approved_item_ids.len();
|
|
cm_db::repo::level_up::mark_applied(
|
|
pool,
|
|
proposal_id,
|
|
workspace_id.as_uuid(),
|
|
approved_by.as_uuid(),
|
|
&actually_applied,
|
|
partial,
|
|
)
|
|
.await
|
|
.map_err(|e| format!("mark applied: {e}"))?;
|
|
Ok(())
|
|
}
|
|
|
|
/// Is autonomous skill authoring on?
|
|
///
|
|
/// Default OFF since 2026-09-20, by operator decision. It shipped default ON,
|
|
/// and in the months since no agent-authored skill was ever delivered to a
|
|
/// mission or scored by the Skill-Use scorer — prod's `level_up_proposals`
|
|
/// held zero rows on the day of the flip. An auto-apply loop whose output has
|
|
/// never been measured is a supply chain of our own making (the shape Cisco
|
|
/// found in OpenClaw's third-party skills), so it waits for a human until
|
|
/// `promoted_from_brain` skills go through the `files` delivery arm and get
|
|
/// a Trigger/Compliance score like the hand-authored ones. Stated at boot
|
|
/// either way: a safety gate that changes state silently is how nobody
|
|
/// notices it changed.
|
|
pub fn self_authoring_enabled() -> bool {
|
|
matches!(
|
|
std::env::var("CLAWMATES_SKILL_SELF_AUTHORING")
|
|
.unwrap_or_default()
|
|
.trim()
|
|
.to_ascii_lowercase()
|
|
.as_str(),
|
|
"1" | "on" | "true"
|
|
)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod self_authoring_flag_tests {
|
|
/// Serialised through one env var; each case restores the prior state.
|
|
fn with(value: Option<&str>, f: impl FnOnce()) {
|
|
let key = "CLAWMATES_SKILL_SELF_AUTHORING";
|
|
let prior = std::env::var(key).ok();
|
|
match value {
|
|
Some(v) => std::env::set_var(key, v),
|
|
None => std::env::remove_var(key),
|
|
}
|
|
f();
|
|
match prior {
|
|
Some(v) => std::env::set_var(key, v),
|
|
None => std::env::remove_var(key),
|
|
}
|
|
}
|
|
|
|
/// Off unless switched on. The previous default was the reverse.
|
|
#[test]
|
|
fn off_by_default_on_by_explicit_opt_in() {
|
|
with(None, || assert!(!super::self_authoring_enabled()));
|
|
with(Some(""), || assert!(!super::self_authoring_enabled()));
|
|
with(Some("0"), || assert!(!super::self_authoring_enabled()));
|
|
with(Some("yes"), || assert!(!super::self_authoring_enabled()));
|
|
with(Some("1"), || assert!(super::self_authoring_enabled()));
|
|
with(Some("on"), || assert!(super::self_authoring_enabled()));
|
|
with(Some("TRUE"), || assert!(super::self_authoring_enabled()));
|
|
}
|
|
}
|
|
|
|
/// Apply a pending proposal's `skill_candidate` items with no human decision.
|
|
///
|
|
/// ONLY `skill_candidate`. The other item kinds are deliberately left to the
|
|
/// human gate: `identity_refinement` rewrites an agent's system prompt and
|
|
/// `brain_consolidation` edits its memory, and both change what the agent IS
|
|
/// rather than adding a procedure it can consult. Self-authoring a skill is
|
|
/// recoverable — the row is workspace-scoped, versioned and revertible, and
|
|
/// cannot take a hand-authored name. Rewriting an identity autonomously is not
|
|
/// the same bet, and it is not the one that was asked for.
|
|
///
|
|
/// The remaining items stay pending, so a human still sees them.
|
|
pub async fn apply_autonomous(
|
|
pool: &PgPool,
|
|
workspace_id: cm_domain::WorkspaceId,
|
|
proposal_id: Uuid,
|
|
) -> Result<Vec<String>, String> {
|
|
let proposal = cm_db::repo::level_up::get(pool, proposal_id, workspace_id.as_uuid())
|
|
.await
|
|
.map_err(|e| format!("load proposal: {e}"))?
|
|
.ok_or_else(|| "proposal not found".to_string())?;
|
|
if proposal.status != "pending" {
|
|
return Err(format!("proposal already {}", proposal.status));
|
|
}
|
|
|
|
let items = proposal
|
|
.payload
|
|
.get("suggested_items")
|
|
.and_then(|v| v.as_array())
|
|
.cloned()
|
|
.unwrap_or_default();
|
|
|
|
let mut applied: Vec<String> = Vec::new();
|
|
let mut candidates = 0usize;
|
|
for item in items {
|
|
let Some(item_id) = item.get("id").and_then(|v| v.as_str()) else {
|
|
continue;
|
|
};
|
|
if item.get("kind").and_then(|v| v.as_str()) != Some("skill_candidate") {
|
|
continue;
|
|
}
|
|
candidates += 1;
|
|
match apply_skill_candidate(pool, &proposal, &item).await {
|
|
Ok(()) => applied.push(item_id.to_string()),
|
|
// A refused draft is a normal outcome (a name collision with a
|
|
// hand-authored skill is the common one), not a failure of the
|
|
// sweep. Said out loud so a refusal is never mistaken for the
|
|
// agent simply not having proposed anything.
|
|
Err(e) => eprintln!(
|
|
"level_up: autonomous apply refused {item_id} for workspace {}: {e}",
|
|
workspace_id.as_uuid()
|
|
),
|
|
}
|
|
}
|
|
|
|
if candidates == 0 {
|
|
return Ok(Vec::new());
|
|
}
|
|
cm_db::repo::level_up::mark_applied_autonomously(
|
|
pool,
|
|
proposal_id,
|
|
workspace_id.as_uuid(),
|
|
&applied,
|
|
applied.len() != candidates,
|
|
)
|
|
.await
|
|
.map_err(|e| format!("mark applied: {e}"))?;
|
|
Ok(applied)
|
|
}
|
|
|
|
// ── Appliers ───────────────────────────────────────────────────
|
|
|
|
async fn apply_identity(
|
|
pool: &PgPool,
|
|
proposal: &cm_db::repo::level_up::LevelUpProposal,
|
|
item: &Value,
|
|
) -> Result<(), String> {
|
|
let Some(agent_id) = proposal.agent_id else {
|
|
return Err("identity_refinement requires an agent proposal".into());
|
|
};
|
|
let Some(new_prompt) = item.get("new_system_prompt").and_then(|v| v.as_str()) else {
|
|
return Err("missing new_system_prompt".into());
|
|
};
|
|
sqlx::query("UPDATE agents SET system_prompt = $1 WHERE id = $2")
|
|
.bind(new_prompt)
|
|
.bind(agent_id)
|
|
.execute(pool)
|
|
.await
|
|
.map_err(|e| format!("update agent: {e}"))?;
|
|
Ok(())
|
|
}
|
|
|
|
async fn apply_skill_add(
|
|
pool: &PgPool,
|
|
proposal: &cm_db::repo::level_up::LevelUpProposal,
|
|
item: &Value,
|
|
) -> Result<(), String> {
|
|
let Some(agent_id) = proposal.agent_id else {
|
|
return Err("skill_add on team proposal — use per-agent override".into());
|
|
};
|
|
let Some(skill_id_str) = item.get("skill_id").and_then(|v| v.as_str()) else {
|
|
return Err("missing skill_id".into());
|
|
};
|
|
let skill_id = Uuid::parse_str(skill_id_str).map_err(|e| format!("parse skill_id: {e}"))?;
|
|
let pin = item
|
|
.get("pin_in_context")
|
|
.and_then(|v| v.as_bool())
|
|
.unwrap_or(false);
|
|
cm_db::repo::skills_catalog::set_agent_skill(
|
|
pool,
|
|
agent_id,
|
|
skill_id,
|
|
true,
|
|
pin,
|
|
proposal.approved_by,
|
|
item.get("rationale").and_then(|v| v.as_str()),
|
|
)
|
|
.await
|
|
.map_err(|e| format!("set agent skill: {e}"))?;
|
|
Ok(())
|
|
}
|
|
|
|
async fn apply_skill_candidate(
|
|
pool: &PgPool,
|
|
proposal: &cm_db::repo::level_up::LevelUpProposal,
|
|
item: &Value,
|
|
) -> Result<(), String> {
|
|
let draft = item
|
|
.get("draft")
|
|
.ok_or_else(|| "missing draft".to_string())?;
|
|
let name = draft
|
|
.get("name")
|
|
.and_then(|v| v.as_str())
|
|
.ok_or_else(|| "draft.name missing".to_string())?;
|
|
let description = draft
|
|
.get("description")
|
|
.and_then(|v| v.as_str())
|
|
.unwrap_or("(no description)");
|
|
let body = draft.get("body").and_then(|v| v.as_str()).unwrap_or("");
|
|
let when_to_use = draft.get("when_to_use").and_then(|v| v.as_str());
|
|
let tags: Vec<String> = draft
|
|
.get("tags")
|
|
.and_then(|v| v.as_array())
|
|
.map(|a| {
|
|
a.iter()
|
|
.filter_map(|v| v.as_str().map(|s| s.to_string()))
|
|
.collect()
|
|
})
|
|
.unwrap_or_default();
|
|
// A draft may never take the name of a hand-authored skill.
|
|
//
|
|
// The row itself is safe — ids are workspace-scoped, so this cannot
|
|
// overwrite a builtin, and bindings resolve by skill_id rather than name,
|
|
// so it cannot shadow one either. What it CAN do is put two different
|
|
// procedures under one name in the same agent's bundle, and then nobody
|
|
// reading a transcript can tell which one the agent followed. That
|
|
// ambiguity is the whole problem in a system where the skill is the
|
|
// standard the behaviour is graded against.
|
|
let collides: Option<Uuid> = sqlx::query_scalar(
|
|
"SELECT id FROM skills WHERE name = $1 AND workspace_id IS NULL",
|
|
)
|
|
.bind(name)
|
|
.fetch_optional(pool)
|
|
.await
|
|
.map_err(|e| format!("check builtin collision: {e}"))?;
|
|
if collides.is_some() {
|
|
return Err(format!(
|
|
"skill name {name:?} is hand-authored — an agent-authored draft \
|
|
cannot take the name of a skill it is graded against"
|
|
));
|
|
}
|
|
|
|
// Workspace-scoped custom skill. Deterministic id per
|
|
// (workspace, name) so re-approving the same draft updates in
|
|
// place rather than duplicating.
|
|
let id = workspace_skill_id(proposal.workspace_id, name);
|
|
|
|
// Versioned, for the same reason builtins are: a self-authored skill that
|
|
// silently replaces its own body has no undo, and the version a run was
|
|
// judged under is the only way to read that run back honestly later.
|
|
let mut tx = pool.begin().await.map_err(|e| format!("begin: {e}"))?;
|
|
let existing: Option<(i32, String)> =
|
|
sqlx::query_as("SELECT current_version, body FROM skills WHERE id = $1")
|
|
.bind(id)
|
|
.fetch_optional(&mut *tx)
|
|
.await
|
|
.map_err(|e| format!("read current skill: {e}"))?;
|
|
let (next_version, bump) = match &existing {
|
|
Some((v, prev)) if prev == body => (*v, false),
|
|
Some((v, _)) => (v + 1, true),
|
|
None => (1, true),
|
|
};
|
|
|
|
sqlx::query(
|
|
"INSERT INTO skills
|
|
(id, name, title, author, description, when_to_use, tags,
|
|
source_kind, workspace_id, current_version, body)
|
|
VALUES ($1,$2,$2,'level_up',$3,$4,$5,'promoted_from_brain',$6,$8,$7)
|
|
ON CONFLICT (id) DO UPDATE SET
|
|
description = EXCLUDED.description,
|
|
when_to_use = EXCLUDED.when_to_use,
|
|
tags = EXCLUDED.tags,
|
|
body = EXCLUDED.body,
|
|
current_version = EXCLUDED.current_version,
|
|
updated_at = now()",
|
|
)
|
|
.bind(id)
|
|
.bind(name)
|
|
.bind(description)
|
|
.bind(when_to_use)
|
|
.bind(&tags)
|
|
.bind(proposal.workspace_id)
|
|
.bind(body)
|
|
.bind(next_version)
|
|
.execute(&mut *tx)
|
|
.await
|
|
.map_err(|e| format!("upsert skill draft: {e}"))?;
|
|
|
|
if bump {
|
|
sqlx::query(
|
|
"INSERT INTO skill_versions
|
|
(skill_id, version, body_md, description, when_to_use)
|
|
VALUES ($1,$2,$3,$4,$5)
|
|
ON CONFLICT DO NOTHING",
|
|
)
|
|
.bind(id)
|
|
.bind(next_version)
|
|
.bind(body)
|
|
.bind(description)
|
|
.bind(when_to_use)
|
|
.execute(&mut *tx)
|
|
.await
|
|
.map_err(|e| format!("record skill version: {e}"))?;
|
|
}
|
|
tx.commit().await.map_err(|e| format!("commit: {e}"))?;
|
|
Ok(())
|
|
}
|
|
|
|
async fn apply_brain_consolidation(
|
|
proposal: &cm_db::repo::level_up::LevelUpProposal,
|
|
item: &Value,
|
|
) -> Result<(), String> {
|
|
let Some(agent_id) = proposal.agent_id else {
|
|
return Err("brain_consolidation requires agent proposal".into());
|
|
};
|
|
let Some(new_agent_md) = item.get("brain_md_diff").and_then(|v| v.as_str()) else {
|
|
return Err("missing brain_md_diff".into());
|
|
};
|
|
// Overwrites agent_md — level-up is the sanctioned path, unlike
|
|
// brain_seed::ingest which skips if agent_md is populated.
|
|
let agent_id_owned = agent_id;
|
|
let md_owned = new_agent_md.to_string();
|
|
tokio::task::spawn_blocking(move || -> Result<(), String> {
|
|
use cm_brain::ClawBrain;
|
|
let dir = std::env::var("CLAWMATES_BRAIN_DIR")
|
|
.ok()
|
|
.map(std::path::PathBuf::from)
|
|
.unwrap_or_else(|| std::path::PathBuf::from("/data/brains"));
|
|
let path = dir.join(format!("claw_{agent_id_owned}.h5"));
|
|
let mut brain = ClawBrain::open_or_create(&path, &agent_id_owned.to_string())
|
|
.map_err(|e| format!("open brain: {e}"))?;
|
|
brain
|
|
.set_agent_md(&md_owned)
|
|
.map_err(|e| format!("set agent_md: {e}"))?;
|
|
brain
|
|
.commit(Some("level_up: brain consolidation"))
|
|
.map_err(|e| format!("commit: {e}"))?;
|
|
Ok(())
|
|
})
|
|
.await
|
|
.map_err(|e| format!("brain task join: {e}"))?
|
|
}
|
|
|
|
// ── Helpers ────────────────────────────────────────────────────
|
|
|
|
async fn load_brain_summary(agent_id: Uuid) -> Value {
|
|
tokio::task::spawn_blocking(move || {
|
|
use cm_brain::ClawBrain;
|
|
let dir = std::env::var("CLAWMATES_BRAIN_DIR")
|
|
.ok()
|
|
.map(std::path::PathBuf::from)
|
|
.unwrap_or_else(|| std::path::PathBuf::from("/data/brains"));
|
|
let path = dir.join(format!("claw_{agent_id}.h5"));
|
|
let Ok(brain) = ClawBrain::open_or_create(&path, &agent_id.to_string()) else {
|
|
return json!({ "available": false });
|
|
};
|
|
let agent_md = brain.agent_md().unwrap_or_default();
|
|
let skills: Vec<Value> = brain
|
|
.skills()
|
|
.into_iter()
|
|
.map(|(n, b)| json!({ "name": n, "body_excerpt": excerpt(&b, 400) }))
|
|
.collect();
|
|
json!({
|
|
"available": true,
|
|
"agent_md_excerpt": excerpt(&agent_md, 2000),
|
|
"agent_md_bytes": agent_md.len(),
|
|
"skills": skills,
|
|
})
|
|
})
|
|
.await
|
|
.unwrap_or_else(|_| json!({ "available": false }))
|
|
}
|
|
|
|
async fn recent_run_summary(pool: &PgPool, agent_id: Uuid, limit: i64) -> Result<Value, String> {
|
|
// Runs the agent participated in — via team_members.claw_id +
|
|
// team_id + mission_id. Cheap best-effort join; missing → empty.
|
|
let rows = sqlx::query(
|
|
"SELECT tr.id, tr.status, tr.error, tr.created_at
|
|
FROM topology_runs tr
|
|
WHERE tr.mission_id IN (
|
|
SELECT m.id FROM missions m
|
|
JOIN team_members tm ON tm.team_id = m.team_id
|
|
WHERE tm.claw_id = $1
|
|
)
|
|
ORDER BY tr.created_at DESC LIMIT $2",
|
|
)
|
|
.bind(agent_id)
|
|
.bind(limit)
|
|
.fetch_all(pool)
|
|
.await
|
|
.map_err(|e| format!("load runs: {e}"))?;
|
|
let items: Vec<Value> = rows
|
|
.into_iter()
|
|
.map(|r| {
|
|
let id: Uuid = r.get("id");
|
|
let status: String = r.get("status");
|
|
let err: Option<String> = r.get("error");
|
|
json!({
|
|
"run_id": id,
|
|
"status": status,
|
|
"error": err,
|
|
})
|
|
})
|
|
.collect();
|
|
Ok(json!(items))
|
|
}
|
|
|
|
async fn call_llm_for_agent(
|
|
runtime: &cm_runtime::Runtime,
|
|
name: &str,
|
|
role: &str,
|
|
system_prompt: &str,
|
|
brain: &Value,
|
|
runs: &Value,
|
|
link: Option<&cm_db::repo::agent_template_link::AgentTemplateLink>,
|
|
) -> Result<Value, String> {
|
|
let template_note = link
|
|
.map(|l| {
|
|
format!(
|
|
"Template lineage: template {} v{} role {}.",
|
|
l.template_id, l.template_version, l.role_slot
|
|
)
|
|
})
|
|
.unwrap_or_else(|| "No template lineage (LLM-derived or manual).".to_string());
|
|
|
|
let system = r#"You review an AI agent's history and propose targeted improvements.
|
|
Return ONLY JSON matching this schema:
|
|
{
|
|
"kind": "agent",
|
|
"current": { "system_prompt": "<current>", "skills": [] },
|
|
"suggested_items": [
|
|
// 0..5 items, each with `id`, `kind`, `rationale`.
|
|
// kind ∈ {identity_refinement, skill_candidate, skill_add, brain_consolidation}
|
|
// identity_refinement: extra `new_system_prompt`
|
|
// skill_candidate: extra `draft: { name, description, when_to_use, body, tags[] }`
|
|
// skill_add: extra `skill_id`
|
|
// brain_consolidation: extra `brain_md_diff` (full replacement text)
|
|
]
|
|
}
|
|
Propose changes only when there's evidence — a clear pattern in the
|
|
brain or a failure in recent runs. Do not propose changes purely for
|
|
the sake of proposing."#;
|
|
|
|
let user = json!({
|
|
"name": name,
|
|
"role_slot": role,
|
|
"system_prompt": system_prompt,
|
|
"brain": brain,
|
|
"recent_runs": runs,
|
|
"template_lineage_note": template_note,
|
|
})
|
|
.to_string();
|
|
|
|
call_llm_json(runtime, system, &user).await
|
|
}
|
|
|
|
async fn call_llm_for_team(
|
|
runtime: &cm_runtime::Runtime,
|
|
members: &[Value],
|
|
) -> Result<Value, String> {
|
|
let system = r#"You review an AI team's roster + recent history and propose
|
|
targeted improvements. Return ONLY JSON:
|
|
{
|
|
"kind": "team",
|
|
"suggested_items": [
|
|
// 0..8 items. Same shapes as agent, plus:
|
|
// roster_change: { op: "add"|"drop"|"rename", slot, rationale }
|
|
// mcp_bundle_change:{ op: "add"|"drop", bundle, rationale }
|
|
// Per-agent items should carry an extra `agent_id` field.
|
|
]
|
|
}
|
|
Prefer removing unused roles over adding new ones. Prefer tightening
|
|
prompts over adding skills. Only add skills when a clear
|
|
"the team keeps getting stuck on <X>" pattern appears."#;
|
|
|
|
let user = json!({ "members": members }).to_string();
|
|
call_llm_json(runtime, system, &user).await
|
|
}
|
|
|
|
/// Ask the configured proposer model for one JSON object.
|
|
///
|
|
/// Goes through the provider registry rather than a vendor's HTTP API, so any
|
|
/// model the platform can already reach works and no single vendor's billing can
|
|
/// take level-up down.
|
|
///
|
|
/// The JSON is extracted rather than assumed: an anthropic-format model is not
|
|
/// bound by Gemini's `response_mime_type: application/json`, and will happily
|
|
/// wrap an object in prose or a ```json fence. Parsing the raw reply worked
|
|
/// against Gemini and would fail on everything else.
|
|
async fn call_llm_json(
|
|
runtime: &cm_runtime::Runtime,
|
|
system: &str,
|
|
user: &str,
|
|
) -> Result<Value, String> {
|
|
use cm_llm::{ChatMessage, ChatRequest, ChatRole, ContentPart, LlmEvent};
|
|
use futures::StreamExt as _;
|
|
|
|
let spec = model_name();
|
|
let (provider, model) = runtime.resolve_provider(&spec);
|
|
let request = ChatRequest {
|
|
system: system.to_string(),
|
|
model: model.to_string(),
|
|
messages: vec![ChatMessage {
|
|
role: ChatRole::User,
|
|
parts: vec![ContentPart::text(user)],
|
|
}],
|
|
tools: vec![],
|
|
max_tokens: 8192,
|
|
web_search: false,
|
|
};
|
|
let mut stream = provider
|
|
.stream(request)
|
|
.await
|
|
.map_err(|e| format!("level-up call ({spec}): {e}"))?;
|
|
let mut text = String::new();
|
|
while let Some(event) = stream.next().await {
|
|
match event {
|
|
Ok(LlmEvent::TextDelta(t)) => text.push_str(&t),
|
|
Ok(_) => {}
|
|
Err(e) => return Err(format!("level-up stream ({spec}): {e}")),
|
|
}
|
|
}
|
|
let body = extract_json_object(&text)
|
|
.ok_or_else(|| format!("no JSON object in {spec} reply: {}", excerpt(&text, 300)))?;
|
|
serde_json::from_str(body).map_err(|e| format!("parse suggestion json: {e}"))
|
|
}
|
|
|
|
/// The outermost `{...}` in a reply, so a fenced or prose-wrapped object parses.
|
|
///
|
|
/// Brace-counting rather than a regex: a nested object would end a lazy match at
|
|
/// the first inner `}`, and these proposals are nested by design (items carry
|
|
/// per-role objects).
|
|
fn extract_json_object(text: &str) -> Option<&str> {
|
|
let start = text.find('{')?;
|
|
let mut depth = 0usize;
|
|
let mut in_string = false;
|
|
let mut escaped = false;
|
|
for (i, c) in text[start..].char_indices() {
|
|
if in_string {
|
|
match c {
|
|
_ if escaped => escaped = false,
|
|
'\\' => escaped = true,
|
|
'"' => in_string = false,
|
|
_ => {}
|
|
}
|
|
continue;
|
|
}
|
|
match c {
|
|
'"' => in_string = true,
|
|
'{' => depth += 1,
|
|
'}' => {
|
|
depth -= 1;
|
|
if depth == 0 {
|
|
return Some(&text[start..start + i + 1]);
|
|
}
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
fn excerpt(s: &str, max: usize) -> String {
|
|
if s.len() <= max {
|
|
s.to_string()
|
|
} else {
|
|
format!("{}...", &s[..max])
|
|
}
|
|
}
|
|
|
|
fn workspace_skill_id(workspace_id: Uuid, name: &str) -> Uuid {
|
|
use sha2::{Digest, Sha256};
|
|
let mut h = Sha256::new();
|
|
h.update(b"clawmates.workspace.skill\x00");
|
|
h.update(workspace_id.as_bytes());
|
|
h.update(name.as_bytes());
|
|
let d = h.finalize();
|
|
let mut bytes = [0u8; 16];
|
|
bytes.copy_from_slice(&d[..16]);
|
|
bytes[6] = (bytes[6] & 0x0f) | 0x40;
|
|
bytes[8] = (bytes[8] & 0x3f) | 0x80;
|
|
Uuid::from_bytes(bytes)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
/// Gemini was asked for `response_mime_type: application/json` and obliged.
|
|
/// Anthropic-format models are under no such obligation and routinely wrap
|
|
/// the object in prose or a fenced block, so the reply is EXTRACTED, not
|
|
/// assumed. Parsing the raw text worked against Gemini and would fail
|
|
/// everywhere else — exactly the shape of bug a provider swap hides until
|
|
/// the first real proposal.
|
|
#[test]
|
|
fn a_json_object_is_extracted_from_however_the_model_wrapped_it() {
|
|
let bare = r#"{"items":[]}"#;
|
|
assert_eq!(super::extract_json_object(bare), Some(bare));
|
|
|
|
let fenced = "Here is my proposal:\n```json\n{\"items\":[1]}\n```\nDone.";
|
|
assert_eq!(super::extract_json_object(fenced), Some(r#"{"items":[1]}"#));
|
|
|
|
// Nested objects: a lazy match would stop at the first inner brace and
|
|
// hand back invalid JSON. These proposals are nested by design.
|
|
let nested = r#"prose {"a":{"b":{"c":1}},"d":2} trailing"#;
|
|
assert_eq!(
|
|
super::extract_json_object(nested),
|
|
Some(r#"{"a":{"b":{"c":1}},"d":2}"#)
|
|
);
|
|
|
|
// A brace inside a string must not close the object.
|
|
let stringy = r#"{"note":"an unmatched } here","ok":true}"#;
|
|
assert_eq!(super::extract_json_object(stringy), Some(stringy));
|
|
|
|
assert_eq!(super::extract_json_object("no object here"), None);
|
|
}
|
|
|
|
/// The default must not be a vendor whose billing already took a feature
|
|
/// down. It is a REGISTRY SPEC (`provider:model`), not a bare model name —
|
|
/// `resolve_provider` needs the provider half.
|
|
#[test]
|
|
fn the_default_proposer_is_a_registry_spec_and_not_gemini() {
|
|
assert!(super::DEFAULT_MODEL.contains(':'), "{}", super::DEFAULT_MODEL);
|
|
assert!(!super::DEFAULT_MODEL.contains("gemini"), "{}", super::DEFAULT_MODEL);
|
|
}
|
|
}
|