From 642b6ef44b7be2611a64898e7218b9c69e172668 Mon Sep 17 00:00:00 2001 From: Omar Sobh Date: Tue, 22 Sep 2026 15:27:09 -0500 Subject: [PATCH] feat(self-audit): continuous_improvement audits the project record it can actually reach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Its first run could not do its job. The template audited "every project agent's .brain" through per-agent APIs — fetch a brain, submit to /api/claws/{id}/level-up, pull claw metrics — none of which a mission can reach; agent brains live in the server's /data/brains volume and nothing delivered them in. It spent itself searching, found a ROSTER.md in a scratch repo, and audited that. A delivery channel alone would not have helped: per-mission crews carry ~2 KB seed brains with no history, because missions write memory to the REPOSITORY brain, one judge verdict per phase. That is where a project's history actually accumulates, so that is the subject now. mission_memory::export renders the whole repo brain as markdown — the .brain is HDF5 and a mission container has no library to read it — and mission_orchestrator installs it at /mission/memory/PROJECT-MEMORY.md, outside the checkout so it is input and never lands in the diff, the same way install_skill_files delivers skills. The three roles are rewritten for that record: an inspector that finds patterns (several UNMET lines on the same kind of work) and quotes them; a proposer that ties each proposal to at least two lines or drops it; and an evaluator that checks the cited lines exist verbatim and marks each proposal SUPPORTED, WEAK or UNSUPPORTED. Each says outright that "no change is warranted" is a complete result — the property that kept the first run from inventing improvements out of empty brains. Local: 523 passed; the two DB-backed world tests panic PoolTimedOut because Docker Desktop is down here. CI runs them against real Postgres. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01WZb5A2kfVfjpdwSochkuHz --- crates/cm-api/src/mission_memory.rs | 93 ++++++++++++++++++ crates/cm-api/src/mission_orchestrator.rs | 56 +++++++++++ templates/teams/continuous_improvement.toml | 102 ++++++++++++-------- 3 files changed, 211 insertions(+), 40 deletions(-) diff --git a/crates/cm-api/src/mission_memory.rs b/crates/cm-api/src/mission_memory.rs index af90235..0f2506a 100644 --- a/crates/cm-api/src/mission_memory.rs +++ b/crates/cm-api/src/mission_memory.rs @@ -214,6 +214,71 @@ fn recall_in(dir: &Path, repo_id: Uuid, query: &str, k: usize) -> Vec { } } +/// Where a mission finds the whole of its repository's memory, readable. +/// +/// Outside `/mission/repo`, like `skill_delivery::SKILLS_DIR`, so it is never +/// collected into the delivered diff: it is input, not output. +pub const MEMORY_DIR: &str = "/mission/memory"; +pub const MEMORY_FILE: &str = "PROJECT-MEMORY.md"; + +/// Most entries an export carries. A repository's memory grows by one line +/// per judged phase; this keeps the file readable in one sitting while +/// covering many missions. +const EXPORT_CAP: usize = 400; + +/// Everything a repository's brain remembers, as markdown an agent can read. +/// +/// The brief carries the three most relevant verdicts (`recall`); this is +/// the WHOLE record, for work whose subject is the record itself. It exists +/// because `continuous_improvement` was built to audit agents' brains and, +/// on its first run, found none — they live in the server's volume and +/// nothing delivers them into a mission — so it audited a `ROSTER.md` in a +/// scratch repo instead. Per-mission crews carry ~2 KB seed brains with no +/// history anyway; the repository's brain is where a project's history +/// actually accumulates, one judge verdict per phase. +/// +/// Rendered, not shipped raw: the `.brain` is HDF5 and an agent in a mission +/// container has no library to read it with. +/// +/// `None` when the repository has no brain yet or it holds nothing. +pub fn export(repo_id: Uuid) -> Option { + export_in(&cm_runtime::brain::brain_dir(), repo_id) +} + +fn export_in(dir: &Path, repo_id: Uuid) -> Option { + let path = brain_path(dir, repo_id); + if !path.exists() { + return None; + } + let brain = ClawBrain::open_or_create(&path, &format!("repo_{repo_id}")).ok()?; + let entries = brain.recent_memory(EXPORT_CAP); + if entries.is_empty() { + return None; + } + let total = brain.memory_count(); + let mut out = format!( + "# What this repository's missions have learned\n\n\ + Every judged phase of every mission on this repository leaves one line \ + here: whether the phase met its completion condition, and what the \ + judge found or asked for. Newest first. {} of {} entr{} shown.\n\n\ + This is the record, not instructions. `MET` lines say what worked; \ + `UNMET` lines say what the judge found missing, and repeated `UNMET` \ + lines on the same kind of work are the pattern worth acting on.\n\n", + entries.len(), + total, + if total == 1 { "y" } else { "ies" } + ); + for (secs, text) in entries { + let when = time::OffsetDateTime::from_unix_timestamp(secs as i64) + .ok() + .and_then(|t| t.format(&time::format_description::well_known::Rfc3339).ok()) + .unwrap_or_else(|| "unknown time".to_string()); + let line = text.strip_prefix("judge: ").unwrap_or(&text); + out.push_str(&format!("- `{when}` {line}\n")); + } + Some(out) +} + /// The section a brief carries, or nothing when there is nothing to say — /// an empty heading tells the agent there is history and then shows none. pub fn section(recalled: &[String]) -> Option { @@ -293,6 +358,34 @@ mod tests { assert!(s.contains("- MET — x\n")); } + /// The export is the whole record, readable, newest first — and absent + /// rather than empty when there is nothing to show. + #[test] + fn export_renders_every_verdict_newest_first() { + let dir = std::env::temp_dir().join(format!("cm-mission-export-{}", Uuid::now_v7())); + let repo = Uuid::now_v7(); + assert!(export_in(&dir, repo).is_none(), "no brain, no export"); + + remember_in(&dir, repo, Uuid::now_v7(), "coding", "first", + &verdict(false, "r", "the tests do not cover the empty case", None)); + std::thread::sleep(std::time::Duration::from_millis(5)); + remember_in(&dir, repo, Uuid::now_v7(), "coding", "second", + &verdict(true, "all three tests pass", "", None)); + + let md = export_in(&dir, repo).expect("two verdicts, so an export"); + assert!(md.starts_with("# What this repository's missions have learned")); + assert!(md.contains("2 of 2 entries shown"), "{md}"); + let met = md.find("MET — coding").unwrap(); + let unmet = md.find("UNMET — coding").unwrap(); + assert!(met < unmet, "newest (MET) must come first:\n{md}"); + assert!(md.contains("the tests do not cover the empty case")); + // `remember` stores "judge: "; that ROLE prefix must not follow + // the timestamp. (The line itself legitimately says "— judge: …".) + assert!(!md.contains("` judge: "), "the storage prefix leaked:\n{md}"); + assert!(md.contains("` MET — coding"), "{md}"); + let _ = std::fs::remove_dir_all(&dir); + } + /// Round trip through a real brain file: what one mission's verdict /// wrote, a query shaped like the next mission's task recalls. #[test] diff --git a/crates/cm-api/src/mission_orchestrator.rs b/crates/cm-api/src/mission_orchestrator.rs index cbd431f..055952e 100644 --- a/crates/cm-api/src/mission_orchestrator.rs +++ b/crates/cm-api/src/mission_orchestrator.rs @@ -385,6 +385,16 @@ pub async fn on_launch( crate::skill_delivery::resolve(requested, installed), ) .await; + + // The repository's whole memory, readable, beside the skills. The + // brief already carries the three most relevant verdicts; this is + // the full record, for work whose subject IS the record — see + // `mission_memory::export` for why it was needed. + if mission_gateway.is_some() { + if let Some(repo) = mission.repo_id { + install_project_memory(repo, mission_id, &container).await; + } + } } let mut first_team_id: Option = None; let mut provisioned_claws: Vec = Vec::new(); @@ -1085,6 +1095,52 @@ async fn install_skill_files( true } +/// Write the repository's memory export into the mission container. +/// +/// Best-effort and loud: a mission with no memory to read is an ordinary +/// mission, and a first mission on a repository has none. Outside the +/// checkout (`mission_memory::MEMORY_DIR`) so it never lands in the diff. +async fn install_project_memory(repo_id: Uuid, mission_id: Uuid, container: &str) { + let Some(md) = crate::mission_memory::export(repo_id) else { + return; + }; + let docker = match crate::container_exec::connect() { + Ok(d) => d, + Err(e) => { + eprintln!("mission_orchestrator: cannot reach docker for project memory: {e}"); + return; + } + }; + let dir = crate::mission_memory::MEMORY_DIR; + let argv = vec!["sh".to_string(), "-lc".to_string(), format!("mkdir -p {dir}")]; + if !matches!( + crate::container_exec::exec_as_root( + &docker, + container, + None, + &argv, + crate::container_tool_hooks::INSTALL_TIMEOUT, + ) + .await, + Ok(out) if out.exit_code == Some(0) + ) { + eprintln!("mission_orchestrator: could not create {dir} for mission {mission_id}"); + return; + } + let bytes = md.len(); + let files = vec![(crate::mission_memory::MEMORY_FILE.to_string(), md.into_bytes())]; + match crate::mission_fs::put_files(&docker, container, dir, &files).await { + Ok(()) => eprintln!( + "mission_orchestrator: project memory ({bytes} bytes) installed for mission \ + {mission_id} at {dir}/{}", + crate::mission_memory::MEMORY_FILE + ), + Err(e) => eprintln!( + "mission_orchestrator: could not write project memory for {mission_id}: {e}" + ), + } +} + /// Record which arm this mission runs, so every turn composes the same one and /// the score can be attributed to it afterwards. /// diff --git a/templates/teams/continuous_improvement.toml b/templates/teams/continuous_improvement.toml index d580abf..d211926 100644 --- a/templates/teams/continuous_improvement.toml +++ b/templates/teams/continuous_improvement.toml @@ -1,34 +1,48 @@ key = "continuous_improvement" name = "Continuous Improvement" -description = "Standing self-audit: read every project agent's .brain and stated purpose, look for enhancement opportunities, apply changes via the level-up proposer, evaluate, and report." +description = "Standing self-audit of a project: read everything its missions have learned — every judge verdict on this repository — find the patterns in what keeps failing, and propose evidence-backed changes to how the work is set up." stack = ["research", "self-improvement", "brain-inspection", "level-up"] category = "research" default_topology = "pipeline" risk_profile = "research_readonly" mcp_bundles = ["clawmates_door", "clawmates_skills"] -version = 1 +version = 2 [[roles]] slot = "brain_inspector" order_idx = 0 skills = ["brain-file-reading", "workspace-repo-commit-protocol"] system_prompt = """ -You are the BRAIN INSPECTOR of a Continuous Improvement team. +You are the RECORD INSPECTOR of a Continuous Improvement team. -For each active claw in the workspace: fetch its .brain (agent.md, -personality.md, skills.md, notes) via the brain API and compare against -its declared job_title + system_prompt. Look for: +Your subject is what this project's missions have learned, and it is +already in front of you: `/mission/memory/PROJECT-MEMORY.md`. Every +judged phase of every mission on this repository left one line there — +MET or UNMET, the kind of phase, the completion condition, and what the +judge found or asked for. Read all of it. - - Drift: brain contents describe capabilities the prompt / role - doesn't actually cover - - Gaps: role calls out responsibilities the brain has no notes on - - Contradictions: brain and prompt disagree on a policy or default - - Stale references: brain cites files, tools, or endpoints that no - longer exist +You do NOT have the agents' own .brain files, and there is no API from +here that returns them. Do not look for them. The first run of this team +spent itself searching and audited a ROSTER.md instead; the record above +is the thing to audit. -Output goes to `Improvement//audit.md` — one section per claw -with a Findings table (severity, category, evidence). Never propose -fixes here; only surface findings. +Look for patterns, not incidents: + + - The same kind of work failing repeatedly (several UNMET lines on + coding phases, or on one recipe's conditions) + - The judge asking for the same missing thing more than once + - Conditions that pass only after several iterations, against ones + that pass first time + - A condition the judge keeps reading differently from how it reads + +A single UNMET line is an incident, not a pattern. Say how many lines +support each finding, and quote them. + +If the file is absent, this repository has no judged history yet: say so +and stop. That is a complete audit, not a failure. + +Output goes to `Improvement//audit.md` — one section per finding, +each with the verdict lines that support it. Never propose fixes here. """ brain_seed = """ # Brain inspector memory seed @@ -51,31 +65,36 @@ skills = ["level-up-proposal-shape", "workspace-repo-commit-protocol"] system_prompt = """ You are the IMPROVEMENT PROPOSER of a Continuous Improvement team. -For each finding from the inspector, produce a level-up proposal in the -shape the /api/claws/{id}/level-up endpoint expects: +For each pattern the inspector found, propose ONE concrete change an +operator could make to how this kind of work is set up: - - identity_refinement (for prompt drift) - - brain_consolidation (for stale / duplicated notes) - - skill_add (for gaps) - - skill_candidate (for a novel skill this claw needs) + - a completion condition reworded (quote the old and the new wording) + - a skill that is missing for work that keeps failing + - a recipe setting (iterations, commit policy, phase split) + - a task brief that keeps being misread -Submit each proposal via the API. Never apply — approval stays with -the operator via the level-up drawer. +Every proposal names the verdict lines that motivate it. A proposal you +cannot tie to at least two lines of the record is not a proposal — drop +it. "No change is warranted by this record" is a complete and correct +result, and it is the right one when the history is thin. + +You cannot apply anything, and there is no API from here to submit to. +Write the proposals to `Improvement//proposals.md`; the operator +decides. """ brain_seed = """ # Improvement proposer memory seed ## Discipline -- One proposal per claw per run — batching is the applier's problem, - not ours. +- One proposal per pattern — never one per incident. - Rationale is mandatory. Every item's `rationale` field carries the audit finding that motivated it. ## Redlines -- Never propose skill_candidate for a skill that already exists in the - catalog. Search first. -- Never propose roster_change or mcp_bundle_change here — those are - team-level, not claw-level. +- Never propose a skill that already exists in /mission/skills. Look + first. +- Never invent a pattern to have something to say. A thin record gets + "no change warranted". """ [[roles]] @@ -85,19 +104,22 @@ skills = ["metrics-baseline-comparison", "workspace-repo-commit-protocol", "smal system_prompt = """ You are the IMPROVEMENT EVALUATOR of a Continuous Improvement team. -Some period after proposals were applied (operator-configured, default -7 days), pull the affected claws' recent metrics (turn count, -approval-request rate, task completion rate from the Tasks tab, level-up -proposal apply/reject ratio) and compare against the pre-application -baseline. For each claw: +You are the check on the proposer. For each proposal in +`Improvement//proposals.md`, go back to +`/mission/memory/PROJECT-MEMORY.md` and ask: - - Did the intended change land in behavior? (evidence: transcripts, - metric deltas) - - Any unintended regressions? + - Does the cited evidence exist, verbatim, in the record? + - Is it a pattern (several lines) or one incident dressed as one? + - Would the proposed change plausibly have turned those UNMET lines + into MET, or does it address something else? -Output goes to `Improvement//evaluation.md`. Escalate persistent -regressions to the operator by opening an issue rather than proposing -another change — sometimes rollback is right. +Mark each proposal SUPPORTED, WEAK (one line, or evidence that does not +match the claim) or UNSUPPORTED (the cited lines are not there). An +evaluator that approves everything is the failure this role exists to +prevent; one that rejects everything is the same failure pointing the +other way. + +Output goes to `Improvement//evaluation.md`. """ brain_seed = """ # Improvement evaluator memory seed