Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b653dbfe72 | ||
|
|
547b5d9987 | ||
|
|
ea0b989b3f | ||
|
|
771092b165 | ||
|
|
113de610ec | ||
|
|
5c2c63f8e8 | ||
|
|
91a6b4e304 | ||
|
|
769e002bb3 | ||
|
|
e3247fee4b | ||
|
|
e4942ce985 | ||
|
|
18dc0b964b | ||
|
|
4358964c05 |
@@ -341,6 +341,11 @@ async fn run() -> Result<(), String> {
|
||||
// for INT-XX markers in event payloads and upserts mission_tasks
|
||||
// rows so the canvas renders a live status timeline.
|
||||
cm_api::task_card_worker::spawn(pool.clone());
|
||||
// Agents apply their own skill drafts. Announced at boot by the spawner
|
||||
// itself, because this flips an approval gate that existed since the
|
||||
// feature shipped — and a safety gate whose state is invisible is one
|
||||
// nobody notices has changed.
|
||||
cm_api::skill_self_authoring::spawn(pool.clone());
|
||||
// Load the workflow recipes now rather than lazily on first mission
|
||||
// create, so a malformed TOML shows up in the boot log instead of
|
||||
// silently yielding a mission with no phase config.
|
||||
@@ -492,6 +497,10 @@ async fn run() -> Result<(), String> {
|
||||
// every consequence — an ungated test suite, a scan that scanned nothing —
|
||||
// looked like a normal result rather than a broken deployment.
|
||||
cm_api::runtime_preflight::report_at_boot();
|
||||
// And whether the gateway those missions drive is configured at all. Both
|
||||
// of its variables are read at FIRST USE, so a deployment missing them
|
||||
// boots clean and fails on the first phase someone runs.
|
||||
cm_api::gateway_preflight::report_at_boot();
|
||||
// Graceful shutdown: on SIGTERM/Ctrl-C, stop accepting, finish in-flight
|
||||
// requests, then DRAIN the sandbox managers so no container is left running.
|
||||
let shutdown = async move {
|
||||
|
||||
@@ -19,6 +19,17 @@ pub enum ApiError {
|
||||
Conflict,
|
||||
#[error("{0}")]
|
||||
Quota(String),
|
||||
/// A 400 whose REASON the caller needs.
|
||||
///
|
||||
/// Same argument as `Unavailable` below, one status code down. The
|
||||
/// proposal decide handlers each computed a precise refusal — "the mission
|
||||
/// is running, not a draft", "no node can boot that backend any more" —
|
||||
/// logged it to stderr, and returned a bare `BadRequest`. The person who
|
||||
/// needed the sentence was the one clicking Approve, and they got
|
||||
/// "bad request". `mission_plan::Refusal` exists and is written as
|
||||
/// human-readable copy; this is how it reaches them.
|
||||
#[error("{0}")]
|
||||
Refused(String),
|
||||
/// A dependency is temporarily refusing work and will accept it later —
|
||||
/// today, the Claude Code subscription's rate limit. Distinct from
|
||||
/// `Internal` because the operator's next action is different: wait and
|
||||
@@ -59,7 +70,7 @@ impl From<cm_auth::AuthError> for ApiError {
|
||||
impl IntoResponse for ApiError {
|
||||
fn into_response(self) -> Response {
|
||||
let status = match self {
|
||||
ApiError::BadRequest => StatusCode::BAD_REQUEST,
|
||||
ApiError::BadRequest | ApiError::Refused(_) => StatusCode::BAD_REQUEST,
|
||||
ApiError::Unauthorized => StatusCode::UNAUTHORIZED,
|
||||
ApiError::Forbidden => StatusCode::FORBIDDEN,
|
||||
ApiError::NotFound => StatusCode::NOT_FOUND,
|
||||
@@ -71,3 +82,31 @@ impl IntoResponse for ApiError {
|
||||
(status, Json(json!({ "error": self.to_string() }))).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::body::to_bytes;
|
||||
|
||||
/// A refusal must carry its reason into the response body.
|
||||
///
|
||||
/// The proposal decide handlers each computed a precise sentence and then
|
||||
/// returned a bare `BadRequest`, so the person clicking Approve saw
|
||||
/// "bad request" while the reason went to a server log they cannot read.
|
||||
#[tokio::test]
|
||||
async fn a_refusal_reaches_the_caller_and_a_bare_bad_request_does_not_pretend_to() {
|
||||
let refused = ApiError::Refused("this mission is running, not a draft".into());
|
||||
let response = refused.into_response();
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
let body = to_bytes(response.into_body(), 64 * 1024).await.unwrap();
|
||||
let text = String::from_utf8_lossy(&body);
|
||||
assert!(
|
||||
text.contains("running, not a draft"),
|
||||
"the reason must be in the body, not only in the server log: {text}"
|
||||
);
|
||||
|
||||
// The bare variant stays as it was — same status, no invented detail.
|
||||
let bare = ApiError::BadRequest.into_response();
|
||||
assert_eq!(bare.status(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
//! Is the mission gateway configured, and is anything listening?
|
||||
//!
|
||||
//! The third sibling of [`crate::runtime_preflight`] and
|
||||
//! [`crate::validator_preflight`], for the same class of failure: the
|
||||
//! configuration is absent or wrong, and nothing says so until a mission pays
|
||||
//! for it.
|
||||
//!
|
||||
//! `ZEROCLAW_GATEWAY_URL` and `ZEROCLAW_TOKEN` have no defaults and are read at
|
||||
//! FIRST USE, inside `ZeroClawDriveExecutor::from_env`. So a deployment missing
|
||||
//! them boots clean, serves every page, lists every mission — and fails the
|
||||
//! first time someone presses run, with an error that surfaces on a phase
|
||||
//! rather than at startup. The information exists the whole time; nobody is
|
||||
//! told until it is expensive.
|
||||
//!
|
||||
//! A report, not a gate, matching its siblings. A server with no gateway should
|
||||
//! still boot: the frontend, the catalogue and every read path work without it,
|
||||
//! and refusing to start would turn a degraded deployment into a dead one.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
const PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// What the preflight found.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Verdict {
|
||||
/// No gateway configured. Missions on the container tier cannot run.
|
||||
NotConfigured { missing: Vec<String> },
|
||||
/// Configured but nothing answered at that address.
|
||||
Unreachable { url: String, error: String },
|
||||
/// Configured and something answered.
|
||||
Reachable { url: String },
|
||||
}
|
||||
|
||||
impl Verdict {
|
||||
/// The line to print at boot.
|
||||
///
|
||||
/// Each names the CONSEQUENCE, not just the state. "ZEROCLAW_TOKEN not set"
|
||||
/// tells an operator what is missing; it does not tell them that every
|
||||
/// container-tier mission they launch will fail on its first phase.
|
||||
pub fn message(&self) -> String {
|
||||
match self {
|
||||
Verdict::NotConfigured { missing } => format!(
|
||||
"gateway_preflight: NOT CONFIGURED ({}) — container-tier missions \
|
||||
cannot run. They will launch, provision a runtime, and fail on \
|
||||
the first turn; the server is otherwise healthy",
|
||||
missing.join(", ")
|
||||
),
|
||||
Verdict::Unreachable { url, error } => format!(
|
||||
"gateway_preflight: {url} is configured but did not answer ({error}) \
|
||||
— container-tier missions will fail on their first turn. The \
|
||||
config is right and the machine is not"
|
||||
),
|
||||
Verdict::Reachable { url } => {
|
||||
format!("gateway_preflight: {url} answered")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Which required variables are absent.
|
||||
///
|
||||
/// Split from the network probe so the rule is testable without a gateway:
|
||||
/// this is the half that is pure, and it is the half that is wrong most often.
|
||||
pub fn missing_config(url: Option<&str>, token: Option<&str>, pairing: Option<&str>) -> Vec<String> {
|
||||
let mut missing = Vec::new();
|
||||
if url.map(str::trim).unwrap_or("").is_empty() {
|
||||
missing.push("ZEROCLAW_GATEWAY_URL".to_string());
|
||||
}
|
||||
// Either credential works: a durable token, or a one-time pairing code the
|
||||
// executor exchanges on first use.
|
||||
let has_token = !token.map(str::trim).unwrap_or("").is_empty();
|
||||
let has_pairing = !pairing.map(str::trim).unwrap_or("").is_empty();
|
||||
if !has_token && !has_pairing {
|
||||
missing.push("ZEROCLAW_TOKEN or ZEROCLAW_PAIRING_CODE".to_string());
|
||||
}
|
||||
missing
|
||||
}
|
||||
|
||||
fn env_opt(key: &str) -> Option<String> {
|
||||
std::env::var(key).ok().filter(|v| !v.trim().is_empty())
|
||||
}
|
||||
|
||||
/// Probe the configured gateway.
|
||||
pub async fn check() -> Verdict {
|
||||
let url = env_opt("ZEROCLAW_GATEWAY_URL");
|
||||
let missing = missing_config(
|
||||
url.as_deref(),
|
||||
env_opt("ZEROCLAW_TOKEN").as_deref(),
|
||||
env_opt("ZEROCLAW_PAIRING_CODE").as_deref(),
|
||||
);
|
||||
if !missing.is_empty() {
|
||||
return Verdict::NotConfigured { missing };
|
||||
}
|
||||
let url = url.expect("checked above");
|
||||
|
||||
// Any HTTP answer proves something is listening and routable, which is the
|
||||
// question this preflight exists to answer. Authenticating here would need
|
||||
// a pairing exchange that BURNS a one-time code — a preflight that costs
|
||||
// the deployment its credential is worse than no preflight.
|
||||
let client = match reqwest::Client::builder().timeout(PROBE_TIMEOUT).build() {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
return Verdict::Unreachable {
|
||||
url,
|
||||
error: e.to_string(),
|
||||
}
|
||||
}
|
||||
};
|
||||
match client.get(&url).send().await {
|
||||
Ok(_) => Verdict::Reachable { url },
|
||||
Err(e) => Verdict::Unreachable {
|
||||
url,
|
||||
error: e.to_string(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Run the probe and print the verdict. Never panics, never blocks boot.
|
||||
pub fn report_at_boot() {
|
||||
tokio::spawn(async {
|
||||
eprintln!("{}", check().await.message());
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_fully_configured_deployment_is_missing_nothing() {
|
||||
assert!(missing_config(Some("http://gw:42617"), Some("tok"), None).is_empty());
|
||||
// A pairing code alone is enough — the executor exchanges it on first use.
|
||||
assert!(missing_config(Some("http://gw:42617"), None, Some("123456")).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_string_counts_as_absent() {
|
||||
// The failure this whole module exists for: `unwrap_or_default` and an
|
||||
// empty env var turn "unconfigured" into "configured with nothing",
|
||||
// which fails later as a 401 rather than now as a missing setting.
|
||||
let missing = missing_config(Some(" "), Some(""), Some(" "));
|
||||
assert_eq!(missing.len(), 2, "both must be reported: {missing:?}");
|
||||
assert!(missing[0].contains("GATEWAY_URL"));
|
||||
assert!(missing[1].contains("ZEROCLAW_TOKEN"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_message_names_the_consequence_not_just_the_state() {
|
||||
let v = Verdict::NotConfigured {
|
||||
missing: vec!["ZEROCLAW_GATEWAY_URL".into()],
|
||||
};
|
||||
let m = v.message();
|
||||
assert!(m.contains("ZEROCLAW_GATEWAY_URL"));
|
||||
assert!(
|
||||
m.contains("cannot run"),
|
||||
"an operator needs to know what stops working, not only what is \
|
||||
unset: {m}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -225,6 +225,91 @@ pub async fn apply(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Is autonomous skill authoring on?
|
||||
///
|
||||
/// Default ON, by operator decision. Stated at boot rather than assumed: this
|
||||
/// flips a human approval gate that has existed since the feature shipped, and
|
||||
/// a safety gate that changes state silently is how nobody notices it changed.
|
||||
pub fn self_authoring_enabled() -> bool {
|
||||
!matches!(
|
||||
std::env::var("CLAWMATES_SKILL_SELF_AUTHORING")
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_ascii_lowercase()
|
||||
.as_str(),
|
||||
"0" | "off" | "false"
|
||||
)
|
||||
}
|
||||
|
||||
/// Apply a pending proposal's `skill_candidate` items with no human decision.
|
||||
///
|
||||
/// ONLY `skill_candidate`. The other item kinds are deliberately left to the
|
||||
/// human gate: `identity_refinement` rewrites an agent's system prompt and
|
||||
/// `brain_consolidation` edits its memory, and both change what the agent IS
|
||||
/// rather than adding a procedure it can consult. Self-authoring a skill is
|
||||
/// recoverable — the row is workspace-scoped, versioned and revertible, and
|
||||
/// cannot take a hand-authored name. Rewriting an identity autonomously is not
|
||||
/// the same bet, and it is not the one that was asked for.
|
||||
///
|
||||
/// The remaining items stay pending, so a human still sees them.
|
||||
pub async fn apply_autonomous(
|
||||
pool: &PgPool,
|
||||
workspace_id: cm_domain::WorkspaceId,
|
||||
proposal_id: Uuid,
|
||||
) -> Result<Vec<String>, String> {
|
||||
let proposal = cm_db::repo::level_up::get(pool, proposal_id, workspace_id.as_uuid())
|
||||
.await
|
||||
.map_err(|e| format!("load proposal: {e}"))?
|
||||
.ok_or_else(|| "proposal not found".to_string())?;
|
||||
if proposal.status != "pending" {
|
||||
return Err(format!("proposal already {}", proposal.status));
|
||||
}
|
||||
|
||||
let items = proposal
|
||||
.payload
|
||||
.get("suggested_items")
|
||||
.and_then(|v| v.as_array())
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut applied: Vec<String> = Vec::new();
|
||||
let mut candidates = 0usize;
|
||||
for item in items {
|
||||
let Some(item_id) = item.get("id").and_then(|v| v.as_str()) else {
|
||||
continue;
|
||||
};
|
||||
if item.get("kind").and_then(|v| v.as_str()) != Some("skill_candidate") {
|
||||
continue;
|
||||
}
|
||||
candidates += 1;
|
||||
match apply_skill_candidate(pool, &proposal, &item).await {
|
||||
Ok(()) => applied.push(item_id.to_string()),
|
||||
// A refused draft is a normal outcome (a name collision with a
|
||||
// hand-authored skill is the common one), not a failure of the
|
||||
// sweep. Said out loud so a refusal is never mistaken for the
|
||||
// agent simply not having proposed anything.
|
||||
Err(e) => eprintln!(
|
||||
"level_up: autonomous apply refused {item_id} for workspace {}: {e}",
|
||||
workspace_id.as_uuid()
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
if candidates == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
cm_db::repo::level_up::mark_applied_autonomously(
|
||||
pool,
|
||||
proposal_id,
|
||||
workspace_id.as_uuid(),
|
||||
&applied,
|
||||
applied.len() != candidates,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("mark applied: {e}"))?;
|
||||
Ok(applied)
|
||||
}
|
||||
|
||||
// ── Appliers ───────────────────────────────────────────────────
|
||||
|
||||
async fn apply_identity(
|
||||
@@ -304,20 +389,61 @@ async fn apply_skill_candidate(
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
// A draft may never take the name of a hand-authored skill.
|
||||
//
|
||||
// The row itself is safe — ids are workspace-scoped, so this cannot
|
||||
// overwrite a builtin, and bindings resolve by skill_id rather than name,
|
||||
// so it cannot shadow one either. What it CAN do is put two different
|
||||
// procedures under one name in the same agent's bundle, and then nobody
|
||||
// reading a transcript can tell which one the agent followed. That
|
||||
// ambiguity is the whole problem in a system where the skill is the
|
||||
// standard the behaviour is graded against.
|
||||
let collides: Option<Uuid> = sqlx::query_scalar(
|
||||
"SELECT id FROM skills WHERE name = $1 AND workspace_id IS NULL",
|
||||
)
|
||||
.bind(name)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(|e| format!("check builtin collision: {e}"))?;
|
||||
if collides.is_some() {
|
||||
return Err(format!(
|
||||
"skill name {name:?} is hand-authored — an agent-authored draft \
|
||||
cannot take the name of a skill it is graded against"
|
||||
));
|
||||
}
|
||||
|
||||
// Workspace-scoped custom skill. Deterministic id per
|
||||
// (workspace, name) so re-approving the same draft updates in
|
||||
// place rather than duplicating.
|
||||
let id = workspace_skill_id(proposal.workspace_id, name);
|
||||
|
||||
// Versioned, for the same reason builtins are: a self-authored skill that
|
||||
// silently replaces its own body has no undo, and the version a run was
|
||||
// judged under is the only way to read that run back honestly later.
|
||||
let mut tx = pool.begin().await.map_err(|e| format!("begin: {e}"))?;
|
||||
let existing: Option<(i32, String)> =
|
||||
sqlx::query_as("SELECT current_version, body FROM skills WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_optional(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| format!("read current skill: {e}"))?;
|
||||
let (next_version, bump) = match &existing {
|
||||
Some((v, prev)) if prev == body => (*v, false),
|
||||
Some((v, _)) => (v + 1, true),
|
||||
None => (1, true),
|
||||
};
|
||||
|
||||
sqlx::query(
|
||||
"INSERT INTO skills
|
||||
(id, name, title, author, description, when_to_use, tags,
|
||||
source_kind, workspace_id, current_version, body)
|
||||
VALUES ($1,$2,$2,'level_up',$3,$4,$5,'promoted_from_brain',$6,1,$7)
|
||||
VALUES ($1,$2,$2,'level_up',$3,$4,$5,'promoted_from_brain',$6,$8,$7)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
description = EXCLUDED.description,
|
||||
when_to_use = EXCLUDED.when_to_use,
|
||||
tags = EXCLUDED.tags,
|
||||
body = EXCLUDED.body,
|
||||
current_version = EXCLUDED.current_version,
|
||||
updated_at = now()",
|
||||
)
|
||||
.bind(id)
|
||||
@@ -327,9 +453,28 @@ async fn apply_skill_candidate(
|
||||
.bind(&tags)
|
||||
.bind(proposal.workspace_id)
|
||||
.bind(body)
|
||||
.execute(pool)
|
||||
.bind(next_version)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| format!("upsert skill draft: {e}"))?;
|
||||
|
||||
if bump {
|
||||
sqlx::query(
|
||||
"INSERT INTO skill_versions
|
||||
(skill_id, version, body_md, description, when_to_use)
|
||||
VALUES ($1,$2,$3,$4,$5)
|
||||
ON CONFLICT DO NOTHING",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(next_version)
|
||||
.bind(body)
|
||||
.bind(description)
|
||||
.bind(when_to_use)
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.map_err(|e| format!("record skill version: {e}"))?;
|
||||
}
|
||||
tx.commit().await.map_err(|e| format!("commit: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -49,9 +49,12 @@ pub mod repo_digest;
|
||||
pub mod root_copy;
|
||||
mod routes;
|
||||
pub mod runtime_preflight;
|
||||
mod runtime_provision;
|
||||
pub mod runtime_provision;
|
||||
pub mod security_scan;
|
||||
pub mod session_executor;
|
||||
pub mod gateway_preflight;
|
||||
pub mod skill_self_authoring;
|
||||
pub mod skill_use;
|
||||
pub mod skills_loader;
|
||||
pub mod subscription;
|
||||
pub mod swarm;
|
||||
@@ -59,11 +62,12 @@ pub mod task_card_parser;
|
||||
pub mod task_card_worker;
|
||||
pub mod team_template_loader;
|
||||
pub mod tool_versions;
|
||||
mod topology_exec;
|
||||
pub mod topology_exec;
|
||||
pub mod topology_worker;
|
||||
pub mod validator_preflight;
|
||||
pub mod vm_placement;
|
||||
pub mod vm_stop_gate;
|
||||
pub mod vm_tool_gate;
|
||||
pub mod vm_tool_tap;
|
||||
pub mod workflow_registry;
|
||||
|
||||
@@ -589,6 +593,10 @@ pub fn router(state: AppState) -> Router {
|
||||
"/api/missions/{id}/phases/{phase_id}/evaluations",
|
||||
get(routes::missions::list_phase_evaluations),
|
||||
)
|
||||
.route(
|
||||
"/api/missions/{id}/skill-use",
|
||||
get(routes::missions::skill_use),
|
||||
)
|
||||
.route(
|
||||
"/api/missions/{id}/teams",
|
||||
get(routes::missions::list_teams),
|
||||
|
||||
@@ -691,10 +691,37 @@ async fn run_inside(
|
||||
// ONE settings document, written once, carrying whichever hooks installed.
|
||||
// Two writers here is the silent clobber `guest_settings` exists to stop:
|
||||
// whichever ran second would erase the other's hook with no error at all.
|
||||
let settings = match (gate_dir, tap_dir) {
|
||||
(None, None) => None,
|
||||
(g, t) => {
|
||||
let doc = crate::vm_tool_tap::guest_settings(g, t);
|
||||
// The PRE-execution gate. Installed on the same terms as the tap: a
|
||||
// failure here degrades to no gate rather than failing the phase, because
|
||||
// a mission that runs ungated is what we have today and a mission that
|
||||
// refuses to run is a regression.
|
||||
let tool_gate_dir = match settings_supported {
|
||||
false => None,
|
||||
true => match vm
|
||||
.exec(
|
||||
&crate::vm_tool_gate::install_command(crate::vm_tool_gate::GUEST_DIR),
|
||||
None,
|
||||
60,
|
||||
&[],
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(o) if o.rc == 0 => Some(crate::vm_tool_gate::GUEST_DIR),
|
||||
other => {
|
||||
eprintln!(
|
||||
"microvm_executor: could not install the tool gate on {} ({other:?}) \
|
||||
— this phase's tool calls will run unchecked",
|
||||
vm.vm_id()
|
||||
);
|
||||
None
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
let settings = match (gate_dir, tap_dir, tool_gate_dir) {
|
||||
(None, None, None) => None,
|
||||
(g, t, tg) => {
|
||||
let doc = crate::vm_tool_tap::guest_settings(g, t, tg);
|
||||
let cmd = crate::vm_tool_tap::settings_command(
|
||||
crate::vm_tool_tap::SETTINGS_PATH,
|
||||
&doc,
|
||||
|
||||
@@ -23,6 +23,36 @@ pub const PHASE_COMPLETED: &str = "phase.completed";
|
||||
pub const TOOL_CALL: &str = "tool.call";
|
||||
/// A tool touched a path. `target` is the path, repo-relative where known.
|
||||
pub const FILE_TOUCH: &str = "file.touch";
|
||||
/// The exact prompt text an agent was given. `detail.text` is the full string,
|
||||
/// `target` is the role or tier that composed it.
|
||||
///
|
||||
/// The durable answer to "what did this agent actually receive". Skills, the
|
||||
/// task, the evaluator's feedback and the tool preamble are assembled from four
|
||||
/// places across three tiers, so re-deriving the prompt after the fact means
|
||||
/// re-running that assembly against data that has since changed. Recording it
|
||||
/// is the only way the question stays answerable.
|
||||
pub const PROMPT_COMPOSED: &str = "prompt.composed";
|
||||
/// The agent's own narrative for a turn. `detail.text`.
|
||||
///
|
||||
/// Written by `topology_worker` and pushed live once by `live_bus`. Until the
|
||||
/// reader below existed, the stored row was never read again by anything: both
|
||||
/// database readers in `routes/world.rs` filter to `tool.call`/`file.touch`,
|
||||
/// and the only other statement touching the table is the GC that deletes it.
|
||||
pub const REASONING: &str = "reasoning";
|
||||
|
||||
/// Kinds the per-phase cap applies to.
|
||||
///
|
||||
/// The cap exists to bound the two unbounded kinds: a coding phase can call
|
||||
/// thousands of tools and touch thousands of paths. The others are bounded by
|
||||
/// the phase's own structure — one start, one completion, one prompt per turn —
|
||||
/// and counting them against the same budget meant a busy phase could push out
|
||||
/// its OWN terminal event, leaving a phase that looks like it never finished.
|
||||
const CAPPED_KINDS: &[&str] = &[TOOL_CALL, FILE_TOUCH];
|
||||
|
||||
/// Does this kind count against, and get dropped by, `PER_PHASE_CAP`?
|
||||
pub fn is_capped(kind: &str) -> bool {
|
||||
CAPPED_KINDS.contains(&kind)
|
||||
}
|
||||
|
||||
/// Most events one phase may record.
|
||||
///
|
||||
@@ -89,12 +119,17 @@ pub async fn record(pool: &PgPool, e: MissionEvent) {
|
||||
} else {
|
||||
e.detail
|
||||
};
|
||||
// The cap is still decided INSIDE the insert (see the test below), and now
|
||||
// only counts the kinds it is meant to bound.
|
||||
let capped = is_capped(&e.kind);
|
||||
let res = sqlx::query(
|
||||
"INSERT INTO mission_events
|
||||
(mission_id, phase_id, run_id, agent_id, kind, target, detail)
|
||||
SELECT $1, $2, $3, $4, $5, $6, $7
|
||||
WHERE $2::uuid IS NULL
|
||||
OR (SELECT count(*) FROM mission_events WHERE phase_id = $2) < $8",
|
||||
OR NOT $9
|
||||
OR (SELECT count(*) FROM mission_events
|
||||
WHERE phase_id = $2 AND kind = ANY($10)) < $8",
|
||||
)
|
||||
.bind(e.mission_id)
|
||||
.bind(e.phase_id)
|
||||
@@ -104,6 +139,8 @@ pub async fn record(pool: &PgPool, e: MissionEvent) {
|
||||
.bind(&e.target)
|
||||
.bind(&detail)
|
||||
.bind(PER_PHASE_CAP)
|
||||
.bind(capped)
|
||||
.bind(CAPPED_KINDS)
|
||||
.execute(pool)
|
||||
.await;
|
||||
if let Err(err) = res {
|
||||
@@ -112,6 +149,40 @@ pub async fn record(pool: &PgPool, e: MissionEvent) {
|
||||
}
|
||||
|
||||
/// Record several events under one round trip's worth of intent.
|
||||
/// Every recorded prompt and narrative for a mission, oldest first.
|
||||
///
|
||||
/// The read side of `PROMPT_COMPOSED` / `REASONING`. Both kinds were write-only
|
||||
/// before this: the prompt was never stored at all, and the narrative was
|
||||
/// stored and then read by nothing. Together they answer "what did this agent
|
||||
/// receive, and what did it say it did", which is the question
|
||||
/// `docs/PROVENANCE-ASSESSMENT.md` records as unanswerable.
|
||||
pub async fn narrative_for_mission(
|
||||
pool: &PgPool,
|
||||
mission_id: Uuid,
|
||||
) -> Result<Vec<(String, Option<Uuid>, Option<String>, String)>, sqlx::Error> {
|
||||
let rows: Vec<(String, Option<Uuid>, Option<String>, Value)> = sqlx::query_as(
|
||||
"SELECT kind, agent_id, target, detail
|
||||
FROM mission_events
|
||||
WHERE mission_id = $1 AND kind = ANY($2)
|
||||
ORDER BY id",
|
||||
)
|
||||
.bind(mission_id)
|
||||
.bind(&[PROMPT_COMPOSED, REASONING][..])
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|(kind, agent, target, detail)| {
|
||||
let text = detail
|
||||
.get("text")
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
(kind, agent, target, text)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn record_all(pool: &PgPool, events: Vec<MissionEvent>) {
|
||||
for e in events {
|
||||
record(pool, e).await;
|
||||
|
||||
@@ -137,12 +137,21 @@ const EVENT_RETENTION_DAYS: i32 = 7;
|
||||
/// deployment that has been accumulating for months would otherwise take a long
|
||||
/// lock on its first sweep after this ships. The sweep runs on a timer, so a
|
||||
/// large backlog simply drains over several passes.
|
||||
async fn reap_mission_events(pool: &PgPool, out: &mut Reclaimed) {
|
||||
pub async fn reap_mission_events(pool: &PgPool, out: &mut Reclaimed) {
|
||||
let res = sqlx::query(
|
||||
"DELETE FROM mission_events
|
||||
WHERE id IN (
|
||||
SELECT id FROM mission_events
|
||||
WHERE created_at < now() - make_interval(days => $1)
|
||||
SELECT e.id FROM mission_events e
|
||||
JOIN missions m ON m.id = e.mission_id
|
||||
WHERE e.created_at < now() - make_interval(days => $1)
|
||||
-- A mission under measurement or investigation keeps its
|
||||
-- events. Without this the evidence a Skill-Use baseline or a
|
||||
-- provenance question depends on expires while the question
|
||||
-- is still open, and the answer degrades silently into
|
||||
-- \"there are no events\" — which reads identically to
|
||||
-- \"nothing happened\".
|
||||
AND (m.retain_events_until IS NULL
|
||||
OR m.retain_events_until < now())
|
||||
LIMIT 10000
|
||||
)",
|
||||
)
|
||||
|
||||
@@ -679,7 +679,12 @@ async fn mint_team_from_template(
|
||||
// out-of-band via MissionRuntimeProvisioner::pin_agent_workspaces.
|
||||
if let Some(p) = provisioner {
|
||||
match p
|
||||
.provision_claw(claw_id, role_model, &template.template.risk_profile)
|
||||
.provision_claw(
|
||||
claw_id,
|
||||
role_model,
|
||||
&template.template.risk_profile,
|
||||
&template.template.mcp_bundles,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => provisioned_claws.push(agent_id),
|
||||
|
||||
@@ -53,6 +53,33 @@ pub const KNOWN_KEYS: &[KnownKey] = &[
|
||||
phase that changes no files still completes; also vm_stop_gate::\
|
||||
StopGate::for_phase, where it drops the in-loop delivery check",
|
||||
},
|
||||
KnownKey {
|
||||
key: "tools",
|
||||
read_by: "security_scan::run — gates which of cargo_audit / gitleaks / \
|
||||
trivy_fs / semgrep run against the phase's checkout; absent \
|
||||
means all four. Listed here as NOT IMPLEMENTED while wired, \
|
||||
which understated the recipe: the key was real, what was \
|
||||
missing was anything that FIRED the scan outside an operator \
|
||||
button — now phase_runner::scan_finished_security_phases",
|
||||
},
|
||||
KnownKey {
|
||||
key: "harness",
|
||||
read_by: "benchmark_runner::harness_from_config — selects criterion / \
|
||||
cargo_bench / vitest_bench / pytest_bench / shell, with \
|
||||
`bench_name` (criterion) and `cmd` (shell) as its arguments. \
|
||||
phase_runner's benchmark sweep runs the baseline through it. \
|
||||
This key was listed as NOT IMPLEMENTED while being fully \
|
||||
wired, which is worse than an unread key: the registry exists \
|
||||
so an operator can trust what a recipe does, and it was wrong",
|
||||
},
|
||||
KnownKey {
|
||||
key: "bench_name",
|
||||
read_by: "benchmark_runner::harness_from_config — the criterion bench target",
|
||||
},
|
||||
KnownKey {
|
||||
key: "cmd",
|
||||
read_by: "benchmark_runner::harness_from_config — the shell harness command line",
|
||||
},
|
||||
KnownKey {
|
||||
key: "done_when_check",
|
||||
read_by: "vm_stop_gate::StopGate::for_phase — a shell command the agent's \
|
||||
@@ -84,14 +111,6 @@ pub const DECLARED_BUT_UNREAD: &[KnownKey] = &[
|
||||
key: "mode",
|
||||
read_by: "NOT IMPLEMENTED — benchmark/refactor mode selection",
|
||||
},
|
||||
KnownKey {
|
||||
key: "harness",
|
||||
read_by: "NOT IMPLEMENTED — benchmark harness selection",
|
||||
},
|
||||
KnownKey {
|
||||
key: "tools",
|
||||
read_by: "NOT IMPLEMENTED — per-phase tool selection",
|
||||
},
|
||||
KnownKey {
|
||||
key: "benchmark",
|
||||
read_by: "NOT IMPLEMENTED — nested benchmark settings",
|
||||
@@ -104,11 +123,6 @@ pub const DECLARED_BUT_UNREAD: &[KnownKey] = &[
|
||||
setting this per phase changes nothing: security_hardening.toml \
|
||||
asks for gitea_forge + security_scan and its phase gets neither",
|
||||
},
|
||||
KnownKey {
|
||||
key: "test_command",
|
||||
read_by: "NOT IMPLEMENTED — mission_delivery::discover_test_command infers \
|
||||
from the repo and does not consult config",
|
||||
},
|
||||
];
|
||||
|
||||
fn is_listed(key: &str, list: &[KnownKey]) -> bool {
|
||||
|
||||
@@ -73,6 +73,8 @@ async fn sweep_once(
|
||||
crate::mission_outputs::capture_repo_less_phases(pool).await?;
|
||||
// Benchmark phases record their baseline once the work exists to measure.
|
||||
baseline_finished_benchmark_phases(pool).await?;
|
||||
// Security phases run the scanners once the checkout exists to scan.
|
||||
scan_finished_security_phases(pool).await?;
|
||||
// A failed phase makes every later phase unreachable, and saying so is what
|
||||
// lets the mission finish at all.
|
||||
skip_unreachable_phases(pool).await?;
|
||||
@@ -222,6 +224,88 @@ async fn capture_finished_coding_phases(pool: &PgPool) -> Result<(), String> {
|
||||
/// "/mission/repo ... is the mission's git checkout" to a mission that had none.
|
||||
/// One agent recorded the contradiction verbatim — "No git repo — file is
|
||||
/// written" — and wrote into a container that was then reaped unread.
|
||||
#[cfg(test)]
|
||||
mod skill_delivery_wiring_tests {
|
||||
/// Every tier without per-turn injection must be handed the task text that
|
||||
/// CARRIES the skills.
|
||||
///
|
||||
/// The behavioural tests in `tests/mission_skill_delivery.rs` prove
|
||||
/// `phase_skills_text` and `compose_turn_prompt` work. They cannot prove
|
||||
/// the three `launch_*` calls pass the composed string rather than the bare
|
||||
/// one — and that substitution is a one-word edit that would silently
|
||||
/// return all three tiers to delivering no skill, with every test still
|
||||
/// green. Same reasoning as `mission_events::the_cap_is_enforced_in_one_statement`.
|
||||
/// The prompt must be recorded by the tier that SENDS it.
|
||||
///
|
||||
/// Recording at the dispatch fork wrote a phase prompt for container
|
||||
/// missions too, and the container tier does not send that text — it sends
|
||||
/// the bare task and appends skills per turn. Observed on a live mission:
|
||||
/// three prompt.composed rows, one of which was never given to anything.
|
||||
#[test]
|
||||
fn every_solo_tier_records_the_prompt_it_actually_sends() {
|
||||
let src = include_str!("phase_runner.rs");
|
||||
let anchor = format!("async fn {}(", "record_phase_prompt");
|
||||
assert!(
|
||||
src.contains(&anchor),
|
||||
"the per-tier recorder is gone; a fork-level record would log \
|
||||
prompts that were never sent"
|
||||
);
|
||||
for launcher in [
|
||||
"launch_composed_microvm_phase",
|
||||
"launch_microvm_phase",
|
||||
"launch_direct_session",
|
||||
] {
|
||||
let body = src
|
||||
.split(&format!("async fn {launcher}("))
|
||||
.nth(1)
|
||||
.and_then(|s| s.split("\nasync fn ").next())
|
||||
.unwrap_or_else(|| panic!("{launcher} not found"));
|
||||
assert!(
|
||||
body.contains("record_phase_prompt("),
|
||||
"{launcher} runs a prompt it never records — that phase becomes \
|
||||
unexplainable after the fact"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_three_solo_tiers_are_handed_the_skill_bearing_task() {
|
||||
let src = include_str!("phase_runner.rs");
|
||||
// Only the dispatch body, so the helper definitions and these tests do
|
||||
// not satisfy the assertion by accident.
|
||||
// Built at runtime, not written as one literal: a literal anchor
|
||||
// appears in THIS test's own source too, and the split then matches
|
||||
// itself instead of the code under test.
|
||||
let anchor = format!("let task_with_skills = match {}(", "phase_skills_text");
|
||||
let body = src
|
||||
.split(&anchor)
|
||||
.nth(1)
|
||||
.and_then(|s| s.split("\nasync fn ").next())
|
||||
.expect("dispatch body");
|
||||
|
||||
for (call, tier) in [
|
||||
("launch_composed_microvm_phase(", "composed microVM"),
|
||||
("launch_microvm_phase(", "solo microVM"),
|
||||
("launch_direct_session(", "direct session"),
|
||||
] {
|
||||
let args = body
|
||||
.split(call)
|
||||
.nth(1)
|
||||
.and_then(|s| s.split(')').next())
|
||||
.unwrap_or_else(|| panic!("{tier}: call site not found"));
|
||||
assert!(
|
||||
args.contains("&task_with_skills"),
|
||||
"{tier} is passed the bare task — that tier has no per-turn \
|
||||
injection, so its agents would receive no skill at all"
|
||||
);
|
||||
assert!(
|
||||
!args.contains("&task,"),
|
||||
"{tier} is passed `&task`, the pre-skills string"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod repo_less_text_tests {
|
||||
use super::*;
|
||||
@@ -352,6 +436,74 @@ async fn baseline_finished_benchmark_phases(pool: &PgPool) -> Result<(), String>
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run the scanners for `security_scan` phases that have finished without one.
|
||||
///
|
||||
/// The same defect the benchmark sweep above was written for, in the other
|
||||
/// half of the platform. `security_scan::run` — which reads `phase.config.tools`
|
||||
/// to pick scanners and upserts each finding as a `mission_task` — was
|
||||
/// reachable only from `POST /api/missions/{id}/security-scan`, an operator
|
||||
/// button. So `security_hardening.toml`, a workflow whose entire first phase is
|
||||
/// a scan, ran an agent that was never told to scan and then never fired the
|
||||
/// scanner either. The phase reported `completed` having scanned nothing, and
|
||||
/// the two facts that would have exposed it — zero findings, and a green
|
||||
/// phase — are exactly what a genuinely clean repository also looks like.
|
||||
///
|
||||
/// Guarded on the marker row rather than on the presence of findings, because
|
||||
/// a clean scan writes no findings: without it, a clean phase would be
|
||||
/// rescanned on every tick for as long as the mission existed.
|
||||
///
|
||||
/// Spawned, not awaited, for the same reason as the benchmark baseline: four
|
||||
/// scanners against a large tree take minutes, and this loop also starts,
|
||||
/// closes and evaluates every phase on the platform.
|
||||
/// The selection half of the sweep, split out so the marker guard is testable
|
||||
/// without a container: the scan itself needs Docker, the guard is the part
|
||||
/// that decides whether it runs twice or never.
|
||||
pub async fn unscanned_security_phases(pool: &PgPool) -> Result<Vec<(Uuid, Uuid)>, String> {
|
||||
let rows = sqlx::query(
|
||||
"SELECT mp.id, mp.mission_id
|
||||
FROM mission_phases mp
|
||||
WHERE mp.kind = 'security_scan'
|
||||
AND mp.status IN ('completed', 'failed')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM mission_tasks t
|
||||
WHERE t.phase_id = mp.id AND t.external_id = $1
|
||||
)
|
||||
ORDER BY mp.completed_at DESC NULLS LAST
|
||||
LIMIT 2",
|
||||
)
|
||||
.bind(crate::security_scan::SCAN_MARKER)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| format!("select security_scan phases: {e}"))?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|r| (r.get("id"), r.get("mission_id")))
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn scan_finished_security_phases(pool: &PgPool) -> Result<(), String> {
|
||||
for (phase_id, mission_id) in unscanned_security_phases(pool).await? {
|
||||
let pool = pool.clone();
|
||||
tokio::spawn(async move {
|
||||
match crate::security_scan::run(&pool, mission_id, phase_id).await {
|
||||
Ok(n) => eprintln!(
|
||||
"phase_runner: security scan recorded {n} finding(s) for phase \
|
||||
{phase_id} of mission {mission_id}"
|
||||
),
|
||||
// Not a phase failure, for the same reason the benchmark
|
||||
// baseline is not: a repo the scanners cannot read is a real
|
||||
// outcome. Said out loud, though — "no findings" must not be
|
||||
// indistinguishable from "never scanned".
|
||||
Err(e) => eprintln!(
|
||||
"phase_runner: security scan did not run for phase {phase_id} \
|
||||
of mission {mission_id}: {e}"
|
||||
),
|
||||
}
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Did this phase finish without delivering the work it exists to produce?
|
||||
///
|
||||
/// A coding phase that changes no files has done nothing, and until now that
|
||||
@@ -712,7 +864,7 @@ async fn launch_phase(
|
||||
crate::runtime_provision::RuntimeProvisioner::for_gateway(ec.endpoint.clone())
|
||||
{
|
||||
let crew = sqlx::query(
|
||||
"SELECT DISTINCT a.id, a.model_binding, t.risk_profile
|
||||
"SELECT DISTINCT a.id, a.model_binding, t.risk_profile, t.mcp_bundles
|
||||
FROM team_members tm
|
||||
JOIN mission_teams mt ON mt.team_id = tm.team_id
|
||||
JOIN teams t ON t.id = tm.team_id
|
||||
@@ -728,11 +880,27 @@ async fn launch_phase(
|
||||
let aid: uuid::Uuid = row.get("id");
|
||||
let model: Option<String> = row.get("model_binding");
|
||||
let risk: Option<String> = row.get("risk_profile");
|
||||
// Re-assert the team's OWN bundles. Passing a constant
|
||||
// here would quietly strip `clawmates_skills` from a
|
||||
// crew that had it, and a re-provision that removes a
|
||||
// capability is worse than one that never ran: the
|
||||
// agent keeps working and simply stops being able to
|
||||
// read its skills, halfway through the mission.
|
||||
let bundles: Vec<String> = row
|
||||
.get::<serde_json::Value, _>("mcp_bundles")
|
||||
.as_array()
|
||||
.map(|a| {
|
||||
a.iter()
|
||||
.filter_map(|v| v.as_str().map(|s| s.to_string()))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
match rp
|
||||
.provision_claw(
|
||||
aid,
|
||||
model.as_deref().unwrap_or("claude"),
|
||||
risk.as_deref().unwrap_or("research_readonly"),
|
||||
&bundles,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -801,6 +969,15 @@ async fn launch_phase(
|
||||
_ => task,
|
||||
};
|
||||
|
||||
// The container tier is deliberately NOT given this: it injects per-turn in
|
||||
// `topology_exec`, with the running node's own role, and appending here too
|
||||
// would put every crew member's skills in every turn twice.
|
||||
let task_with_skills = match phase_skills_text(pool, mission_id).await {
|
||||
Some(skills) => crate::topology_exec::compose_turn_prompt(&task, Some(&skills)),
|
||||
None => task.clone(),
|
||||
};
|
||||
|
||||
|
||||
// Direct-session executor: run the whole phase as ONE `claude -p` session
|
||||
// against the mission checkout, instead of driving turns through ZeroClaw.
|
||||
//
|
||||
@@ -937,7 +1114,7 @@ async fn launch_phase(
|
||||
phase_id,
|
||||
workspace_id,
|
||||
iteration,
|
||||
&task,
|
||||
&task_with_skills,
|
||||
team,
|
||||
purposes,
|
||||
)
|
||||
@@ -951,7 +1128,7 @@ async fn launch_phase(
|
||||
kind,
|
||||
workspace_id,
|
||||
iteration,
|
||||
&task,
|
||||
&task_with_skills,
|
||||
p.backend,
|
||||
chosen_node,
|
||||
p.team_engine,
|
||||
@@ -962,7 +1139,14 @@ async fn launch_phase(
|
||||
}
|
||||
|
||||
if crate::session_executor::direct_mode() {
|
||||
return launch_direct_session(pool, mission_id, phase_id, workspace_id, iteration, &task)
|
||||
return launch_direct_session(
|
||||
pool,
|
||||
mission_id,
|
||||
phase_id,
|
||||
workspace_id,
|
||||
iteration,
|
||||
&task_with_skills,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -1058,6 +1242,7 @@ async fn launch_composed_microvm_phase(
|
||||
teams: Vec<(Uuid, serde_json::Value)>,
|
||||
purposes: &[&str],
|
||||
) -> Result<(), String> {
|
||||
record_phase_prompt(pool, mission_id, phase_id, "composed_microvm", task).await;
|
||||
sqlx::query(
|
||||
"DELETE FROM topology_runs
|
||||
WHERE mission_phase_id = $1 AND status IN ('failed', 'cancelled')",
|
||||
@@ -1198,6 +1383,7 @@ async fn launch_microvm_phase(
|
||||
// `VmPhase::has_repo`.
|
||||
has_repo: bool,
|
||||
) -> Result<(), String> {
|
||||
record_phase_prompt(pool, mission_id, phase_id, "microvm", task).await;
|
||||
sqlx::query(
|
||||
"DELETE FROM topology_runs
|
||||
WHERE mission_phase_id = $1 AND status IN ('failed', 'cancelled')",
|
||||
@@ -1431,6 +1617,7 @@ async fn launch_direct_session(
|
||||
iteration: i32,
|
||||
task: &str,
|
||||
) -> Result<(), String> {
|
||||
record_phase_prompt(pool, mission_id, phase_id, "session", task).await;
|
||||
sqlx::query(
|
||||
"DELETE FROM topology_runs
|
||||
WHERE mission_phase_id = $1 AND status IN ('failed', 'cancelled')",
|
||||
@@ -1480,16 +1667,52 @@ async fn launch_direct_session(
|
||||
summary.chars().take(200).collect::<String>()
|
||||
);
|
||||
let status = if ok { "completed" } else { "failed" };
|
||||
|
||||
// The agent's account, kept. Until now this tier wrote NO checkpoint
|
||||
// record — the same defect the solo microVM path was fixed for, in the
|
||||
// one remaining tier: `summary` went to stderr above and nowhere else,
|
||||
// so the Live and Output tabs were empty for a session that did real
|
||||
// work, and nothing downstream could read what the agent said it did.
|
||||
//
|
||||
// Shape matches the microVM path exactly, so the two existing readers
|
||||
// (`/api/missions/{id}/documents` and `/api/topology-runs/{id}/events`)
|
||||
// need no change.
|
||||
let record = serde_json::json!({
|
||||
"records": [{
|
||||
"node_id": "n0",
|
||||
"role": "session",
|
||||
"phase": "work",
|
||||
"output": summary,
|
||||
"tokens": 0,
|
||||
"gated": [],
|
||||
}]
|
||||
});
|
||||
// Also as a durable event, so the narrative is queryable per mission
|
||||
// rather than only by walking topology_runs JSON.
|
||||
let mut ev = crate::mission_events::MissionEvent::new(
|
||||
mission_id,
|
||||
crate::mission_events::REASONING,
|
||||
);
|
||||
ev.phase_id = Some(phase_id);
|
||||
ev.run_id = Some(run_id);
|
||||
ev.target = Some("session".to_string());
|
||||
ev.detail = serde_json::json!({ "text": summary });
|
||||
crate::mission_events::record(&pool, ev).await;
|
||||
|
||||
// Never overwrite a cancellation. The operator asking to stop is a decision;
|
||||
// this task reporting how the VM turned out is an observation, and it may
|
||||
// land minutes later. Without the guard a cancelled run silently reappears
|
||||
// as completed or failed.
|
||||
if let Err(e) = sqlx::query(
|
||||
"UPDATE topology_runs SET status = $2, updated_at = now()
|
||||
"UPDATE topology_runs
|
||||
SET status = $2,
|
||||
checkpoint = COALESCE(checkpoint, '{}'::jsonb) || $3::jsonb,
|
||||
updated_at = now()
|
||||
WHERE id = $1 AND status <> 'cancelled'",
|
||||
)
|
||||
.bind(run_id)
|
||||
.bind(status)
|
||||
.bind(&record)
|
||||
.execute(&pool)
|
||||
.await
|
||||
{
|
||||
@@ -1501,6 +1724,103 @@ async fn launch_direct_session(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The pinned skills for a phase's crew, rendered for the task text.
|
||||
///
|
||||
/// The container tier gets skills per TURN (`topology_exec::pinned_skills_text`),
|
||||
/// where each node carries its own claw alias and therefore its own role's
|
||||
/// skills. The microVM and direct-session tiers have no per-turn alias — the
|
||||
/// phase runs as one `claude -p` session — so their skills have to be resolved
|
||||
/// per PHASE and appended to the task instead. Without this, those three tiers
|
||||
/// deliver no skill at all, which is what they did until now.
|
||||
///
|
||||
/// Union across the crew, deduplicated. A solo tier does not know which crew
|
||||
/// member's turn it is running, and a procedure that applies to the role doing
|
||||
/// the work still applies when one agent does all of it. Erring toward the
|
||||
/// union is safe here in a way it would not be on the container tier, where
|
||||
/// per-role precision is available and used.
|
||||
/// Record the prompt a phase is about to run with.
|
||||
///
|
||||
/// Called from inside each tier rather than at the dispatch point, because the
|
||||
/// tiers do not all send the same text: the three solo tiers send the
|
||||
/// skills-bearing task, and the container tier sends the bare task and appends
|
||||
/// skills per turn in `topology_exec`. Recording at the fork wrote a `solo`
|
||||
/// prompt for container missions too — a prompt that was composed and never
|
||||
/// sent. A provenance record of something that did not happen is worse than no
|
||||
/// record: it is the wrong answer, delivered confidently.
|
||||
async fn record_phase_prompt(
|
||||
pool: &PgPool,
|
||||
mission_id: Uuid,
|
||||
phase_id: Uuid,
|
||||
tier: &str,
|
||||
text: &str,
|
||||
) {
|
||||
let mut ev = crate::mission_events::MissionEvent::new(
|
||||
mission_id,
|
||||
crate::mission_events::PROMPT_COMPOSED,
|
||||
);
|
||||
ev.phase_id = Some(phase_id);
|
||||
ev.target = Some(tier.to_string());
|
||||
ev.detail = serde_json::json!({ "text": text, "tier": tier });
|
||||
crate::mission_events::record(pool, ev).await;
|
||||
}
|
||||
|
||||
pub async fn phase_skills_text(pool: &PgPool, mission_id: Uuid) -> Option<String> {
|
||||
let crew = sqlx::query(
|
||||
"SELECT DISTINCT a.id
|
||||
FROM team_members tm
|
||||
JOIN mission_teams mt ON mt.team_id = tm.team_id
|
||||
JOIN agents a ON a.id = tm.claw_id
|
||||
WHERE mt.mission_id = $1 AND a.deleted_at IS NULL",
|
||||
)
|
||||
.bind(mission_id)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
let mut seen: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
|
||||
let mut out = String::new();
|
||||
for row in &crew {
|
||||
let agent_id: Uuid = row.get("id");
|
||||
let link = cm_db::repo::agent_template_link::get(pool, agent_id)
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
let (tpl_id, slot) = link
|
||||
.as_ref()
|
||||
.map(|l| (Some(l.template_id), Some(l.role_slot.as_str())))
|
||||
.unwrap_or((None, None));
|
||||
let Ok(bindings) =
|
||||
cm_db::repo::skills_catalog::effective_for_agent(pool, agent_id, tpl_id, slot).await
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
for b in bindings.iter().filter(|b| b.pin_in_context) {
|
||||
if !seen.insert(b.skill.name.clone()) {
|
||||
continue;
|
||||
}
|
||||
// Bounded, and truncation is STATED. A silently clipped procedure is
|
||||
// worse than an absent one: the agent follows the half it can see
|
||||
// and reports success against a rule it never read.
|
||||
if out.len() + b.skill.body.len() > crate::topology_exec::MAX_PINNED_SKILL_BYTES {
|
||||
out.push_str(&format!(
|
||||
"\n[skill \"{}\" omitted — the pinned set exceeded {} bytes]\n",
|
||||
b.skill.name,
|
||||
crate::topology_exec::MAX_PINNED_SKILL_BYTES
|
||||
));
|
||||
continue;
|
||||
}
|
||||
out.push_str(&crate::topology_exec::render_pinned_skill(
|
||||
&b.skill.name,
|
||||
&b.skill.body,
|
||||
));
|
||||
}
|
||||
}
|
||||
if seen.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
fn phase_task_text(
|
||||
kind: &str,
|
||||
title: &str,
|
||||
|
||||
@@ -48,7 +48,12 @@ async fn repo_digest(pool: &sqlx::PgPool, mission_id: uuid::Uuid) -> String {
|
||||
return "(this mission has no repository)".to_string();
|
||||
};
|
||||
let branch = branch.unwrap_or_else(|| "main".to_string());
|
||||
// Distinguish "no credential" from "the forge said no". Both used to
|
||||
// arrive as the same "(could not be read)" string, so an unconfigured
|
||||
// deployment looked identical to a private repo — and the planner, told
|
||||
// only that the read failed, cannot say which.
|
||||
let token = std::env::var("GITEA_TOKEN").unwrap_or_default();
|
||||
let unauthenticated = token.trim().is_empty();
|
||||
let Ok(client) = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(20))
|
||||
.build()
|
||||
@@ -70,6 +75,11 @@ async fn repo_digest(pool: &sqlx::PgPool, mission_id: uuid::Uuid) -> String {
|
||||
);
|
||||
let tree: serde_json::Value = match auth(client.get(&tree_url)).send().await {
|
||||
Ok(r) if r.status().is_success() => r.json().await.unwrap_or_default(),
|
||||
_ if unauthenticated => {
|
||||
return "(the repository tree could not be read: GITEA_TOKEN is unset, \
|
||||
so this read was unauthenticated)"
|
||||
.to_string()
|
||||
}
|
||||
_ => return "(the repository tree could not be read)".to_string(),
|
||||
};
|
||||
let entries: Vec<crate::repo_digest::FileEntry> = tree
|
||||
@@ -304,8 +314,11 @@ pub async fn decide(
|
||||
.map_err(|_| ApiError::Internal)?
|
||||
.ok_or(ApiError::NotFound)?;
|
||||
if mission.status != "draft" {
|
||||
eprintln!("mission {id}: plan approval refused — mission is {}", mission.status);
|
||||
return Err(ApiError::BadRequest);
|
||||
return Err(ApiError::Refused(format!(
|
||||
"this mission is {} — a {} can only be approved while it is a draft, \
|
||||
because approving one rewrites how the mission will run",
|
||||
mission.status, "plan"
|
||||
)));
|
||||
}
|
||||
|
||||
let plan: Plan = serde_json::from_value(proposal.plan.clone()).map_err(|e| {
|
||||
@@ -317,6 +330,7 @@ pub async fn decide(
|
||||
// before a deploy could name a kind this build no longer dispatches.
|
||||
if let Err(why) = plan.validate() {
|
||||
eprintln!("mission {id}: plan {pid} is no longer runnable: {why}");
|
||||
let reason = why.to_string();
|
||||
let _ = cm_db::repo::mission_plan_proposals::decide(
|
||||
&state.pool,
|
||||
pid,
|
||||
@@ -326,7 +340,12 @@ pub async fn decide(
|
||||
Some(user.user_id.as_uuid().to_owned()),
|
||||
)
|
||||
.await;
|
||||
return Err(ApiError::BadRequest);
|
||||
// `Refusal` is already written as human-readable copy — it names the
|
||||
// constraint and why it exists. It was going to stderr only.
|
||||
return Err(ApiError::Refused(format!(
|
||||
"this plan is no longer runnable on the current build, so it was \
|
||||
rejected: {reason}"
|
||||
)));
|
||||
}
|
||||
|
||||
let phases = plan.phases();
|
||||
|
||||
@@ -253,8 +253,11 @@ pub async fn decide(
|
||||
.map_err(|_| ApiError::Internal)?
|
||||
.ok_or(ApiError::NotFound)?;
|
||||
if mission.status != "draft" {
|
||||
eprintln!("mission {id}: roster approval refused — mission is {}", mission.status);
|
||||
return Err(ApiError::BadRequest);
|
||||
return Err(ApiError::Refused(format!(
|
||||
"this mission is {} — a {} can only be approved while it is a draft, \
|
||||
because approving one rewrites how the mission will run",
|
||||
mission.status, "roster"
|
||||
)));
|
||||
}
|
||||
|
||||
let roster: Roster = serde_json::from_value(proposal.roster.clone()).map_err(|e| {
|
||||
@@ -269,6 +272,7 @@ pub async fn decide(
|
||||
.map_err(|_| ApiError::Internal)?;
|
||||
if let Err(why) = roster.validate(&available) {
|
||||
eprintln!("mission {id}: roster {pid} is no longer applicable: {why}");
|
||||
let reason = why.to_string();
|
||||
let _ = cm_db::repo::mission_team_proposals::decide(
|
||||
&state.pool,
|
||||
pid,
|
||||
@@ -278,7 +282,13 @@ pub async fn decide(
|
||||
Some(user.user_id.as_uuid().to_owned()),
|
||||
)
|
||||
.await;
|
||||
return Err(ApiError::BadRequest);
|
||||
// The proposal has just been auto-rejected, so the caller is about to
|
||||
// re-read a list where it says "rejected" with no visible cause. The
|
||||
// reason is the whole content of this response.
|
||||
return Err(ApiError::Refused(format!(
|
||||
"this roster no longer applies to the fleet as it is now, so it was \
|
||||
rejected: {reason}"
|
||||
)));
|
||||
}
|
||||
|
||||
let graph = roster.graph().map_err(|e| {
|
||||
|
||||
@@ -1223,6 +1223,42 @@ pub async fn retry_phase(
|
||||
/// One row per pass. The `reason` is the operator-facing explanation of why a
|
||||
/// phase iterated (or stopped), and is the same text fed back to the agents as
|
||||
/// guidance for the following pass.
|
||||
/// Skill-Use scores for a mission: did the skills we delivered change what the
|
||||
/// agent did?
|
||||
///
|
||||
/// Reads only what was recorded — the prompts the agent received and the
|
||||
/// narratives it returned. An empty result means the evidence is gone (events
|
||||
/// are reaped after 7 days unless `retain_events_until` is set), NOT that no
|
||||
/// skill was followed, and the caller has to present it that way.
|
||||
pub async fn skill_use(
|
||||
State(state): State<AppState>,
|
||||
Authed(user): Authed,
|
||||
Path(id): Path<Uuid>,
|
||||
) -> Result<Json<Value>, ApiError> {
|
||||
let _ = cm_db::repo::missions::get(&state.pool, id, user.workspace_id.as_uuid())
|
||||
.await?
|
||||
.ok_or(ApiError::NotFound)?;
|
||||
let scores = crate::skill_use::score_mission(&state.pool, id)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
eprintln!("skill_use: scoring mission {id} failed: {e}");
|
||||
ApiError::Internal
|
||||
})?;
|
||||
Ok(Json(serde_json::json!({
|
||||
"mission_id": id,
|
||||
"skills": scores,
|
||||
// Said in the payload rather than left for the reader to infer: an
|
||||
// empty list has two very different causes and they must not look the
|
||||
// same to whoever consumes this.
|
||||
"evidence": if scores.is_empty() {
|
||||
"no delivered skills found in the recorded prompts — either none \
|
||||
were delivered, or the events have been reaped"
|
||||
} else {
|
||||
"scored from recorded prompt.composed and reasoning events"
|
||||
},
|
||||
})))
|
||||
}
|
||||
|
||||
pub async fn list_phase_evaluations(
|
||||
State(state): State<AppState>,
|
||||
Authed(user): Authed,
|
||||
|
||||
@@ -83,6 +83,21 @@ pub(crate) async fn build_team(
|
||||
.await
|
||||
}
|
||||
|
||||
/// MCP bundles for a team built by the wizard or the planner rather than from a
|
||||
/// team template.
|
||||
///
|
||||
/// These teams have no template, so there is no `mcp_bundles` list to inherit —
|
||||
/// which previously meant they were provisioned with the door alone and could
|
||||
/// not reach the skills catalogue at all. `mcp_skills` scopes what it lists to
|
||||
/// the caller's workspace, so an agent with no template link still sees the
|
||||
/// global skills, which is the useful half for an ad-hoc team.
|
||||
fn adhoc_bundles() -> Vec<String> {
|
||||
vec![
|
||||
"clawmates_door".to_string(),
|
||||
"clawmates_skills".to_string(),
|
||||
]
|
||||
}
|
||||
|
||||
/// Same as `build_team` but with an explicit `lifecycle` (`permanent` |
|
||||
/// `ephemeral`). Ephemeral teams are torn down by the topology_worker after
|
||||
/// their last run terminates — used by the Scheduled + Triggered planner modes.
|
||||
@@ -140,7 +155,7 @@ pub(crate) async fn build_team_with_lifecycle(
|
||||
// Ad-hoc team-wizard teams aren't mission-bound, so they use the
|
||||
// default per-agent workspace under <install>/agents/<alias>/workspace/.
|
||||
provisioner
|
||||
.provision_claw(claw_id, &m.model, risk)
|
||||
.provision_claw(claw_id, &m.model, risk, &adhoc_bundles())
|
||||
.await
|
||||
.map_err(|e| {
|
||||
eprintln!("teams: provision claw {claw_id} failed: {e}");
|
||||
@@ -574,7 +589,7 @@ pub struct AutoProvisionRequest {
|
||||
#[serde(default)]
|
||||
pub risk_profile: Option<String>,
|
||||
/// MCP bundle aliases — same fall-back rule applies (always
|
||||
/// clawmates_door; gitea_forge when a repo is bound; deep-research
|
||||
/// clawmates_door + clawmates_skills; deep-research
|
||||
/// skill for research profiles).
|
||||
#[serde(default)]
|
||||
pub mcp_bundles: Vec<String>,
|
||||
@@ -656,11 +671,13 @@ pub async fn auto_provision(
|
||||
let mut mcp_bundles = body.mcp_bundles.clone();
|
||||
if mcp_bundles.is_empty() {
|
||||
mcp_bundles.push("clawmates_door".to_string());
|
||||
// gitea_forge is scoped to teams that will touch repos; the
|
||||
// wizard's downstream repo-binding step is what earns it.
|
||||
// Always safe to add now — the MCP layer no-ops when the token
|
||||
// isn't present in the container env.
|
||||
mcp_bundles.push("gitea_forge".to_string());
|
||||
// No `gitea_forge`: it was named in nine places and defined in none,
|
||||
// and agents reach the forge through `git` over HTTPS with the ambient
|
||||
// GITEA_TOKEN (mission_workspace::with_ambient_auth) — which is why
|
||||
// nothing ever broke. It was harmless while provision_claw ignored the
|
||||
// bundle list; now that the list is honoured, an undefined name is a
|
||||
// capability an agent is told it has and does not.
|
||||
mcp_bundles.push("clawmates_skills".to_string());
|
||||
}
|
||||
|
||||
// 1) LLM plan pass → roster JSON.
|
||||
|
||||
@@ -109,14 +109,17 @@ pub async fn compare_topologies(
|
||||
Json(req): Json<CompareRequest>,
|
||||
) -> Result<Json<Comparison>, ApiError> {
|
||||
// Execution turns run on the exec model (default = configured model, e.g.
|
||||
// sonnet); the judge uses the judge model (default claude-opus-4-8). Either
|
||||
// sonnet); the judge uses the judge model (cm_runtime::judge_model). Either
|
||||
// can name a registry provider as "<name>:<model>" (e.g. "glm:glm-4.6",
|
||||
// "kimi:kimi-k2") to run on GLM/Kimi instead.
|
||||
let exec_spec = std::env::var("CLAWMATES_TOPOLOGY_EXEC_MODEL")
|
||||
.unwrap_or_else(|_| state.runtime.model().to_string());
|
||||
let (exec_provider, exec_model) = state.runtime.resolve_provider(&exec_spec);
|
||||
let judge_spec =
|
||||
std::env::var("CLAWMATES_JUDGE_MODEL").unwrap_or_else(|_| "claude-opus-4-8".to_string());
|
||||
// `cm_runtime::judge_model()`, not a second read of the same variable: this
|
||||
// line and that function disagreed on the default (opus-4-8 vs opus-5), so
|
||||
// an unconfigured deployment scored topology comparisons on a different
|
||||
// model than the door governor and nothing recorded which.
|
||||
let judge_spec = cm_runtime::judge_model();
|
||||
let (judge_provider, judge_model) = state.runtime.resolve_provider(&judge_spec);
|
||||
let executor = ProviderExecutor::new(exec_provider, exec_model, state.runtime.max_tokens());
|
||||
let scorer = JudgeScorer::new(judge_provider, judge_model, 16);
|
||||
|
||||
@@ -14,6 +14,24 @@
|
||||
use uuid::Uuid;
|
||||
|
||||
/// The runtime agent alias for a claw id.
|
||||
/// The bundles an agent is provisioned with: whatever the template asked for,
|
||||
/// plus `clawmates_door`, always.
|
||||
///
|
||||
/// The door is not optional. It carries the §15 approval gate, so an agent
|
||||
/// provisioned without it is not a restricted agent, it is an ungated one —
|
||||
/// and a template that simply forgot to list it would silently get that.
|
||||
fn with_door(bundles: &[String]) -> Vec<String> {
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
out.push("clawmates_door".to_string());
|
||||
for b in bundles {
|
||||
let b = b.trim();
|
||||
if !b.is_empty() && !out.iter().any(|x| x == b) {
|
||||
out.push(b.to_string());
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub fn claw_alias(claw_id: Uuid) -> String {
|
||||
format!("claw_{}", claw_id.simple())
|
||||
}
|
||||
@@ -227,13 +245,26 @@ impl RuntimeProvisioner {
|
||||
}
|
||||
}
|
||||
|
||||
/// Create `claw_<id>` as a live runtime agent bound to `model_alias`,
|
||||
/// Create `claw_<id>` as a live runtime agent bound to `model_alias`,
|
||||
/// `risk_profile` (from the team template — controls which tools this
|
||||
/// agent gets: `toolfree` = nothing, `research_readonly` = file_read +
|
||||
/// content_search + glob_search, `coding_readwrite` = adds file_edit +
|
||||
/// git_operations + shell, etc.; see the `[risk_profiles.*]` allowlists
|
||||
/// in `deploy/clawmates-runtime/agent.config.example.toml`), and the
|
||||
/// `clawmates_door` MCP bundle.
|
||||
/// in `deploy/clawmates-runtime/agent.config.example.toml`), and the MCP
|
||||
/// bundles the team template asked for.
|
||||
///
|
||||
/// `bundles` used to be the constant `["clawmates_door"]`, which is how
|
||||
/// every skill in the catalogue became unreachable from a mission. The
|
||||
/// skills are delivered by ONE channel — the `clawmates_skills` MCP server
|
||||
/// (`mcp_skills.rs`) — a template that does not receive that bundle cannot
|
||||
/// list or read a single skill, and 5 of 11 templates ask for it. Two
|
||||
/// separate doc comments in `cm-runtime` describe the mission path as
|
||||
/// already having this, which is why nobody looked: the belief was written
|
||||
/// down twice and checked zero times.
|
||||
///
|
||||
/// `clawmates_door` is always included regardless of what is passed. It
|
||||
/// carries the §15 approval gate, and an agent provisioned without it does
|
||||
/// not become safer, it becomes ungated.
|
||||
///
|
||||
/// NOTE ON WORKSPACE PINNING: `[agents.<alias>.workspace.path]` is an
|
||||
/// `Option<PathBuf>` field that the ZeroClaw config prop-schema does NOT
|
||||
@@ -251,6 +282,7 @@ impl RuntimeProvisioner {
|
||||
claw_id: Uuid,
|
||||
model: &str,
|
||||
risk_profile: &str,
|
||||
bundles: &[String],
|
||||
) -> Result<String, String> {
|
||||
let alias = claw_alias(claw_id);
|
||||
let model_alias = provider_alias_for(model);
|
||||
@@ -285,7 +317,7 @@ impl RuntimeProvisioner {
|
||||
.await?;
|
||||
self.set_prop(
|
||||
&format!("agents.{alias}.mcp_bundles"),
|
||||
serde_json::json!(["clawmates_door"]),
|
||||
serde_json::json!(with_door(bundles)),
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -36,6 +36,9 @@ use uuid::Uuid;
|
||||
|
||||
use cm_db::repo::missions::UpsertTask;
|
||||
|
||||
/// `external_id` of the marker row proving a scan ran against a phase.
|
||||
pub const SCAN_MARKER: &str = "security_scan:complete";
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Finding {
|
||||
pub external_id: String,
|
||||
@@ -92,6 +95,32 @@ pub async fn run(pool: &PgPool, mission_id: Uuid, phase_id: Uuid) -> Result<usiz
|
||||
}
|
||||
}
|
||||
|
||||
// A completion marker, always written — including for a scan that found
|
||||
// nothing. Without it "we scanned and the repo is clean" and "no scan ever
|
||||
// ran" are both zero rows, and the sweep in `phase_runner` that fires this
|
||||
// would have no way to tell whether it had already run: a clean phase would
|
||||
// be rescanned on every tick, forever. It is also the answer to the
|
||||
// question an operator actually asks, which is not "how many findings"
|
||||
// but "was this looked at, by what, and when".
|
||||
let scanned_with = tools.join(", ");
|
||||
cm_db::repo::missions::upsert_task(
|
||||
pool,
|
||||
UpsertTask {
|
||||
mission_id,
|
||||
phase_id,
|
||||
external_id: SCAN_MARKER,
|
||||
title: &format!(
|
||||
"security scan complete — ran [{scanned_with}], {} finding(s)",
|
||||
all_findings.len()
|
||||
),
|
||||
assigned_agent_id: None,
|
||||
status: "created",
|
||||
run_id: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("upsert scan marker: {e}"))?;
|
||||
|
||||
for f in &all_findings {
|
||||
cm_db::repo::missions::upsert_task(
|
||||
pool,
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
//! Applies agents' own skill drafts, with no human decision.
|
||||
//!
|
||||
//! `level_up` has generated complete skill drafts from a model since it
|
||||
//! shipped; the only thing between a draft and the catalogue was an operator
|
||||
//! ticking a checkbox in `LevelUpDrawer`. This worker removes the checkbox, by
|
||||
//! operator decision.
|
||||
//!
|
||||
//! What is deliberately NOT removed is the record. Every write stays
|
||||
//! workspace-scoped and versioned, cannot take the name of a hand-authored
|
||||
//! skill, and lands with `approved_by = NULL` — so "an agent decided this" is
|
||||
//! distinguishable from "a person decided this" forever after, which is the
|
||||
//! property that makes the change reversible instead of merely fast.
|
||||
//!
|
||||
//! Only `skill_candidate` items apply here. `identity_refinement` and
|
||||
//! `brain_consolidation` still wait for a human: they change what an agent IS
|
||||
//! rather than adding a procedure it can consult.
|
||||
|
||||
use sqlx::{PgPool, Row};
|
||||
use std::time::Duration;
|
||||
|
||||
/// How often to sweep for pending drafts.
|
||||
///
|
||||
/// Proposals arrive when someone runs a level-up, not continuously, so this is
|
||||
/// slow on purpose — the work is bounded by how often an agent reflects, and
|
||||
/// polling faster would only add load.
|
||||
const SWEEP_INTERVAL: Duration = Duration::from_secs(120);
|
||||
|
||||
/// Start the sweep, unless self-authoring is switched off.
|
||||
pub fn spawn(pool: PgPool) {
|
||||
if !crate::level_up::self_authoring_enabled() {
|
||||
eprintln!(
|
||||
"skill_self_authoring: DISABLED (CLAWMATES_SKILL_SELF_AUTHORING) — \
|
||||
agent skill drafts wait for a human in the level-up drawer"
|
||||
);
|
||||
return;
|
||||
}
|
||||
eprintln!(
|
||||
"skill_self_authoring: ENABLED — agents apply their own skill drafts \
|
||||
without human approval. Writes are workspace-scoped, versioned, and \
|
||||
cannot take a hand-authored skill's name; each lands with no approver \
|
||||
recorded. Set CLAWMATES_SKILL_SELF_AUTHORING=0 to restore the gate."
|
||||
);
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
if let Err(e) = sweep(&pool).await {
|
||||
eprintln!("skill_self_authoring: sweep failed: {e}");
|
||||
}
|
||||
tokio::time::sleep(SWEEP_INTERVAL).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Apply every pending proposal's skill candidates. Returns how many skills landed.
|
||||
pub async fn sweep(pool: &PgPool) -> Result<usize, String> {
|
||||
// Bounded per pass: a backlog drains over several sweeps rather than
|
||||
// holding the pool for as long as it takes to apply all of it.
|
||||
let rows = sqlx::query(
|
||||
"SELECT id, workspace_id FROM level_up_proposals
|
||||
WHERE status = 'pending'
|
||||
ORDER BY created_at
|
||||
LIMIT 20",
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| format!("select pending proposals: {e}"))?;
|
||||
|
||||
let mut applied = 0usize;
|
||||
for row in &rows {
|
||||
let id: uuid::Uuid = row.get("id");
|
||||
let workspace_id: uuid::Uuid = row.get("workspace_id");
|
||||
match crate::level_up::apply_autonomous(
|
||||
pool,
|
||||
cm_domain::WorkspaceId::from(workspace_id),
|
||||
id,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(items) if !items.is_empty() => {
|
||||
applied += items.len();
|
||||
eprintln!(
|
||||
"skill_self_authoring: applied {} skill draft(s) from proposal {id} \
|
||||
with no human approval",
|
||||
items.len()
|
||||
);
|
||||
}
|
||||
// A proposal with no skill candidates is left pending on purpose —
|
||||
// its identity/memory items still belong to the human gate.
|
||||
Ok(_) => {}
|
||||
Err(e) => eprintln!("skill_self_authoring: proposal {id}: {e}"),
|
||||
}
|
||||
}
|
||||
Ok(applied)
|
||||
}
|
||||
@@ -0,0 +1,511 @@
|
||||
//! Did a skill we delivered actually change what the agent did?
|
||||
//!
|
||||
//! Scored on the three axes from `Skill-Use` (arXiv, 2026-08-05): **Trigger**
|
||||
//! (did the agent reach for the skill), **Compliance** (did it follow the
|
||||
//! procedure), **Boundary** (did it avoid what the skill forbids).
|
||||
//!
|
||||
//! ## One axis does not survive the translation, and saying so is the finding
|
||||
//!
|
||||
//! The paper measures agents under *progressive disclosure*: the agent sees a
|
||||
//! name and a description, and must decide to retrieve the body. The retrieval
|
||||
//! is the Trigger event, and it is observable because it is a tool call.
|
||||
//!
|
||||
//! We do not deliver skills that way on the mission path. `pinned_skills_text`
|
||||
//! inlines full bodies into the prompt, because mission claws run on
|
||||
//! `claude_cli`, which cannot surface a tool call at all — there is nothing to
|
||||
//! retrieve *with*. So the agent never "reaches for" a skill; it is simply
|
||||
//! holding one.
|
||||
//!
|
||||
//! Trigger is therefore **not observable on the mission path**, and this module
|
||||
//! reports it as `NotObservable` with the reason attached rather than scoring
|
||||
//! it zero. A zero would read as "the agents ignore their skills" when it
|
||||
//! actually means "the question does not apply to how we deliver them" — the
|
||||
//! precise confusion that made 55 empty skill bindings invisible for months.
|
||||
//!
|
||||
//! Compliance and Boundary are observable, because they are properties of the
|
||||
//! output rather than of the retrieval.
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
/// The outcome of one axis for one skill.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case", tag = "verdict", content = "why")]
|
||||
pub enum Verdict {
|
||||
Pass,
|
||||
Fail(String),
|
||||
/// The skill says nothing this axis can check.
|
||||
NotApplicable,
|
||||
/// The axis cannot be measured here, for a stated structural reason.
|
||||
///
|
||||
/// Distinct from `Fail` on purpose. Collapsing the two is how a
|
||||
/// measurement reports a system defect as an agent defect.
|
||||
NotObservable(String),
|
||||
}
|
||||
|
||||
impl Verdict {
|
||||
pub fn label(&self) -> &'static str {
|
||||
match self {
|
||||
Verdict::Pass => "pass",
|
||||
Verdict::Fail(_) => "FAIL",
|
||||
Verdict::NotApplicable => "n/a",
|
||||
Verdict::NotObservable(_) => "not observable",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One skill's score for one phase.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct SkillUse {
|
||||
pub skill: String,
|
||||
/// `builtin` (hand-authored) or `promoted_from_brain` (agent-authored).
|
||||
///
|
||||
/// Carried into the report because an agent that authors its own skill can
|
||||
/// raise its own compliance score without changing what it does. A rising
|
||||
/// number on agent-authored skills has to be visible as such rather than
|
||||
/// averaged in with the rest.
|
||||
pub source_kind: String,
|
||||
pub trigger: Verdict,
|
||||
pub compliance: Verdict,
|
||||
pub boundary: Verdict,
|
||||
}
|
||||
|
||||
/// The skills a prompt actually delivered.
|
||||
///
|
||||
/// Delegates to the delivery layer's own parser, so the reader cannot drift
|
||||
/// from the writer. This function originally matched `## <name>` itself, and
|
||||
/// skill bodies are markdown full of `##` headings — a live mission duly
|
||||
/// scored "Sizing heuristic" and "The output shape" as skills. Parsed from the
|
||||
/// recorded prompt rather than re-derived from the catalogue, because the
|
||||
/// catalogue changes, and now that agents author their own skills it changes
|
||||
/// by itself.
|
||||
pub fn skills_in_prompt(prompt: &str) -> Vec<String> {
|
||||
crate::topology_exec::skill_names_in(prompt)
|
||||
}
|
||||
|
||||
/// Score every skill a phase's prompt delivered, against what the agent produced.
|
||||
pub fn score(prompt: &str, output: &str, source_kinds: &dyn Fn(&str) -> String) -> Vec<SkillUse> {
|
||||
skills_in_prompt(prompt)
|
||||
.into_iter()
|
||||
.map(|skill| {
|
||||
let (compliance, boundary) = check(&skill, output);
|
||||
SkillUse {
|
||||
source_kind: source_kinds(&skill),
|
||||
trigger: Verdict::NotObservable(
|
||||
"skills are inlined into the prompt, not retrieved — there is \
|
||||
no retrieval event to observe on the mission path"
|
||||
.into(),
|
||||
),
|
||||
compliance,
|
||||
boundary,
|
||||
skill,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Per-skill mechanical checks.
|
||||
///
|
||||
/// Only skills whose procedure has a machine-checkable consequence are checked.
|
||||
/// Everything else returns `NotApplicable` rather than a guess: a heuristic
|
||||
/// that scores prose by keyword overlap produces a number that looks like a
|
||||
/// measurement and is not one.
|
||||
fn check(skill: &str, output: &str) -> (Verdict, Verdict) {
|
||||
match skill {
|
||||
"int-xx-marker-protocol" => (marker_compliance(output), marker_boundary(output)),
|
||||
"arxiv-daily" => (Verdict::NotApplicable, arxiv_boundary(output)),
|
||||
"workspace-repo-commit-protocol" => (Verdict::NotApplicable, workspace_boundary(output)),
|
||||
_ => (Verdict::NotApplicable, Verdict::NotApplicable),
|
||||
}
|
||||
}
|
||||
|
||||
/// The markers must be ones `task_card_parser` actually parses.
|
||||
///
|
||||
/// Checked by running the real parser rather than a copy of its rules — a
|
||||
/// second implementation of the contract would drift from the first, and then
|
||||
/// the measurement would pass while the mission loop still stalled.
|
||||
fn marker_compliance(output: &str) -> Verdict {
|
||||
if crate::task_card_parser::parse(output).is_empty() {
|
||||
// Only a failure if the output looks like it TRIED. A turn with no
|
||||
// marker-shaped line was probably not a coding turn at all.
|
||||
if output.lines().any(|l| looks_like_marker_attempt(l)) {
|
||||
return Verdict::Fail(
|
||||
"emitted marker-shaped lines that the parser does not accept — \
|
||||
the mission loop will not advance"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
return Verdict::NotApplicable;
|
||||
}
|
||||
Verdict::Pass
|
||||
}
|
||||
|
||||
/// Rule 1: exactly one INT id per marker line.
|
||||
///
|
||||
/// Two shapes violate it, and they fail differently:
|
||||
///
|
||||
/// `COMPLETED: INT-05, INT-06` — the parser takes the first and drops the
|
||||
/// rest, so an item is silently never closed.
|
||||
/// `PLAN_COMPLETE: INT-01..02` — the range form. The parser now REJECTS a
|
||||
/// malformed id, so this is caught by the
|
||||
/// compliance check above as a marker the
|
||||
/// parser does not accept.
|
||||
///
|
||||
/// The second was found by running this measurement against a live mission. It
|
||||
/// is the worse of the two, because a dropped marker leaves a gap and a
|
||||
/// malformed one leaves a plausible-looking row.
|
||||
fn marker_boundary(output: &str) -> Verdict {
|
||||
for line in output.lines() {
|
||||
let t = line.trim();
|
||||
if !looks_like_marker_attempt(t) {
|
||||
continue;
|
||||
}
|
||||
let ids = t.matches("INT-").count();
|
||||
if ids > 1 {
|
||||
return Verdict::Fail(format!(
|
||||
"{ids} INT ids on one marker line — only the first parses, so \
|
||||
the rest are silently dropped: {t:?}"
|
||||
));
|
||||
}
|
||||
}
|
||||
Verdict::NotApplicable
|
||||
}
|
||||
|
||||
fn looks_like_marker_attempt(line: &str) -> bool {
|
||||
const KINDS: &[&str] = &[
|
||||
"TASK:", "WORK:", "HANDOFF:", "TEST_PASS:", "TEST_FAIL:", "REVIEW_APPROVE:",
|
||||
"REVIEW_BLOCK:", "COMPLETED:", "PLAN_COMPLETE:",
|
||||
];
|
||||
let t = line.trim().trim_start_matches(['*', '#', '-', '`', ' ']);
|
||||
KINDS.iter().any(|k| t.starts_with(k))
|
||||
}
|
||||
|
||||
/// The mission checkout is `/mission/repo`. Working anywhere else is not
|
||||
/// delivered.
|
||||
///
|
||||
/// Scored as a BOUNDARY rather than compliance: the skill's positive
|
||||
/// instruction ("cd there at the start of every substantive turn") has no
|
||||
/// reliable trace in the output, but writing source somewhere that is never
|
||||
/// collected does, and it is the failure that costs a whole phase.
|
||||
///
|
||||
/// This check exists because the skill itself was wrong. It taught
|
||||
/// `/workspace/repo` — a path the platform does not mount — while the same
|
||||
/// prompt told the agent `/mission/repo`. An agent that obeyed the skill wrote
|
||||
/// into a directory nothing collects. Corrected 2026-08-19, and
|
||||
/// `skills_loader::contradiction_tests` now holds it.
|
||||
fn workspace_boundary(output: &str) -> Verdict {
|
||||
// Only paths that look like a REPO root the agent chose to work in. A
|
||||
// mention of /tmp is normal; `cd /workspace/repo` is not.
|
||||
const WRONG_ROOTS: &[&str] = &["/workspace/repo", "~/workspace/repo"];
|
||||
for root in WRONG_ROOTS {
|
||||
if output.contains(root) {
|
||||
return Verdict::Fail(format!(
|
||||
"worked in {root} — the mission checkout is /mission/repo, so \
|
||||
anything written there is never collected and the phase \
|
||||
delivers nothing"
|
||||
));
|
||||
}
|
||||
}
|
||||
Verdict::Pass
|
||||
}
|
||||
|
||||
/// `arxiv-daily` forbids searching arXiv — the harvest already ran.
|
||||
///
|
||||
/// This is the one boundary we have watched an agent cross in production, so it
|
||||
/// is checked against the query endpoint rather than the word "arxiv", which
|
||||
/// appears legitimately all over a research turn.
|
||||
fn arxiv_boundary(output: &str) -> Verdict {
|
||||
const QUERY_MARKERS: &[&str] = &[
|
||||
"export.arxiv.org/api/query",
|
||||
"arxiv.org/api/query",
|
||||
"http://export.arxiv.org",
|
||||
];
|
||||
for m in QUERY_MARKERS {
|
||||
if output.contains(m) {
|
||||
return Verdict::Fail(format!(
|
||||
"queried the arXiv API ({m}) — the harvest already ran, and \
|
||||
shelving papers outside it corrupts the seen-set"
|
||||
));
|
||||
}
|
||||
}
|
||||
Verdict::Pass
|
||||
}
|
||||
|
||||
/// Score every skill delivered during a mission, from what was recorded.
|
||||
///
|
||||
/// Reads `prompt.composed` and `reasoning` rows. Both are per-mission and
|
||||
/// ordered, so the prompts say what was delivered and the narratives say what
|
||||
/// came back. Scoring is against the CONCATENATED output for the mission rather
|
||||
/// than turn-by-turn: a procedure can be followed in a later turn than the one
|
||||
/// that carried it, and pairing strictly by turn would score that as a failure.
|
||||
///
|
||||
/// Returns an empty vec for a mission whose events have already been reaped —
|
||||
/// which is why `missions.retain_events_until` exists. An empty result means
|
||||
/// "no evidence", never "no compliance", and the report has to say so.
|
||||
pub async fn score_mission(
|
||||
pool: &sqlx::PgPool,
|
||||
mission_id: uuid::Uuid,
|
||||
) -> Result<Vec<SkillUse>, String> {
|
||||
let rows = crate::mission_events::narrative_for_mission(pool, mission_id)
|
||||
.await
|
||||
.map_err(|e| format!("read narrative: {e}"))?;
|
||||
|
||||
let mut prompts = String::new();
|
||||
let mut outputs = String::new();
|
||||
for (kind, _, _, text) in &rows {
|
||||
if kind == crate::mission_events::PROMPT_COMPOSED {
|
||||
prompts.push_str(text);
|
||||
prompts.push('\n');
|
||||
} else {
|
||||
outputs.push_str(text);
|
||||
outputs.push('\n');
|
||||
}
|
||||
}
|
||||
|
||||
// One lookup for every delivered name, so the report can separate
|
||||
// hand-authored skills from the ones agents wrote for themselves.
|
||||
let names = skills_in_prompt(&prompts);
|
||||
let kinds: std::collections::HashMap<String, String> = sqlx::query_as::<_, (String, String)>(
|
||||
"SELECT name, source_kind FROM skills WHERE name = ANY($1)",
|
||||
)
|
||||
.bind(&names)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| format!("read skill sources: {e}"))?
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
Ok(score(&prompts, &outputs, &|name| {
|
||||
kinds
|
||||
.get(name)
|
||||
.cloned()
|
||||
// A skill in a prompt with no catalogue row was delivered and then
|
||||
// deleted. Naming that explicitly beats defaulting it to builtin.
|
||||
.unwrap_or_else(|| "unknown (no catalogue row)".to_string())
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn builtin(_: &str) -> String {
|
||||
"builtin".to_string()
|
||||
}
|
||||
|
||||
/// A prompt exactly as the delivery layer renders it.
|
||||
fn rendered(skills: &[(&str, &str)]) -> String {
|
||||
let body: String = skills
|
||||
.iter()
|
||||
.map(|(n, b)| crate::topology_exec::render_pinned_skill(n, b))
|
||||
.collect();
|
||||
crate::topology_exec::compose_turn_prompt("Task: do the thing", Some(&body))
|
||||
}
|
||||
|
||||
/// The prompt is the record of what was delivered, so parsing it must match
|
||||
/// exactly what the delivery layer writes.
|
||||
#[test]
|
||||
fn the_delivered_skills_are_read_back_out_of_the_prompt() {
|
||||
let prompt = rendered(&[
|
||||
("arxiv-daily", "Do not re-search."),
|
||||
("int-xx-marker-protocol", "Emit markers."),
|
||||
]);
|
||||
assert_eq!(
|
||||
skills_in_prompt(&prompt),
|
||||
vec!["arxiv-daily", "int-xx-marker-protocol"],
|
||||
"the scorer reads what the delivery layer wrote — if these drift, \
|
||||
every score is attributed to the wrong skill"
|
||||
);
|
||||
}
|
||||
|
||||
/// Skill bodies are markdown and contain their own headings.
|
||||
///
|
||||
/// The first delimiter was `## <name>`, so every section of every body
|
||||
/// counted as a separate skill. Caught on a live mission, which scored
|
||||
/// "Sizing heuristic" and "The output shape" — both of them subheadings
|
||||
/// inside `decompose-int-items` — as skills with no catalogue row.
|
||||
#[test]
|
||||
fn headings_inside_a_skill_body_are_not_skills() {
|
||||
let body = "# Decomposing into INT-XX items\n\n\
|
||||
## Sizing heuristic\nOne unit of work.\n\n\
|
||||
## The output shape\nTASK: INT-NN — title\n";
|
||||
let prompt = rendered(&[("decompose-int-items", body)]);
|
||||
assert_eq!(
|
||||
skills_in_prompt(&prompt),
|
||||
vec!["decompose-int-items"],
|
||||
"a body's own headings must not be counted as skills — every one \
|
||||
would be scored against a catalogue row that does not exist"
|
||||
);
|
||||
}
|
||||
|
||||
/// A body that quotes the marker text must not fabricate a skill either.
|
||||
#[test]
|
||||
fn a_body_mentioning_the_marker_does_not_create_a_skill() {
|
||||
let body = format!(
|
||||
"Prompts introduce a skill with a line beginning `{}`.\n",
|
||||
crate::topology_exec::SKILL_MARKER.trim()
|
||||
);
|
||||
let prompt = rendered(&[("prompt-anatomy", &body)]);
|
||||
assert_eq!(
|
||||
skills_in_prompt(&prompt),
|
||||
vec!["prompt-anatomy"],
|
||||
"the marker is matched at line start; an inline mention is prose"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_prompt_with_no_skills_yields_no_scores() {
|
||||
assert!(skills_in_prompt("Task: do the thing").is_empty());
|
||||
assert!(score("Task: do the thing", "did it", &builtin).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trigger_is_reported_as_unobservable_not_as_a_failure() {
|
||||
let prompt = rendered(&[("arxiv-daily", "x")]);
|
||||
let scored = score(&prompt, "I read the manifest.", &builtin);
|
||||
assert!(
|
||||
matches!(scored[0].trigger, Verdict::NotObservable(_)),
|
||||
"scoring Trigger zero would report a delivery-model property as an \
|
||||
agent failure — the exact confusion this module exists to avoid"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn markers_the_real_parser_accepts_are_compliant() {
|
||||
let prompt = rendered(&[("int-xx-marker-protocol", "Emit markers.")]);
|
||||
let good = "I implemented the parser.\nCOMPLETED: INT-07 — wire the loop";
|
||||
let scored = score(&prompt, good, &builtin);
|
||||
assert_eq!(scored[0].compliance, Verdict::Pass);
|
||||
}
|
||||
|
||||
/// The failure the skill exists to prevent: marker-shaped lines that the
|
||||
/// parser rejects, so the mission silently never advances.
|
||||
#[test]
|
||||
fn marker_shaped_lines_the_parser_rejects_are_a_failure() {
|
||||
let prompt = rendered(&[("int-xx-marker-protocol", "Emit markers.")]);
|
||||
// Bold, which the skill explicitly forbids, and the parser will not take.
|
||||
let bad = "**COMPLETED: INT-07**";
|
||||
let scored = score(&prompt, bad, &builtin);
|
||||
assert!(
|
||||
matches!(scored[0].compliance, Verdict::Fail(_)),
|
||||
"a marker the parser rejects must score as a failure — that is the \
|
||||
whole consequence the skill is written to avoid. Got {:?}",
|
||||
scored[0].compliance
|
||||
);
|
||||
|
||||
// A turn that never tried is not a violation.
|
||||
let unrelated = score(&prompt, "I read three files and wrote a summary.", &builtin);
|
||||
assert_eq!(unrelated[0].compliance, Verdict::NotApplicable);
|
||||
}
|
||||
|
||||
/// Found on a live mission: `PLAN_COMPLETE: INT-01..02`.
|
||||
///
|
||||
/// Two defects in one line, both since fixed in the parser rather than
|
||||
/// worked around here:
|
||||
///
|
||||
/// 1. `PLAN_COMPLETE` was documented in `int-xx-marker-protocol` and never
|
||||
/// implemented, so an agent following the skill exactly was ignored. It
|
||||
/// is implemented now.
|
||||
/// 2. The range form parsed into the id `INT-01..02`, matching no real
|
||||
/// item — a task card for something that did not exist. Ids are now
|
||||
/// strictly `INT-<digits>`, so the marker is REJECTED instead, which is
|
||||
/// visible where a plausible-looking row was not.
|
||||
#[test]
|
||||
fn plan_complete_is_now_a_real_marker() {
|
||||
let parsed = crate::task_card_parser::parse("PLAN_COMPLETE: INT-01");
|
||||
assert_eq!(parsed.len(), 1, "the skill documents it; the parser must accept it");
|
||||
assert_eq!(parsed[0].int_id, "INT-01");
|
||||
|
||||
let prompt = rendered(&[("int-xx-marker-protocol", "Emit markers.")]);
|
||||
assert_eq!(
|
||||
score(&prompt, "PLAN_COMPLETE: INT-01", &builtin)[0].compliance,
|
||||
Verdict::Pass
|
||||
);
|
||||
}
|
||||
|
||||
/// The exact line a live planner emitted.
|
||||
#[test]
|
||||
fn a_range_marker_is_rejected_rather_than_creating_a_phantom_item() {
|
||||
assert!(
|
||||
crate::task_card_parser::parse("PLAN_COMPLETE: INT-01..02").is_empty(),
|
||||
"a range must not parse — it produced a task card for an item that \
|
||||
does not exist while INT-01 and INT-02 stayed open"
|
||||
);
|
||||
assert!(crate::task_card_parser::parse("COMPLETED: INT-01..02").is_empty());
|
||||
|
||||
// And the agent is told, because the marker it emitted did nothing.
|
||||
let prompt = rendered(&[("int-xx-marker-protocol", "Emit markers.")]);
|
||||
match &score(&prompt, "PLAN_COMPLETE: INT-01..02", &builtin)[0].compliance {
|
||||
Verdict::Fail(why) => assert!(why.contains("does not accept")),
|
||||
other => panic!("an ignored marker must not read as success; got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_int_ids_on_one_marker_line_cross_the_boundary() {
|
||||
let prompt = rendered(&[("int-xx-marker-protocol", "Emit markers.")]);
|
||||
let scored = score(&prompt, "COMPLETED: INT-05, INT-06", &builtin);
|
||||
match &scored[0].boundary {
|
||||
Verdict::Fail(why) => assert!(why.contains("silently dropped")),
|
||||
other => panic!("the second id is silently dropped by the parser; got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn querying_arxiv_crosses_the_boundary_but_naming_it_does_not() {
|
||||
let prompt = rendered(&[("arxiv-daily", "x")]);
|
||||
|
||||
let violating = score(
|
||||
&prompt,
|
||||
"curl 'http://export.arxiv.org/api/query?search_query=all:agents'",
|
||||
&builtin,
|
||||
);
|
||||
assert!(matches!(violating[0].boundary, Verdict::Fail(_)));
|
||||
|
||||
// The word appears legitimately in every research turn. Scoring on it
|
||||
// would make the metric fire constantly and mean nothing.
|
||||
let fine = score(
|
||||
&prompt,
|
||||
"I read the arXiv notes in the manifest and summarised three of them.",
|
||||
&builtin,
|
||||
);
|
||||
assert_eq!(fine[0].boundary, Verdict::Pass);
|
||||
}
|
||||
|
||||
/// Agent-authored skills must stay visible as such in the report.
|
||||
#[test]
|
||||
fn writing_outside_the_mission_checkout_crosses_the_boundary() {
|
||||
let prompt = rendered(&[("workspace-repo-commit-protocol", "Work in /mission/repo.")]);
|
||||
|
||||
let wrong = score(
|
||||
&prompt,
|
||||
"cd /workspace/repo && git add -A && git commit -m 'INT-01 done'",
|
||||
&builtin,
|
||||
);
|
||||
match &wrong[0].boundary {
|
||||
Verdict::Fail(why) => assert!(
|
||||
why.contains("never collected"),
|
||||
"the failure must name the consequence — a phase that delivers \
|
||||
nothing — not just the wrong path: {why}"
|
||||
),
|
||||
other => panic!("the wrong checkout must be caught; got {other:?}"),
|
||||
}
|
||||
|
||||
let right = score(
|
||||
&prompt,
|
||||
"cd /mission/repo && git add -A && git commit -m 'INT-01 done'",
|
||||
&builtin,
|
||||
);
|
||||
assert_eq!(right[0].boundary, Verdict::Pass);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_source_of_a_skill_is_carried_into_its_score() {
|
||||
let prompt = rendered(&[("self-made", "x")]);
|
||||
let scored = score(&prompt, "done", &|_| "promoted_from_brain".to_string());
|
||||
assert_eq!(
|
||||
scored[0].source_kind, "promoted_from_brain",
|
||||
"an agent that writes its own skill can raise its own score against \
|
||||
it; that has to be legible in the report rather than averaged in"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -170,3 +170,107 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod contradiction_tests {
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn skills_root() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../skills")
|
||||
.canonicalize()
|
||||
.expect("skills dir")
|
||||
}
|
||||
|
||||
fn all_skills() -> Vec<(String, String)> {
|
||||
fn walk(dir: &std::path::Path, out: &mut Vec<(String, String)>) {
|
||||
for e in std::fs::read_dir(dir).expect("read skills dir") {
|
||||
let p = e.expect("entry").path();
|
||||
if p.is_dir() {
|
||||
walk(&p, out);
|
||||
} else if p.extension().and_then(|x| x.to_str()) == Some("md") {
|
||||
out.push((
|
||||
p.file_name().unwrap().to_string_lossy().to_string(),
|
||||
std::fs::read_to_string(&p).expect("read skill"),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
walk(&skills_root(), &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
/// No skill may teach a workspace path the platform does not use.
|
||||
///
|
||||
/// `workspace-repo-commit-protocol` told agents that `/workspace/repo` was
|
||||
/// "the ONLY path where source-modifying edits belong". The platform mounts
|
||||
/// and advertises `/mission/repo` — in 26 places — and `/workspace/repo`
|
||||
/// appears nowhere in the code. The skill is pinned on 29 role bindings and
|
||||
/// was delivered twice in a single measured run, so agents received the
|
||||
/// platform's real path and a skill contradicting it in the SAME prompt.
|
||||
#[test]
|
||||
fn no_skill_teaches_a_repo_path_the_platform_does_not_mount() {
|
||||
let mut offenders = Vec::new();
|
||||
for (name, body) in all_skills() {
|
||||
if body.contains("/workspace/repo") {
|
||||
offenders.push(name);
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
offenders.is_empty(),
|
||||
"{} skill(s) name /workspace/repo; the mission checkout is \
|
||||
/mission/repo, so an agent following them writes somewhere that is \
|
||||
never delivered: {}",
|
||||
offenders.len(),
|
||||
offenders.join(", ")
|
||||
);
|
||||
}
|
||||
|
||||
/// No skill may instruct an agent to call a tool it does not have.
|
||||
///
|
||||
/// Every mission turn ends in `claude -p`, so the tools are Claude Code's
|
||||
/// (`Read`/`Edit`/`Write`/`Bash`/`Glob`/`Grep`). `phase_task_text` used to
|
||||
/// advertise ZeroClaw's names and was fixed after five agents spent 7.4k
|
||||
/// tokens on one mission describing the mismatch instead of working — and
|
||||
/// the same wrong names survived inside a pinned skill.
|
||||
///
|
||||
/// Matched as a backticked instruction, not as bare words: a skill may
|
||||
/// legitimately DISCUSS these names, as this one now does when warning
|
||||
/// against them.
|
||||
#[test]
|
||||
fn no_skill_instructs_an_agent_to_call_a_zeroclaw_tool() {
|
||||
const ZEROCLAW_TOOLS: &[&str] = &[
|
||||
"`file_read`",
|
||||
"`file_write`",
|
||||
"`file_edit`",
|
||||
"`content_search`",
|
||||
"`glob_search`",
|
||||
];
|
||||
let mut offenders = Vec::new();
|
||||
for (name, body) in all_skills() {
|
||||
// The line has to READ as an instruction. "Do not reach for
|
||||
// `file_read`" is the correction, not the defect.
|
||||
for line in body.lines() {
|
||||
let l = line.to_ascii_lowercase();
|
||||
if l.contains("do not")
|
||||
|| l.contains("never")
|
||||
|| l.contains("instead of")
|
||||
|| l.contains("not what")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if ZEROCLAW_TOOLS.iter().any(|t| line.contains(t)) {
|
||||
offenders.push(format!("{name}: {}", line.trim()));
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
offenders.is_empty(),
|
||||
"{} skill line(s) tell an agent to use a tool its subprocess does \
|
||||
not expose:\n {}",
|
||||
offenders.len(),
|
||||
offenders.join("\n ")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,6 +39,7 @@ pub struct Marker {
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum MarkerKind {
|
||||
Task,
|
||||
PlanComplete,
|
||||
Work,
|
||||
Handoff,
|
||||
TestPass,
|
||||
@@ -53,7 +54,11 @@ impl MarkerKind {
|
||||
/// motion — the UPSERT layer may still overwrite prior states.
|
||||
pub fn status(&self) -> &'static str {
|
||||
match self {
|
||||
MarkerKind::Task => "created",
|
||||
// The planner finished specifying; no work has started, so the item
|
||||
// is in the same state a fresh TASK leaves it in. A distinct status
|
||||
// would need a column value the UI does not render, and inventing
|
||||
// one to look complete is how a status stops meaning anything.
|
||||
MarkerKind::Task | MarkerKind::PlanComplete => "created",
|
||||
MarkerKind::Work => "working",
|
||||
MarkerKind::Handoff | MarkerKind::TestPass | MarkerKind::ReviewApprove => "validating",
|
||||
MarkerKind::TestFail | MarkerKind::ReviewBlock => "failed",
|
||||
@@ -75,12 +80,27 @@ pub fn parse(text: &str) -> Vec<Marker> {
|
||||
out
|
||||
}
|
||||
|
||||
/// `INT-` followed by at least one digit and nothing else.
|
||||
fn is_int_id(id: &str) -> bool {
|
||||
match id.strip_prefix("INT-") {
|
||||
Some(rest) => !rest.is_empty() && rest.chars().all(|c| c.is_ascii_digit()),
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_line(line: &str) -> Option<Marker> {
|
||||
// Match `<KIND>: INT-NN` (rest optional). Strict on the colon and
|
||||
// the INT- prefix — anything laxer starts matching prose.
|
||||
let (kind_str, rest) = line.split_once(':')?;
|
||||
let kind = match kind_str.trim() {
|
||||
"TASK" => MarkerKind::Task,
|
||||
// Documented in `skills/foundation/int-xx-marker-protocol.md` since the
|
||||
// skill was written, and never implemented here. Agents that followed
|
||||
// the skill exactly emitted it and were silently ignored — observed on
|
||||
// a live mission, found by the Skill-Use measurement. Implemented
|
||||
// rather than removed from the skill: the planner needs a way to say
|
||||
// it is done specifying, and agents already emit this one.
|
||||
"PLAN_COMPLETE" => MarkerKind::PlanComplete,
|
||||
"WORK" => MarkerKind::Work,
|
||||
"HANDOFF" => MarkerKind::Handoff,
|
||||
"TEST_PASS" => MarkerKind::TestPass,
|
||||
@@ -95,10 +115,16 @@ fn parse_line(line: &str) -> Option<Marker> {
|
||||
Some((a, b)) => (a, Some(b.trim())),
|
||||
None => (rest, None),
|
||||
};
|
||||
if !id_tok.starts_with("INT-") {
|
||||
let int_id = id_tok.trim_end_matches(&[',', ';', '.'][..]).to_string();
|
||||
// Strictly `INT-<digits>`. `starts_with("INT-")` alone accepted range forms
|
||||
// like `INT-01..02`, which parse into an id matching no real item — so a
|
||||
// task card appeared for something that did not exist while the two items
|
||||
// it was meant to cover stayed open. Observed live. Rejecting is right:
|
||||
// the marker is ignored, which is visible, instead of creating a plausible
|
||||
// row, which is not.
|
||||
if !is_int_id(&int_id) {
|
||||
return None;
|
||||
}
|
||||
let int_id = id_tok.trim_end_matches(&[',', ';', '.'][..]).to_string();
|
||||
// Title: after the id + any of ` — / – / - ` separators
|
||||
let title = tail.and_then(|t| {
|
||||
let t = t.trim_start_matches(['—', '–', '-', ':'].as_slice()).trim();
|
||||
|
||||
@@ -306,6 +306,34 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// EVERY referenced name must resolve to an authored skill.
|
||||
///
|
||||
/// The other direction, and the one that was missing. Both existing tests
|
||||
/// assert `authored ⊆ referenced` — true of all 30 authored skills, so both
|
||||
/// passed while 55 of 85 bindings resolved to nothing and ten roles ran with
|
||||
/// an empty context bundle.
|
||||
///
|
||||
/// The old comment on the test below called the gap "deliberately
|
||||
/// aspirational". An aspirational binding is indistinguishable at runtime
|
||||
/// from a typo: `get_by_name` returns Ok(None), the loader logs a line
|
||||
/// nobody reads, and the role ships without the instructions its prompt
|
||||
/// assumes it has. If a skill is worth naming it is worth authoring, and if
|
||||
/// it is not, the name should not be in the template.
|
||||
#[test]
|
||||
fn every_referenced_skill_resolves_to_an_authored_one() {
|
||||
let mut authored = HashSet::new();
|
||||
authored_skill_names(&repo_root().join("skills"), &mut authored);
|
||||
let referenced = referenced_skill_names();
|
||||
let mut missing: Vec<_> = referenced.difference(&authored).cloned().collect();
|
||||
missing.sort();
|
||||
assert!(
|
||||
missing.is_empty(),
|
||||
"{} referenced skill(s) bind to nothing — the role gets no instructions \
|
||||
and nothing errors: {missing:#?}",
|
||||
missing.len()
|
||||
);
|
||||
}
|
||||
|
||||
/// A referenced name that matches no authored skill binds to nothing. Some
|
||||
/// are deliberately aspirational, so this asserts the *resolvable* ones
|
||||
/// stay resolvable rather than demanding every name exist.
|
||||
@@ -322,3 +350,86 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod bundle_tests {
|
||||
use std::collections::HashSet;
|
||||
|
||||
fn repo() -> std::path::PathBuf {
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../..")
|
||||
.canonicalize()
|
||||
.expect("repo root")
|
||||
}
|
||||
|
||||
/// Every `mcp_bundles` name a template asks for must be one the runtime
|
||||
/// config actually defines.
|
||||
///
|
||||
/// This was harmless while `provision_claw` wrote a constant bundle list
|
||||
/// and ignored the templates. It is not harmless now that the list is
|
||||
/// honoured: an undefined name is a capability the agent is told it has and
|
||||
/// does not, which is the same failure as an unresolved skill binding one
|
||||
/// layer down. `gitea_forge` was named by seven team templates, one
|
||||
/// workflow recipe, the auto-provision path and a user-selectable dropdown,
|
||||
/// and defined nowhere.
|
||||
#[test]
|
||||
fn every_named_mcp_bundle_is_defined_by_the_runtime_config() {
|
||||
let cfg = std::fs::read_to_string(
|
||||
repo().join("deploy/clawmates-runtime/agent.config.example.toml"),
|
||||
)
|
||||
.expect("runtime config");
|
||||
let defined: HashSet<String> = cfg
|
||||
.lines()
|
||||
.filter_map(|l| l.trim().strip_prefix("[mcp_bundles."))
|
||||
.filter_map(|r| r.strip_suffix(']'))
|
||||
.map(|s| s.to_string())
|
||||
.collect();
|
||||
assert!(
|
||||
defined.contains("clawmates_door"),
|
||||
"parsed no bundles from the runtime config — the parser, not the \
|
||||
templates, is what broke"
|
||||
);
|
||||
|
||||
let mut missing: Vec<String> = Vec::new();
|
||||
for dir in ["templates/teams", "templates/workflows"] {
|
||||
for entry in std::fs::read_dir(repo().join(dir)).expect("template dir") {
|
||||
let path = entry.expect("entry").path();
|
||||
if path.extension().and_then(|e| e.to_str()) != Some("toml") {
|
||||
continue;
|
||||
}
|
||||
let body = std::fs::read_to_string(&path).expect("read template");
|
||||
for line in body.lines() {
|
||||
let t = line.trim();
|
||||
// Skip comments: several deliberately NAME a bundle while
|
||||
// explaining that it is not delivered.
|
||||
if t.starts_with('#') || !t.starts_with("mcp_bundles") {
|
||||
continue;
|
||||
}
|
||||
let Some(inner) = t.split_once('[').and_then(|(_, r)| r.rsplit_once(']'))
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
for name in inner.0.split(',') {
|
||||
let name = name.trim().trim_matches('"');
|
||||
if !name.is_empty() && !defined.contains(name) {
|
||||
missing.push(format!(
|
||||
"{}: {name}",
|
||||
path.file_name().unwrap().to_string_lossy()
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
missing.sort();
|
||||
missing.dedup();
|
||||
assert!(
|
||||
missing.is_empty(),
|
||||
"{} template(s) name an MCP bundle the runtime does not define, so \
|
||||
the agent is provisioned with a capability that resolves to \
|
||||
nothing:\n {}",
|
||||
missing.len(),
|
||||
missing.join("\n ")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,11 +7,22 @@
|
||||
//! gateway, opens `/ws/chat?agent=<alias>`, sends the role+task+context prompt,
|
||||
//! and streams the turn's events back into a [`TurnOutcome`].
|
||||
//!
|
||||
//! **§15 by construction:** the agents are provisioned tool-free (every
|
||||
//! sensitive capability is a gated Clawmates MCP tool — the "door"), so a turn
|
||||
//! takes no sandbox-leaving action here. If the gateway nonetheless emits an
|
||||
//! `approval_request`, we record it as a **blocked** `GatedAction` and end the
|
||||
//! turn — we never auto-approve.
|
||||
//! **These agents are NOT tool-free.** That claim stood here for months and is
|
||||
//! false — see `docs/TOOL-CALL-ARCHITECTURE.md`. It was inferred from a frame
|
||||
//! stream that carried no tool events, and the emptiness has a different cause:
|
||||
//! `claude_cli` runs `claude -p --output-format json`, which returns a single
|
||||
//! final result object, and the provider hardcodes `tool_calls: Vec::new()`.
|
||||
//! The agent calls Claude Code's own tools; the transport discards them.
|
||||
//! `--output-format stream-json` emits `tool_use`/`tool_result` blocks —
|
||||
//! verified against the deployed Claude Code 2.1.228.
|
||||
//!
|
||||
//! The door-shaped provider that WOULD make this true (`--mcp-config` +
|
||||
//! `--disallowedTools` on the natives) is built and documented in
|
||||
//! `agent.config.example.toml`, and is not deployed: mission claws bind to
|
||||
//! `claude_cli.default`, which sets none of it.
|
||||
//!
|
||||
//! If the gateway emits an `approval_request` we still record it as a
|
||||
//! **blocked** `GatedAction` and end the turn — we never auto-approve.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
@@ -42,6 +53,42 @@ use tokio_tungstenite::tungstenite::Message;
|
||||
const TURN_TIMEOUT: Duration = Duration::from_secs(3600);
|
||||
|
||||
/// Drives ZeroClaw role-agents (in one container) to execute topology turns.
|
||||
/// Cap on the pinned-skill text injected into one mission turn.
|
||||
///
|
||||
/// Skill bodies average ~3.5 KB and pinning is `idx < 2 || foundation`, so a
|
||||
/// role lands near 7-10 KB. The cap exists for the role that grows a long
|
||||
/// foundation set, and it is stated in the prompt when it fires.
|
||||
pub(crate) const MAX_PINNED_SKILL_BYTES: usize = 24_000;
|
||||
|
||||
/// The line that introduces each skill in a prompt.
|
||||
///
|
||||
/// NOT a markdown heading. The first version used `## <name>`, and skill bodies
|
||||
/// are markdown that contain their own `##` headings — so anything reading the
|
||||
/// prompt back counted every section of every body as a separate skill. A live
|
||||
/// mission scored "Sizing heuristic" and "The output shape" as skills, which is
|
||||
/// what surfaced it.
|
||||
///
|
||||
/// This marker cannot occur inside a body, so the prompt stays parseable by
|
||||
/// whatever reads it later. Skills are written by one function
|
||||
/// ([`render_pinned_skill`]) for the same reason: two renderers would drift and
|
||||
/// the reader would silently match only one.
|
||||
pub const SKILL_MARKER: &str = "--- SKILL: ";
|
||||
|
||||
/// One skill, rendered for a prompt.
|
||||
pub fn render_pinned_skill(name: &str, body: &str) -> String {
|
||||
format!("\n{SKILL_MARKER}{name} ---\n{body}\n")
|
||||
}
|
||||
|
||||
/// The skill names a rendered prompt delivered.
|
||||
pub fn skill_names_in(prompt: &str) -> Vec<String> {
|
||||
prompt
|
||||
.lines()
|
||||
.filter_map(|l| l.trim().strip_prefix(SKILL_MARKER))
|
||||
.map(|rest| rest.trim_end_matches(" ---").trim().to_string())
|
||||
.filter(|n| !n.is_empty())
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub struct ZeroClawDriveExecutor {
|
||||
/// Gateway base URL, e.g. `http://127.0.0.1:42617`.
|
||||
gateway_url: String,
|
||||
@@ -100,9 +147,11 @@ pub struct ToolTrace {
|
||||
/// `chunk`, `done` and `session_start` and no tool frames at all. That is
|
||||
/// not a protocol mismatch — `tool_call` is in the deployed binary
|
||||
/// (`zeroclaw-gateway/src/ws.rs` emits `{"type":"tool_call","id","name",
|
||||
/// "args"}`) — it is §15: these agents are provisioned tool-free behind the
|
||||
/// MCP door, so they call nothing. The histogram is what let us tell those
|
||||
/// two apart, which was its whole purpose.
|
||||
/// "args"}`) — and it is NOT that the agents are tool-free, which is what
|
||||
/// this comment used to say. `claude_cli` asks for `--output-format json`,
|
||||
/// so the subprocess's tool calls never reach the gateway to be framed.
|
||||
/// The histogram still does its job: it distinguishes "no frames" from
|
||||
/// "frames we do not recognise", and the answer was the former.
|
||||
pub unmatched: std::collections::BTreeMap<String, u32>,
|
||||
}
|
||||
|
||||
@@ -276,6 +325,61 @@ impl ZeroClawDriveExecutor {
|
||||
Ok(token)
|
||||
}
|
||||
|
||||
/// The pinned skills for the claw behind `alias`, rendered for the prompt.
|
||||
///
|
||||
/// Missions had NO path to a skill. The catalogue's only delivery channel
|
||||
/// is the `clawmates_skills` MCP server, and a mission agent cannot reach
|
||||
/// it for three independent reasons: `provision_claw` wrote a constant
|
||||
/// bundle list, the runtime config defines no such bundle, and mission
|
||||
/// claws run on `claude_cli`, which is text-only and cannot surface a tool
|
||||
/// call at all. Two doc comments in `cm-runtime` describe the mission path
|
||||
/// as already having this contract. It never did — so every skill authored
|
||||
/// for a mission role was unreachable prose, and no measurement of whether
|
||||
/// skills fire could have returned anything but zero.
|
||||
///
|
||||
/// Bodies, not an index. The chat path lists names and lets the claw call
|
||||
/// `skills.read`; there is no such tool here, so an index would advertise a
|
||||
/// capability that does not exist — the exact failure this whole change is
|
||||
/// about. Pinned only (`pin_in_context`), because everything else would go
|
||||
/// in unbounded and unread.
|
||||
pub async fn pinned_skills_text(&self, alias: &str) -> Option<String> {
|
||||
let tap = self.tap.as_ref()?;
|
||||
let agent_id = crate::runtime_provision::claw_from_alias(alias)?;
|
||||
let link = cm_db::repo::agent_template_link::get(&tap.pool, agent_id)
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
let (tpl_id, slot) = link
|
||||
.as_ref()
|
||||
.map(|l| (Some(l.template_id), Some(l.role_slot.as_str())))
|
||||
.unwrap_or((None, None));
|
||||
let bindings =
|
||||
cm_db::repo::skills_catalog::effective_for_agent(&tap.pool, agent_id, tpl_id, slot)
|
||||
.await
|
||||
.ok()?;
|
||||
|
||||
let mut out = String::new();
|
||||
let mut n = 0usize;
|
||||
for b in bindings.iter().filter(|b| b.pin_in_context) {
|
||||
// Bounded, and truncation is STATED. A silently clipped procedure
|
||||
// is worse than an absent one: the agent follows the half it can
|
||||
// see and reports success against a rule it never read.
|
||||
if out.len() + b.skill.body.len() > MAX_PINNED_SKILL_BYTES {
|
||||
out.push_str(&format!(
|
||||
"\n[skill \"{}\" omitted — the pinned set exceeded {} bytes]\n",
|
||||
b.skill.name, MAX_PINNED_SKILL_BYTES
|
||||
));
|
||||
continue;
|
||||
}
|
||||
out.push_str(&render_pinned_skill(&b.skill.name, &b.skill.body));
|
||||
n += 1;
|
||||
}
|
||||
if n == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// Mirror of `ProviderExecutor`'s prompt, flattened to one `content` string
|
||||
/// (the gateway `message` envelope carries a single content field).
|
||||
fn build_prompt(req: &TurnRequest) -> String {
|
||||
@@ -674,11 +778,48 @@ impl TurnExecutor for ZeroClawDriveExecutor {
|
||||
);
|
||||
fallback
|
||||
});
|
||||
let prompt = Self::build_prompt(&req);
|
||||
let prompt = compose_turn_prompt(
|
||||
&Self::build_prompt(&req),
|
||||
self.pinned_skills_text(&alias).await.as_deref(),
|
||||
);
|
||||
// Record what this agent is ACTUALLY about to receive, before driving.
|
||||
// Re-deriving it later would re-run the skill lookup against a
|
||||
// catalogue that may have changed — and once agents author their own
|
||||
// skills, it certainly will have.
|
||||
if let Some(tap) = self.tap.as_ref() {
|
||||
let mut ev = crate::mission_events::MissionEvent::new(
|
||||
tap.mission_id,
|
||||
crate::mission_events::PROMPT_COMPOSED,
|
||||
);
|
||||
ev.phase_id = tap.phase_id;
|
||||
ev.run_id = tap.run_id;
|
||||
ev.agent_id = crate::runtime_provision::claw_from_alias(&alias);
|
||||
ev.target = Some(req.role.clone());
|
||||
ev.detail = serde_json::json!({ "text": prompt, "tier": "container" });
|
||||
crate::mission_events::record(&tap.pool, ev).await;
|
||||
}
|
||||
self.drive(&alias, &prompt).await
|
||||
}
|
||||
}
|
||||
|
||||
/// The base turn prompt with the agent's pinned skills appended, if it has any.
|
||||
///
|
||||
/// Split out from `run_turn` so the wiring is testable: `pinned_skills_text`
|
||||
/// working and `run_turn` actually calling it are different claims, and the
|
||||
/// second is the one that was false for every skill in the catalogue.
|
||||
pub fn compose_turn_prompt(base: &str, skills: Option<&str>) -> String {
|
||||
let Some(skills) = skills.map(str::trim).filter(|s| !s.is_empty()) else {
|
||||
// No heading when there is nothing under it. An empty "Your skills"
|
||||
// section tells the model it has skills and then shows it none, which
|
||||
// is worse than silence.
|
||||
return base.to_string();
|
||||
};
|
||||
format!(
|
||||
"{base}\n\n# Your skills\n\nThese are procedures you are expected to follow for \
|
||||
this kind of work. Where one applies to what you are about to do, follow it.\n\n{skills}"
|
||||
)
|
||||
}
|
||||
|
||||
/// Parse `role=alias,role=alias` into a map (blank/malformed entries skipped).
|
||||
fn parse_agent_map(s: &str) -> HashMap<String, String> {
|
||||
s.split(',')
|
||||
|
||||
@@ -464,7 +464,7 @@ mod tests {
|
||||
assert!(!install.contains(write), "{install}");
|
||||
}
|
||||
assert_eq!(
|
||||
crate::vm_tool_tap::guest_settings(Some(GATE_DIR), None)["hooks"]["Stop"][0]["hooks"]
|
||||
crate::vm_tool_tap::guest_settings(Some(GATE_DIR), None, None)["hooks"]["Stop"][0]["hooks"]
|
||||
[0]["command"],
|
||||
json!("/root/gate/stop-gate.sh")
|
||||
);
|
||||
|
||||
@@ -0,0 +1,456 @@
|
||||
//! A **pre-execution** gate on mission tool calls.
|
||||
//!
|
||||
//! Everything else that watches a mission agent watches it too late.
|
||||
//! [`crate::vm_tool_tap`] is a `PostToolUse` hook — it fires after the tool has
|
||||
//! already run, and `exit 0`s unconditionally because a non-zero `PostToolUse`
|
||||
//! talks back to the model. It is telemetry and says so.
|
||||
//!
|
||||
//! So until now a mission agent's `Bash` call was gated by nothing, anywhere.
|
||||
//! `GatePolicy` — the §15 door — has exactly one enforcement site, the chat
|
||||
//! loop, and its approvals key on `(session_id, message_id)`, which no mission
|
||||
//! phase can produce. Meanwhile the three solo tiers run `claude -p` with
|
||||
//! `--permission-mode acceptEdits`: `Read`, `Edit`, `Write`, `Bash`,
|
||||
//! pre-approved.
|
||||
//!
|
||||
//! `PreToolUse` fires under `claude -p` in this image — measured by
|
||||
//! [`crate::vm_stop_gate`], which proved the hook mechanism and the exit-2
|
||||
//! contract — and had **no callers at all**. This module is that hook.
|
||||
//!
|
||||
//! # What this is, and what it is not
|
||||
//!
|
||||
//! It is a deterministic policy gate: a small deny list of actions that are
|
||||
//! destructive or exfiltrating regardless of intent, blocked before they run,
|
||||
//! with the reason handed back to the model so it can choose differently.
|
||||
//!
|
||||
//! It is **not** the §15 human approval gate. A hook blocks the agent's process
|
||||
//! while it runs, and a human decision takes minutes to hours — waiting inside
|
||||
//! the hook would wedge the turn. Making mission work suspendable for human
|
||||
//! approval is a larger change (the approval key alone has no mission-shaped
|
||||
//! form). This closes the gap between "nothing" and "something", and it should
|
||||
//! not be described as more than that.
|
||||
//!
|
||||
//! # Why the deny list is short
|
||||
//!
|
||||
//! A gate that blocks legitimate work is worse than none: the agent cannot ask
|
||||
//! a human, so it either works around the block — which is how you get an agent
|
||||
//! doing something stranger than what you denied — or it burns the turn. Every
|
||||
//! entry here is an action with no legitimate form inside a mission checkout.
|
||||
|
||||
use serde_json::{json, Value};
|
||||
|
||||
/// Where the gate lives in the guest. Under `/root`, never the repository —
|
||||
/// anything written into the checkout would show up in the delivered diff.
|
||||
pub const GUEST_DIR: &str = "/root/toolgate";
|
||||
|
||||
/// The file the gate appends a line to for every denial.
|
||||
pub const DENIED_FILE: &str = "denied.jsonl";
|
||||
|
||||
/// How a rule's needle is matched.
|
||||
#[derive(PartialEq, Eq, Clone, Copy)]
|
||||
enum Match {
|
||||
/// The needle must START a command segment. `grep -rn 'rm -rf /' docs/`
|
||||
/// searches for the string and must not be denied; `rm -rf / …` runs it.
|
||||
/// A plain substring test cannot tell those apart, and the first version
|
||||
/// of this gate denied the grep — caught by its own test.
|
||||
Command,
|
||||
/// A flag anywhere in the segment, unless the segment is a text tool that
|
||||
/// is plainly reading or printing the flag rather than passing it.
|
||||
Flag,
|
||||
}
|
||||
|
||||
/// One denial rule.
|
||||
struct Rule {
|
||||
/// Matched against the Bash command line, case-insensitively.
|
||||
needle: &'static str,
|
||||
how: Match,
|
||||
/// Given to the model verbatim. It says what to do instead, because a bare
|
||||
/// refusal makes an agent retry the same thing with different quoting.
|
||||
reason: &'static str,
|
||||
}
|
||||
|
||||
/// Actions with no legitimate form inside a mission.
|
||||
///
|
||||
/// Deliberately not a general-purpose sandbox. The container and microVM
|
||||
/// boundaries do that job; this catches the specific commands that damage the
|
||||
/// mission itself or move its contents off the machine.
|
||||
const RULES: &[Rule] = &[
|
||||
Rule {
|
||||
needle: "rm -rf /",
|
||||
how: Match::Command,
|
||||
reason: "Refusing `rm -rf /`. Delete specific paths under the checkout \
|
||||
instead; nothing in a mission needs to remove a filesystem root.",
|
||||
},
|
||||
Rule {
|
||||
needle: "git push --force",
|
||||
how: Match::Command,
|
||||
reason: "Refusing a force push. It rewrites history other phases and \
|
||||
the reviewer rely on. Push normally, or if history genuinely \
|
||||
must change, say so in your output and stop.",
|
||||
},
|
||||
Rule {
|
||||
needle: "git push -f ",
|
||||
how: Match::Command,
|
||||
reason: "Refusing a force push. It rewrites history other phases and \
|
||||
the reviewer rely on. Push normally, or if history genuinely \
|
||||
must change, say so in your output and stop.",
|
||||
},
|
||||
Rule {
|
||||
needle: "git reset --hard origin",
|
||||
how: Match::Command,
|
||||
reason: "Refusing to hard-reset onto the remote. That discards the \
|
||||
work this phase was asked to produce. If the checkout is \
|
||||
wrong, report it rather than resetting it away.",
|
||||
},
|
||||
Rule {
|
||||
needle: "curl -x post",
|
||||
how: Match::Command,
|
||||
reason: "Refusing an outbound POST. Reading is fine; sending mission \
|
||||
content off the machine goes through the platform, not curl.",
|
||||
},
|
||||
Rule {
|
||||
needle: "--dangerously-skip-permissions",
|
||||
how: Match::Flag,
|
||||
reason: "Refusing to relaunch without permission checks. You already \
|
||||
hold the tools this phase is meant to use.",
|
||||
},
|
||||
];
|
||||
|
||||
/// The reason a command is denied, or `None` to allow it.
|
||||
///
|
||||
/// Pure so the policy is testable without a VM — the half most likely to be
|
||||
/// wrong is the matching, and it is the half that needs no guest to exercise.
|
||||
pub fn deny_reason(tool: &str, command: &str) -> Option<&'static str> {
|
||||
// Only Bash carries arbitrary commands. Read/Edit/Write are bounded by the
|
||||
// filesystem the tier already isolates, and blocking them on substrings
|
||||
// would deny a file whose CONTENTS mention a denied string.
|
||||
if !tool.eq_ignore_ascii_case("bash") {
|
||||
return None;
|
||||
}
|
||||
let lower = command.to_ascii_lowercase();
|
||||
for segment in segments(&lower) {
|
||||
let segment = segment.trim();
|
||||
if segment.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for rule in RULES {
|
||||
let hit = match rule.how {
|
||||
Match::Command => segment.starts_with(rule.needle),
|
||||
Match::Flag => segment.contains(rule.needle) && !is_text_tool(segment),
|
||||
};
|
||||
if hit {
|
||||
return Some(rule.reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Split a command line on shell separators, so each piece can be tested as a
|
||||
/// command in its own right.
|
||||
///
|
||||
/// Not a shell parser, and it does not need to be: a determined agent can
|
||||
/// defeat any string-matching gate (base64, a variable, a here-doc), and this
|
||||
/// one is aimed at accidents and obvious cases rather than at an adversary.
|
||||
/// Saying so is better than implying a guarantee it cannot make — the real
|
||||
/// isolation is the container and microVM boundary.
|
||||
fn segments(command: &str) -> Vec<&str> {
|
||||
// `&` and `|` cover `&&`/`||` too — splitting on the single character
|
||||
// leaves an empty piece between them, which the caller skips.
|
||||
command
|
||||
.split(|c| matches!(c, ';' | '|' | '&' | '\n'))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Is this segment a tool that reads or prints its arguments rather than
|
||||
/// executing them?
|
||||
fn is_text_tool(segment: &str) -> bool {
|
||||
const TEXT_TOOLS: &[&str] = &[
|
||||
"grep", "rg", "ag", "echo", "printf", "cat", "less", "head", "tail",
|
||||
"sed", "awk", "comm", "diff",
|
||||
];
|
||||
let first = segment.split_whitespace().next().unwrap_or("");
|
||||
TEXT_TOOLS.contains(&first)
|
||||
}
|
||||
|
||||
/// The guest hook script.
|
||||
///
|
||||
/// The hook is handed the tool-use event as JSON on stdin, so it must extract
|
||||
/// `tool_name` and `tool_input.command` before it can match anything. The first
|
||||
/// version matched the raw JSON text and therefore could never anchor a rule to
|
||||
/// the start of a command — `case` saw `{"tool_name":"bash",...` every time.
|
||||
///
|
||||
/// Parsing uses `node`, not `jq` (absent from the image) and not a `sed`
|
||||
/// pipeline (JSON escaping). `node` is guaranteed present: Claude Code is a
|
||||
/// node program, so any image that can run `claude` can run this.
|
||||
///
|
||||
/// Every failure path allows. A gate that fails closed on a parse error blocks
|
||||
/// every tool call in the phase, which is precisely what a `case`-syntax bug
|
||||
/// did here before a test ran the script under a real shell.
|
||||
pub fn hook_script(dir: &str) -> String {
|
||||
let mut checks = String::new();
|
||||
for r in RULES {
|
||||
// The literal half is DOUBLE-QUOTED. A `case` pattern is shell words,
|
||||
// so an unquoted needle containing a space (`rm -rf /`) is a syntax
|
||||
// error — and a syntax error makes the whole script exit non-zero,
|
||||
// which as a PreToolUse hook denies EVERY call.
|
||||
let pattern = match r.how {
|
||||
Match::Command => format!("\"{}\"*", shell_pattern(r.needle)),
|
||||
Match::Flag => format!("*\"{}\"*", shell_pattern(r.needle)),
|
||||
};
|
||||
checks.push_str(&format!(
|
||||
" case \"$seg\" in\n\
|
||||
\x20 {pattern})\n\
|
||||
\x20 printf '%s\\n' {reason} >&2\n\
|
||||
\x20 printf '%s\\n' \"$payload\" >> {dir}/{denied} 2>/dev/null\n\
|
||||
\x20 exit 2\n\
|
||||
\x20 ;;\n\
|
||||
\x20 esac\n",
|
||||
pattern = pattern,
|
||||
reason = shell_quote(r.reason),
|
||||
dir = dir,
|
||||
denied = DENIED_FILE,
|
||||
));
|
||||
}
|
||||
format!(
|
||||
"#!/bin/sh\n\
|
||||
# Pre-execution tool gate. See cm-api/src/vm_tool_gate.rs.\n\
|
||||
mkdir -p {dir} 2>/dev/null\n\
|
||||
payload=$(cat)\n\
|
||||
# Tool name on line 1, command on line 2. Anything unparseable prints\n\
|
||||
# nothing and the gate allows — never fail closed here.\n\
|
||||
info=$(printf '%s' \"$payload\" | node -e '{extract}' 2>/dev/null)\n\
|
||||
tool=$(printf '%s\\n' \"$info\" | sed -n 1p)\n\
|
||||
cmd=$(printf '%s\\n' \"$info\" | sed -n 2p)\n\
|
||||
# Only Bash carries arbitrary commands.\n\
|
||||
[ \"$tool\" = bash ] || exit 0\n\
|
||||
[ -n \"$cmd\" ] || exit 0\n\
|
||||
lower=$(printf '%s' \"$cmd\" | tr '[:upper:]' '[:lower:]')\n\
|
||||
# Split on shell separators and test each piece as its own command,\n\
|
||||
# so `grep 'rm -rf /' docs` is searching, not running.\n\
|
||||
old_ifs=$IFS\n\
|
||||
# A LITERAL newline. `IFS='\\n'` in POSIX sh sets IFS to backslash and\n\
|
||||
# the letter n, not a newline — so nothing split, and only commands\n\
|
||||
# with no separator at all were ever tested.\n\
|
||||
IFS='\n'\n\
|
||||
for seg in $(printf '%s' \"$lower\" | tr ';|&' '\\n'); do\n\
|
||||
\x20 seg=$(printf '%s' \"$seg\" | sed 's/^ *//')\n\
|
||||
{checks}\
|
||||
done\n\
|
||||
IFS=$old_ifs\n\
|
||||
# Nothing matched. Exit 0 ALLOWS the call.\n\
|
||||
exit 0\n",
|
||||
extract = NODE_EXTRACT,
|
||||
)
|
||||
}
|
||||
|
||||
/// Reads the hook event on stdin and prints `tool_name` then the command.
|
||||
///
|
||||
/// Lowercases the tool name so the shell comparison is exact. Silent on any
|
||||
/// error: the caller treats empty output as "allow".
|
||||
const NODE_EXTRACT: &str = r#"let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{try{const j=JSON.parse(s);const n=String(j.tool_name||"").toLowerCase();const c=String((j.tool_input&&j.tool_input.command)||"").replace(/\n/g," ");process.stdout.write(n+"\n"+c+"\n")}catch(e){}})"#;
|
||||
|
||||
/// A needle as a `case` pattern: glob metacharacters escaped.
|
||||
fn shell_pattern(needle: &str) -> String {
|
||||
// Inside double quotes a glob metacharacter is already literal; what must
|
||||
// not appear raw is a quote or a backslash.
|
||||
needle.replace('\\', "\\\\").replace('"', "\\\"")
|
||||
}
|
||||
|
||||
/// Single-quote for `sh`, closing and reopening around any embedded quote.
|
||||
fn shell_quote(s: &str) -> String {
|
||||
format!("'{}'", s.replace('\'', "'\\''"))
|
||||
}
|
||||
|
||||
/// The `PreToolUse` entry for the guest settings document.
|
||||
///
|
||||
/// Returned rather than written, because [`crate::vm_tool_tap::guest_settings`]
|
||||
/// is the single writer of that document and must stay so: each feature writing
|
||||
/// its own `settings.json` is a silent clobber, and the stop gate disappearing
|
||||
/// is how a coding phase completes having written nothing.
|
||||
pub fn settings_hook(dir: &str) -> Value {
|
||||
json!([{ "hooks": [{ "type": "command", "command": format!("{dir}/tool-gate.sh") }] }])
|
||||
}
|
||||
|
||||
/// One shell command that installs the gate.
|
||||
pub fn install_command(dir: &str) -> String {
|
||||
format!(
|
||||
"mkdir -p {dir} && cat > {dir}/tool-gate.sh <<'CM_GATE_EOF'\n{}\nCM_GATE_EOF\nchmod +x {dir}/tool-gate.sh",
|
||||
hook_script(dir)
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn destructive_commands_are_denied_with_a_reason_that_says_what_to_do() {
|
||||
let why = deny_reason("Bash", "rm -rf / --no-preserve-root").expect("must deny");
|
||||
assert!(
|
||||
why.contains("instead"),
|
||||
"a bare refusal makes the agent retry with different quoting: {why}"
|
||||
);
|
||||
assert!(deny_reason("Bash", "git push --force origin main").is_some());
|
||||
assert!(deny_reason("Bash", "git reset --hard origin/main").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matching_is_case_insensitive() {
|
||||
assert!(deny_reason("Bash", "GIT PUSH --FORCE origin main").is_some());
|
||||
assert!(deny_reason("bash", "RM -RF /").is_some());
|
||||
}
|
||||
|
||||
/// The gate must not become a general-purpose linter. Every one of these is
|
||||
/// ordinary mission work, and denying any of them would make an agent work
|
||||
/// around the block — which is worse than not gating.
|
||||
#[test]
|
||||
fn ordinary_mission_work_is_allowed() {
|
||||
for cmd in [
|
||||
"cargo test --workspace",
|
||||
"git add -A && git commit -m 'INT-01 done'",
|
||||
"git push origin mission-branch",
|
||||
"rm -rf target/debug",
|
||||
"rm -rf ./node_modules",
|
||||
"curl -s https://export.arxiv.org/abs/2401.00001",
|
||||
"grep -rn 'rm -rf /' docs/",
|
||||
] {
|
||||
assert_eq!(
|
||||
deny_reason("Bash", cmd),
|
||||
None,
|
||||
"denied ordinary work: {cmd}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Only Bash carries arbitrary commands. Matching a file's CONTENTS against
|
||||
/// the deny list would refuse to read a document that merely mentions one.
|
||||
#[test]
|
||||
fn non_bash_tools_are_not_matched_on_their_arguments() {
|
||||
assert_eq!(deny_reason("Read", "/mission/repo/docs/rm -rf / notes.md"), None);
|
||||
assert_eq!(deny_reason("Write", "git push --force"), None);
|
||||
}
|
||||
|
||||
/// The generated shell must agree with the Rust predicate. Two
|
||||
/// implementations of one policy is how a gate passes its unit tests and
|
||||
/// denies something else in the guest.
|
||||
#[test]
|
||||
fn the_script_carries_every_rule() {
|
||||
let script = hook_script(GUEST_DIR);
|
||||
for rule in RULES {
|
||||
assert!(
|
||||
script.contains(&shell_pattern(rule.needle)),
|
||||
"rule {:?} is enforced in Rust and missing from the guest script",
|
||||
rule.needle
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_script_denies_with_exit_2_and_allows_by_falling_through_to_exit_0() {
|
||||
let script = hook_script(GUEST_DIR);
|
||||
assert!(script.contains("exit 2"), "denial must block the call");
|
||||
assert!(
|
||||
script.trim_end().ends_with("exit 0"),
|
||||
"the last statement must be an allow — no path may fail open into a \
|
||||
non-zero exit and block legitimate work"
|
||||
);
|
||||
assert!(script.contains(">&2"), "the reason must reach the model");
|
||||
}
|
||||
|
||||
/// A reason containing an apostrophe must not break out of its quoting.
|
||||
#[test]
|
||||
fn reasons_are_shell_quoted() {
|
||||
let q = shell_quote("don't do that");
|
||||
assert_eq!(q, "'don'\\''t do that'");
|
||||
}
|
||||
}
|
||||
|
||||
/// The generated script run against a real `sh`.
|
||||
///
|
||||
/// The unit tests above check the Rust predicate and the script's TEXT. Neither
|
||||
/// proves the shell behaves: a quoting slip, a `case` pattern that never
|
||||
/// matches, or an `IFS` mistake all pass those and allow everything in the
|
||||
/// guest. The stop gate learned this the same way, which is why it has the
|
||||
/// equivalent test.
|
||||
#[cfg(test)]
|
||||
mod shell_tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
use std::process::{Command, Stdio};
|
||||
|
||||
/// Run the hook with `payload` on stdin. Returns (exit code, stderr).
|
||||
fn run(payload: &str) -> (i32, String) {
|
||||
// Unique per invocation: these tests run in parallel and each removes
|
||||
// its directory afterwards, so a shared path has them deleting the
|
||||
// script out from under each other.
|
||||
static N: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
|
||||
let seq = N.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
let dir = std::env::temp_dir().join(format!("cm-gate-{}-{seq}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let script = dir.join("tool-gate.sh");
|
||||
std::fs::write(&script, hook_script(&dir.to_string_lossy())).unwrap();
|
||||
|
||||
let mut child = Command::new("sh")
|
||||
.arg(&script)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.expect("spawn sh");
|
||||
child
|
||||
.stdin
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.write_all(payload.as_bytes())
|
||||
.unwrap();
|
||||
let out = child.wait_with_output().expect("wait");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
(
|
||||
out.status.code().unwrap_or(-1),
|
||||
String::from_utf8_lossy(&out.stderr).to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_shell_blocks_a_force_push_with_exit_2_and_a_reason() {
|
||||
let payload = r#"{"tool_name":"Bash","tool_input":{"command":"git push --force origin main"}}"#;
|
||||
let (code, stderr) = run(payload);
|
||||
assert_eq!(code, 2, "exit 2 is what blocks the call; stderr={stderr}");
|
||||
assert!(
|
||||
stderr.contains("force push"),
|
||||
"the model must be told why: {stderr}"
|
||||
);
|
||||
}
|
||||
|
||||
/// The false positive the Rust predicate was fixed for, proven in the shell
|
||||
/// too — the two implementations have to agree.
|
||||
#[test]
|
||||
fn the_shell_allows_grepping_for_a_denied_string() {
|
||||
let payload = r#"{"tool_name":"Bash","tool_input":{"command":"grep -rn 'rm -rf /' docs/"}}"#;
|
||||
let (code, stderr) = run(payload);
|
||||
assert_eq!(code, 0, "searching for the string is not running it: {stderr}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_shell_allows_ordinary_work() {
|
||||
for cmd in [
|
||||
"cargo test --workspace",
|
||||
"git add -A && git commit -m 'INT-01 done'",
|
||||
"rm -rf target/debug",
|
||||
] {
|
||||
let payload = format!(
|
||||
r#"{{"tool_name":"Bash","tool_input":{{"command":"{cmd}"}}}}"#
|
||||
);
|
||||
let (code, stderr) = run(&payload);
|
||||
assert_eq!(code, 0, "denied ordinary work {cmd:?}: {stderr}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_shell_blocks_a_destructive_delete_reached_after_a_cd() {
|
||||
let payload =
|
||||
r#"{"tool_name":"Bash","tool_input":{"command":"cd /tmp && rm -rf / --no-preserve-root"}}"#;
|
||||
let (code, _) = run(payload);
|
||||
assert_eq!(code, 2, "a separator must not smuggle the command past the gate");
|
||||
}
|
||||
}
|
||||
@@ -103,8 +103,12 @@ pub fn hook_script(dir: &str) -> String {
|
||||
/// gate exists to catch. One writer, one document, one test that both hooks
|
||||
/// survive it.
|
||||
///
|
||||
/// `None` for either half means that hook is simply absent.
|
||||
pub fn guest_settings(gate_dir: Option<&str>, tap_dir: Option<&str>) -> Value {
|
||||
/// `None` for any part means that hook is simply absent.
|
||||
pub fn guest_settings(
|
||||
gate_dir: Option<&str>,
|
||||
tap_dir: Option<&str>,
|
||||
tool_gate_dir: Option<&str>,
|
||||
) -> Value {
|
||||
let mut hooks = serde_json::Map::new();
|
||||
if let Some(dir) = gate_dir {
|
||||
hooks.insert(
|
||||
@@ -118,6 +122,12 @@ pub fn guest_settings(gate_dir: Option<&str>, tap_dir: Option<&str>) -> Value {
|
||||
json!([{ "hooks": [{ "type": "command", "command": format!("{dir}/tap.sh") }] }]),
|
||||
);
|
||||
}
|
||||
if let Some(dir) = tool_gate_dir {
|
||||
// PRE-execution, unlike the tap above. Composed here rather than
|
||||
// written by `vm_tool_gate` itself for the same reason everything else
|
||||
// is: one writer, one document.
|
||||
hooks.insert("PreToolUse".into(), crate::vm_tool_gate::settings_hook(dir));
|
||||
}
|
||||
json!({ "hooks": Value::Object(hooks) })
|
||||
}
|
||||
|
||||
@@ -195,7 +205,7 @@ mod tests {
|
||||
/// failure the gate was built to catch.
|
||||
#[test]
|
||||
fn both_hooks_survive_one_settings_document() {
|
||||
let s = guest_settings(Some("/root/gate"), Some(TAP_DIR));
|
||||
let s = guest_settings(Some("/root/gate"), Some(TAP_DIR), None);
|
||||
let hooks = s.get("hooks").expect("hooks");
|
||||
assert_eq!(
|
||||
hooks["Stop"][0]["hooks"][0]["command"],
|
||||
@@ -211,11 +221,11 @@ mod tests {
|
||||
/// Either half absent leaves the other exactly as it was.
|
||||
#[test]
|
||||
fn one_hook_alone_is_a_valid_document() {
|
||||
let gate_only = guest_settings(Some("/root/gate"), None);
|
||||
let gate_only = guest_settings(Some("/root/gate"), None, None);
|
||||
assert!(gate_only["hooks"].get("Stop").is_some());
|
||||
assert!(gate_only["hooks"].get("PostToolUse").is_none());
|
||||
|
||||
let tap_only = guest_settings(None, Some(TAP_DIR));
|
||||
let tap_only = guest_settings(None, Some(TAP_DIR), None);
|
||||
assert!(tap_only["hooks"].get("Stop").is_none());
|
||||
assert!(tap_only["hooks"].get("PostToolUse").is_some());
|
||||
}
|
||||
@@ -329,3 +339,44 @@ mod tests {
|
||||
assert!(parse(r#"{"tool_name":" ","tool_input":{}}"#).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod three_hook_tests {
|
||||
use super::*;
|
||||
|
||||
/// All three hooks must survive one document.
|
||||
///
|
||||
/// The tap and the stop gate already shared it; the pre-execution gate is
|
||||
/// the third, and the clobber this function exists to prevent gets more
|
||||
/// likely with each one. A missing `Stop` lets a phase finish having
|
||||
/// written nothing; a missing `PreToolUse` runs every command unchecked.
|
||||
#[test]
|
||||
fn the_document_carries_the_stop_gate_the_tap_and_the_pre_execution_gate() {
|
||||
let s = guest_settings(
|
||||
Some("/root/gate"),
|
||||
Some(TAP_DIR),
|
||||
Some(crate::vm_tool_gate::GUEST_DIR),
|
||||
);
|
||||
let hooks = s["hooks"].as_object().expect("hooks object");
|
||||
assert!(hooks.contains_key("Stop"), "stop gate lost");
|
||||
assert!(hooks.contains_key("PostToolUse"), "tap lost");
|
||||
assert!(hooks.contains_key("PreToolUse"), "pre-execution gate lost");
|
||||
assert_eq!(hooks.len(), 3, "an unexpected hook appeared: {hooks:?}");
|
||||
|
||||
// And the pre-execution hook points at the gate's own script, not the
|
||||
// tap's — pointing PreToolUse at tap.sh would exit 0 on everything and
|
||||
// read as a gate that allows all.
|
||||
let cmd = s["hooks"]["PreToolUse"][0]["hooks"][0]["command"]
|
||||
.as_str()
|
||||
.expect("command");
|
||||
assert!(cmd.ends_with("tool-gate.sh"), "wrong script: {cmd}");
|
||||
}
|
||||
|
||||
/// The gate alone must still produce a usable document.
|
||||
#[test]
|
||||
fn the_gate_can_be_installed_without_the_others() {
|
||||
let s = guest_settings(None, None, Some(crate::vm_tool_gate::GUEST_DIR));
|
||||
assert_eq!(s["hooks"].as_object().unwrap().len(), 1);
|
||||
assert!(s["hooks"]["PreToolUse"].is_array());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
//! Do a mission agent's pinned skills actually reach its prompt?
|
||||
//!
|
||||
//! Before this test the honest answer was no, for every skill and every role.
|
||||
//! The catalogue's only delivery channel was the `clawmates_skills` MCP server,
|
||||
//! and a mission claw could not reach it: `provision_claw` wrote a constant
|
||||
//! bundle list, the runtime config defines no such bundle, and mission claws
|
||||
//! run on `claude_cli`, which is text-only and cannot surface a tool call.
|
||||
//!
|
||||
//! So the skills were authored, bound, listed in the boot log as bound — and
|
||||
//! structurally unreadable. That is why this is a test and not a comment: the
|
||||
//! failure produced no error anywhere, and every layer reported success.
|
||||
|
||||
use cm_api::topology_exec::{MissionTap, ZeroClawDriveExecutor};
|
||||
use cm_domain::{Agent, AgentId, AgentStatus, Role, User, UserId, Workspace, WorkspaceId};
|
||||
|
||||
/// `agents.managed_by` is a real FK, so the owner has to exist.
|
||||
async fn seed_user(pool: &sqlx::PgPool, ws: WorkspaceId) -> UserId {
|
||||
let user = User {
|
||||
id: UserId::new(),
|
||||
workspace_id: ws,
|
||||
email: format!("owner-{}@example.com", Uuid::now_v7().simple()),
|
||||
role: Role::Owner,
|
||||
display_name: "Owner".into(),
|
||||
created_at: time::OffsetDateTime::UNIX_EPOCH,
|
||||
};
|
||||
cm_db::repo::users::insert(pool, &user).await.unwrap();
|
||||
user.id
|
||||
}
|
||||
use std::collections::HashMap;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// A claw with one template-bound, pinned skill. Returns its runtime alias.
|
||||
async fn seed_claw_with_pinned_skill(pool: &sqlx::PgPool, body: &str) -> (String, WorkspaceId) {
|
||||
let ws = Workspace {
|
||||
id: WorkspaceId::new(),
|
||||
name: "Skill Delivery Test".into(),
|
||||
plan: "team".into(),
|
||||
};
|
||||
cm_db::repo::workspaces::insert(pool, &ws).await.unwrap();
|
||||
|
||||
let user = seed_user(pool, ws.id).await;
|
||||
let agent = Agent {
|
||||
id: AgentId::new(),
|
||||
workspace_id: ws.id,
|
||||
name: "Scout".into(),
|
||||
job_title: "researcher".into(),
|
||||
system_prompt: String::new(),
|
||||
avatar: String::new(),
|
||||
accent: String::new(),
|
||||
wallpaper: String::new(),
|
||||
managed_by: user,
|
||||
status: AgentStatus::Online,
|
||||
};
|
||||
cm_db::repo::agents::insert(pool, &agent, &cm_domain::AccessPolicy::default())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// A template with one role, and a skill pinned to it.
|
||||
let template_id = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO team_templates (id, key, name, description, category, stack,
|
||||
default_topology, risk_profile, mcp_bundles, version)
|
||||
VALUES ($1, $2, 'Delivery Test', 'test', 'research', '{}',
|
||||
'pipeline', 'research_readonly', '{}', 1)",
|
||||
)
|
||||
.bind(template_id)
|
||||
.bind(format!("delivery_test_{}", template_id.simple()))
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
sqlx::query(
|
||||
"INSERT INTO template_roles (template_id, slot, order_idx, system_prompt)
|
||||
VALUES ($1, 'researcher', 0, 'you research')",
|
||||
)
|
||||
.bind(template_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let skill_id = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO skills
|
||||
(id, workspace_id, name, title, author, description, when_to_use,
|
||||
tags, source_kind, current_version, body)
|
||||
VALUES ($1, NULL, $2, $2, 'system',
|
||||
'a procedure the agent must follow', 'always', '{}',
|
||||
'builtin', 1, $3)",
|
||||
)
|
||||
.bind(skill_id)
|
||||
.bind(format!("delivery-test-skill-{}", skill_id.simple()))
|
||||
.bind(body)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
sqlx::query(
|
||||
"INSERT INTO template_role_skills (template_id, slot, skill_id, pin_in_context, order_idx)
|
||||
VALUES ($1, 'researcher', $2, true, 0)",
|
||||
)
|
||||
.bind(template_id)
|
||||
.bind(skill_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
cm_db::repo::agent_template_link::upsert(
|
||||
pool,
|
||||
agent.id.as_uuid(),
|
||||
template_id,
|
||||
1,
|
||||
"researcher",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
(
|
||||
cm_api::runtime_provision::claw_alias(agent.id.as_uuid()),
|
||||
ws.id,
|
||||
)
|
||||
}
|
||||
|
||||
fn executor(pool: &sqlx::PgPool, workspace_id: WorkspaceId) -> ZeroClawDriveExecutor {
|
||||
ZeroClawDriveExecutor::new(
|
||||
"http://127.0.0.1:1".into(),
|
||||
"unused".into(),
|
||||
HashMap::new(),
|
||||
"default".into(),
|
||||
)
|
||||
.with_tap(MissionTap {
|
||||
pool: pool.clone(),
|
||||
workspace_id: workspace_id.as_uuid(),
|
||||
mission_id: Uuid::now_v7(),
|
||||
phase_id: None,
|
||||
run_id: None,
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_pinned_skill_body_reaches_the_turn_prompt() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
const MARKER: &str = "Never review a paper from its title alone.";
|
||||
let (alias, ws) = seed_claw_with_pinned_skill(&pool, MARKER).await;
|
||||
|
||||
let text = executor(&pool, ws)
|
||||
.pinned_skills_text(&alias)
|
||||
.await
|
||||
.expect("a claw with a pinned template skill must produce skill text");
|
||||
|
||||
assert!(
|
||||
text.contains(MARKER),
|
||||
"the skill BODY must be present, not just its name — there is no \
|
||||
`skills.read` tool on the mission path, so an index would name a \
|
||||
procedure the agent has no way to fetch. Got:\n{text}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_agent_with_no_pinned_skills_adds_nothing() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let ws = Workspace {
|
||||
id: WorkspaceId::new(),
|
||||
name: "No Skills".into(),
|
||||
plan: "team".into(),
|
||||
};
|
||||
cm_db::repo::workspaces::insert(&pool, &ws).await.unwrap();
|
||||
let user = seed_user(&pool, ws.id).await;
|
||||
let agent = Agent {
|
||||
id: AgentId::new(),
|
||||
workspace_id: ws.id,
|
||||
name: "Bare".into(),
|
||||
job_title: "researcher".into(),
|
||||
system_prompt: String::new(),
|
||||
avatar: String::new(),
|
||||
accent: String::new(),
|
||||
wallpaper: String::new(),
|
||||
managed_by: user,
|
||||
status: AgentStatus::Online,
|
||||
};
|
||||
cm_db::repo::agents::insert(&pool, &agent, &cm_domain::AccessPolicy::default())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let alias = cm_api::runtime_provision::claw_alias(agent.id.as_uuid());
|
||||
assert!(
|
||||
executor(&pool, ws.id)
|
||||
.pinned_skills_text(&alias)
|
||||
.await
|
||||
.is_none(),
|
||||
"an agent with no bound skills must add no section at all — an empty \
|
||||
`# Your skills` heading tells the model it has skills and then shows \
|
||||
it none"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_composed_prompt_carries_the_skill_and_omits_the_heading_when_empty() {
|
||||
let base = "You are the \"researcher\" agent.\n\nTask: read the papers";
|
||||
|
||||
let with = cm_api::topology_exec::compose_turn_prompt(base, Some("## arxiv-daily\nDo not re-search."));
|
||||
assert!(with.contains("Task: read the papers"), "the base turn must survive");
|
||||
assert!(with.contains("Do not re-search."), "the skill body must be in the prompt");
|
||||
assert!(with.contains("# Your skills"), "the section needs a heading");
|
||||
|
||||
for empty in [None, Some(""), Some(" \n ")] {
|
||||
let without = cm_api::topology_exec::compose_turn_prompt(base, empty);
|
||||
assert_eq!(
|
||||
without, base,
|
||||
"with no skills the prompt must be byte-identical to the base — an \
|
||||
empty heading announces skills the agent does not have"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── The other three tiers ───────────────────────────────────────────
|
||||
//
|
||||
// `topology_exec` injects per TURN and covers only the container tier. The
|
||||
// composed-microVM, solo-microVM and direct-session paths in `phase_runner`
|
||||
// share one task string built by `phase_task_text`, and until now that string
|
||||
// carried no skill at all — so a mission on any of those tiers ran with the
|
||||
// catalogue unreachable, exactly as the container tier did before e4942ce.
|
||||
|
||||
/// Bind a claw to a mission's crew so `phase_skills_text` can find it.
|
||||
async fn seed_mission_with_crew(pool: &sqlx::PgPool, ws: WorkspaceId, agent: AgentId) -> Uuid {
|
||||
let mission = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO missions (id, workspace_id, title, template_kind, status)
|
||||
VALUES ($1, $2, 'skill delivery', 'research_only', 'running')",
|
||||
)
|
||||
.bind(mission)
|
||||
.bind(ws.as_uuid())
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let team = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO teams (id, workspace_id, name, kind, lifecycle, graph)
|
||||
VALUES ($1, $2, 'crew', 'pipeline', 'permanent', '{}'::jsonb)",
|
||||
)
|
||||
.bind(team)
|
||||
.bind(ws.as_uuid())
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO team_members (team_id, claw_id, node_id, role) VALUES ($1, $2, 'researcher', 'researcher')")
|
||||
.bind(team)
|
||||
.bind(agent.as_uuid())
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
sqlx::query("INSERT INTO mission_teams (mission_id, team_id, purpose) VALUES ($1, $2, 'mission')")
|
||||
.bind(mission)
|
||||
.bind(team)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
mission
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_microvm_and_session_tiers_get_the_skill_in_their_task_text() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
const MARKER: &str = "Never review a paper from its title alone.";
|
||||
let (alias, ws) = seed_claw_with_pinned_skill(&pool, MARKER).await;
|
||||
let agent = AgentId::from(cm_api::runtime_provision::claw_from_alias(&alias).unwrap());
|
||||
let mission = seed_mission_with_crew(&pool, ws, agent).await;
|
||||
|
||||
let skills = cm_api::phase_runner::phase_skills_text(&pool, mission)
|
||||
.await
|
||||
.expect("a mission whose crew holds a pinned skill must produce skill text");
|
||||
assert!(
|
||||
skills.contains(MARKER),
|
||||
"the pinned BODY must reach the phase task — these tiers run one \
|
||||
`claude -p` session with no per-turn injection, so this string is the \
|
||||
agent's only route to the procedure. Got:\n{skills}"
|
||||
);
|
||||
|
||||
// All three tiers share this composition, so testing it once covers them.
|
||||
let composed = cm_api::topology_exec::compose_turn_prompt("Task: read the papers", Some(&skills));
|
||||
assert!(composed.contains(MARKER));
|
||||
assert!(composed.contains("Task: read the papers"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_mission_whose_crew_has_no_skills_adds_nothing() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let ws = Workspace {
|
||||
id: WorkspaceId::new(),
|
||||
name: "Bare Crew".into(),
|
||||
plan: "team".into(),
|
||||
};
|
||||
cm_db::repo::workspaces::insert(&pool, &ws).await.unwrap();
|
||||
let user = seed_user(&pool, ws.id).await;
|
||||
let agent = Agent {
|
||||
id: AgentId::new(),
|
||||
workspace_id: ws.id,
|
||||
name: "Bare".into(),
|
||||
job_title: "researcher".into(),
|
||||
system_prompt: String::new(),
|
||||
avatar: String::new(),
|
||||
accent: String::new(),
|
||||
wallpaper: String::new(),
|
||||
managed_by: user,
|
||||
status: AgentStatus::Online,
|
||||
};
|
||||
cm_db::repo::agents::insert(&pool, &agent, &cm_domain::AccessPolicy::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let mission = seed_mission_with_crew(&pool, ws.id, agent.id).await;
|
||||
|
||||
assert!(
|
||||
cm_api::phase_runner::phase_skills_text(&pool, mission)
|
||||
.await
|
||||
.is_none(),
|
||||
"a crew with no pinned skills must add no section — the empty-heading \
|
||||
rule has to hold on this path too"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
//! What an agent received, and what it said it did, must survive the phase.
|
||||
//!
|
||||
//! `docs/PROVENANCE-ASSESSMENT.md` records "why did the agent say X?" as
|
||||
//! unanswerable. The first two things it needs are the prompt and the
|
||||
//! narrative. Before this, the prompt was never stored at all, and the
|
||||
//! narrative was stored and then read by nothing — both database readers in
|
||||
//! `routes/world.rs` filter to `tool.call`/`file.touch`, and the only other
|
||||
//! statement touching the table is the GC that deletes it.
|
||||
|
||||
use cm_api::mission_events::{self, MissionEvent, PER_PHASE_CAP, PROMPT_COMPOSED, REASONING, TOOL_CALL};
|
||||
use cm_domain::{Workspace, WorkspaceId};
|
||||
use uuid::Uuid;
|
||||
|
||||
async fn seed_phase(pool: &sqlx::PgPool) -> (Uuid, Uuid) {
|
||||
let ws = Workspace {
|
||||
id: WorkspaceId::new(),
|
||||
name: "Provenance Test".into(),
|
||||
plan: "team".into(),
|
||||
};
|
||||
cm_db::repo::workspaces::insert(pool, &ws).await.unwrap();
|
||||
let mission = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO missions (id, workspace_id, title, template_kind, status)
|
||||
VALUES ($1, $2, 'provenance', 'research_only', 'running')",
|
||||
)
|
||||
.bind(mission)
|
||||
.bind(ws.id.as_uuid())
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let phase = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO mission_phases (id, mission_id, kind, order_idx, status)
|
||||
VALUES ($1, $2, 'research', 0, 'running')",
|
||||
)
|
||||
.bind(phase)
|
||||
.bind(mission)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
(mission, phase)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_prompt_and_the_narrative_are_both_readable_after_the_fact() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (mission, phase) = seed_phase(&pool).await;
|
||||
|
||||
let mut prompt = MissionEvent::new(mission, PROMPT_COMPOSED);
|
||||
prompt.phase_id = Some(phase);
|
||||
prompt.target = Some("researcher".into());
|
||||
prompt.detail = serde_json::json!({ "text": "Task: read the papers\n## arxiv-daily\nDo not re-search." });
|
||||
mission_events::record(&pool, prompt).await;
|
||||
|
||||
let mut said = MissionEvent::new(mission, REASONING);
|
||||
said.phase_id = Some(phase);
|
||||
said.detail = serde_json::json!({ "text": "I read the manifest and wrote analysis.md." });
|
||||
mission_events::record(&pool, said).await;
|
||||
|
||||
let narrative = mission_events::narrative_for_mission(&pool, mission)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let prompt_text = narrative
|
||||
.iter()
|
||||
.find(|(kind, ..)| kind == PROMPT_COMPOSED)
|
||||
.map(|(.., text)| text.clone())
|
||||
.expect("the prompt must be recoverable — re-deriving it later re-runs \
|
||||
the skill lookup against a catalogue that will have changed");
|
||||
assert!(prompt_text.contains("Do not re-search."));
|
||||
assert!(
|
||||
prompt_text.contains("Task: read the papers"),
|
||||
"the whole composed prompt, not just the skills half"
|
||||
);
|
||||
|
||||
assert!(
|
||||
narrative
|
||||
.iter()
|
||||
.any(|(kind, .., text)| kind == REASONING && text.contains("analysis.md")),
|
||||
"the agent's own account must come back out of the database"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_busy_phase_cannot_push_out_its_own_provenance() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (mission, phase) = seed_phase(&pool).await;
|
||||
|
||||
// Fill the phase past the cap with the kind the cap exists to bound.
|
||||
for i in 0..(PER_PHASE_CAP + 20) {
|
||||
let mut ev = MissionEvent::new(mission, TOOL_CALL);
|
||||
ev.phase_id = Some(phase);
|
||||
ev.target = Some(format!("tool_{i}"));
|
||||
mission_events::record(&pool, ev).await;
|
||||
}
|
||||
|
||||
let tool_rows: i64 = sqlx::query_scalar(
|
||||
"SELECT count(*) FROM mission_events WHERE phase_id = $1 AND kind = $2",
|
||||
)
|
||||
.bind(phase)
|
||||
.bind(TOOL_CALL)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
tool_rows, PER_PHASE_CAP,
|
||||
"the cap must still bound the kind it was written for"
|
||||
);
|
||||
|
||||
// The prompt arrives AFTER the flood, which is the real ordering: a coding
|
||||
// phase calls its tools and then the next turn is composed.
|
||||
let mut prompt = MissionEvent::new(mission, PROMPT_COMPOSED);
|
||||
prompt.phase_id = Some(phase);
|
||||
prompt.detail = serde_json::json!({ "text": "the next turn's prompt" });
|
||||
mission_events::record(&pool, prompt).await;
|
||||
|
||||
let narrative = mission_events::narrative_for_mission(&pool, mission)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
narrative.iter().any(|(.., text)| text == "the next turn's prompt"),
|
||||
"a phase that called {} tools dropped its own prompt — the cap counted \
|
||||
provenance against a budget meant for the two unbounded kinds, so the \
|
||||
busier the phase, the less of it is explainable",
|
||||
PER_PHASE_CAP + 20
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_mission_under_measurement_keeps_its_events_past_the_window() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (kept, kept_phase) = seed_phase(&pool).await;
|
||||
let (reaped, reaped_phase) = seed_phase(&pool).await;
|
||||
|
||||
// Only one of them is held.
|
||||
sqlx::query("UPDATE missions SET retain_events_until = now() + interval '30 days' WHERE id = $1")
|
||||
.bind(kept)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
for (mission, phase) in [(kept, kept_phase), (reaped, reaped_phase)] {
|
||||
let mut ev = MissionEvent::new(mission, PROMPT_COMPOSED);
|
||||
ev.phase_id = Some(phase);
|
||||
ev.detail = serde_json::json!({ "text": "the prompt" });
|
||||
mission_events::record(&pool, ev).await;
|
||||
}
|
||||
// Age both beyond the global window.
|
||||
sqlx::query("UPDATE mission_events SET created_at = now() - interval '90 days'")
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut out = cm_api::mission_gc::Reclaimed::default();
|
||||
cm_api::mission_gc::reap_mission_events(&pool, &mut out).await;
|
||||
|
||||
assert!(
|
||||
!mission_events::narrative_for_mission(&pool, kept)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty(),
|
||||
"a mission held for measurement lost its events — the evidence expires \
|
||||
while the question is still open, and 'no events' reads exactly like \
|
||||
'nothing happened'"
|
||||
);
|
||||
assert!(
|
||||
mission_events::narrative_for_mission(&pool, reaped)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty(),
|
||||
"an unheld mission must still be reaped — an exemption that applies to \
|
||||
everything is not an exemption, it is a raised global bound"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
//! The guard on the security-scan sweep.
|
||||
//!
|
||||
//! `phase_runner::scan_finished_security_phases` fires `security_scan::run`
|
||||
//! for finished `security_scan` phases. The scan needs Docker; the part that
|
||||
//! decides whether it runs twice, once, or never is pure SQL, and it is the
|
||||
//! part that fails silently in both directions — rescanning forever, or never
|
||||
//! scanning at all and leaving a phase that looks identical to a clean repo.
|
||||
|
||||
use cm_domain::{Workspace, WorkspaceId};
|
||||
use uuid::Uuid;
|
||||
|
||||
async fn seed_mission(pool: &sqlx::PgPool) -> Uuid {
|
||||
let ws = Workspace {
|
||||
id: WorkspaceId::new(),
|
||||
name: "Security Sweep Test".into(),
|
||||
plan: "team".into(),
|
||||
};
|
||||
cm_db::repo::workspaces::insert(pool, &ws).await.unwrap();
|
||||
let id = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO missions (id, workspace_id, title, template_kind, status)
|
||||
VALUES ($1, $2, 'security test', 'security_hardening', 'running')",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(ws.id.as_uuid())
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
id
|
||||
}
|
||||
|
||||
async fn seed_phase(
|
||||
pool: &sqlx::PgPool,
|
||||
mission_id: Uuid,
|
||||
kind: &str,
|
||||
status: &str,
|
||||
order_idx: i32,
|
||||
) -> Uuid {
|
||||
let id = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO mission_phases (id, mission_id, kind, order_idx, status, completed_at)
|
||||
VALUES ($1, $2, $3, $4, $5, now())",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(mission_id)
|
||||
.bind(kind)
|
||||
.bind(order_idx)
|
||||
.bind(status)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
id
|
||||
}
|
||||
|
||||
async fn mark_scanned(pool: &sqlx::PgPool, mission_id: Uuid, phase_id: Uuid) {
|
||||
cm_db::repo::missions::upsert_task(
|
||||
pool,
|
||||
cm_db::repo::missions::UpsertTask {
|
||||
mission_id,
|
||||
phase_id,
|
||||
external_id: cm_api::security_scan::SCAN_MARKER,
|
||||
title: "security scan complete — ran [gitleaks], 0 finding(s)",
|
||||
assigned_agent_id: None,
|
||||
status: "created",
|
||||
run_id: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_finished_security_phase_is_selected_until_it_carries_a_scan_marker() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let mission = seed_mission(&pool).await;
|
||||
let phase = seed_phase(&pool, mission, "security_scan", "completed", 0).await;
|
||||
|
||||
let selected = cm_api::phase_runner::unscanned_security_phases(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
selected.iter().any(|(p, _)| *p == phase),
|
||||
"a finished security_scan phase with no marker must be selected — \
|
||||
otherwise the scan never runs and the phase reports completed having \
|
||||
scanned nothing"
|
||||
);
|
||||
|
||||
// A clean scan writes NO findings, so the marker is the only evidence the
|
||||
// scan happened. This is the case that would otherwise rescan forever.
|
||||
mark_scanned(&pool, mission, phase).await;
|
||||
|
||||
let selected = cm_api::phase_runner::unscanned_security_phases(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
!selected.iter().any(|(p, _)| *p == phase),
|
||||
"a phase carrying the scan marker must not be selected again, even \
|
||||
though it has zero findings"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn only_finished_security_phases_are_selected() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let mission = seed_mission(&pool).await;
|
||||
|
||||
let running = seed_phase(&pool, mission, "security_scan", "running", 0).await;
|
||||
let coding = seed_phase(&pool, mission, "coding", "completed", 1).await;
|
||||
let failed = seed_phase(&pool, mission, "security_scan", "failed", 2).await;
|
||||
|
||||
let selected: Vec<Uuid> = cm_api::phase_runner::unscanned_security_phases(&pool)
|
||||
.await
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|(p, _)| p)
|
||||
.collect();
|
||||
|
||||
assert!(
|
||||
!selected.contains(&running),
|
||||
"scanning a phase still running would scan a half-written checkout"
|
||||
);
|
||||
assert!(!selected.contains(&coding), "only security_scan phases scan");
|
||||
assert!(
|
||||
selected.contains(&failed),
|
||||
"a FAILED security phase is exactly the one worth scanning — the \
|
||||
scanners are how we find out what state it left behind"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
//! Every skill a team template names must survive the trip through the
|
||||
//! database and come back out as a real binding.
|
||||
//!
|
||||
//! `team_template_loader`'s unit test checks names against the files in
|
||||
//! `skills/`. That is not the same question: bindings are resolved by
|
||||
//! `skills_catalog::get_by_name` against rows that `skills_loader` wrote, so a
|
||||
//! skill file that exists but fails to ingest (bad frontmatter, a name that
|
||||
//! does not match its filename) still leaves the role with an empty bundle —
|
||||
//! the exact silent-empty failure the loader's own comment describes.
|
||||
//!
|
||||
//! This runs both loaders in boot order and asserts the bindings landed.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
fn repo_root() -> std::path::PathBuf {
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../..")
|
||||
.canonicalize()
|
||||
.expect("canonicalize repo root")
|
||||
}
|
||||
|
||||
/// Every `skills = [...]` name in every team template, deduplicated per role
|
||||
/// the same way the loader binds them (slot + name).
|
||||
fn referenced_bindings() -> HashSet<(String, String, String)> {
|
||||
let mut out = HashSet::new();
|
||||
let dir = repo_root().join("templates/teams");
|
||||
for entry in std::fs::read_dir(&dir).expect("read templates/teams") {
|
||||
let path = entry.expect("dir entry").path();
|
||||
if path.extension().and_then(|e| e.to_str()) != Some("toml") {
|
||||
continue;
|
||||
}
|
||||
let key = path.file_stem().unwrap().to_string_lossy().to_string();
|
||||
let body = std::fs::read_to_string(&path).expect("read template");
|
||||
let mut slot = String::new();
|
||||
for line in body.lines() {
|
||||
let t = line.trim();
|
||||
if let Some(rest) = t.strip_prefix("slot") {
|
||||
if let Some(v) = rest.split('"').nth(1) {
|
||||
slot = v.to_string();
|
||||
}
|
||||
}
|
||||
if t.starts_with("skills") {
|
||||
if let Some(inner) = t.split_once('[').and_then(|(_, r)| r.rsplit_once(']')) {
|
||||
for name in inner.0.split(',') {
|
||||
let name = name.trim().trim_matches('"');
|
||||
if !name.is_empty() {
|
||||
out.insert((key.clone(), slot.clone(), name.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn every_template_role_skill_binds_through_the_database() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
|
||||
// Both loaders resolve their directory relative to the process cwd, which
|
||||
// for an integration test is the crate, not the repo.
|
||||
let root = repo_root();
|
||||
std::env::set_var("CLAWMATES_SKILLS_DIR", root.join("skills"));
|
||||
std::env::set_var("CLAWMATES_TEAM_TEMPLATES_DIR", root.join("templates/teams"));
|
||||
|
||||
// Boot order, from clawmates-server/src/main.rs: skills first, then the
|
||||
// templates that reference them.
|
||||
let n_skills = cm_api::skills_loader::load_builtins(&pool).await;
|
||||
assert!(n_skills > 0, "skills_loader ingested nothing");
|
||||
cm_api::team_template_loader::load_builtins(&pool).await;
|
||||
|
||||
let bound: HashSet<(String, String, String)> = sqlx::query_as::<_, (String, String, String)>(
|
||||
"SELECT t.key, trs.slot, s.name \
|
||||
FROM template_role_skills trs \
|
||||
JOIN team_templates t ON t.id = trs.template_id \
|
||||
JOIN skills s ON s.id = trs.skill_id",
|
||||
)
|
||||
.fetch_all(&pool)
|
||||
.await
|
||||
.expect("read template_role_skills")
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
let referenced = referenced_bindings();
|
||||
let mut missing: Vec<String> = referenced
|
||||
.difference(&bound)
|
||||
.map(|(k, slot, name)| format!("{k}.{slot} → {name}"))
|
||||
.collect();
|
||||
missing.sort();
|
||||
|
||||
assert!(
|
||||
missing.is_empty(),
|
||||
"{} template role skill binding(s) named in TOML never reached the \
|
||||
database — those roles run with a smaller context bundle than their \
|
||||
prompt assumes:\n {}",
|
||||
missing.len(),
|
||||
missing.join("\n ")
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
//! Agents author their own skills, with no human in the loop.
|
||||
//!
|
||||
//! The operator's decision. The machinery already existed — `level_up` has
|
||||
//! generated full skill drafts from a model since it shipped — and the only
|
||||
//! thing between propose and apply was an operator ticking a checkbox.
|
||||
//!
|
||||
//! What replaces that checkbox is not another gate but three properties, and
|
||||
//! these tests are what hold them: the write is workspace-scoped and can never
|
||||
//! take a hand-authored skill's name, every change appends a version so it can
|
||||
//! be read back and reverted, and a proposal applied with no human carries no
|
||||
//! human's name in its approval trail.
|
||||
|
||||
use cm_domain::{Agent, AgentId, AgentStatus, Role, User, UserId, Workspace, WorkspaceId};
|
||||
use serde_json::json;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// A workspace with one agent. `level_up_target_one` requires a proposal to
|
||||
/// name exactly one of agent_id / team_id, so the agent is not optional here.
|
||||
async fn seed_workspace(pool: &sqlx::PgPool) -> (WorkspaceId, AgentId) {
|
||||
let ws = Workspace {
|
||||
id: WorkspaceId::new(),
|
||||
name: "Self Authoring".into(),
|
||||
plan: "team".into(),
|
||||
};
|
||||
cm_db::repo::workspaces::insert(pool, &ws).await.unwrap();
|
||||
|
||||
let user = User {
|
||||
id: UserId::new(),
|
||||
workspace_id: ws.id,
|
||||
email: format!("owner-{}@example.com", Uuid::now_v7().simple()),
|
||||
role: Role::Owner,
|
||||
display_name: "Owner".into(),
|
||||
created_at: time::OffsetDateTime::UNIX_EPOCH,
|
||||
};
|
||||
cm_db::repo::users::insert(pool, &user).await.unwrap();
|
||||
|
||||
let agent = Agent {
|
||||
id: AgentId::new(),
|
||||
workspace_id: ws.id,
|
||||
name: "Scribe".into(),
|
||||
job_title: "researcher".into(),
|
||||
system_prompt: String::new(),
|
||||
avatar: String::new(),
|
||||
accent: String::new(),
|
||||
wallpaper: String::new(),
|
||||
managed_by: user.id,
|
||||
status: AgentStatus::Online,
|
||||
};
|
||||
cm_db::repo::agents::insert(pool, &agent, &cm_domain::AccessPolicy::default())
|
||||
.await
|
||||
.unwrap();
|
||||
(ws.id, agent.id)
|
||||
}
|
||||
|
||||
/// A pending proposal carrying one `skill_candidate` draft.
|
||||
async fn seed_proposal(
|
||||
pool: &sqlx::PgPool,
|
||||
ws: WorkspaceId,
|
||||
agent: AgentId,
|
||||
name: &str,
|
||||
body: &str,
|
||||
) -> Uuid {
|
||||
let payload = json!({
|
||||
"suggested_items": [{
|
||||
"id": "item-1",
|
||||
"kind": "skill_candidate",
|
||||
"draft": {
|
||||
"name": name,
|
||||
"description": "a procedure the agent wrote for itself",
|
||||
"when_to_use": "when the situation arises",
|
||||
"body": body,
|
||||
"tags": ["self-authored"],
|
||||
}
|
||||
}]
|
||||
});
|
||||
cm_db::repo::level_up::insert(
|
||||
pool,
|
||||
cm_db::repo::level_up::NewProposal {
|
||||
workspace_id: ws.as_uuid(),
|
||||
agent_id: Some(agent.as_uuid()),
|
||||
team_id: None,
|
||||
payload: &payload,
|
||||
model: Some("glm:glm-4.7"),
|
||||
created_by: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_agent_applies_its_own_skill_with_no_human_and_it_is_versioned() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (ws, agent) = seed_workspace(&pool).await;
|
||||
|
||||
let p1 = seed_proposal(&pool, ws, agent, "vault-note-shape", "First: write the date line.").await;
|
||||
let applied = cm_api::level_up::apply_autonomous(&pool, ws, p1)
|
||||
.await
|
||||
.expect("autonomous apply must succeed");
|
||||
assert_eq!(applied, vec!["item-1".to_string()]);
|
||||
|
||||
let (id, source_kind, workspace, version): (Uuid, String, Option<Uuid>, i32) = sqlx::query_as(
|
||||
"SELECT id, source_kind, workspace_id, current_version FROM skills WHERE name = $1",
|
||||
)
|
||||
.bind("vault-note-shape")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("the skill must exist with no human approval");
|
||||
assert_eq!(source_kind, "promoted_from_brain", "self-authored skills must stay distinguishable from builtins in one query");
|
||||
assert_eq!(workspace, Some(ws.as_uuid()), "must be workspace-scoped, never global");
|
||||
assert_eq!(version, 1);
|
||||
|
||||
// The approval trail must not name a human who did not approve.
|
||||
let approved_by: Option<Uuid> =
|
||||
sqlx::query_scalar("SELECT approved_by FROM level_up_proposals WHERE id = $1")
|
||||
.bind(p1)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
approved_by.is_none(),
|
||||
"an autonomously applied proposal must record NO approver — putting a \
|
||||
user id here would attribute a decision to someone who never made it"
|
||||
);
|
||||
|
||||
// A revision bumps the version and keeps the old body readable.
|
||||
let p2 = seed_proposal(&pool, ws, agent, "vault-note-shape", "First: write the date line. Then the source.").await;
|
||||
cm_api::level_up::apply_autonomous(&pool, ws, p2).await.unwrap();
|
||||
|
||||
let versions: Vec<(i32, String)> =
|
||||
sqlx::query_as("SELECT version, body_md FROM skill_versions WHERE skill_id = $1 ORDER BY version")
|
||||
.bind(id)
|
||||
.fetch_all(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
versions.len(),
|
||||
2,
|
||||
"each self-authored revision must append a version — without history \
|
||||
there is no revert, and no way to read back which text a past run was \
|
||||
actually judged under"
|
||||
);
|
||||
assert_eq!(versions[0].1, "First: write the date line.");
|
||||
assert!(versions[1].1.contains("Then the source."));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_draft_cannot_take_a_hand_authored_skills_name() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (ws, agent) = seed_workspace(&pool).await;
|
||||
|
||||
// A builtin, as `skills_loader` writes them: global, workspace_id NULL.
|
||||
let builtin = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO skills
|
||||
(id, name, title, author, description, when_to_use, tags,
|
||||
source_kind, workspace_id, current_version, body)
|
||||
VALUES ($1,'arxiv-daily','arxiv-daily','system','the real one','always',
|
||||
'{}','builtin',NULL,1,'Do NOT re-search arXiv.')",
|
||||
)
|
||||
.bind(builtin)
|
||||
.execute(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let p = seed_proposal(&pool, ws, agent, "arxiv-daily", "Actually, re-searching arXiv is fine.").await;
|
||||
let applied = cm_api::level_up::apply_autonomous(&pool, ws, p).await.unwrap();
|
||||
assert!(
|
||||
applied.is_empty(),
|
||||
"a draft taking a hand-authored name must be refused: two procedures \
|
||||
under one name means nobody reading a transcript can tell which the \
|
||||
agent followed — and this one inverts the rule it shadows"
|
||||
);
|
||||
|
||||
let body: String = sqlx::query_scalar("SELECT body FROM skills WHERE id = $1")
|
||||
.bind(builtin)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
body, "Do NOT re-search arXiv.",
|
||||
"the hand-authored skill must be untouched"
|
||||
);
|
||||
let n: i64 = sqlx::query_scalar("SELECT count(*) FROM skills WHERE name = 'arxiv-daily'")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(n, 1, "no second row may exist under that name");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn autonomous_apply_leaves_identity_and_memory_items_for_a_human() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (ws, agent) = seed_workspace(&pool).await;
|
||||
|
||||
let payload = json!({
|
||||
"suggested_items": [
|
||||
{ "id": "skill-1", "kind": "skill_candidate",
|
||||
"draft": { "name": "commit-message-shape", "description": "d",
|
||||
"body": "Say what changed and why.", "tags": [] } },
|
||||
{ "id": "identity-1", "kind": "identity_refinement",
|
||||
"new_system_prompt": "You are now a different agent." }
|
||||
]
|
||||
});
|
||||
let p = cm_db::repo::level_up::insert(
|
||||
&pool,
|
||||
cm_db::repo::level_up::NewProposal {
|
||||
workspace_id: ws.as_uuid(),
|
||||
agent_id: Some(agent.as_uuid()),
|
||||
team_id: None,
|
||||
payload: &payload,
|
||||
model: Some("glm:glm-4.7"),
|
||||
created_by: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let applied = cm_api::level_up::apply_autonomous(&pool, ws, p).await.unwrap();
|
||||
assert_eq!(
|
||||
applied,
|
||||
vec!["skill-1".to_string()],
|
||||
"only skill_candidate items apply autonomously — an identity rewrite \
|
||||
changes what the agent IS rather than adding a procedure it can \
|
||||
consult, and that is a different bet than the one that was taken"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_sweep_applies_pending_drafts_and_leaves_nothing_pending_twice() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (ws, agent) = seed_workspace(&pool).await;
|
||||
seed_proposal(&pool, ws, agent, "swept-skill", "Say what changed and why.").await;
|
||||
|
||||
let n = cm_api::skill_self_authoring::sweep(&pool).await.unwrap();
|
||||
assert_eq!(n, 1, "the sweep must apply the pending draft with no human");
|
||||
|
||||
let body: String = sqlx::query_scalar("SELECT body FROM skills WHERE name = 'swept-skill'")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.expect("the swept draft must be in the catalogue");
|
||||
assert_eq!(body, "Say what changed and why.");
|
||||
|
||||
// Idempotent: the proposal is no longer pending, so a second pass is a
|
||||
// no-op rather than a duplicate apply or a version bump for no change.
|
||||
let again = cm_api::skill_self_authoring::sweep(&pool).await.unwrap();
|
||||
assert_eq!(again, 0, "a swept proposal must not be applied twice");
|
||||
|
||||
let versions: i64 =
|
||||
sqlx::query_scalar("SELECT count(*) FROM skill_versions sv JOIN skills s ON s.id = sv.skill_id WHERE s.name = 'swept-skill'")
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(versions, 1, "an unchanged body must not append a version");
|
||||
}
|
||||
@@ -147,9 +147,19 @@ async fn mention_round_trip_verifies_runs_and_gates_the_reply() {
|
||||
"event": {"type": "app_mention", "text": "summarize [[scenario:mention]]"}
|
||||
})
|
||||
.to_string();
|
||||
// A CURRENT timestamp. It used to be the literal "12345" — a 1970 date —
|
||||
// which passed only because nothing checked freshness. The broker now
|
||||
// enforces Slack's 5-minute replay window, so a fixture that never moves
|
||||
// starts failing the moment the guard is real.
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs()
|
||||
.to_string();
|
||||
|
||||
let forged = client
|
||||
.post(format!("{base}/api/slack/events"))
|
||||
.header("x-slack-request-timestamp", "12345")
|
||||
.header("x-slack-request-timestamp", &now)
|
||||
.header("x-slack-signature", "v0=deadbeef")
|
||||
.body(body.clone())
|
||||
.send()
|
||||
@@ -161,10 +171,10 @@ async fn mention_round_trip_verifies_runs_and_gates_the_reply() {
|
||||
let challenge_body = json!({"type": "url_verification", "challenge": "abc123"}).to_string();
|
||||
let challenge = client
|
||||
.post(format!("{base}/api/slack/events"))
|
||||
.header("x-slack-request-timestamp", "12345")
|
||||
.header("x-slack-request-timestamp", &now)
|
||||
.header(
|
||||
"x-slack-signature",
|
||||
sign(signing_secret, "12345", &challenge_body),
|
||||
sign(signing_secret, &now, &challenge_body),
|
||||
)
|
||||
.body(challenge_body.clone())
|
||||
.send()
|
||||
@@ -176,12 +186,31 @@ async fn mention_round_trip_verifies_runs_and_gates_the_reply() {
|
||||
"abc123"
|
||||
);
|
||||
|
||||
// A correctly signed request from outside the window is refused. The
|
||||
// signature is genuine — that is the point of a replay: an attacker holding
|
||||
// one captured request must not be able to use it forever.
|
||||
let stale_ts = (now.parse::<u64>().unwrap() - 3600).to_string();
|
||||
let replayed = client
|
||||
.post(format!("{base}/api/slack/events"))
|
||||
.header("x-slack-request-timestamp", &stale_ts)
|
||||
.header("x-slack-signature", sign(signing_secret, &stale_ts, &body))
|
||||
.body(body.clone())
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
replayed.status(),
|
||||
401,
|
||||
"a validly signed but hour-old request must be refused — otherwise one \
|
||||
captured request authenticates forever"
|
||||
);
|
||||
|
||||
// A properly signed mention starts a run in the '💬 Slack' session and
|
||||
// the agent's reply is intercepted by the approval gate.
|
||||
let mention = client
|
||||
.post(format!("{base}/api/slack/events"))
|
||||
.header("x-slack-request-timestamp", "12345")
|
||||
.header("x-slack-signature", sign(signing_secret, "12345", &body))
|
||||
.header("x-slack-request-timestamp", &now)
|
||||
.header("x-slack-signature", sign(signing_secret, &now, &body))
|
||||
.body(body)
|
||||
.send()
|
||||
.await
|
||||
|
||||
@@ -137,3 +137,44 @@ pub async fn usage_last_7_days(
|
||||
.await?;
|
||||
Ok((row.tokens_in, row.tokens_out, row.credits))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod pricing_tests {
|
||||
use super::*;
|
||||
|
||||
/// Pricing is the one place a rounding slip bills a real person.
|
||||
///
|
||||
/// Pure, cheap to test, and previously untested — the crate's four tests
|
||||
/// all exercise the database path, so the arithmetic underneath them was
|
||||
/// never checked directly.
|
||||
#[test]
|
||||
fn credits_round_up_and_never_charge_zero() {
|
||||
// A run that used tokens always costs at least one credit; charging
|
||||
// zero for real work is how usage silently stops being metered.
|
||||
assert_eq!(credits_for_tokens(1), 1);
|
||||
assert_eq!(credits_for_tokens(TOKENS_PER_CREDIT - 1), 1);
|
||||
assert_eq!(credits_for_tokens(TOKENS_PER_CREDIT), 1);
|
||||
// Round UP, not to nearest: one token into the next bracket is a
|
||||
// whole credit, which is the documented contract.
|
||||
assert_eq!(credits_for_tokens(TOKENS_PER_CREDIT + 1), 2);
|
||||
assert_eq!(credits_for_tokens(TOKENS_PER_CREDIT * 3), 3);
|
||||
assert_eq!(credits_for_tokens(TOKENS_PER_CREDIT * 3 + 1), 4);
|
||||
}
|
||||
|
||||
/// Zero tokens is the odd case: the `.max(1)` floor means it still costs a
|
||||
/// credit. That is deliberate, and worth pinning so a future "fix" to it
|
||||
/// is a decision rather than an accident.
|
||||
#[test]
|
||||
fn a_zero_token_run_still_costs_one_credit() {
|
||||
assert_eq!(credits_for_tokens(0), 1);
|
||||
}
|
||||
|
||||
/// The cast to i64 must not wrap into a negative charge — a negative
|
||||
/// credit is a refund, and a refund granted by an overflow is the worst
|
||||
/// shape this bug could take.
|
||||
#[test]
|
||||
fn an_absurd_token_count_does_not_wrap_negative() {
|
||||
assert!(credits_for_tokens(u64::MAX / 2) > 0);
|
||||
assert!(credits_for_tokens(u64::MAX) > 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -122,6 +122,35 @@ pub async fn mark_applied(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Mark a proposal applied with NO approving user.
|
||||
///
|
||||
/// `approved_by` stays NULL, which is the truthful record when a proposal was
|
||||
/// applied autonomously. Reusing `mark_applied` with some stand-in user id
|
||||
/// would put a human's name on a decision no human made — and the approval
|
||||
/// trail is one of the few things in this system that has to be exactly true.
|
||||
pub async fn mark_applied_autonomously(
|
||||
pool: &PgPool,
|
||||
id: Uuid,
|
||||
workspace_id: Uuid,
|
||||
applied_items: &[String],
|
||||
partial: bool,
|
||||
) -> Result<(), DbError> {
|
||||
let status = if partial { "partial" } else { "applied" };
|
||||
sqlx::query(
|
||||
"UPDATE level_up_proposals
|
||||
SET status = $3, applied_items = $4, approved_by = NULL,
|
||||
applied_at = now()
|
||||
WHERE id = $1 AND workspace_id = $2 AND status = 'pending'",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(workspace_id)
|
||||
.bind(status)
|
||||
.bind(applied_items)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn mark_rejected(
|
||||
pool: &PgPool,
|
||||
id: Uuid,
|
||||
|
||||
@@ -515,7 +515,15 @@ pub async fn upsert_task(pool: &PgPool, t: UpsertTask<'_>) -> Result<Uuid, DbErr
|
||||
assigned_agent_id, status, run_id, completed_at)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,
|
||||
CASE WHEN $9 THEN now() ELSE NULL END)
|
||||
ON CONFLICT (phase_id, external_id)
|
||||
-- The WHERE clause is REQUIRED, not decoration: the index it infers
|
||||
-- (mission_tasks_external_uniq) is PARTIAL, and Postgres will not
|
||||
-- match a partial index to an ON CONFLICT target unless the statement
|
||||
-- repeats its predicate. Without it every call raised 42P10, no
|
||||
-- unique or exclusion constraint matching the ON CONFLICT
|
||||
-- specification, so both callers -- the task-card parser and the
|
||||
-- security scanner -- failed on their first row and surfaced it as a
|
||||
-- log line rather than as a broken feature.
|
||||
ON CONFLICT (phase_id, external_id) WHERE external_id IS NOT NULL
|
||||
DO UPDATE SET
|
||||
title = EXCLUDED.title,
|
||||
assigned_agent_id = COALESCE(EXCLUDED.assigned_agent_id, mission_tasks.assigned_agent_id),
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
//! `upsert_task` against the partial unique index it depends on.
|
||||
//!
|
||||
//! `mission_tasks_external_uniq` is declared `WHERE external_id IS NOT NULL`.
|
||||
//! Postgres will not match a partial index to an `ON CONFLICT` target unless
|
||||
//! the statement repeats that predicate, so the upsert raised 42P10 on its
|
||||
//! first row for every caller — the task-card parser (INT markers) and the
|
||||
//! security scanner. Both map the error to a string their caller logs, so the
|
||||
//! feature was broken and the platform stayed green.
|
||||
//!
|
||||
//! The insert half and the update half are tested separately because they fail
|
||||
//! differently: a missing conflict target breaks both, but a wrong DO UPDATE
|
||||
//! breaks only the second write, which is the one that happens on re-run.
|
||||
|
||||
use cm_db::repo::missions::{upsert_task, UpsertTask};
|
||||
use cm_domain::{Workspace, WorkspaceId};
|
||||
use uuid::Uuid;
|
||||
|
||||
async fn seed_phase(pool: &sqlx::PgPool) -> (Uuid, Uuid) {
|
||||
let ws = Workspace {
|
||||
id: WorkspaceId::new(),
|
||||
name: "Task Upsert Test".into(),
|
||||
plan: "team".into(),
|
||||
};
|
||||
cm_db::repo::workspaces::insert(pool, &ws).await.unwrap();
|
||||
let mission = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO missions (id, workspace_id, title, template_kind, status)
|
||||
VALUES ($1, $2, 'upsert test', 'security_hardening', 'running')",
|
||||
)
|
||||
.bind(mission)
|
||||
.bind(ws.id.as_uuid())
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let phase = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO mission_phases (id, mission_id, kind, order_idx, status)
|
||||
VALUES ($1, $2, 'security_scan', 0, 'completed')",
|
||||
)
|
||||
.bind(phase)
|
||||
.bind(mission)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
(mission, phase)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_same_external_id_updates_in_place_instead_of_erroring() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (mission_id, phase_id) = seed_phase(&pool).await;
|
||||
|
||||
let first = upsert_task(
|
||||
&pool,
|
||||
UpsertTask {
|
||||
mission_id,
|
||||
phase_id,
|
||||
external_id: "RUSTSEC-2026-0001",
|
||||
title: "advisory: something",
|
||||
assigned_agent_id: None,
|
||||
status: "created",
|
||||
run_id: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("first upsert must succeed — 42P10 here means the ON CONFLICT \
|
||||
target no longer matches the partial unique index");
|
||||
|
||||
// The re-run. A scanner that finds the same advisory twice must not
|
||||
// duplicate the row, and must not fail.
|
||||
let second = upsert_task(
|
||||
&pool,
|
||||
UpsertTask {
|
||||
mission_id,
|
||||
phase_id,
|
||||
external_id: "RUSTSEC-2026-0001",
|
||||
title: "advisory: something (rescanned)",
|
||||
assigned_agent_id: None,
|
||||
status: "complete",
|
||||
run_id: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("second upsert must succeed");
|
||||
|
||||
assert_eq!(first, second, "the same (phase_id, external_id) must be one row");
|
||||
|
||||
let (title, status, completed): (String, String, Option<time::OffsetDateTime>) =
|
||||
sqlx::query_as("SELECT title, status, completed_at FROM mission_tasks WHERE id = $1")
|
||||
.bind(first)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(title, "advisory: something (rescanned)");
|
||||
assert_eq!(status, "complete");
|
||||
assert!(
|
||||
completed.is_some(),
|
||||
"a task upserted as `complete` must get its completed_at stamped"
|
||||
);
|
||||
|
||||
let n: i64 = sqlx::query_scalar("SELECT count(*) FROM mission_tasks WHERE phase_id = $1")
|
||||
.bind(phase_id)
|
||||
.fetch_one(&pool)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(n, 1, "two upserts of one external_id left {n} rows");
|
||||
}
|
||||
@@ -28,7 +28,7 @@ const APPROVAL_TTL: time::Duration = time::Duration::hours(24);
|
||||
|
||||
/// Model used for LLM-as-judge roles (the door governor and the topology
|
||||
/// comparison scorer). Judges use the strongest model — default
|
||||
/// `claude-opus-4-8` — while everything else runs on the configured default
|
||||
/// `claude-opus-5` — while everything else runs on the configured default
|
||||
/// model (`claude-sonnet-4-6`). Override with `CLAWMATES_JUDGE_MODEL`.
|
||||
pub fn judge_model() -> String {
|
||||
std::env::var("CLAWMATES_JUDGE_MODEL").unwrap_or_else(|_| "claude-opus-5".to_string())
|
||||
|
||||
@@ -20,12 +20,58 @@ fn credential_field(secret: &str, field: &str) -> String {
|
||||
}
|
||||
|
||||
/// Slack request signing: v0=hex(HMAC-SHA256(secret, "v0:{ts}:{body}")).
|
||||
/// How stale a Slack request may be before the broker refuses it.
|
||||
///
|
||||
/// Slack's documented replay window. Without it a signature stays valid
|
||||
/// forever: the timestamp is an input to the HMAC, so an old request's
|
||||
/// signature verifies exactly as well as a fresh one. Anyone who captured a
|
||||
/// single signed request — a proxy log, a mirrored packet, a leaked webhook
|
||||
/// body — could replay it indefinitely and every replay would authenticate.
|
||||
const SLACK_MAX_AGE_SECS: i64 = 5 * 60;
|
||||
|
||||
pub(crate) fn slack_signature_valid(
|
||||
signing_secret: &str,
|
||||
timestamp: &str,
|
||||
body: &str,
|
||||
signature: &str,
|
||||
) -> bool {
|
||||
slack_signature_valid_at(
|
||||
signing_secret,
|
||||
timestamp,
|
||||
body,
|
||||
signature,
|
||||
// std, not the `time` crate: `time` is a DEV-dependency here, so a
|
||||
// reference to it compiles under `cargo test` and breaks the library
|
||||
// build — which is exactly how this line shipped broken for one run.
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs() as i64)
|
||||
.unwrap_or(0),
|
||||
)
|
||||
}
|
||||
|
||||
/// [`slack_signature_valid`] with the clock injected, so freshness is testable
|
||||
/// without sleeping or mocking the process clock.
|
||||
pub(crate) fn slack_signature_valid_at(
|
||||
signing_secret: &str,
|
||||
timestamp: &str,
|
||||
body: &str,
|
||||
signature: &str,
|
||||
now_unix: i64,
|
||||
) -> bool {
|
||||
// Freshness FIRST, and in the broker rather than the caller: the broker
|
||||
// does not trust its caller (§15), and a check the caller can forget to
|
||||
// make is one that will eventually be forgotten.
|
||||
let Ok(ts) = timestamp.trim().parse::<i64>() else {
|
||||
return false;
|
||||
};
|
||||
// Symmetric window — a timestamp far in the FUTURE is equally suspect, and
|
||||
// allowing it would let an attacker mint a request valid for as long as
|
||||
// they chose.
|
||||
if (now_unix - ts).abs() > SLACK_MAX_AGE_SECS {
|
||||
return false;
|
||||
}
|
||||
|
||||
use hmac::{Hmac, Mac};
|
||||
let Ok(mut mac) = Hmac::<sha2::Sha256>::new_from_slice(signing_secret.as_bytes()) else {
|
||||
return false;
|
||||
@@ -41,6 +87,110 @@ pub(crate) fn slack_signature_valid(
|
||||
== 0
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod slack_signature_tests {
|
||||
use super::*;
|
||||
use hmac::{Hmac, Mac};
|
||||
|
||||
const SECRET: &str = "8f742231b10e8888abcd99yyyzzz85a5";
|
||||
const BODY: &str = "token=xyz&team_id=T1&text=hello";
|
||||
const NOW: i64 = 1_700_000_000;
|
||||
|
||||
fn sign(ts: i64, body: &str) -> String {
|
||||
let mut mac = Hmac::<sha2::Sha256>::new_from_slice(SECRET.as_bytes()).unwrap();
|
||||
mac.update(format!("v0:{ts}:{body}").as_bytes());
|
||||
format!("v0={}", hex::encode(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_correctly_signed_fresh_request_is_accepted() {
|
||||
let sig = sign(NOW, BODY);
|
||||
assert!(slack_signature_valid_at(
|
||||
SECRET,
|
||||
&NOW.to_string(),
|
||||
BODY,
|
||||
&sig,
|
||||
NOW
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tampered_body_is_rejected() {
|
||||
let sig = sign(NOW, BODY);
|
||||
assert!(!slack_signature_valid_at(
|
||||
SECRET,
|
||||
&NOW.to_string(),
|
||||
"token=xyz&team_id=T1&text=goodbye",
|
||||
&sig,
|
||||
NOW
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_wrong_secret_is_rejected() {
|
||||
let sig = sign(NOW, BODY);
|
||||
assert!(!slack_signature_valid_at(
|
||||
"not-the-signing-secret",
|
||||
&NOW.to_string(),
|
||||
BODY,
|
||||
&sig,
|
||||
NOW
|
||||
));
|
||||
}
|
||||
|
||||
/// The replay this window exists to stop.
|
||||
///
|
||||
/// The signature is still perfectly valid — that is the point. Anyone who
|
||||
/// captured one signed request could otherwise replay it forever.
|
||||
#[test]
|
||||
fn a_correctly_signed_but_stale_request_is_rejected() {
|
||||
let old = NOW - SLACK_MAX_AGE_SECS - 1;
|
||||
let sig = sign(old, BODY);
|
||||
assert!(
|
||||
slack_signature_valid_at(SECRET, &old.to_string(), BODY, &sig, old),
|
||||
"the signature itself is valid at its own time"
|
||||
);
|
||||
assert!(
|
||||
!slack_signature_valid_at(SECRET, &old.to_string(), BODY, &sig, NOW),
|
||||
"a validly signed request older than the window must still be refused"
|
||||
);
|
||||
}
|
||||
|
||||
/// A timestamp far in the future would otherwise mint a long-lived request.
|
||||
#[test]
|
||||
fn a_far_future_timestamp_is_rejected() {
|
||||
let future = NOW + SLACK_MAX_AGE_SECS + 1;
|
||||
let sig = sign(future, BODY);
|
||||
assert!(!slack_signature_valid_at(
|
||||
SECRET,
|
||||
&future.to_string(),
|
||||
BODY,
|
||||
&sig,
|
||||
NOW
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn small_clock_skew_in_either_direction_is_tolerated() {
|
||||
for delta in [-SLACK_MAX_AGE_SECS + 1, -30, 0, 30, SLACK_MAX_AGE_SECS - 1] {
|
||||
let ts = NOW + delta;
|
||||
let sig = sign(ts, BODY);
|
||||
assert!(
|
||||
slack_signature_valid_at(SECRET, &ts.to_string(), BODY, &sig, NOW),
|
||||
"delta {delta}s must be inside the window — a strict clock would \
|
||||
reject legitimate traffic"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_non_numeric_timestamp_is_rejected_rather_than_defaulting() {
|
||||
let sig = sign(NOW, BODY);
|
||||
assert!(!slack_signature_valid_at(SECRET, "not-a-time", BODY, &sig, NOW));
|
||||
assert!(!slack_signature_valid_at(SECRET, "", BODY, &sig, NOW));
|
||||
}
|
||||
}
|
||||
|
||||
/// The broker daemon: listens on a unix socket reachable only by the
|
||||
/// server process (never mounted into agent sandboxes).
|
||||
pub struct BrokerServer {
|
||||
|
||||
@@ -204,6 +204,35 @@ headers = { Authorization = "Bearer REPLACE_WITH_DOOR_TOKEN" }
|
||||
[mcp_bundles.clawmates_door]
|
||||
servers = ["clawmates"]
|
||||
|
||||
# The skills catalogue (cm-api::mcp_skills). Templates name this bundle and
|
||||
# `provision_claw` now honours what they name — but a bundle an agent is
|
||||
# assigned and this file does not define resolves to nothing, so the entry
|
||||
# below is what makes the assignment mean something.
|
||||
#
|
||||
# NOTE, and it is the important one: this channel only works for a provider
|
||||
# that can surface tool calls. Mission claws run on `claude_cli`, which is
|
||||
# TEXT-ONLY (see the note further down this file), so they cannot call an MCP
|
||||
# server at all. Their skills arrive as text in the turn prompt instead
|
||||
# (`topology_exec::pinned_skills_text`). This entry is for tool-capable agents
|
||||
# — and so that an agent assigned the bundle is not assigned a name that
|
||||
# resolves to nothing.
|
||||
[[mcp.servers]]
|
||||
name = "clawmates_skills"
|
||||
transport = "http"
|
||||
url = "http://clawmates_server_1:8080/mcp/skills"
|
||||
tool_timeout_secs = 60
|
||||
headers = { Authorization = "Bearer REPLACE_WITH_DOOR_TOKEN" }
|
||||
|
||||
[mcp_bundles.clawmates_skills]
|
||||
servers = ["clawmates_skills"]
|
||||
|
||||
# NOT DEFINED: `gitea_forge`. Six team templates name it and it resolves to
|
||||
# nothing here. Left undefined rather than invented — a plausible-looking
|
||||
# definition pointing at the wrong URL would turn a name that resolves to
|
||||
# nothing into a server that fails at call time, which is harder to notice, not
|
||||
# easier. Agents reach the forge through `git` over HTTPS with the ambient
|
||||
# GITEA_TOKEN (see cm-api::mission_workspace), which is why nothing broke.
|
||||
|
||||
# ── A2A ingress (Phase 2) ───────────────────────────────────────────────────
|
||||
# ZeroClaw's Agent2Agent server. These props are set at RUNTIME by cm-api
|
||||
# (runtime_provision::enable_a2a_server / publish_claw) when a workspace opts in
|
||||
@@ -228,9 +257,13 @@ servers = ["clawmates"]
|
||||
# public_base_url. Ingress throttle: CLAWMATES_A2A_POLICY=deny (kill switch),
|
||||
# CLAWMATES_A2A_RATE_LIMIT=<n>/hour.
|
||||
|
||||
# NOTE: `claude_cli` is a TEXT-ONLY provider — `claude -p` doesn't surface
|
||||
# tool-calls back to ZeroClaw, so claude_cli agents reason but can't invoke the
|
||||
# door. An agent that ACTS through the door needs a tool-capable provider
|
||||
# NOTE: `claude_cli` agents DO call tools — Claude Code's own — and they can
|
||||
# invoke the door too, via `mcp_config` (see [providers.models.claude_cli.door]
|
||||
# above). What `--output-format json` does not do is surface those calls BACK to
|
||||
# ZeroClaw: it returns one final result object, so the provider reports no tool
|
||||
# calls even when the agent made several. `--output-format stream-json` emits
|
||||
# `tool_use`/`tool_result` blocks (verified on Claude Code 2.1.228).
|
||||
# See docs/TOOL-CALL-ARCHITECTURE.md. An agent that ACTS through the door needs a tool-capable provider
|
||||
# (groq/anthropic/openai). Example actor agent (uncomment + provide groq creds
|
||||
# via env ZEROCLAW_providers__models__groq__default__{model,api_key}):
|
||||
# [providers.models.groq.default]
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
# ClawMates capability review
|
||||
|
||||
*2026-08-19. What exists, what is wired, what was repaired in this pass, and
|
||||
what is deferred with reasons.*
|
||||
|
||||
## Inventory
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Crates | 19 libraries + 4 binaries (`clawmates-server`, `-broker`, `-node`, `fcagent`) |
|
||||
| HTTP | 181 routes / 207 method+handler pairs across 39 route modules |
|
||||
| Workers | 21 background loops, 2s (approval resume) to 24h (tool versions) |
|
||||
| Database | 79 migrations, ~90 tables |
|
||||
| Templates | 6 workflow recipes, 11 team templates, **53 authored skills** (was 30) |
|
||||
| Frontend | 6 tiers (VIZ · MISSIONS · AGENT · PODCAST · REPOS · INFRA), ~57 dashboard components |
|
||||
| Execution | 5 paths: container/ZeroClaw, microVM solo, composed microVM graph, local herdr, direct headless session |
|
||||
|
||||
Dead-code hygiene is genuinely excellent: no `todo!`, no `unimplemented!`, no
|
||||
stray `TODO` in production paths. **Every defect found in this review was a
|
||||
wiring defect** — code that was correct, reachable by nothing, and reported as
|
||||
working.
|
||||
|
||||
## Where skills come from
|
||||
|
||||
Asked directly, and worth recording because it was not answerable from any
|
||||
document before now.
|
||||
|
||||
- **All 53 catalogue skills are hand-authored markdown** committed to
|
||||
`skills/**/*.md`, ingested at boot by `skills_loader` with
|
||||
`source_kind='builtin'`, `workspace_id = NULL`. None are generated at runtime
|
||||
and none are pulled from anywhere.
|
||||
- **An LLM-authoring path exists and is fully wired**: `level_up.rs:435` calls a
|
||||
model (default `glm:glm-4.7`) whose prompt asks for `skill_candidate` items
|
||||
carrying a complete `draft.body`. Nothing is written to `skills` at propose
|
||||
time — the payload sits in `level_up_proposals` as `pending`.
|
||||
- **Application WAS gated on a human click**, and as of 2026-08-19 is not.
|
||||
`apply()` still honours `approved_item_ids` for the manual path, but
|
||||
`skill_self_authoring` now sweeps pending proposals every two minutes and
|
||||
applies their `skill_candidate` items autonomously, by operator decision.
|
||||
What replaces the gate is not another gate but four properties, each held by
|
||||
a test in `tests/skill_self_authoring.rs`:
|
||||
- the write is **workspace-scoped** (`workspace_id` set, never NULL), so it
|
||||
can never modify a hand-authored skill;
|
||||
- a draft **cannot take a hand-authored skill's name** — ids are scoped and
|
||||
bindings resolve by `skill_id`, so it could not overwrite or shadow one
|
||||
anyway, but two procedures under one name means nobody reading a transcript
|
||||
can tell which the agent followed;
|
||||
- every revision **appends a `skill_versions` row**, so a self-authored skill
|
||||
can be reverted and a past run can be read back against the text it was
|
||||
actually judged under;
|
||||
- the proposal lands with **`approved_by = NULL`** — an agent's decision is
|
||||
never attributed to a person who did not make it.
|
||||
|
||||
`identity_refinement` and `brain_consolidation` still wait for a human: they
|
||||
change what an agent *is* rather than adding a procedure it can consult.
|
||||
Disable with `CLAWMATES_SKILL_SELF_AUTHORING=0`; the state is announced at
|
||||
boot either way.
|
||||
- **No external registry feeds the catalogue.** ClawBrainHub trades `.brain`
|
||||
files and never touches the `skills` table. `cm_db::repo::skills::create` —
|
||||
the only path that would produce `source_kind='hand_authored'` — has no
|
||||
production caller.
|
||||
|
||||
So the only way a skill enters the catalogue today is a committed markdown file
|
||||
or an operator approving an agent's draft.
|
||||
|
||||
## The theme
|
||||
|
||||
Everything below is one shape, the project's own "silent-success" class: a
|
||||
capability is built, a second layer does not connect to it, and every layer
|
||||
reports success. It is not carelessness. In four of the six cases the *belief*
|
||||
that the wire existed was written down in a doc comment, and no test asked.
|
||||
|
||||
That is the actionable lesson: **a comment asserting a connection is not
|
||||
evidence of one.** Two of the audit's own findings that fed this plan were
|
||||
themselves wrong in the same way, and the registry whose job is to record which
|
||||
config keys are read was inaccurate in both directions.
|
||||
|
||||
## What was repaired
|
||||
|
||||
### 1. Skill bindings — 55 of 85 resolved to nothing
|
||||
|
||||
Only 30 skills were authored; 10 of 11 templates referenced names that did not
|
||||
exist. Three roles bound **zero** skills while their prompts described
|
||||
procedures to follow.
|
||||
|
||||
Invisible because both existing tests assert `authored ⊆ referenced` (30/30,
|
||||
green) and one explicitly declines to check the other direction.
|
||||
|
||||
Fixed: 23 skills authored, renames onto real skills, 22 aspirational references
|
||||
deleted. Two tests now hold it — one against the files, one against the database
|
||||
(different questions: resolution goes through catalogue rows, so a file that
|
||||
exists but fails to ingest still leaves the role empty).
|
||||
|
||||
### 2. Skills could not reach a mission agent at all
|
||||
|
||||
The larger finding, and the reason #1 was never noticed. The catalogue's only
|
||||
delivery channel was the `clawmates_skills` MCP server, and a mission claw could
|
||||
not reach it for **three independent reasons**:
|
||||
|
||||
- `provision_claw` wrote the constant `["clawmates_door"]`, ignoring the
|
||||
template's `mcp_bundles` — which `mission_orchestrator` had already stored.
|
||||
- The runtime config defines no `clawmates_skills` bundle. (The live local
|
||||
config defines **no bundles at all**, not even the door.)
|
||||
- Mission claws run on `claude_cli`, which the runtime's own config comments
|
||||
document as text-only: it cannot surface a tool call, so no MCP server is
|
||||
reachable from a mission turn regardless.
|
||||
|
||||
And a mission turn's entire system context is two sentences synthesised from the
|
||||
role slot (`topology_exec::build_prompt`) — the template's role prose is not
|
||||
used either, which `mission_orchestrator` documents.
|
||||
|
||||
So every skill authored for a mission role was unreachable prose, and the
|
||||
Skill-Use measurement this review planned could only ever have returned zero.
|
||||
|
||||
Fixed **on the container/ZeroClaw tier only**: `provision_claw` now provisions
|
||||
the template's bundles (door always added), and the pinned skills are injected
|
||||
as **bodies** into the turn prompt — not an index, because there is no
|
||||
`skills.read` tool on this path and an index would advertise a capability that
|
||||
does not exist.
|
||||
|
||||
The correction matters, because the first version of this document said
|
||||
"mission agents" without qualifying the tier. `compose_turn_prompt` /
|
||||
`pinned_skills_text` have exactly one production caller
|
||||
(`topology_exec.rs:745`), which `topology_worker` drives. `phase_runner`'s three
|
||||
other paths do not call it:
|
||||
|
||||
| Path | Entry point | Skills reach the agent? |
|
||||
|---|---|---|
|
||||
| Container / ZeroClaw (the default) | `topology_worker` → `ZeroClawDriveExecutor` | **yes** |
|
||||
| Composed microVM | `phase_runner::launch_composed_microvm_phase` | not yet |
|
||||
| Solo microVM | `microvm_executor::run_phase_in_vm` | not yet |
|
||||
| Direct session | `phase_runner::launch_direct_session` | not yet |
|
||||
|
||||
### 3. `upsert_task` raised 42P10 on every call
|
||||
|
||||
`mission_tasks_external_uniq` is a **partial** unique index. Postgres will not
|
||||
match a partial index to an `ON CONFLICT` target unless the statement repeats
|
||||
the predicate. Both callers — the task-card parser that turns INT markers into
|
||||
tasks, and the security scanner — map the error to a string their caller logs.
|
||||
|
||||
Two features were broken for as long as they have existed, and nothing was red.
|
||||
|
||||
### 4. The security scan phase never scanned
|
||||
|
||||
`security_scan::run` was reachable only from an operator button. So
|
||||
`security_hardening.toml` — a workflow whose entire first phase is a scan — ran
|
||||
an agent that was never told to scan, and never fired the scanner either. Now
|
||||
swept by `phase_runner`, mirroring the benchmark baseline sweep added earlier
|
||||
for the identical defect, guarded on a completion marker (a clean scan writes no
|
||||
findings, so a findings-guard would rescan forever).
|
||||
|
||||
### 5. Two recipes could not fail
|
||||
|
||||
`security_hardening.toml` and `benchmark.toml` carried no `task` and no
|
||||
`done_when` on any phase. Without `done_when` a phase never enters `evaluating`,
|
||||
is never judged, and reports `completed` whatever it did. Both now state the
|
||||
work and the condition.
|
||||
|
||||
### 6. Smaller wiring
|
||||
|
||||
- `CLAWMATES_JUDGE_MODEL` had two different defaults and a doc comment naming a
|
||||
third; one source now.
|
||||
- `GITEA_TOKEN` absence is stated rather than degrading into the same message a
|
||||
private repo produces.
|
||||
- `phase_config`'s registry corrected: `harness` and `tools` were listed NOT
|
||||
IMPLEMENTED while fully read; `bench_name`/`cmd` added; `test_command`
|
||||
deleted (no reader **and** no writer).
|
||||
|
||||
## Verification
|
||||
|
||||
- Full workspace suite green.
|
||||
- Every fix has a test, and **every test was negative-controlled**: the skills
|
||||
test failed naming all 55; reverting the `ON CONFLICT` predicate reproduces
|
||||
42P10 exactly; reverting one skill name fails naming that exact role.
|
||||
- Local stack boots; test-server path (`scripts/test-server.sh up`) is required
|
||||
for the integration suite, otherwise `cm-testkit` falls back to a
|
||||
testcontainer that times out.
|
||||
|
||||
## Deferred, with reasons
|
||||
|
||||
- ~~**`mission_plan` / `mission_roster` have no frontend.**~~ **Shipped
|
||||
2026-08-19.** `MissionProposalDrawer` reaches both flows from a mission's
|
||||
SETUP tab. Verified end to end against the live backend: a model proposed a
|
||||
roster, approval flipped the mission to the `composed` engine, and approval on
|
||||
a non-draft mission is refused.
|
||||
|
||||
Building it surfaced a defect in the backend it consumes: every refusal path
|
||||
computed a precise reason, logged it to stderr, and returned a bare
|
||||
`{"error":"bad request"}`. The person who needed the sentence was the one
|
||||
clicking Approve. `ApiError::Refused(String)` now carries it — the same
|
||||
argument `ApiError::Unavailable` was added for, one status code down.
|
||||
- **Skill-Use measurement (Trigger / Compliance / Boundary).** Deferred not for
|
||||
cost but because it was **unmeasurable until this pass**: with no delivery
|
||||
channel, trigger rate was structurally zero. It is now worth running, and it
|
||||
is the natural next step.
|
||||
- **Provenance layer.** Assessed only, by decision — see
|
||||
[PROVENANCE-ASSESSMENT.md](PROVENANCE-ASSESSMENT.md).
|
||||
- **Graph memory / `clawhdf5-agent`.** Declared in the workspace manifest, used
|
||||
by no crate. The matched study (`MemoryLake on MemoryArena`) shows structured
|
||||
memory winning modestly on low absolute numbers, and `Harness the Memory`
|
||||
finds excessive retrieval actively harms agent decisions. Measure against a
|
||||
baseline before migrating.
|
||||
- **Test coverage, re-examined.** The original entry ranked crates by raw test
|
||||
count. That was a shallow metric and it was misleading: `cm-safety`'s seven
|
||||
tests already cover the CAS on decide, grant double-consume, expiry, and the
|
||||
approved/rejected split, and the `audit_log` immutability trigger is tested in
|
||||
`cm-db`. Counting tests found the wrong crates.
|
||||
|
||||
Reading the API surface against the tests found the right ones:
|
||||
- **`verify_slack_signature` had no replay protection** — fixed, see below.
|
||||
- **`credits_for_tokens` was untested** — pure pricing arithmetic, now pinned
|
||||
including the round-up contract and an overflow case.
|
||||
|
||||
Genuinely still thin: `cm-brain`, where 6 of 9 tests need live
|
||||
`clawbrainhub.com` so the whole hub client is unexercised offline. Stubbing it
|
||||
means reproducing an external registry protocol we have no spec for, which is
|
||||
its own piece of work rather than a coverage chore.
|
||||
|
||||
- **A Slack request could be replayed forever.** `slack_signature_valid`
|
||||
verified the HMAC correctly and nothing anywhere checked the timestamp's age —
|
||||
it was only an input to the basestring, so an old request's signature verified
|
||||
exactly as well as a fresh one. Anyone holding one captured signed request
|
||||
could replay it indefinitely. Slack's documented 5-minute window is now
|
||||
enforced **in the broker**, not the caller, because the broker does not trust
|
||||
its caller and a check the caller can forget will eventually be forgotten.
|
||||
Symmetric, so a far-future timestamp cannot mint a long-lived request.
|
||||
|
||||
- ~~**`ZEROCLAW_GATEWAY_URL` / `_TOKEN` fail at *first use*, not boot.**~~
|
||||
**Fixed 2026-08-19.** `gateway_preflight` reports at startup, the third
|
||||
sibling of `runtime_preflight` and `validator_preflight`. A report, not a
|
||||
gate — every read path works without a gateway, and refusing to boot would
|
||||
turn a degraded deployment into a dead one. The message names the
|
||||
consequence ("container-tier missions cannot run") rather than just the
|
||||
missing variable.
|
||||
- ~~**`gitea_forge` resolves to nothing.**~~ **Resolved 2026-08-19 by removing
|
||||
the name.** It was harmless while `provision_claw` ignored the bundle list;
|
||||
once the list was honoured, an undefined name became a capability an agent is
|
||||
told it has and does not. Removed from seven team templates, one workflow
|
||||
recipe, the auto-provision path and a user-selectable dropdown. `web_fetch`
|
||||
went the same way — and a new test
|
||||
(`team_template_loader::bundle_tests`) now asserts every bundle a template
|
||||
names is defined in the runtime config, which is what found `web_fetch` in
|
||||
two templates I had missed by hand. Agents reach the forge through `git` over
|
||||
HTTPS with the ambient `GITEA_TOKEN`, which is why nothing ever broke.
|
||||
- **The deployed runtime config is not the example.** `[mcp_bundles.clawmates_skills]`
|
||||
is now in `agent.config.example.toml`, but the live local config carries no
|
||||
bundle definitions at all — a fresh deploy needs the example's blocks. The
|
||||
prompt-injection path does not depend on this, which is why it was the fix
|
||||
chosen for the mission tier.
|
||||
|
||||
## The one process change worth making
|
||||
|
||||
Every defect above was found by **checking a claim instead of reading it**. The
|
||||
existing `runtime_preflight` module is the pattern that works: it asks the
|
||||
running container what it actually has and says so at boot, because "the code is
|
||||
right and the machine is not" produced no error anywhere.
|
||||
|
||||
The equivalent check for this pass — does a provisioned agent actually receive
|
||||
the bundles and skills its template names — does not exist yet, and is the
|
||||
cheapest guard against all of this recurring.
|
||||
@@ -0,0 +1,104 @@
|
||||
# Where this left off — 2026-08-20
|
||||
|
||||
Read `CAPABILITY-REVIEW.md` for the system picture,
|
||||
`TOOL-CALL-ARCHITECTURE.md` for the current investigation, and
|
||||
`SKILL-USE-BASELINE.md` for the measurement.
|
||||
|
||||
## State of the tree
|
||||
|
||||
**11 unpushed commits on `main`** (`origin/main` is 11 behind). Nothing has been
|
||||
pushed and nothing is deployed to gw-04. Full suite green: 106 test binaries,
|
||||
zero build errors, frontend builds.
|
||||
|
||||
**1 unpushed commit in the fork** at `/Users/quantum/projects/zeroclaw`, branch
|
||||
`merge/upstream-v0.8.4`: `db1c50966 feat(claude_cli): stream-json …`.
|
||||
|
||||
## The thing to do first
|
||||
|
||||
**A runtime image with the `stream-json` fix is built and NOT deployed.**
|
||||
|
||||
```
|
||||
clawmates-runtime:streamjson # built 2026-08-19, never run
|
||||
```
|
||||
|
||||
Nothing has yet confirmed end to end that real `tool.call` rows land in
|
||||
`mission_events` from a live mission. That confirmation is the whole point of
|
||||
the change and it is the one step not taken. Until it runs, treat the fix as
|
||||
plausible rather than proven — the parser is unit-tested against a captured
|
||||
real stream, but no mission has driven it.
|
||||
|
||||
To do it: deploy the image as the mission runtime, launch a mission, then
|
||||
|
||||
```sql
|
||||
SELECT kind, target, count(*) FROM mission_events
|
||||
WHERE mission_id = '<id>' GROUP BY 1,2;
|
||||
```
|
||||
|
||||
Expect `tool.call` rows on the container tier for the first time. If they do not
|
||||
appear, the next suspects are `topology_exec`'s frame parser and whether the
|
||||
gateway forwards the provider's `tool_calls` at all — neither has been exercised
|
||||
with a non-empty list.
|
||||
|
||||
## Then, in order
|
||||
|
||||
1. **Prove the PreToolUse gate in a real VM.** `vm_tool_gate` is unit-tested and
|
||||
shell-tested on the host; it has never run in a guest. Four of its bugs were
|
||||
found only by executing the generated script, and two of them (syntax error,
|
||||
`IFS`) produce a gate that looks installed while being fully closed or fully
|
||||
open. Check `/root/toolgate/denied.jsonl` after a mission that tries
|
||||
something denied.
|
||||
|
||||
2. **Give the direct-session tier a tap.** It is the one tier with no
|
||||
observability at all — `session_executor::run_session` is a single
|
||||
`container_exec::exec` returning a string. Same settings document, written
|
||||
into a container instead of a VM.
|
||||
|
||||
3. **Deploy the door we already built** (`docs/TOOL-CALL-ARCHITECTURE.md` §3).
|
||||
Create `/zeroclaw-data/clawmates-mcp.json`, add a door-shaped provider alias,
|
||||
bind mission claws to it. This is config, not code — the provider feature is
|
||||
ours and shipped. It would also make `clawmates_skills` genuinely reachable,
|
||||
which is the precondition for moving skills from inlined bodies to
|
||||
progressive disclosure and making Skill-Use **Trigger** measurable the way
|
||||
the paper defines it.
|
||||
|
||||
4. **Pull upstream's egress policy** — `0db7d999a feat(plugins): add shared
|
||||
egress policy foundation (#9137)`. We are 218 commits behind; this is the one
|
||||
item identified as worth taking, and it is defence for a problem we have not
|
||||
solved.
|
||||
|
||||
## Open decisions that are yours
|
||||
|
||||
- **Self-authoring scope.** Agents now apply their own `skill_candidate` items
|
||||
with no human click (`CLAWMATES_SKILL_SELF_AUTHORING=0` restores the gate).
|
||||
`identity_refinement` and `brain_consolidation` still wait for a human,
|
||||
because they change what an agent IS rather than adding a procedure it can
|
||||
consult. Say if you want those autonomous too.
|
||||
- **Skill-Use Compliance coverage.** Most skills still score `not_applicable` —
|
||||
we cannot tell whether they changed anything. `small-focused-commits` and
|
||||
`tdd-red-green-refactor` are the next candidates and both need the repository
|
||||
diff rather than the turn text.
|
||||
|
||||
## Deliberately not done
|
||||
|
||||
- **The mission executor swap** (running turns through `ProviderExecutor` or the
|
||||
chat `Runtime`). The blockers are structural, not wiring: `cm-runtime`'s
|
||||
`files` tool rejects absolute paths *by construction*, `shell` runs in a
|
||||
per-agent sandbox with no mission mount, `ToolContext` carries no path or VM
|
||||
handle, and approvals key on `(session_id, message_id)`. The cheap fixes
|
||||
deliver what it was wanted for.
|
||||
- **`cm-brain` offline tests** — 6 of 9 need live `clawbrainhub.com`. Stubbing
|
||||
means reproducing an external registry protocol we have no spec for.
|
||||
- **Graph memory / `clawhdf5-agent`** — in the workspace manifest, used by no
|
||||
crate. Measure against a baseline before migrating.
|
||||
|
||||
## Two corrections made this session, worth remembering
|
||||
|
||||
- **"Missions can't call tools at all" was wrong.** They call `Bash` and `Write`
|
||||
with permissions pre-accepted. The gap was observing and gating, not having.
|
||||
- **Raw test counts are a bad coverage metric.** They pointed at `cm-safety`,
|
||||
whose seven tests already covered its critical paths, and missed a Slack
|
||||
replay hole that let one captured request authenticate forever.
|
||||
|
||||
The recurring shape, now seven times over: **a claim in a comment or a doc,
|
||||
believed and never checked.** Every significant finding this session came from
|
||||
running the thing rather than reading about it.
|
||||
@@ -0,0 +1,166 @@
|
||||
# Provenance: what we can answer today, and what we cannot
|
||||
|
||||
*Assessment only. No schema, no migration, nothing built. Written 2026-08-19.*
|
||||
|
||||
The question this exists to answer is narrow and practical:
|
||||
|
||||
> An agent said something. Where did it come from?
|
||||
|
||||
That is the question a provenance layer has to make answerable. Everything
|
||||
below is measured against it.
|
||||
|
||||
## The short answer
|
||||
|
||||
We can reconstruct **what an agent did**, on both execution paths, for seven
|
||||
days. We cannot reconstruct **why it said what it said** on any path, at any
|
||||
retention, because no store links a statement to the evidence that produced it.
|
||||
There is no claim as a first-class object anywhere in the system.
|
||||
|
||||
That is a design gap, not a bug. Nothing is broken; the edge was never built.
|
||||
|
||||
## What each store actually holds
|
||||
|
||||
### `mission_events` — the action record
|
||||
|
||||
The main one. Structured rows for the mission path: tool calls, phase
|
||||
transitions, agent lifecycle.
|
||||
|
||||
- **Retention: 7 days** (`EVENT_RETENTION_DAYS`, `mission_gc.rs`). This is the
|
||||
single most consequential fact in this document. The richest signal we have
|
||||
expires before most retrospectives happen, and a level-up proposal citing an
|
||||
event id older than a week points at nothing.
|
||||
- `reasoning` rows **are written** (`topology_worker.rs`) and **are read by
|
||||
nothing**. `routes/world.rs` explicitly excludes them, with a comment saying
|
||||
so. The model's stated rationale is recorded and then discarded unread.
|
||||
- No foreign key to `run_id` — `0075` dropped it deliberately. Events reference
|
||||
runs by convention, so nothing enforces that the reference resolves.
|
||||
|
||||
### `steps` — the rich record, on the wrong path
|
||||
|
||||
Structurally the best provenance we have: `kind`, `tool_name`, `input`,
|
||||
`output`, **and `taint[]`** — per tool call, with the taint sources carried
|
||||
through.
|
||||
|
||||
It is **chat-path only**, and not by omission. `steps.message_id` references a
|
||||
chat message, and `runtime::record_step` is the sole writer. A mission phase
|
||||
has no message, so it cannot write a step even in principle.
|
||||
|
||||
The consequence is worth stating plainly: the execution path that does the
|
||||
substantial work — missions — produces the *poorer* provenance record, and the
|
||||
path that produces the good one is the conversational one. Any real provenance
|
||||
work starts by resolving that asymmetry, and it is a schema change, not a call
|
||||
site.
|
||||
|
||||
### `audit_log` — genuinely immutable, narrowly used
|
||||
|
||||
A `BEFORE UPDATE OR DELETE` trigger (`0001_init.sql`) makes rows append-only for
|
||||
real, not by convention. It is the only tamper-evident store in the system.
|
||||
|
||||
It is used mostly as a rate-limit counter. The mechanism we would want for
|
||||
provenance already exists here and holds almost none of the content we would
|
||||
want in it.
|
||||
|
||||
### Approvals — the best-shaped record we have
|
||||
|
||||
Approval rows carry `taint_sources[]` and the exact preview a human was shown.
|
||||
For the narrow slice of actions that pass a gate, we can answer "what was the
|
||||
human told, what did they decide, and what data influenced it" — completely.
|
||||
|
||||
This is the shape to copy. It is also, currently, an island.
|
||||
|
||||
### The `.brain` — memory, versioned, chat-only
|
||||
|
||||
Full `.onion` revision history with `commit`/`revisions`/`rollback`, so a
|
||||
definition's evolution is fully recoverable. Memory is BM25 keyword recall and
|
||||
is written on the **chat** path; mission work writes none, so an agent that ran
|
||||
missions for a week has an empty memory section.
|
||||
|
||||
`cm-brain/src/lib.rs:234-239` exposes `set_provenance` / `provenance` — a wired
|
||||
slot with **zero callers** in the entire workspace. It is a place to put this,
|
||||
already plumbed, already versioned, already per-agent.
|
||||
|
||||
### `mission_phase_summaries` — prose, and overwritten
|
||||
|
||||
A model's post-hoc narrative of a phase, **overwritten on retry**. A summary is
|
||||
a model's later account of its own behaviour, and the retry that changed the
|
||||
outcome erases the account of the attempt that failed — which is precisely the
|
||||
one worth reading.
|
||||
|
||||
## The questions we cannot answer
|
||||
|
||||
Concretely, with what breaks each:
|
||||
|
||||
| Question | Why not |
|
||||
|---|---|
|
||||
| "Why did the agent claim X?" | No claim object; no edge from a statement to a tool result |
|
||||
| "Which tool output led to this file change?" | Mission path writes no `steps`, so no input/output pair exists |
|
||||
| "What did the agent believe when it decided?" | `reasoning` rows are written and read by nothing |
|
||||
| "Was this conclusion derived from tainted input?" | `taint[]` exists only on the chat path |
|
||||
| "What happened on the attempt that failed?" | Phase summaries are overwritten on retry |
|
||||
| "Why did this mission go wrong last month?" | 7-day retention |
|
||||
|
||||
The recurring shape: **the pieces exist, individually, on the wrong path or
|
||||
unread.** This is not a system that lacks provenance primitives. It is one where
|
||||
they were never connected into something that answers a question.
|
||||
|
||||
## Two candidate paths
|
||||
|
||||
### A. Postgres claim/edge tables
|
||||
|
||||
Add a claim as a first-class row, and edges from claim → evidence (tool call,
|
||||
file, event). Extend `steps` off the mission path so mission tool calls record
|
||||
input/output/taint the way chat ones do.
|
||||
|
||||
- **For:** stays in the store we already query, back up and migrate; the
|
||||
approvals record already proves the shape works here; no new dependency.
|
||||
- **Against:** a real schema addition on a 79-migration database; retention
|
||||
policy must be decided deliberately (7 days makes the whole thing pointless);
|
||||
graph queries in SQL get awkward exactly when they get interesting.
|
||||
|
||||
### B. Adopt `clawhdf5-agent::knowledge` + `::provenance`
|
||||
|
||||
Already in the dependency graph via the clawsync patch — declared in the
|
||||
workspace `Cargo.toml`, and **no crate depends on it**. Roughly 21k lines:
|
||||
typed entities, relations including `RelationType::Causal`, BFS and spreading
|
||||
activation; `provenance.rs` with `MemorySource` / `content_hash` /
|
||||
`session_id`.
|
||||
|
||||
- **For:** the causal structure is the thing we lack, and it is written; it is
|
||||
per-agent and per-file, matching the `.brain` model; `set_provenance` is
|
||||
already the slot it would fill.
|
||||
- **Against:** 21k lines of unexercised code entering a critical path; it is
|
||||
file-local, so cross-agent queries need a second mechanism; the `.brain` is
|
||||
reaped with the agent, which is the wrong lifetime for an audit record.
|
||||
|
||||
### What the research says about choosing
|
||||
|
||||
`MemoryLake on MemoryArena` (2026-08-14) compared memory backends with the
|
||||
backend as the only changed component. Structured beat vector RAG and
|
||||
long-context — on success rates of 9/40, 12/20 and 4/20, with every system
|
||||
scoring zero somewhere. `Harness the Memory` (08-15) found no substrate
|
||||
dominates, and that *excessive* retrieval actively harms agent decision-making
|
||||
even while helping factual QA.
|
||||
|
||||
Read together: a substrate swap is not where the win is, and adopting a graph
|
||||
memory because it is more sophisticated is not supported by the evidence. That
|
||||
argues for **A first** — make the record complete and correctly retained on the
|
||||
path that matters — and to treat B as a question to measure later, against a
|
||||
baseline captured beforehand.
|
||||
|
||||
`D²ACCI` (08-18) is the sharper finding for us: "end-to-end evaluation reveals
|
||||
that an error occurred, but not which stage caused it", and its **DCR** metric
|
||||
grades whether failures stay *localizable*. That is this project's recurring
|
||||
defect class stated as a research problem, and localizability — not
|
||||
completeness — is the property a provenance layer here should be judged on.
|
||||
|
||||
## The cheapest thing that would help, if we do nothing else
|
||||
|
||||
1. **Read the `reasoning` rows we already write.** They exist. Nothing consumes
|
||||
them. This is a query, not a schema.
|
||||
2. **Raise retention for a subset.** Seven days is right for volume, wrong for
|
||||
audit. The distinction is which rows, not how long.
|
||||
3. **Stop overwriting phase summaries on retry.** The erased attempt is the
|
||||
informative one.
|
||||
|
||||
None of these is the provenance layer. All three are cheap, and each closes a
|
||||
question we currently cannot answer at all.
|
||||
@@ -0,0 +1,94 @@
|
||||
# Research sweep — papers from the fortnight to 2026-08-19
|
||||
|
||||
Scoped to what could change ClawMates: agent runtimes, sub-agent structure,
|
||||
memory substrates, agent performance measurement, and provenance.
|
||||
|
||||
Each entry says what we did about it. "Nothing" is a legitimate outcome and is
|
||||
recorded as such — a sweep where every paper is actionable is a sweep that
|
||||
stopped judging.
|
||||
|
||||
---
|
||||
|
||||
## Skill-Use (2026-08-05) — *acted on*
|
||||
|
||||
Benchmarks whether an agent actually **uses** a skill under progressive
|
||||
disclosure: it sees a name and description and must retrieve the procedure.
|
||||
Three scores:
|
||||
|
||||
- **Trigger** — did it invoke the skill at all
|
||||
- **Compliance** — did it follow the procedure
|
||||
- **Boundary** — did it avoid what the skill forbids
|
||||
|
||||
That is exactly our skills shape, and reading it is what prompted asking
|
||||
whether ours fire. The answer turned out to be structural rather than
|
||||
behavioural: **they could not**. 55 of 85 bindings resolved to nothing, and the
|
||||
catalogue had no delivery channel to a mission agent at all. Both are now fixed
|
||||
(see [CAPABILITY-REVIEW.md](CAPABILITY-REVIEW.md)).
|
||||
|
||||
The measurement itself is now the obvious next piece of work, and it is worth
|
||||
doing on the paper's three axes rather than as a pass/fail — the useful output
|
||||
is *which* of our 53 skills are inert prose.
|
||||
|
||||
## MemoryLake on MemoryArena (2026-08-14) — *held as a finding*
|
||||
|
||||
A matched comparison: same framework, same model, same tasks, with the memory
|
||||
backend the only changed component. Structured memory beat vector RAG and
|
||||
long-context.
|
||||
|
||||
The numbers are the point. Success rates were **9/40, 12/20 and 4/20**, and
|
||||
every system scored zero on some task category. A useful antidote to expecting
|
||||
a memory swap to be transformative.
|
||||
|
||||
**What we did: nothing, deliberately.** It is the strongest argument against
|
||||
adopting graph memory on enthusiasm, and the strongest argument for capturing a
|
||||
baseline first.
|
||||
|
||||
## Harness the Memory (2026-08-15) — *held as a finding*
|
||||
|
||||
No memory substrate dominates. Broad retrieval helps factual QA, while
|
||||
**excessive retrieval actively harms agent decision-making**.
|
||||
|
||||
Directly relevant to a decision we made this pass: mission prompts now carry
|
||||
pinned skill *bodies*. This paper is the reason that is bounded
|
||||
(`MAX_PINNED_SKILL_BYTES`) and restricted to pinned skills rather than "give the
|
||||
agent everything it might need".
|
||||
|
||||
## D²ACCI (2026-08-18) — *the sharpest one for us*
|
||||
|
||||
> "End-to-end evaluation reveals that an error occurred, but not which stage
|
||||
> caused it."
|
||||
|
||||
That sentence is this project's recurring defect class stated as a research
|
||||
problem. The paper proposes **DCR**, a graded metric for whether failures stay
|
||||
*localizable*.
|
||||
|
||||
This reframed the provenance assessment: the property to optimise is not
|
||||
completeness of the record but **localizability of failure**. A system that
|
||||
records everything and cannot tell you which stage broke has not solved the
|
||||
problem. See [PROVENANCE-ASSESSMENT.md](PROVENANCE-ASSESSMENT.md).
|
||||
|
||||
---
|
||||
|
||||
## On clawhdf5 as the provenance layer
|
||||
|
||||
The specific question asked. `clawhdf5-agent` (~21k lines: typed entities,
|
||||
relations including `RelationType::Causal`, BFS and spreading activation;
|
||||
`provenance.rs` with `MemorySource` / `content_hash` / `session_id`) is
|
||||
**already in the workspace manifest and used by no crate**. `cm-brain` exposes
|
||||
`set_provenance` / `provenance` — a wired slot with **zero callers**.
|
||||
|
||||
So the adoption cost is lower than it looks. The assessment still recommends
|
||||
against leading with it, for reasons that are about our system rather than its
|
||||
quality: the `.brain` is reaped with the agent, which is the wrong lifetime for
|
||||
an audit record, and it is file-local, so cross-agent questions need a second
|
||||
mechanism anyway.
|
||||
|
||||
The honest sequencing is the one the memory papers argue for: make the record
|
||||
complete and correctly retained on the path that matters, capture a baseline,
|
||||
then measure whether the graph substrate earns its place.
|
||||
|
||||
## Not found
|
||||
|
||||
Nothing in the fortnight materially changes our **runtime** design (microVM
|
||||
boot, vsock RPC, per-backend egress) or the sub-agent/topology model. Recording
|
||||
that so the next sweep does not re-search the same ground.
|
||||
@@ -0,0 +1,174 @@
|
||||
# Skill-Use baseline
|
||||
|
||||
*First measurement of whether ClawMates' skills change what agents do.
|
||||
2026-08-19.*
|
||||
|
||||
Scored on the three axes from `Skill-Use` (arXiv, 2026-08-05): **Trigger** (did
|
||||
the agent reach for the skill), **Compliance** (did it follow the procedure),
|
||||
**Boundary** (did it avoid what the skill forbids).
|
||||
|
||||
Read the method before the numbers. A measurement whose limits are not stated
|
||||
is worse than none, because it gets quoted without them.
|
||||
|
||||
## Why there was no baseline before today
|
||||
|
||||
Not because nobody ran it. Because **it could not have returned anything but
|
||||
zero**, for two structural reasons that had nothing to do with agent behaviour:
|
||||
|
||||
1. 55 of 85 role skill bindings resolved to skills that were never authored.
|
||||
2. Even resolved skills had no delivery channel to a mission agent — the
|
||||
catalogue's only route was an MCP server that mission claws cannot reach.
|
||||
|
||||
Both were fixed in the two commits preceding this document. Anyone who had run
|
||||
this measurement in July would have concluded "our agents ignore their skills",
|
||||
which would have been false and expensive.
|
||||
|
||||
## Method, and what it cannot see
|
||||
|
||||
Scored by `cm_api::skill_use` from what the platform records: the
|
||||
`prompt.composed` event (the exact bytes an agent received) and the `reasoning`
|
||||
events (what it said it did). No re-derivation from the catalogue — the
|
||||
catalogue changes, and now that agents author their own skills it changes by
|
||||
itself.
|
||||
|
||||
### Trigger is not observable here, and that is a finding
|
||||
|
||||
The paper measures agents under **progressive disclosure**: the agent sees a
|
||||
name and description and must decide to retrieve the body. That retrieval is a
|
||||
tool call, which makes Trigger observable.
|
||||
|
||||
We do not deliver skills that way. `pinned_skills_text` inlines full bodies into
|
||||
the prompt, because mission claws run on `claude_cli`, which cannot surface a
|
||||
tool call — there is nothing to retrieve *with*. The agent never reaches for a
|
||||
skill; it is simply holding one.
|
||||
|
||||
So Trigger is reported as `not_observable` with the reason attached, **never as
|
||||
zero**. Scoring it zero would report a delivery-model property as an agent
|
||||
failure — the same confusion that kept 55 empty bindings invisible.
|
||||
|
||||
### Compliance and Boundary are checked mechanically, or not at all
|
||||
|
||||
Only skills whose procedure has a machine-checkable consequence are scored.
|
||||
Everything else returns `not_applicable` rather than a guess: a heuristic that
|
||||
scores prose by keyword overlap produces a number that looks like a measurement
|
||||
and is not one.
|
||||
|
||||
Compliance for `int-xx-marker-protocol` is checked by running the **real**
|
||||
`task_card_parser`, not a copy of its rules — a second implementation would
|
||||
drift, and then the score would pass while the mission loop still stalled.
|
||||
|
||||
## The runs
|
||||
|
||||
Two missions on the container/ZeroClaw tier, local stack, `research_only`.
|
||||
|
||||
| | run 1 | run 2 |
|
||||
|---|---|---|
|
||||
| distinct skills delivered | 3 | **9** |
|
||||
| total deliveries (per role prompt) | 3 | 14 |
|
||||
| phantom "skills" scored | **2** | 0 |
|
||||
|
||||
Run 1's phantom entries are the finding of the run, described below.
|
||||
|
||||
Run 2, per skill (all `source_kind=builtin`; no agent-authored skill has been
|
||||
delivered yet):
|
||||
|
||||
| skill | deliveries | compliance | boundary |
|
||||
|---|---|---|---|
|
||||
| `int-xx-marker-protocol` | 1 | **pass** | n/a |
|
||||
| `small-focused-commits` | 4 | n/a | n/a |
|
||||
| `cargo-test-driven-development` | 2 | n/a | n/a |
|
||||
| `workspace-repo-commit-protocol` | 2 | n/a | n/a |
|
||||
| `decompose-int-items` | 1 | n/a | n/a |
|
||||
| `write-rust-current-edition` | 1 | n/a | n/a |
|
||||
| `code-review-checklist` | 1 | n/a | n/a |
|
||||
| `criterion-benchmarking` | 1 | n/a | n/a |
|
||||
| `tdd-red-green-refactor` | 1 | n/a | n/a |
|
||||
|
||||
**n = 2 runs. No spread is reported because two runs cannot establish one.**
|
||||
This is a baseline in the sense of "the first honest number", not in the sense
|
||||
of `metrics-baseline-comparison.md`, which requires enough runs to see the noise
|
||||
floor before any change is judged against it. Do not compare a future number to
|
||||
this one without first establishing that floor.
|
||||
|
||||
## What the measurement found
|
||||
|
||||
Four defects, none of which any test or log would have surfaced.
|
||||
|
||||
### 1. The prompt format made its own record unparseable
|
||||
|
||||
Skills were introduced with `## <name>`, and skill bodies are markdown full of
|
||||
`##` headings. Run 1 duly scored **"Sizing heuristic"** and **"The output
|
||||
shape"** — both subheadings inside `decompose-int-items` — as skills with no
|
||||
catalogue row.
|
||||
|
||||
Fixed with an unambiguous `--- SKILL: <name> ---` marker, and both writers now
|
||||
share one renderer so the reader cannot drift from the writer.
|
||||
|
||||
### 2. A prompt was recorded that was never sent
|
||||
|
||||
The phase prompt was recorded at the dispatch fork, before the tier was chosen.
|
||||
The container tier does not send that text — it sends the bare task and appends
|
||||
skills per turn. So every container mission logged a `solo` prompt that reached
|
||||
no agent.
|
||||
|
||||
A provenance record of something that did not happen is worse than no record: it
|
||||
is the wrong answer, delivered confidently. Recording now happens inside each
|
||||
tier, and a test asserts every launcher records the prompt it actually sends.
|
||||
|
||||
### 3. A pinned skill contradicted the platform in the same prompt
|
||||
|
||||
`workspace-repo-commit-protocol` told agents that **`/workspace/repo`** was "the
|
||||
ONLY path where source-modifying edits belong". The platform mounts and
|
||||
advertises **`/mission/repo`** — 26 references in the code; `/workspace/repo`
|
||||
appears in none. The skill is bound on **29 role bindings** and was delivered
|
||||
twice in run 2, so agents received the real path in the tool preamble and a
|
||||
skill contradicting it a few hundred tokens later.
|
||||
|
||||
It also instructed `file_read` / `file_write` / `shell` — ZeroClaw's tool names,
|
||||
the exact ones `phase_task_text` was fixed to stop advertising after five agents
|
||||
on one mission spent 7.4k tokens describing the mismatch instead of working.
|
||||
|
||||
An agent that obeyed this skill wrote source into a directory nothing collects,
|
||||
and reached for tools its subprocess does not expose. Rewritten against what the
|
||||
code actually does, with two guards in `skills_loader::contradiction_tests`: no
|
||||
skill may name a repo path the platform does not mount, and none may instruct a
|
||||
tool the agent does not have. Both negative-controlled.
|
||||
|
||||
This is the same shape as the finding below and it is worth stating as a class:
|
||||
**the skills were never checked against the platform they describe.** Nothing
|
||||
compared them, so a skill could contradict the prompt it ships inside and stay
|
||||
that way indefinitely.
|
||||
|
||||
### 4. The skill documents a marker the platform never implemented
|
||||
|
||||
`int-xx-marker-protocol` lists `PLAN_COMPLETE: INT-NN` in its ladder.
|
||||
`task_card_parser` has **no such kind** and never has. An agent following the
|
||||
skill exactly emits a marker that is silently ignored.
|
||||
|
||||
Observed live: run 2's planner emitted `PLAN_COMPLETE: INT-01..02`, which is
|
||||
also the range form — on the kinds that *are* parsed, that yields the id
|
||||
`INT-01..02`, a task card for an item that does not exist while the two real
|
||||
items stay open.
|
||||
|
||||
**This is a skill/implementation mismatch, not an agent failure**, and it is
|
||||
precisely what this measurement exists to find: the agent did what it was told,
|
||||
and what it was told was wrong. Both shapes are now scored as failures; the
|
||||
underlying reconciliation — implement `PLAN_COMPLETE` or remove it from the
|
||||
skill — is deliberately left as a decision rather than guessed at here.
|
||||
|
||||
## Honest limits
|
||||
|
||||
- **Two runs, one tier, one workflow.** Nothing here generalises to the microVM
|
||||
or session tiers yet, and this document should not be read as if it does.
|
||||
- **Most skills score `not_applicable`** on both observable axes. That is not a
|
||||
pass. It means we cannot currently tell whether those skills changed anything.
|
||||
`workspace-repo-commit-protocol` now has a Boundary check (writing outside
|
||||
`/mission/repo`); `small-focused-commits` and `tdd-red-green-refactor` remain
|
||||
candidates, and both need the repository diff rather than the turn text.
|
||||
- **No agent-authored skill has been measured.** Self-authoring shipped in the
|
||||
same pass; `source_kind` is carried through the scorer specifically so a
|
||||
rising score on agent-authored skills is visible rather than averaged in.
|
||||
- **Evidence expires.** Mission events are reaped after 7 days unless
|
||||
`retain_events_until` is set. Both runs here are held for 90 days. An empty
|
||||
score means "no evidence", never "no compliance", and the API says so in its
|
||||
payload rather than leaving the caller to infer it.
|
||||
@@ -0,0 +1,171 @@
|
||||
# Mission tool calls: what is actually true, and what to do
|
||||
|
||||
*Research, 2026-08-19. Supersedes the claim "missions can't call tools at all",
|
||||
which I wrote and which is wrong.*
|
||||
|
||||
## The claim was wrong, and the truth is worse
|
||||
|
||||
Mission agents **do** call tools. Three of the four execution paths end in
|
||||
`claude -p` with Claude Code's own toolset and permissions pre-accepted:
|
||||
|
||||
| Path | Command | Tools | Permission |
|
||||
|---|---|---|---|
|
||||
| Solo microVM | `microvm_executor.rs:341` | `Read Edit Write Bash Agent` | `--permission-mode acceptEdits` |
|
||||
| Composed microVM | same, per node | same | same |
|
||||
| Direct session | `session_executor.rs:118` | `Read Edit Write Bash` | `acceptEdits` |
|
||||
| Container / ZeroClaw | `claude -p --output-format json` | see below | n/a |
|
||||
|
||||
So the real position is not "no tools". It is:
|
||||
|
||||
> **Mission agents run `Bash` and `Write` with permissions pre-accepted, and
|
||||
> nothing in this platform can gate them.**
|
||||
|
||||
That is a stronger finding than the one it replaces. "Can't call tools" sounds
|
||||
like a missing feature. "Calls tools freely, ungated, and mostly unobserved" is
|
||||
a security posture, and it is the one we have.
|
||||
|
||||
## Observe versus gate — they are different, and both are partial
|
||||
|
||||
| Path | Has tools | We observe | We gate |
|
||||
|---|---|---|---|
|
||||
| Solo microVM | yes | **yes** — `vm_tool_tap` | no |
|
||||
| Composed microVM | yes | **yes** — same tap | no |
|
||||
| Direct session | yes | **no mechanism at all** | no |
|
||||
| Container / ZeroClaw | yes (see below) | mechanism exists, receives nothing | no |
|
||||
|
||||
`vm_tool_tap` installs a **`PostToolUse`** hook, which fires *after* the tool has
|
||||
already run, and `exit 0`s unconditionally because a non-zero `PostToolUse`
|
||||
talks back to the model. It is telemetry and says so. It is structurally
|
||||
incapable of gating.
|
||||
|
||||
The §15 `GatePolicy` has exactly **one** enforcement site — `Runtime::drive`,
|
||||
the chat loop — and its approvals are keyed to `(session_id, message_id)`, which
|
||||
no mission phase can produce. A mission agent's `Bash` call is gated by nothing,
|
||||
anywhere.
|
||||
|
||||
## Why the container tier looked tool-free
|
||||
|
||||
`claude_cli` builds `claude -p … --output-format json`, which returns a single
|
||||
final result object, and the provider then hardcodes `tool_calls: Vec::new()`.
|
||||
The tool calls happen; the transport discards them.
|
||||
|
||||
That is why `ToolTrace.calls` was empty on gw-04, and the comment in
|
||||
`topology_exec.rs` reads that emptiness as "§15 by construction: agents are
|
||||
provisioned tool-free". It is not by construction. It is the output format.
|
||||
|
||||
**Proven, not assumed.** Against the live runtime (Claude Code 2.1.228):
|
||||
|
||||
```
|
||||
claude -p "Run the bash command: echo hello" \
|
||||
--output-format stream-json --verbose --allowedTools Bash
|
||||
```
|
||||
|
||||
emits exactly what we need:
|
||||
|
||||
```
|
||||
assistant block: tool_use Bash
|
||||
user block: tool_result
|
||||
assistant block: text
|
||||
event: result success
|
||||
```
|
||||
|
||||
The calls are fully observable. We ask for the wrong output format.
|
||||
|
||||
## The door already exists, and we never plugged it in
|
||||
|
||||
`claude_cli.rs` is **ours** — upstream `zeroclaw-labs/zeroclaw` has no such file.
|
||||
So is the feature that solves this, our own commit
|
||||
`88eef99d4 feat(providers): claude_cli --mcp-config + allow/disallow tools (act via door)`.
|
||||
|
||||
The provider already accepts:
|
||||
|
||||
- `mcp_config` → `claude -p --mcp-config <file> --strict-mcp-config`, so Claude
|
||||
Code's **own** MCP client connects to our door;
|
||||
- `tools` → `--allowedTools`, auto-approving door tools;
|
||||
- `disallowed_tools` → `--disallowedTools`, locking out `Bash`, `Write`, `Edit`,
|
||||
`Read`, … so **the gated door is the only actuator**.
|
||||
|
||||
`deploy/clawmates-runtime/agent.config.example.toml` documents the whole shape
|
||||
as `[providers.models.claude_cli.door]`.
|
||||
|
||||
And in the live runtime: `/zeroclaw-data/clawmates-mcp.json` **does not exist**,
|
||||
the config has no `[providers.*]` block at all, and every mission claw is bound
|
||||
to `claude_cli.default` — which sets none of the three fields.
|
||||
|
||||
So my earlier conclusion that "`claude_cli` cannot reach MCP, therefore the
|
||||
skills server is unreachable" was wrong in its reasoning. The capability is
|
||||
built, documented by us, and simply never deployed.
|
||||
|
||||
Related: we have been setting `agents.<alias>.mcp_bundles`, which configures
|
||||
**ZeroClaw's own** MCP client for its native agent loop. A `claude_cli` agent's
|
||||
actuator is the claude subprocess, which reads `mcp_config` on the **provider**.
|
||||
We were turning a knob connected to a loop that does not run.
|
||||
|
||||
## Upstream: nothing that solves this, one thing worth taking
|
||||
|
||||
We are **218 commits behind** `upstream/master`. Scanning for anything relevant:
|
||||
|
||||
- **No upstream work on `claude_cli`** — the file is ours; upstream has none.
|
||||
- **ACP** (Agent Client Protocol) exists in the fork already
|
||||
(`zeroclaw-gateway/src/acp.rs`, `zeroclaw-channels/src/acp_channel.rs`); the
|
||||
three upstream commits since our merge are workspace-default and
|
||||
tool-approval-localization fixes, not new capability. ACP *does* surface tool
|
||||
calls natively and is a credible long-term transport, but it is a bigger move
|
||||
than the two fixes below and buys the same observability.
|
||||
- **`feat(plugins): add shared egress policy foundation (#9137)`** — a network
|
||||
guard in `zeroclaw-infra::net_guard` with DNS pinning, IPv4-mapped metadata
|
||||
blocking, proxy-conflict surfacing. This is the one upstream item genuinely
|
||||
worth pulling: it is defence for the egress problem we have not solved, and
|
||||
it is hardening we would otherwise write ourselves.
|
||||
|
||||
## What to do, cheapest first
|
||||
|
||||
### 1. Observability — switch `claude_cli` to `stream-json` *(small, proven)*
|
||||
|
||||
In our fork: `--output-format stream-json --verbose`, parse `tool_use` /
|
||||
`tool_result` blocks into `ChatResponse.tool_calls` instead of `Vec::new()`.
|
||||
|
||||
Unblocks: real `tool.call` events on the container tier; `ToolTrace.calls`
|
||||
non-empty; Skill-Use **Trigger** becomes observable on that tier for the first
|
||||
time. Risk: the parser must handle a stream rather than one object, and
|
||||
`--verbose` is required alongside it.
|
||||
|
||||
### 2. A real gate — use the `PreToolUse` hook *(small, mechanism already proven)*
|
||||
|
||||
`vm_stop_gate.rs` records that `PreToolUse` **fires** under `claude -p` in our
|
||||
image, and it is used nowhere — one doc-comment mention, zero call sites. Same
|
||||
install pattern as the tap, but exit-2-to-deny instead of exit-0.
|
||||
|
||||
This is the only pre-execution gate available to the microVM and direct-session
|
||||
paths, and it is the one thing that would let §15 mean something for missions.
|
||||
It must be added to `vm_tool_tap::guest_settings`, the single settings writer,
|
||||
or it will clobber `Stop` and `PostToolUse`.
|
||||
|
||||
Also: give the direct-session path the tap at all. It is the same settings file,
|
||||
written into a container instead of a VM, and today that tier is completely dark.
|
||||
|
||||
### 3. Deploy the door we already built *(config, not code)*
|
||||
|
||||
Create `/zeroclaw-data/clawmates-mcp.json`, define a door-shaped provider alias,
|
||||
and bind mission claws to it. Gets the container tier a gated actuator and makes
|
||||
the `clawmates_skills` MCP server genuinely reachable — which would let skills
|
||||
move from inlined bodies to progressive disclosure, and make Trigger measurable
|
||||
the way the paper defines it.
|
||||
|
||||
Sequencing note: (1) and (2) are independent of (3) and worth doing first,
|
||||
because they make the container tier legible before we change what it can do.
|
||||
|
||||
### Not recommended yet
|
||||
|
||||
Replacing the mission executor with `ProviderExecutor` or the chat `Runtime`.
|
||||
The blockers are real and structural, not wiring: `Runtime::send_message`
|
||||
requires a chat session row, an agent row, two message rows and an `agent_runs`
|
||||
row, and its approval key is `(session_id, message_id)`; `ToolContext` carries
|
||||
no path, checkout, container or VM handle; `cm-runtime`'s `files` tool is a blob
|
||||
store that **rejects absolute paths by construction**, and its `shell` tool
|
||||
executes in a per-agent sandbox with zero egress and no mission mount. Every
|
||||
mission-shaped concept would have to be invented first.
|
||||
|
||||
The cheap fixes above deliver the observability and the gate. The executor swap
|
||||
is a different project, and it should be justified by something other than tool
|
||||
calls — which, it turns out, we already have.
|
||||
@@ -45,6 +45,7 @@ import { MissionLiveEvents } from "./MissionLiveEvents";
|
||||
import { MissionLivePane } from "./MissionLivePane";
|
||||
import { MissionOutputReader } from "./MissionOutputReader";
|
||||
import { MissionTeamTab } from "./MissionTeamTab";
|
||||
import { MissionProposalDrawer } from "./MissionProposalDrawer";
|
||||
import { MissionWizard } from "./MissionWizard";
|
||||
import { PhaseGoalStrip } from "./PhaseGoalStrip";
|
||||
import { PhaseRunsList } from "./PhaseRunsList";
|
||||
@@ -127,6 +128,9 @@ export function MissionCanvas({
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [tab, setTab] = useState<Tab>("run");
|
||||
// The propose→review→approve gate. Backend has been complete since it
|
||||
// shipped; this is the first way for a human to reach the review step.
|
||||
const [proposalsOpen, setProposalsOpen] = useState(false);
|
||||
const [runSub, setRunSub] = useState<RunSub>("phases");
|
||||
const [outputSub, setOutputSub] = useState<OutputSub>("documents");
|
||||
const [setupSub, setSetupSub] = useState<SetupSub>("overview");
|
||||
@@ -977,11 +981,27 @@ export function MissionCanvas({
|
||||
)}
|
||||
|
||||
{tab === "setup" && setupSub === "team" && (
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 10 }}>
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 10 }}>
|
||||
<button
|
||||
onClick={() => setProposalsOpen(true)}
|
||||
style={secondaryBtn}
|
||||
title="Review a model-proposed plan or roster before it is applied"
|
||||
>
|
||||
Review proposals
|
||||
</button>
|
||||
<span style={{ fontSize: 11, color: "#8b8b96" }}>
|
||||
{mission.status === "draft"
|
||||
? "a proposed plan or roster can be approved while this mission is a draft"
|
||||
: `approval applies to drafts only — this mission is ${mission.status}`}
|
||||
</span>
|
||||
</div>
|
||||
<MissionTeamTab
|
||||
missionId={mission.id}
|
||||
teamId={mission.team_id}
|
||||
onOpenClaw={onOpenClaw}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{tab === "run" && runSub === "live" && (
|
||||
@@ -1132,6 +1152,15 @@ export function MissionCanvas({
|
||||
)}
|
||||
</MissionTabScroller>
|
||||
)}
|
||||
|
||||
{proposalsOpen && (
|
||||
<MissionProposalDrawer
|
||||
missionId={mission.id}
|
||||
missionStatus={mission.status}
|
||||
onClose={() => setProposalsOpen(false)}
|
||||
onChanged={onChanged}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,424 @@
|
||||
"use client";
|
||||
|
||||
// MissionProposalDrawer — the human review step for a mission's proposed
|
||||
// PLAN (what phases will run) and ROSTER (who will run them).
|
||||
//
|
||||
// Both backends have been complete and reachable by curl since they
|
||||
// shipped; neither had any user interface. That matters more than a
|
||||
// missing screen usually would, because the decide step is not a
|
||||
// convenience — it IS the safety mechanism. Approving a plan replaces the
|
||||
// mission's phases; approving a roster flips it to the composed engine.
|
||||
// A gate nobody can reach is a gate that is always open or always shut.
|
||||
//
|
||||
// Modelled on LevelUpDrawer, which already does load → review → decide.
|
||||
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import {
|
||||
decidePlanProposal,
|
||||
decideRosterProposal,
|
||||
listPlanProposals,
|
||||
listRosterProposals,
|
||||
suggestPlan,
|
||||
suggestRoster,
|
||||
type MissionPlanProposal,
|
||||
type MissionTeamProposal,
|
||||
} from "@/lib/api/missions";
|
||||
|
||||
const mono =
|
||||
"ui-monospace, SFMono-Regular, SF Mono, Menlo, Monaco, Consolas, monospace";
|
||||
|
||||
export type ProposalTab = "plan" | "roster";
|
||||
|
||||
const STATUS_COLOR: Record<string, string> = {
|
||||
proposed: "#f0c264",
|
||||
approved: "#7fd0a0",
|
||||
rejected: "#ff8a7a",
|
||||
};
|
||||
|
||||
export function MissionProposalDrawer({
|
||||
missionId,
|
||||
missionStatus,
|
||||
onClose,
|
||||
onChanged,
|
||||
}: {
|
||||
missionId: string;
|
||||
/** Approval only applies to a draft; the server returns 400 otherwise. */
|
||||
missionStatus: string;
|
||||
onClose: () => void;
|
||||
onChanged?: () => void;
|
||||
}) {
|
||||
const [tab, setTab] = useState<ProposalTab>("plan");
|
||||
const [plans, setPlans] = useState<MissionPlanProposal[]>([]);
|
||||
const [rosters, setRosters] = useState<MissionTeamProposal[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [busy, setBusy] = useState<string | null>(null);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [note, setNote] = useState("");
|
||||
|
||||
const isDraft = missionStatus === "draft";
|
||||
|
||||
const load = useCallback(async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const [p, r] = await Promise.all([
|
||||
listPlanProposals(missionId),
|
||||
listRosterProposals(missionId),
|
||||
]);
|
||||
setPlans(p);
|
||||
setRosters(r);
|
||||
setError(null);
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : "load failed");
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [missionId]);
|
||||
|
||||
useEffect(() => {
|
||||
void load();
|
||||
}, [load]);
|
||||
|
||||
// `suggest` calls a model AND reads the repo tree from the forge, so it
|
||||
// takes many seconds. Without a real pending state this reads as a hang.
|
||||
const doSuggest = useCallback(async () => {
|
||||
setBusy("suggest");
|
||||
setError(null);
|
||||
try {
|
||||
if (tab === "plan") await suggestPlan(missionId);
|
||||
else await suggestRoster(missionId);
|
||||
await load();
|
||||
} catch (e) {
|
||||
setError(e instanceof Error ? e.message : "suggest failed");
|
||||
} finally {
|
||||
setBusy(null);
|
||||
}
|
||||
}, [tab, missionId, load]);
|
||||
|
||||
const doDecide = useCallback(
|
||||
async (proposalId: string, status: "approved" | "rejected") => {
|
||||
setBusy(proposalId);
|
||||
setError(null);
|
||||
try {
|
||||
if (tab === "plan") {
|
||||
await decidePlanProposal(missionId, proposalId, status, note || undefined);
|
||||
} else {
|
||||
await decideRosterProposal(missionId, proposalId, status, note || undefined);
|
||||
}
|
||||
setNote("");
|
||||
await load();
|
||||
onChanged?.();
|
||||
} catch (e) {
|
||||
// The server's refusal text is written for a human to read — it
|
||||
// names which constraint failed and why. Replacing it with a
|
||||
// generic message throws away the only useful part.
|
||||
setError(e instanceof Error ? e.message : "decide failed");
|
||||
} finally {
|
||||
setBusy(null);
|
||||
}
|
||||
},
|
||||
[tab, missionId, note, load, onChanged],
|
||||
);
|
||||
|
||||
const proposals: (MissionPlanProposal | MissionTeamProposal)[] =
|
||||
tab === "plan" ? plans : rosters;
|
||||
|
||||
return (
|
||||
<div
|
||||
role="dialog"
|
||||
aria-modal
|
||||
aria-label="Mission proposals"
|
||||
onClick={onClose}
|
||||
style={{
|
||||
position: "fixed",
|
||||
inset: 0,
|
||||
background: "rgba(0,0,0,.55)",
|
||||
display: "flex",
|
||||
justifyContent: "flex-end",
|
||||
zIndex: 1000,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
onClick={(e) => e.stopPropagation()}
|
||||
style={{
|
||||
width: "min(620px, 100vw)",
|
||||
height: "100vh",
|
||||
background: "#141419",
|
||||
borderLeft: "1px solid rgba(255,255,255,.08)",
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
}}
|
||||
>
|
||||
<header
|
||||
style={{
|
||||
padding: "14px 18px",
|
||||
borderBottom: "1px solid rgba(255,255,255,.06)",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 12,
|
||||
}}
|
||||
>
|
||||
<strong style={{ color: "#e8e8ee", fontSize: 14 }}>Review proposals</strong>
|
||||
<div style={{ display: "flex", gap: 6, marginLeft: 8 }}>
|
||||
{(["plan", "roster"] as ProposalTab[]).map((t) => (
|
||||
<button
|
||||
key={t}
|
||||
onClick={() => setTab(t)}
|
||||
style={{
|
||||
padding: "4px 10px",
|
||||
fontSize: 12,
|
||||
borderRadius: 6,
|
||||
cursor: "pointer",
|
||||
background: tab === t ? "rgba(255,255,255,.10)" : "transparent",
|
||||
color: tab === t ? "#e8e8ee" : "#8b8b96",
|
||||
border: "1px solid rgba(255,255,255,.10)",
|
||||
}}
|
||||
>
|
||||
{t === "plan" ? "Plan" : "Roster"}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<button
|
||||
onClick={onClose}
|
||||
aria-label="Close"
|
||||
style={{
|
||||
marginLeft: "auto",
|
||||
background: "transparent",
|
||||
border: "none",
|
||||
color: "#8b8b96",
|
||||
cursor: "pointer",
|
||||
fontSize: 18,
|
||||
}}
|
||||
>
|
||||
×
|
||||
</button>
|
||||
</header>
|
||||
|
||||
{!isDraft && (
|
||||
<p
|
||||
style={{
|
||||
margin: 0,
|
||||
padding: "10px 18px",
|
||||
fontSize: 12,
|
||||
color: "#f0c264",
|
||||
background: "rgba(240,194,100,.08)",
|
||||
borderBottom: "1px solid rgba(255,255,255,.06)",
|
||||
}}
|
||||
>
|
||||
This mission is <strong>{missionStatus}</strong>. Proposals can only be
|
||||
approved while it is a draft — approving now would be refused.
|
||||
</p>
|
||||
)}
|
||||
|
||||
<div style={{ flex: 1, overflowY: "auto", padding: "14px 18px" }}>
|
||||
{error && (
|
||||
<p
|
||||
style={{
|
||||
margin: "0 0 12px",
|
||||
padding: "8px 10px",
|
||||
borderRadius: 6,
|
||||
fontSize: 12,
|
||||
fontFamily: mono,
|
||||
whiteSpace: "pre-wrap",
|
||||
color: "#ff8a7a",
|
||||
background: "rgba(255,138,122,.08)",
|
||||
}}
|
||||
>
|
||||
{error}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{loading ? (
|
||||
<p style={{ color: "#8b8b96", fontSize: 13 }}>Loading…</p>
|
||||
) : proposals.length === 0 ? (
|
||||
<p style={{ color: "#8b8b96", fontSize: 13 }}>
|
||||
No {tab} proposals yet. Ask a model for one below — it reads the
|
||||
repository first, so it takes a few seconds.
|
||||
</p>
|
||||
) : (
|
||||
proposals.map((p) => (
|
||||
<article
|
||||
key={p.id}
|
||||
style={{
|
||||
marginBottom: 14,
|
||||
border: "1px solid rgba(255,255,255,.08)",
|
||||
borderRadius: 8,
|
||||
overflow: "hidden",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 8,
|
||||
padding: "8px 12px",
|
||||
background: "rgba(255,255,255,.03)",
|
||||
}}
|
||||
>
|
||||
<span
|
||||
style={{
|
||||
fontSize: 11,
|
||||
padding: "2px 8px",
|
||||
borderRadius: 999,
|
||||
color: "#141419",
|
||||
background: STATUS_COLOR[p.status] ?? "#8b8b96",
|
||||
}}
|
||||
>
|
||||
{p.status}
|
||||
</span>
|
||||
<span style={{ fontSize: 11, color: "#8b8b96", fontFamily: mono }}>
|
||||
{p.author_model ?? "unknown model"}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div style={{ padding: "10px 12px" }}>
|
||||
{"plan" in p ? (
|
||||
<PlanBody plan={p.plan} />
|
||||
) : (
|
||||
<RosterBody roster={p.roster} />
|
||||
)}
|
||||
{p.note && (
|
||||
<p style={{ marginTop: 8, fontSize: 12, color: "#8b8b96" }}>
|
||||
Note: {p.note}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{p.status === "proposed" && (
|
||||
<div
|
||||
style={{
|
||||
display: "flex",
|
||||
gap: 8,
|
||||
padding: "10px 12px",
|
||||
borderTop: "1px solid rgba(255,255,255,.06)",
|
||||
}}
|
||||
>
|
||||
<button
|
||||
disabled={busy !== null || !isDraft}
|
||||
onClick={() => doDecide(p.id, "approved")}
|
||||
style={btn("#7fd0a0", busy !== null || !isDraft)}
|
||||
>
|
||||
{busy === p.id ? "Working…" : "Approve"}
|
||||
</button>
|
||||
<button
|
||||
disabled={busy !== null}
|
||||
onClick={() => doDecide(p.id, "rejected")}
|
||||
style={btn("#ff8a7a", busy !== null)}
|
||||
>
|
||||
Reject
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</article>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
|
||||
<footer
|
||||
style={{
|
||||
padding: "12px 18px",
|
||||
borderTop: "1px solid rgba(255,255,255,.06)",
|
||||
display: "flex",
|
||||
gap: 8,
|
||||
}}
|
||||
>
|
||||
<input
|
||||
value={note}
|
||||
onChange={(e) => setNote(e.target.value)}
|
||||
placeholder="Note for the decision (optional)"
|
||||
aria-label="Decision note"
|
||||
style={{
|
||||
flex: 1,
|
||||
padding: "7px 10px",
|
||||
fontSize: 12,
|
||||
borderRadius: 6,
|
||||
background: "rgba(255,255,255,.04)",
|
||||
border: "1px solid rgba(255,255,255,.10)",
|
||||
color: "#e8e8ee",
|
||||
}}
|
||||
/>
|
||||
<button
|
||||
disabled={busy !== null}
|
||||
onClick={doSuggest}
|
||||
style={btn("#7cd6e0", busy !== null)}
|
||||
>
|
||||
{busy === "suggest" ? "Asking a model…" : `Propose ${tab}`}
|
||||
</button>
|
||||
</footer>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function btn(color: string, disabled: boolean): React.CSSProperties {
|
||||
return {
|
||||
padding: "6px 14px",
|
||||
fontSize: 12,
|
||||
borderRadius: 6,
|
||||
cursor: disabled ? "not-allowed" : "pointer",
|
||||
opacity: disabled ? 0.5 : 1,
|
||||
color: "#141419",
|
||||
background: color,
|
||||
border: "none",
|
||||
};
|
||||
}
|
||||
|
||||
function PlanBody({ plan }: { plan: { phases: PlanPhase[] } }) {
|
||||
if (!plan?.phases?.length) {
|
||||
return <p style={{ fontSize: 12, color: "#8b8b96" }}>No phases proposed.</p>;
|
||||
}
|
||||
return (
|
||||
<ol style={{ margin: 0, paddingLeft: 18, display: "grid", gap: 10 }}>
|
||||
{plan.phases.map((ph, i) => (
|
||||
<li key={i} style={{ fontSize: 12, color: "#c9c9d4" }}>
|
||||
<strong style={{ color: "#e8e8ee", fontFamily: mono }}>{ph.kind}</strong>
|
||||
<p style={{ margin: "4px 0", whiteSpace: "pre-wrap" }}>{ph.task}</p>
|
||||
{/* done_when is what makes a phase judgeable at all — a phase
|
||||
without one reports completed whatever it did, so its absence
|
||||
is worth showing rather than hiding. */}
|
||||
{ph.done_when ? (
|
||||
<p style={{ margin: 0, color: "#7fd0a0" }}>done when: {ph.done_when}</p>
|
||||
) : (
|
||||
<p style={{ margin: 0, color: "#f0c264" }}>
|
||||
no completion condition — this phase cannot fail
|
||||
</p>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
</ol>
|
||||
);
|
||||
}
|
||||
|
||||
function RosterBody({ roster }: { roster: { topology_kind: string; members: RosterM[] } }) {
|
||||
if (!roster?.members?.length) {
|
||||
return <p style={{ fontSize: 12, color: "#8b8b96" }}>No members proposed.</p>;
|
||||
}
|
||||
return (
|
||||
<div>
|
||||
<p style={{ margin: "0 0 8px", fontSize: 12, color: "#8b8b96" }}>
|
||||
topology: <span style={{ fontFamily: mono, color: "#e8e8ee" }}>{roster.topology_kind}</span>
|
||||
</p>
|
||||
<ul style={{ margin: 0, paddingLeft: 18, display: "grid", gap: 8 }}>
|
||||
{roster.members.map((m, i) => (
|
||||
<li key={i} style={{ fontSize: 12, color: "#c9c9d4" }}>
|
||||
<strong style={{ color: "#e8e8ee", fontFamily: mono }}>{m.role}</strong>
|
||||
{m.backend && (
|
||||
<span style={{ color: "#8b8b96" }}> · {m.backend}</span>
|
||||
)}
|
||||
{m.rationale && <p style={{ margin: "3px 0 0" }}>{m.rationale}</p>}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
interface PlanPhase {
|
||||
kind: string;
|
||||
task: string;
|
||||
done_when?: string | null;
|
||||
}
|
||||
interface RosterM {
|
||||
role: string;
|
||||
backend?: string | null;
|
||||
rationale?: string | null;
|
||||
}
|
||||
@@ -44,7 +44,9 @@ export function TeamRunsModal({ teamId, teamName, onClose }: { teamId: string; t
|
||||
{ v: "research_web_readonly", label: "research_web_readonly — + web fetch" },
|
||||
{ v: "coding_readwrite", label: "coding_readwrite — writes to /workspace/repo" },
|
||||
];
|
||||
const BUNDLE_OPTIONS = ["clawmates_door", "gitea_forge"];
|
||||
// Only bundles the runtime actually defines. `gitea_forge` was selectable
|
||||
// here and defined nowhere, so picking it granted nothing.
|
||||
const BUNDLE_OPTIONS = ["clawmates_door", "clawmates_skills"];
|
||||
|
||||
useEffect(() => {
|
||||
(async () => {
|
||||
|
||||
@@ -569,3 +569,102 @@ export const recipeToPreset = (r: WorkflowRecipe): TemplatePreset => ({
|
||||
.map((p) => ({ kind: p.kind, order_idx: p.order_idx })),
|
||||
defaultTeamTemplate: r.default_team_template ?? null,
|
||||
});
|
||||
|
||||
// ── Plan + roster proposals ─────────────────────────────────────────
|
||||
//
|
||||
// A model proposes; a human decides. The decide step IS the safety
|
||||
// mechanism for both flows — a plan approval replaces the mission's phases
|
||||
// and a roster approval flips it to the composed engine — and until now it
|
||||
// had no user interface at all, on a fully working backend.
|
||||
//
|
||||
// Both only apply while the mission is a DRAFT. Approving a running mission
|
||||
// returns 400, and the server's refusal text is written for a human, so it
|
||||
// is surfaced verbatim rather than replaced with "something went wrong".
|
||||
|
||||
export type ProposalDecision = "approved" | "rejected";
|
||||
export type MissionProposalStatus = "proposed" | ProposalDecision;
|
||||
|
||||
export interface PlannedPhase {
|
||||
kind: string;
|
||||
task: string;
|
||||
done_when?: string | null;
|
||||
done_when_check?: string | null;
|
||||
allow_empty?: boolean | null;
|
||||
}
|
||||
|
||||
export interface MissionPlan {
|
||||
phases: PlannedPhase[];
|
||||
}
|
||||
|
||||
export interface RosterMember {
|
||||
role: string;
|
||||
/** Which per-CLI rootfs this member boots. Validated against the fleet. */
|
||||
backend?: string | null;
|
||||
rationale?: string | null;
|
||||
}
|
||||
|
||||
export interface MissionRoster {
|
||||
topology_kind: string;
|
||||
members: RosterMember[];
|
||||
}
|
||||
|
||||
export interface MissionPlanProposal {
|
||||
id: string;
|
||||
mission_id: string;
|
||||
plan: MissionPlan;
|
||||
author_model: string | null;
|
||||
status: MissionProposalStatus;
|
||||
note?: string | null;
|
||||
created_at?: string;
|
||||
decided_at?: string | null;
|
||||
}
|
||||
|
||||
export interface MissionTeamProposal {
|
||||
id: string;
|
||||
mission_id: string;
|
||||
roster: MissionRoster;
|
||||
author_model: string | null;
|
||||
status: MissionProposalStatus;
|
||||
note?: string | null;
|
||||
created_at?: string;
|
||||
decided_at?: string | null;
|
||||
}
|
||||
|
||||
/** Ask a model for a plan. Slow: reads the repo tree from the forge. */
|
||||
export const suggestPlan = (missionId: string) =>
|
||||
api<MissionPlanProposal>(`/api/missions/${missionId}/plan-proposals`, {
|
||||
method: "POST",
|
||||
});
|
||||
|
||||
export const listPlanProposals = (missionId: string) =>
|
||||
api<MissionPlanProposal[]>(`/api/missions/${missionId}/plan-proposals`);
|
||||
|
||||
export const decidePlanProposal = (
|
||||
missionId: string,
|
||||
proposalId: string,
|
||||
status: ProposalDecision,
|
||||
note?: string,
|
||||
) =>
|
||||
api<{ status: string; phases?: { kind: string; order_idx: number }[] }>(
|
||||
`/api/missions/${missionId}/plan-proposals/${proposalId}/decide`,
|
||||
{ method: "POST", body: JSON.stringify({ status, note }) },
|
||||
);
|
||||
|
||||
export const suggestRoster = (missionId: string) =>
|
||||
api<MissionTeamProposal>(`/api/missions/${missionId}/team-proposals`, {
|
||||
method: "POST",
|
||||
});
|
||||
|
||||
export const listRosterProposals = (missionId: string) =>
|
||||
api<MissionTeamProposal[]>(`/api/missions/${missionId}/team-proposals`);
|
||||
|
||||
export const decideRosterProposal = (
|
||||
missionId: string,
|
||||
proposalId: string,
|
||||
status: ProposalDecision,
|
||||
note?: string,
|
||||
) =>
|
||||
api<{ status: string; team_engine?: string; nodes?: number }>(
|
||||
`/api/missions/${missionId}/team-proposals/${proposalId}/decide`,
|
||||
{ method: "POST", body: JSON.stringify({ status, note }) },
|
||||
);
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
-- Keep a mission's events past the global retention window.
|
||||
--
|
||||
-- `mission_gc` deletes mission_events older than EVENT_RETENTION_DAYS (7). That
|
||||
-- bound is right for volume — a single busy coding phase adds hundreds of rows
|
||||
-- — and wrong for anything that needs to be re-read later: a Skill-Use
|
||||
-- measurement, a provenance question, an incident review. All three ask about a
|
||||
-- specific mission, so the exemption is per-mission rather than a raised global.
|
||||
--
|
||||
-- NULL (the default) means the mission obeys the global window, so this changes
|
||||
-- nothing for existing rows.
|
||||
ALTER TABLE missions
|
||||
ADD COLUMN IF NOT EXISTS retain_events_until TIMESTAMPTZ;
|
||||
|
||||
COMMENT ON COLUMN missions.retain_events_until IS
|
||||
'While in the future, mission_gc will not reap this mission''s events. Set when a mission is under measurement or investigation.';
|
||||
|
||||
-- The sweep joins on this, and the vast majority of rows are NULL.
|
||||
CREATE INDEX IF NOT EXISTS missions_retain_events_idx
|
||||
ON missions (retain_events_until)
|
||||
WHERE retain_events_until IS NOT NULL;
|
||||
@@ -23,7 +23,17 @@ case "${1:-up}" in
|
||||
exit 0
|
||||
fi
|
||||
docker rm -f "$CONTAINER" >/dev/null 2>&1 || true
|
||||
# --shm-size: Docker defaults /dev/shm to 64MB. Postgres allocates parallel
|
||||
# query segments there, and the suite runs many tests at once against many
|
||||
# databases, so the default is exhausted mid-run — surfacing as
|
||||
# `could not resize shared memory segment ... No space left on device`
|
||||
# during MIGRATIONS, which reads like a schema fault and is not one.
|
||||
#
|
||||
# Space exhaustion has a second cause with the same symptom: cm-testkit
|
||||
# creates a database per test and drops none, so they accumulate across
|
||||
# runs (779 of them, once). `$0 clean` sweeps those.
|
||||
docker run -d --name "$CONTAINER" --restart unless-stopped \
|
||||
--shm-size=1g \
|
||||
-e POSTGRES_PASSWORD=postgres \
|
||||
-e POSTGRES_DB=postgres \
|
||||
-p "${PORT}:5432" \
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
---
|
||||
name: ast-grep-repo-index
|
||||
description: Building a structural map of an unfamiliar repository — entry points, module boundaries, and where the real logic lives.
|
||||
when_to_use: You are mapping a codebase you did not write, before changing anything in it.
|
||||
tags: [analysis, codebase]
|
||||
---
|
||||
|
||||
# Map the shape before reading the code
|
||||
|
||||
An unfamiliar repository is mostly scaffolding. The goal of a first pass is to
|
||||
find the small fraction that matters and ignore the rest deliberately.
|
||||
|
||||
## Start at the edges, not the top
|
||||
|
||||
```bash
|
||||
rg -l 'fn main\(|async fn main' --type rust # entry points
|
||||
rg -n 'route\(|\.get\(|\.post\(' | head -50 # HTTP surface
|
||||
ls migrations/ | tail -20 # what the data recently grew
|
||||
```
|
||||
|
||||
Entry points, the request surface and the schema tell you what the system *does*
|
||||
in about ten minutes. Reading `lib.rs` top-down tells you how it is organised,
|
||||
which is a different and less useful question early on.
|
||||
|
||||
## Size is a signal
|
||||
|
||||
```bash
|
||||
find . -name '*.rs' -not -path './target/*' | xargs wc -l | sort -rn | head -20
|
||||
```
|
||||
|
||||
The largest files are either the core logic or an accumulation nobody split.
|
||||
Both are worth knowing before you touch anything nearby.
|
||||
|
||||
## Structural search beats text search for call graphs
|
||||
|
||||
Text grep for a function name matches its definition, its calls, its doc
|
||||
comments and any string containing it. When you need call sites specifically,
|
||||
search the shape — `ast-grep --pattern 'foo($$$)'` — or at minimum anchor the
|
||||
text: `rg '\bfoo\('.
|
||||
|
||||
## Follow one request end to end
|
||||
|
||||
The single most valuable early exercise: pick one endpoint and trace it from
|
||||
route registration to database and back. It crosses every layer the codebase
|
||||
has, in the order the codebase thinks about them, and it tells you more than any
|
||||
architecture document. See `request-lifecycle-tracing`.
|
||||
|
||||
## Write down the map
|
||||
|
||||
A map that lives only in your head has to be rebuilt by the next person. Record
|
||||
entry points, the three or four modules with the real logic, and — most
|
||||
valuable — what you *expected* to find and did not. The gaps between a reasonable
|
||||
mental model and the actual structure are where future bugs live.
|
||||
@@ -0,0 +1,55 @@
|
||||
---
|
||||
name: git-log-forensics
|
||||
description: Reading a repository's history to find when behaviour changed and why, rather than guessing from the current tree.
|
||||
when_to_use: You are investigating why code is the way it is, or when a behaviour was introduced.
|
||||
tags: [analysis, git]
|
||||
---
|
||||
|
||||
# The history answers questions the tree cannot
|
||||
|
||||
The current tree shows what is true. It does not show what was tried, what was
|
||||
reverted, or which line was load-bearing enough to be touched forty times.
|
||||
|
||||
## The four commands that answer most questions
|
||||
|
||||
```bash
|
||||
git log -S'<string>' --oneline -- <path> # when did this string appear/vanish
|
||||
git log -L'<start>,<end>:<file>' # every change to these lines
|
||||
git log --follow -- <file> # survives renames
|
||||
git bisect start <bad> <good> # find the commit that changed it
|
||||
```
|
||||
|
||||
`-S` (the "pickaxe") is the most under-used and the most powerful: it searches
|
||||
for commits where the *count* of a string changed, so it finds the commit that
|
||||
introduced a call, not every commit that mentions it. `-L` gives the biography
|
||||
of a specific function.
|
||||
|
||||
## Churn marks risk
|
||||
|
||||
```bash
|
||||
git log --format= --name-only | sort | uniq -c | sort -rn | head -20
|
||||
```
|
||||
|
||||
Files at the top are either the project's core or its problem area, and the
|
||||
commit messages tell you which. A file changed in 200 commits by 15 authors is
|
||||
where the next bug will be, whatever the current code looks like.
|
||||
|
||||
## Read the message, then distrust it
|
||||
|
||||
A commit message states intent. The diff states what happened. When they
|
||||
disagree — "small refactor" touching thirty files, "fix typo" changing a
|
||||
condition — the diff is the truth, and the disagreement is itself a finding
|
||||
worth recording.
|
||||
|
||||
## Blame points at the last toucher, not the author
|
||||
|
||||
`git blame` shows who last modified a line, which after a reformat, a rename or
|
||||
a lint pass is whoever ran the tool. Use `-w` (ignore whitespace) and
|
||||
`-C` (detect moved code) before drawing any conclusion, and prefer `log -L` when
|
||||
you want the line's history rather than its current owner.
|
||||
|
||||
## What to report
|
||||
|
||||
A forensic finding is a commit hash, a date, and the reason the change was made
|
||||
if the message or its PR gives one. "This check was added in `a1b2c3d` after an
|
||||
incident" is actionable. "This code looks defensive" is not.
|
||||
@@ -0,0 +1,53 @@
|
||||
---
|
||||
name: request-lifecycle-tracing
|
||||
description: Following one request from entry to storage and back, so a change can be reasoned about across layers.
|
||||
when_to_use: You need to understand how a specific operation works, or where to make a change that crosses layers.
|
||||
tags: [analysis, codebase]
|
||||
---
|
||||
|
||||
# One request, every layer, in order
|
||||
|
||||
Tracing a single path end to end is the fastest way to learn a system, and the
|
||||
only reliable way to know where a cross-cutting change must land.
|
||||
|
||||
## The trace
|
||||
|
||||
```
|
||||
route registration which handler, which method, what middleware
|
||||
extractors / auth what must be true before the handler body runs
|
||||
handler validation, then the call into real logic
|
||||
domain / service the actual decision
|
||||
persistence the query, the transaction boundary
|
||||
response what is serialised, what is deliberately omitted
|
||||
side effects events, jobs, background work
|
||||
```
|
||||
|
||||
The last line is the one most often missed and the most likely to break: an
|
||||
operation that also enqueues a job, writes an event or invalidates a cache has
|
||||
consequences the response does not mention.
|
||||
|
||||
## Follow the data, not the call stack
|
||||
|
||||
Call stacks show structure; data flow shows behaviour. For each step ask what
|
||||
changed shape and what was dropped. A field silently discarded between the
|
||||
handler and the query is a bug that no test asserting the response will catch.
|
||||
|
||||
## Find the transaction boundary explicitly
|
||||
|
||||
Where does the transaction begin and commit? Anything outside it is not atomic
|
||||
with the write, and that is where "the row exists but the event never fired"
|
||||
comes from. Note it during the trace — reconstructing it later, mid-incident, is
|
||||
much harder.
|
||||
|
||||
## Note what happens on failure
|
||||
|
||||
Walk it a second time asking what happens when each step fails. Errors that are
|
||||
logged and swallowed are where silent failures live: the operation reports
|
||||
success while a step did nothing. If a step's failure produces no observable
|
||||
signal, that is a finding, not a detail.
|
||||
|
||||
## Record the trace with file:line
|
||||
|
||||
The output is a short document with a line per layer and the file and line for
|
||||
each. That is what makes the next change — or the next incident — cheap, and it
|
||||
is checkable by whoever reads it next.
|
||||
@@ -0,0 +1,50 @@
|
||||
---
|
||||
name: openapi-contract-first
|
||||
description: Designing an HTTP API as a written contract before the handler exists, and keeping the document true afterwards.
|
||||
when_to_use: You are the API designer adding or changing an endpoint.
|
||||
tags: [backend, api]
|
||||
---
|
||||
|
||||
# The contract is the deliverable; the handler implements it
|
||||
|
||||
Writing the schema first surfaces the disagreements while they are still cheap —
|
||||
what is optional, what the error shape is, what happens on conflict.
|
||||
|
||||
## Decide these five before writing code
|
||||
|
||||
1. **Resource and method.** `POST /missions` creates; `PATCH /missions/{id}`
|
||||
partially updates. If you need a verb, the resource is probably wrong.
|
||||
2. **The error shape, once, for the whole API.** One envelope everywhere. A
|
||||
client that must handle three error shapes will handle one and log the others.
|
||||
3. **Which fields are optional, and what absent means.** Absent, `null` and empty
|
||||
are three different things and clients will discover the difference in
|
||||
production.
|
||||
4. **The status codes you actually use.** 201 with a Location for creates, 409
|
||||
for conflicts, 422 for validation. Returning 200 with `{"error": ...}` makes
|
||||
every client parse the body to learn what happened.
|
||||
5. **Pagination, from the first endpoint.** Retrofitting it is a breaking
|
||||
change — see `api-pagination-day-1`.
|
||||
|
||||
## The document must stay true
|
||||
|
||||
A stale spec is worse than none: clients trust it and are wrong. Generate it
|
||||
from the types where the framework allows, and where it is hand-written, make a
|
||||
test fail when handler and document disagree.
|
||||
|
||||
The failure mode to design against is a field added to the response and never
|
||||
to the spec. That is invisible until an integrator asks why the documented shape
|
||||
does not match, which is much later and much more expensive.
|
||||
|
||||
## Compatibility rules
|
||||
|
||||
Additive is safe: new optional request fields, new response fields. Everything
|
||||
else is a breaking change, including narrowing an enum, making an optional field
|
||||
required, and changing a field's type — even `int` to `string` for an id.
|
||||
|
||||
If a break is genuinely needed, version the path. Silently changing a shape and
|
||||
telling clients in a changelog is how integrations fail on a Friday.
|
||||
|
||||
## Examples are part of the contract
|
||||
|
||||
One realistic request and response per endpoint answers more questions than the
|
||||
schema does, and a wrong example is caught immediately by anyone who tries it.
|
||||
@@ -0,0 +1,61 @@
|
||||
---
|
||||
name: postgres-explain-analyze
|
||||
description: Reading an EXPLAIN ANALYZE plan to find why a query is slow, rather than adding indexes hopefully.
|
||||
when_to_use: A query is slow, or you want to confirm an index is actually used.
|
||||
tags: [backend, postgres]
|
||||
---
|
||||
|
||||
# Read the plan; do not guess at indexes
|
||||
|
||||
An index added without a plan is as likely to be unused as to help, and each one
|
||||
costs write throughput forever.
|
||||
|
||||
## Always ANALYZE, and usually BUFFERS
|
||||
|
||||
```sql
|
||||
EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT) SELECT ...;
|
||||
```
|
||||
|
||||
`EXPLAIN` alone shows the planner's *estimate*. `ANALYZE` executes and shows
|
||||
what happened, which is the only thing worth reading. `BUFFERS` shows whether
|
||||
the data came from cache or disk — a "slow" query that is entirely
|
||||
`shared read` is an I/O problem, not a plan problem.
|
||||
|
||||
Note that `ANALYZE` actually runs the statement. Wrap a mutation in a
|
||||
transaction you roll back.
|
||||
|
||||
## Read it inside out, and look for two things
|
||||
|
||||
Plans nest; the innermost node runs first. Scan for:
|
||||
|
||||
1. **The largest `actual time`,** not the largest estimate. That is where the
|
||||
time went.
|
||||
2. **Estimate versus actual rows.** `rows=10` with `actual rows=48000` is the
|
||||
planner being wrong, and a wrong estimate is usually the *cause* of a bad
|
||||
plan — it picked a nested loop because it expected ten rows. Fix the
|
||||
statistics (`ANALYZE <table>`, or raise the statistics target) before touching
|
||||
the query.
|
||||
|
||||
## What the node types tell you
|
||||
|
||||
- **Seq Scan** on a large table with a selective filter → a missing index, or a
|
||||
filter the index cannot serve (a function on the column, a leading wildcard).
|
||||
- **Nested Loop** with a large outer side → usually the wrong-estimate problem
|
||||
above; correct rows would have produced a hash join.
|
||||
- **Sort** with `Sort Method: external merge Disk` → `work_mem` too small, or an
|
||||
index could provide the order for free.
|
||||
- **Bitmap Heap Scan** with high `Rows Removed by Filter` → the index found
|
||||
candidates the table had to reject; consider a composite or partial index.
|
||||
|
||||
## Confirm the index is used, not just present
|
||||
|
||||
After adding one, re-run the plan. An index that does not appear is dead weight:
|
||||
it slows every write and helps nothing. Common causes are a type mismatch, a
|
||||
function on the column, or a column order that does not match the predicate —
|
||||
see `postgres-index-selection`.
|
||||
|
||||
## Test against realistic data volume
|
||||
|
||||
Plans change with size. Every plan is a Seq Scan on a thousand rows, and the
|
||||
planner is right to choose it. Validate on production-shaped data or the
|
||||
exercise is theatre.
|
||||
@@ -0,0 +1,51 @@
|
||||
---
|
||||
name: postgres-integration-testing
|
||||
description: Testing against a real Postgres rather than a mock, with isolation that survives parallel runs.
|
||||
when_to_use: You are the tester covering code that issues SQL.
|
||||
tags: [backend, testing, postgres]
|
||||
---
|
||||
|
||||
# Mock the network, never the database
|
||||
|
||||
A mocked database agrees with whatever the code believed. Constraints, cascades,
|
||||
transaction visibility, type coercion and the SQL itself are exactly the things
|
||||
that break, and a mock asserts none of them.
|
||||
|
||||
## Isolation is the whole problem
|
||||
|
||||
Parallel tests sharing a database interfere in ways that look like flaky code.
|
||||
Three workable strategies:
|
||||
|
||||
1. **A fresh migrated database per test.** Cleanest and slowest. Right when the
|
||||
suite is small or the schema is the thing under test.
|
||||
2. **A transaction per test, rolled back.** Fast and well isolated, but the code
|
||||
under test cannot manage its own transactions — which rules it out for
|
||||
anything testing commit behaviour.
|
||||
3. **Unique keys per test.** Every row keyed by a per-test uuid, no cleanup.
|
||||
Scales well, and the leftover rows are useful when something fails.
|
||||
|
||||
Pick one per suite and say which. Mixing them produces the failures each was
|
||||
meant to prevent.
|
||||
|
||||
## Test what only a real database can tell you
|
||||
|
||||
- **Constraints fire.** A unique violation, a FK failure, a check constraint —
|
||||
assert the error, not just the happy path.
|
||||
- **Cascades do what you think.** `ON DELETE CASCADE` reaching further than
|
||||
intended is a data-loss bug that only a real delete reveals.
|
||||
- **The migration applies to a populated table.** A migration tested on an empty
|
||||
database has not been tested. Insert rows first, then migrate.
|
||||
- **Concurrent claims are atomic.** For any `FOR UPDATE SKIP LOCKED` queue,
|
||||
run two workers and assert the row was claimed once.
|
||||
|
||||
## Assert on the database, not only the return value
|
||||
|
||||
A handler can return 200 while writing nothing. Read the row back and check it.
|
||||
The class of bug where the code "succeeded" and the data did not change is
|
||||
invisible to a test that only inspects the response.
|
||||
|
||||
## Keep it fast enough to run
|
||||
|
||||
An integration suite nobody runs protects nothing. Share one container across
|
||||
the suite rather than per test, run in parallel with proper isolation, and keep
|
||||
fixtures small — realistic in shape, not in volume.
|
||||
@@ -1,54 +1,77 @@
|
||||
---
|
||||
name: workspace-repo-commit-protocol
|
||||
description: How to interact with /workspace/repo — the mission's checked-out codebase — and how to commit + push meaningful changes back.
|
||||
description: How to work inside /mission/repo — the mission's checked-out codebase — and how to commit meaningful changes back.
|
||||
when_to_use: You are a coder, committer, or any role that edits code. Pin this at turn start so you never lose orientation.
|
||||
tags: [foundation, coding, git]
|
||||
---
|
||||
|
||||
# Workspace repo + commit protocol
|
||||
# Mission repo + commit protocol
|
||||
|
||||
You are running inside a mission's team container. The clawhdf5/backend/whatever repository the mission targets is bind-mounted at **`/workspace/repo`**. That is the ONLY path where source-modifying edits belong.
|
||||
The repository this mission targets is checked out at **`/mission/repo`**. That
|
||||
is the only path where source-modifying edits belong.
|
||||
|
||||
## Ground rules
|
||||
|
||||
1. **`cd /workspace/repo` at the start of every substantive turn.** If you `pwd` and it isn't `/workspace/repo`, cd there first — your default CWD is the ZeroClaw agent workspace (`~/workspace`), which is scratch storage, not the codebase.
|
||||
2. **All `file_read` / `file_write` / `shell` calls that touch source use paths under `/workspace/repo`.** Anything else is scratch — the mission will not persist it.
|
||||
3. **Never write files outside `/workspace/repo` and expect them to survive** — the agent workspace resets, `/tmp` is per-container ephemeral.
|
||||
1. **`cd /mission/repo` at the start of every substantive turn.** If you `pwd`
|
||||
and it is somewhere else, cd there first.
|
||||
2. **Every read and write that touches source uses a path under
|
||||
`/mission/repo`.** Anything else is scratch and will not be delivered.
|
||||
3. On a mission with **no** repository, `/mission/repo` still exists and is
|
||||
writable — it is a scratch workspace, every file you leave there is
|
||||
collected when the phase ends and published as a mission artifact, and there
|
||||
is nothing to commit or push. Your task text says which kind of mission this
|
||||
is; believe it over any assumption.
|
||||
|
||||
## Use the tool names your prompt gives you
|
||||
|
||||
Your turn runs through Claude Code, so the tools are `Read`, `Edit`, `Write`,
|
||||
`Bash`, `Glob`, `Grep`. Your prompt lists them explicitly — use those names.
|
||||
|
||||
Do not reach for `file_read`, `file_write`, `content_search` or `shell`. Those
|
||||
are ZeroClaw's names, they are not what your subprocess exposes, and agents that
|
||||
tried them spent whole turns describing the mismatch instead of working.
|
||||
|
||||
## Commit protocol
|
||||
|
||||
When (and only when) you have a meaningful, tested change:
|
||||
When, and only when, you have a meaningful, tested change:
|
||||
|
||||
```
|
||||
cd /workspace/repo
|
||||
git status # sanity-check what you touched
|
||||
git diff --stat # confirm scope matches the plan
|
||||
cd /mission/repo
|
||||
git status # what did you actually touch
|
||||
git diff --stat # does the scope match the plan
|
||||
git add -A
|
||||
git commit -m "<INT-NN> <one-line title>
|
||||
|
||||
<one-paragraph rationale — WHY, not what>
|
||||
<one paragraph on WHY, not what>
|
||||
|
||||
Refs: INT-NN
|
||||
"
|
||||
git push
|
||||
```
|
||||
|
||||
- **Commit message uses the mission's INT-XX marker** on the subject line — the mission's task-card parser (Slice 5) advances state on it.
|
||||
- **One INT per commit** unless the change genuinely can't be split; split-when-in-doubt.
|
||||
- **Never `--force`, never rewrite pushed history** without an explicit `HANDOFF: safe to force-push` from the reviewer.
|
||||
- **Put the INT-XX marker on the subject line.** The task-card parser advances
|
||||
mission state on it.
|
||||
- **One INT per commit** unless the change genuinely cannot be split. Split when
|
||||
in doubt: a commit covering three items cannot be reverted for one of them.
|
||||
- **Never `--force`, never rewrite pushed history** without an explicit
|
||||
`HANDOFF: safe to force-push` from the reviewer.
|
||||
- Push only if your task says to. Many missions deliver by having the platform
|
||||
diff your checkout, and a phase that pushes when it should not is harder to
|
||||
undo than one that did not push.
|
||||
|
||||
## When NOT to commit
|
||||
|
||||
- Tests failing → fix or revert; never commit red.
|
||||
- Reviewer emitted `REVIEW_BLOCK: INT-NN — <reason>` for the current item.
|
||||
- The change is a WIP or exploratory — that lives in the agent's scratch workspace, not the repo.
|
||||
- Tests failing. Fix or revert; never commit red.
|
||||
- The reviewer emitted `REVIEW_BLOCK: INT-NN` for the current item.
|
||||
- The change is exploratory. That is not what the mission branch is for.
|
||||
|
||||
## Emit the completion marker
|
||||
|
||||
After a successful push, on a line by itself:
|
||||
After the work is genuinely done, on a line by itself:
|
||||
|
||||
```
|
||||
COMPLETED: INT-NN
|
||||
```
|
||||
|
||||
The mission loop advances on that marker. If you didn't push, don't emit it.
|
||||
Exactly one INT id, no bold, no code fence — the parser takes the literal line
|
||||
and rejects anything else. The mission loop advances on it, so emitting one you
|
||||
cannot back up desynchronizes the mission from the repository.
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
---
|
||||
name: a11y-checklist
|
||||
description: The accessibility checks that catch most real failures, and the ones automated tools cannot make for you.
|
||||
when_to_use: You are designing or testing a UI component or screen.
|
||||
tags: [frontend, accessibility]
|
||||
---
|
||||
|
||||
# Automated checks find about a third
|
||||
|
||||
Run axe, then do the four manual checks it cannot do. A clean axe report on an
|
||||
unusable interface is the normal outcome, not a rare one.
|
||||
|
||||
## The four manual checks
|
||||
|
||||
1. **Tab through it.** Every interactive element reachable, in a sensible order,
|
||||
with a visible focus ring. If focus disappears into an offscreen element or a
|
||||
closed menu, the page is unusable by keyboard — the single most common real
|
||||
failure.
|
||||
2. **Operate it without a mouse.** Open the menu, pick an item, close it with
|
||||
Escape. Dialogs trap focus while open and return it to the trigger on close.
|
||||
3. **Zoom to 200%.** Text reflows, nothing is clipped, nothing overlaps. This is
|
||||
also the fastest way to find fixed-height containers that will break on a
|
||||
phone.
|
||||
4. **Read it with a screen reader once.** VoiceOver on macOS, NVDA on Windows.
|
||||
Five minutes on the main flow finds unlabelled buttons and images whose alt
|
||||
text reads as a filename.
|
||||
|
||||
## What axe does catch
|
||||
|
||||
Colour contrast, missing form labels, missing alt attributes, ARIA misuse,
|
||||
duplicate ids, heading-level jumps. Run it in CI — these regress constantly and
|
||||
are cheap to fix at the point of change.
|
||||
|
||||
## The rules that prevent most issues
|
||||
|
||||
- **A button is a `<button>`.** A clickable `<div>` needs role, tabindex and
|
||||
key handlers to be equivalent, and will get at least one of them wrong.
|
||||
- **Every input has a `<label>`** with `for`, not just a placeholder. A
|
||||
placeholder disappears on focus, which is when it was needed.
|
||||
- **Never remove the focus outline without replacing it.** `outline: none` with
|
||||
no substitute makes keyboard use impossible. Style it instead.
|
||||
- **Icon-only buttons need an accessible name** — `aria-label`. Otherwise the
|
||||
screen reader announces "button".
|
||||
- **Don't announce with colour alone.** A red border with no text says nothing
|
||||
to a colourblind user or a screen reader.
|
||||
|
||||
## Motion
|
||||
|
||||
Honour `prefers-reduced-motion`. Vestibular disorders are common and large
|
||||
parallax or transform animations genuinely make people ill. One media query
|
||||
disables the offending animations.
|
||||
@@ -0,0 +1,60 @@
|
||||
---
|
||||
name: playwright-e2e-patterns
|
||||
description: Writing end-to-end tests that fail only when the product is broken — selectors, waiting, and isolation.
|
||||
when_to_use: You are the tester on a frontend team writing or fixing browser tests.
|
||||
tags: [frontend, testing]
|
||||
---
|
||||
|
||||
# A flaky E2E test is worse than no E2E test
|
||||
|
||||
It trains the team to re-run rather than investigate, and the day it catches a
|
||||
real bug, nobody believes it. Everything below is about determinism.
|
||||
|
||||
## Select by what the user sees
|
||||
|
||||
```ts
|
||||
page.getByRole("button", { name: "Save" }) // best
|
||||
page.getByLabel("Email")
|
||||
page.getByTestId("mission-card") // when semantics genuinely absent
|
||||
page.locator(".btn-primary > span:nth-child(2)") // never
|
||||
```
|
||||
|
||||
Role and label selectors survive restyling and break when the UI genuinely
|
||||
changes meaning — which is exactly the failure you want. CSS-path selectors
|
||||
break on every refactor and pass while the button is invisible.
|
||||
|
||||
## Never sleep
|
||||
|
||||
```ts
|
||||
await page.waitForTimeout(2000); // flaky, always
|
||||
await expect(page.getByText("Saved")).toBeVisible(); // waits for the condition
|
||||
```
|
||||
|
||||
Playwright's assertions retry until timeout. A fixed sleep is either longer than
|
||||
needed (slow suite) or shorter than needed on a loaded CI box (flake). There is
|
||||
no correct constant.
|
||||
|
||||
For network, wait on the response rather than a duration:
|
||||
```ts
|
||||
const done = page.waitForResponse(r => r.url().includes("/api/missions"));
|
||||
await page.getByRole("button", { name: "Create" }).click();
|
||||
await done;
|
||||
```
|
||||
|
||||
## Each test creates what it needs
|
||||
|
||||
Tests that share a fixture pass in isolation and fail in parallel — or worse,
|
||||
pass in the order you wrote them and fail when one is skipped. Create the data
|
||||
in the test, key it with a unique id, and let cleanup be optional.
|
||||
|
||||
## Assert the user-visible outcome
|
||||
|
||||
Assert the success message and the row in the table, not the POST that fired.
|
||||
A test asserting the request passes while the response is silently discarded —
|
||||
which is precisely the bug the test existed to catch.
|
||||
|
||||
## When a test fails, look at the trace
|
||||
|
||||
`--trace on` records DOM snapshots, network and console for every step. Reading
|
||||
the trace takes a minute and answers the question; re-running takes a minute and
|
||||
answers nothing.
|
||||
@@ -0,0 +1,55 @@
|
||||
---
|
||||
name: criterion-benchmarking
|
||||
description: Writing benchmarks whose numbers mean something — warmup, distributions, and the changes that are noise.
|
||||
when_to_use: You are asked to benchmark a change, or to show a performance claim is real.
|
||||
tags: [gpu, rust, benchmarking]
|
||||
---
|
||||
|
||||
# A benchmark is an experiment
|
||||
|
||||
Most performance claims fail not because the code is slow but because the
|
||||
measurement cannot support the claim.
|
||||
|
||||
## What criterion does for you
|
||||
|
||||
`criterion` runs the routine many times, discards warmup, and reports a
|
||||
confidence interval rather than a single number. Take that seriously: if the
|
||||
intervals for before and after overlap, **you have not measured an improvement**,
|
||||
whatever the point estimates say.
|
||||
|
||||
```rust
|
||||
fn bench_search(c: &mut Criterion) {
|
||||
let index = build_index(10_000); // setup OUTSIDE the timed closure
|
||||
c.bench_function("search/10k", |b| {
|
||||
b.iter(|| index.search(black_box(&query), 10))
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
Two mistakes this shape avoids:
|
||||
- **Setup inside `iter`** measures the setup. If the setup must be per-iteration,
|
||||
use `iter_batched` so it is excluded.
|
||||
- **A missing `black_box`** lets the optimiser delete the work entirely. A
|
||||
benchmark that got 400× faster after a refactor usually got deleted, not
|
||||
optimised.
|
||||
|
||||
## Report the shape, not the headline
|
||||
|
||||
"34.7% faster" invites a follow-up question the number cannot answer. Give the
|
||||
distribution, the input size, and the machine. A change that is 30% faster at
|
||||
10k elements and 5% slower at 10M is a trade-off, and only the sweep shows it.
|
||||
|
||||
## What is noise
|
||||
|
||||
On a laptop, expect 5-10% run-to-run variance from thermal state and other
|
||||
processes alone. Treat anything under that as unmeasured. If a change is
|
||||
genuinely small but real, prove it by increasing iterations rather than by
|
||||
asserting it — and if that is too expensive, say the change was too small to
|
||||
measure rather than reporting the point estimate as fact.
|
||||
|
||||
## Benchmarks are also regression tests
|
||||
|
||||
The value is in the baseline. Record it (`--save-baseline`), compare against it
|
||||
(`--baseline`), and keep the numbers with the commit that produced them.
|
||||
`benchmark_snapshots` exists for exactly this — a benchmark whose history is
|
||||
lost measures nothing the next time.
|
||||
@@ -0,0 +1,60 @@
|
||||
---
|
||||
name: gpu-kernel-authoring
|
||||
description: Writing the same kernel across CUDA, Metal and ROCm without three divergent implementations, and the FFI boundary back into Rust.
|
||||
when_to_use: You are the kernel author on a GPU team, adding or changing a compute kernel.
|
||||
tags: [gpu, kernels]
|
||||
---
|
||||
|
||||
# One kernel, three dialects
|
||||
|
||||
The three targets differ less than they appear. Thread indexing, memory scopes
|
||||
and barriers all map onto each other; what differs is spelling and launch
|
||||
configuration. Write the algorithm once, then port the spelling.
|
||||
|
||||
| Concept | CUDA | Metal | ROCm/HIP |
|
||||
|---|---|---|---|
|
||||
| thread id | `threadIdx.x` | `thread_position_in_threadgroup` | `hipThreadIdx_x` |
|
||||
| block id | `blockIdx.x` | `threadgroup_position_in_grid` | `hipBlockIdx_x` |
|
||||
| shared mem | `__shared__` | `threadgroup` | `__shared__` |
|
||||
| barrier | `__syncthreads()` | `threadgroup_barrier(mem_flags::mem_threadgroup)` | `__syncthreads()` |
|
||||
| warp/wave | 32 | 32 (SIMD-group) | **64** on CDNA, 32 on RDNA |
|
||||
|
||||
**The wave-size difference is the one that silently produces wrong answers.**
|
||||
Any kernel that assumes 32 lanes — a warp-shuffle reduction, a ballot, an
|
||||
implicit intra-warp sync — is incorrect on CDNA. Query it (`warpSize`,
|
||||
`[[threads_per_simdgroup]]`) rather than hardcoding, and never rely on implicit
|
||||
lockstep: it was never guaranteed on CUDA either since Volta's independent
|
||||
thread scheduling.
|
||||
|
||||
## Get it correct on one target first
|
||||
|
||||
Write the scalar CPU reference, then the first GPU version, and diff the outputs
|
||||
before porting anywhere. A kernel ported three ways from an unverified original
|
||||
gives you three wrong answers and no baseline to find them with.
|
||||
|
||||
Compare with a tolerance derived from the operation, not a guess: float addition
|
||||
is not associative, so a parallel reduction legitimately differs from a
|
||||
sequential one. `1e-6` on an accumulation over a million elements is a failing
|
||||
test that is not a bug.
|
||||
|
||||
## The FFI boundary
|
||||
|
||||
Kernels are reached from Rust through `extern "C"`. Three rules that prevent the
|
||||
majority of crashes at this seam:
|
||||
|
||||
- **Own the allocation on one side.** Device memory allocated in C and freed in
|
||||
Rust's `Drop` — or the reverse — is a lifetime nobody can read. Wrap the
|
||||
device pointer in a Rust type whose `Drop` calls the same allocator's free.
|
||||
- **Every launch returns a status; check it.** A kernel launch failure is
|
||||
asynchronous and surfaces on the *next* synchronising call, so an unchecked
|
||||
launch reports its error somewhere unrelated. Check after launch AND after
|
||||
synchronise.
|
||||
- **`#[repr(C)]` on every struct crossing the boundary.** Rust's default layout
|
||||
is unspecified and does change.
|
||||
|
||||
## What to write down
|
||||
|
||||
A kernel's launch geometry is a decision, not a constant: record why the block
|
||||
size is what it is (occupancy target, shared-memory budget, register pressure)
|
||||
next to it. The next person to change the shared-memory allocation needs to know
|
||||
the block size was chosen against it.
|
||||
@@ -0,0 +1,50 @@
|
||||
---
|
||||
name: gpu-profiling-workflow
|
||||
description: Using Nsight, rocprof and Metal frame capture to find where a kernel actually spends time, instead of guessing.
|
||||
when_to_use: You are the bench engineer on a GPU team and need to explain or improve a kernel's runtime.
|
||||
tags: [gpu, profiling]
|
||||
---
|
||||
|
||||
# Measure the machine, not your model of it
|
||||
|
||||
GPU intuition is unusually unreliable: the bottleneck is far more often memory
|
||||
movement or occupancy than arithmetic. Profile before changing anything.
|
||||
|
||||
## Order of questions
|
||||
|
||||
1. **Is the GPU busy at all?** Kernel time versus wall time. A "slow kernel"
|
||||
that occupies 8% of wall time is a host-side or transfer problem, and no
|
||||
amount of kernel tuning will show up.
|
||||
2. **Memory or compute bound?** Achieved bandwidth against the device peak, and
|
||||
achieved FLOPs against peak. See `roofline-model` — the roofline tells you
|
||||
which ceiling you are under, and therefore which optimisations can possibly
|
||||
help.
|
||||
3. **Occupancy?** Only after 1 and 2. Occupancy is a means, not a goal: a
|
||||
kernel at 40% occupancy saturating bandwidth is finished, and raising
|
||||
occupancy will not make it faster.
|
||||
|
||||
## The tools
|
||||
|
||||
- **Nsight Compute** (`ncu`) — per-kernel counters. Start with
|
||||
`--set full` on ONE kernel invocation, not the whole run; it serialises and
|
||||
replays kernels, so a full-application profile takes minutes and changes the
|
||||
timing you were trying to measure.
|
||||
- **Nsight Systems** (`nsys`) — the timeline. This is where question 1 is
|
||||
answered: gaps between kernels, host-device copies, stream serialisation.
|
||||
Use it *first*; `ncu` optimises a kernel that `nsys` may show is irrelevant.
|
||||
- **rocprof** — the ROCm equivalent. `--stats` for the summary,
|
||||
`--hip-trace`/`--hsa-trace` for the timeline.
|
||||
- **Metal frame capture** — Xcode's GPU capture. Per-encoder timings and the
|
||||
shader profiler's per-line cost. It is a *frame* capture: for compute work,
|
||||
bracket the dispatch in a capture scope explicitly or you get nothing.
|
||||
|
||||
## Warm up, and say what you measured
|
||||
|
||||
First-call timings include JIT compilation, allocator growth and page faults —
|
||||
routinely 10-100× the steady state. Discard warmup iterations and report a
|
||||
distribution, not a single number: a median with a spread tells the reader
|
||||
whether the change is real. See `criterion-benchmarking` for the statistics.
|
||||
|
||||
Record the device, driver version and clock state alongside the number. GPUs
|
||||
throttle; a measurement without its conditions cannot be compared to the one
|
||||
you take next month.
|
||||
@@ -0,0 +1,54 @@
|
||||
---
|
||||
name: mobile-e2e-and-simulators
|
||||
description: Running mobile tests that mean something — simulator versus device, and what only fails on real hardware.
|
||||
when_to_use: You are the tester on a mobile team setting up or debugging end-to-end runs.
|
||||
tags: [mobile, testing]
|
||||
---
|
||||
|
||||
# The simulator is not the device
|
||||
|
||||
Simulators are excellent for layout and flow, and actively misleading for
|
||||
anything touching hardware, permissions or performance. Know which question you
|
||||
are answering.
|
||||
|
||||
## What the simulator answers honestly
|
||||
|
||||
Layout across screen sizes, navigation flows, most business logic, accessibility
|
||||
labels, localisation. Run these in CI on simulators — they are fast, parallel and
|
||||
deterministic.
|
||||
|
||||
## What only a real device answers
|
||||
|
||||
- **Performance.** A simulator uses your host CPU and GPU. Frame rate, memory
|
||||
pressure and battery there are meaningless.
|
||||
- **Permissions and their denial paths.** The interesting flow is "user said no",
|
||||
and simulator permission behaviour differs.
|
||||
- **Camera, GPS, biometrics, push.** Approximated or absent.
|
||||
- **Network transitions.** Wi-Fi to cellular, offline, captive portals. This is
|
||||
where the network layer's assumptions surface.
|
||||
- **Keyboard behaviour.** Third-party keyboards and predictive input change
|
||||
layout in ways the simulator's keyboard does not.
|
||||
|
||||
## Determinism in E2E
|
||||
|
||||
The mobile equivalents of the browser rules: select by accessibility id rather
|
||||
than by position, wait on conditions rather than durations, and reset app state
|
||||
between tests. A test depending on the previous test's leftover state is the
|
||||
most common cause of a suite that passes locally and fails in CI.
|
||||
|
||||
Cold start versus warm start matters more than on web: a test that only ever
|
||||
runs warm never exercises launch-path initialisation, which is where a
|
||||
surprising share of crashes live.
|
||||
|
||||
## Keep the CI matrix honest
|
||||
|
||||
Two simulators — the smallest supported and the newest — catch most layout
|
||||
regressions cheaply. Add one physical device for the release candidate, not for
|
||||
every commit; a device farm on every push is expensive and rarely finds what the
|
||||
simulator missed except on the axes listed above.
|
||||
|
||||
## Record which it was
|
||||
|
||||
A test result without its target is unactionable. "Passed on iPhone 15 simulator,
|
||||
iOS 18" and "passed on a physical Pixel 6" are different claims, and only the
|
||||
second one says anything about performance.
|
||||
@@ -0,0 +1,51 @@
|
||||
---
|
||||
name: mobile-platform-conventions
|
||||
description: Where iOS and Android genuinely differ in expected behaviour, and where a shared design is fine.
|
||||
when_to_use: You are designing a screen or flow that ships on both iOS and Android.
|
||||
tags: [mobile, design]
|
||||
---
|
||||
|
||||
# Share the layout, respect the platform where it is muscle memory
|
||||
|
||||
Most of a design ports unchanged. A small number of behaviours are so deeply
|
||||
learned that violating them reads as a bug rather than a style.
|
||||
|
||||
## The differences that actually matter
|
||||
|
||||
| | iOS | Android |
|
||||
|---|---|---|
|
||||
| Back | swipe from left edge; back button top-left | system back gesture/button — **must** work |
|
||||
| Primary nav | tab bar, bottom | bottom nav bar (or drawer) |
|
||||
| Destructive confirm | action sheet from bottom | dialog, centred |
|
||||
| Text input done | "Done" / return key | often the system back dismisses |
|
||||
| Sharing | share sheet | share intent |
|
||||
|
||||
**System back is the one to get right.** On Android, back must always do
|
||||
something sensible: close the sheet, pop the screen, and at the root, exit.
|
||||
A screen that traps back is broken in a way users will not report — they will
|
||||
just leave.
|
||||
|
||||
## Safe areas are not optional
|
||||
|
||||
Notches, dynamic islands, home indicators and rounded corners all intrude.
|
||||
Anything within 16pt of an edge needs safe-area insets, not fixed padding. Test
|
||||
on a device with a notch and one without.
|
||||
|
||||
## Touch targets
|
||||
|
||||
44pt minimum on iOS, 48dp on Android. This is the most frequently violated rule
|
||||
in dense interfaces and the most frequently reported as "the app feels
|
||||
unreliable" rather than as a size problem.
|
||||
|
||||
## What NOT to differentiate
|
||||
|
||||
Content layout, spacing scale, typography hierarchy, colour and iconography can
|
||||
and should be shared. Building two visual designs doubles the work and the bugs
|
||||
for a difference users do not notice, and it is the usual reason a "platform
|
||||
conventions" pass runs over.
|
||||
|
||||
## Test on the smallest supported device
|
||||
|
||||
Layouts are usually designed on a large phone and break on a small one, not the
|
||||
reverse. The smallest supported screen at the largest system font size is the
|
||||
worst case, and it takes one simulator run to check.
|
||||
@@ -0,0 +1,53 @@
|
||||
---
|
||||
name: brain-file-reading
|
||||
description: What is actually inside a claw's .brain, which sections reach a prompt, and which are stored but never read.
|
||||
when_to_use: You are inspecting an agent's brain to judge whether its definition matches what it does.
|
||||
tags: [platform, brain]
|
||||
---
|
||||
|
||||
# The brain is one HDF5 file, and most of it is not in the prompt
|
||||
|
||||
A `.brain` is a key/value store (`cm-brain`) holding an agent's definition. The
|
||||
important thing to know before drawing conclusions from one: **stored is not the
|
||||
same as used**.
|
||||
|
||||
## The sections
|
||||
|
||||
| key | reaches the model? |
|
||||
|---|---|
|
||||
| `identity/system_prompt` | **yes** — the system prompt |
|
||||
| `identity/persona` | no |
|
||||
| `identity/agent_md` | no — "how I operate", stored only |
|
||||
| `identity/soul_md` | only as a fallback when `system_prompt` is empty |
|
||||
| `skills/<name>` | via the role's bound skills, not from here |
|
||||
| `tools/<name>` | state (`gated`/`blocked`), not prose |
|
||||
| `memory/<ts_nanos>` | **yes, on the chat path** — recalled by keyword |
|
||||
| `runtime/clawmates` | no — opaque JSON |
|
||||
| `provenance/clawmates` | no — **a wired slot with no callers** |
|
||||
|
||||
`persona` and `agent_md` being stored-but-unused is deliberate and easy to
|
||||
misread: an agent whose `agent_md` describes careful behaviour it does not
|
||||
exhibit is not disobeying — it never saw it.
|
||||
|
||||
## Memory is chat-only
|
||||
|
||||
`remember`/`recall` are driven from the chat turn path. **Mission and phase work
|
||||
never touches the brain**, so an agent that did a week of mission work has an
|
||||
empty memory section. Do not conclude from that it did nothing.
|
||||
|
||||
Recall is BM25 keyword, not semantic: a memory phrased differently from the
|
||||
query will not surface. "The agent forgot" is more often "the query did not
|
||||
share words with the memory".
|
||||
|
||||
## Read the revisions, not just the current state
|
||||
|
||||
The brain is versioned (`.onion` sidecar, `commit`/`revisions`/`rollback`). The
|
||||
history shows what was changed and by what — seeding, a level-up consolidation,
|
||||
a human edit. A brain whose only revision is the seed has never been improved,
|
||||
which is a finding about the loop, not about the agent.
|
||||
|
||||
## What to report
|
||||
|
||||
Whether the definition and the behaviour agree, and which section is responsible.
|
||||
"The system prompt says X, the agent did Y" is actionable. "The brain contains
|
||||
X" is not, until you have said whether X reaches the model at all.
|
||||
@@ -0,0 +1,56 @@
|
||||
---
|
||||
name: level-up-proposal-shape
|
||||
description: Writing an agent-improvement proposal that a human can accept or reject on the evidence, not on the prose.
|
||||
when_to_use: You are proposing a change to an agent's definition, skills or model.
|
||||
tags: [platform, brain]
|
||||
---
|
||||
|
||||
# A proposal is an argument with evidence attached
|
||||
|
||||
Level-up proposals go to a human review gate. The gate is the safety mechanism,
|
||||
so a proposal that cannot be judged quickly either gets rubber-stamped or
|
||||
ignored — both failures.
|
||||
|
||||
## The shape
|
||||
|
||||
```
|
||||
OBSERVATION what the agent did, with a run or event to point at
|
||||
DIAGNOSIS which part of the definition caused it
|
||||
CHANGE the exact edit — the new text, not a description of it
|
||||
EXPECTED what will differ next time, observably
|
||||
RISK what this could make worse
|
||||
```
|
||||
|
||||
`EXPECTED` is what makes the proposal checkable later. "Better research quality"
|
||||
cannot be verified; "will cite the file it changed, as `paper-to-project-relevance`
|
||||
requires" can.
|
||||
|
||||
## Point at evidence that will still exist
|
||||
|
||||
`mission_events` is retained for **seven days**. A proposal citing an event
|
||||
older than that points at nothing by the time anyone re-reads it. Quote the
|
||||
relevant text inline rather than referencing an id alone.
|
||||
|
||||
## Prefer the smallest edit that could work
|
||||
|
||||
A rewritten system prompt is unreviewable — a human cannot tell which of forty
|
||||
changed lines caused the next difference in behaviour. One change per proposal,
|
||||
so the outcome attributes to something.
|
||||
|
||||
## Say which layer
|
||||
|
||||
The same symptom has different fixes at different layers, and naming the wrong
|
||||
one wastes a review cycle:
|
||||
|
||||
- **System prompt** — the agent's standing identity and constraints.
|
||||
- **Skill** — a procedure it should follow when a situation arises.
|
||||
- **Model** — capability, when the agent understood the task and could not do it.
|
||||
- **Nothing** — the task was ambiguous and the agent behaved reasonably.
|
||||
|
||||
"Nothing" is a legitimate proposal outcome and should be written when it is true.
|
||||
|
||||
## Reject-worthy proposals
|
||||
|
||||
If you cannot name the observation, or the change is "be more careful", it is
|
||||
not a proposal. Vague instructions do not change behaviour, and they accumulate
|
||||
in the prompt where they crowd out the specific ones that do.
|
||||
@@ -0,0 +1,53 @@
|
||||
---
|
||||
name: metrics-baseline-comparison
|
||||
description: Judging whether an agent or system change actually improved anything, against a baseline that existed first.
|
||||
when_to_use: You are evaluating whether a change to an agent, prompt or model made things better.
|
||||
tags: [platform, evaluation]
|
||||
---
|
||||
|
||||
# Without a baseline there is no comparison, only an anecdote
|
||||
|
||||
The most common failure in agent evaluation is measuring after the change and
|
||||
comparing against a memory of before.
|
||||
|
||||
## Record the baseline before changing anything
|
||||
|
||||
Whatever the metric — task success, tokens per task, wall clock, human
|
||||
corrections — capture it on the current system first, over enough runs to see
|
||||
the spread. Agent runs are high variance: two runs of the same task on the same
|
||||
prompt can differ enormously, so a single before and a single after tells you
|
||||
nothing.
|
||||
|
||||
## Compare like with like
|
||||
|
||||
Hold constant everything you are not testing: the task set, the model, the
|
||||
runtime tier, the repository state. `MemoryLake on MemoryArena` is the shape to
|
||||
copy — same framework, same model alias, same task samples, same scoring code,
|
||||
with the memory backend the intentionally changed component.
|
||||
|
||||
If two things changed, the result attributes to neither.
|
||||
|
||||
## Variance first, effect second
|
||||
|
||||
Run the *unchanged* system several times to learn the noise floor. An
|
||||
improvement smaller than the run-to-run spread has not been demonstrated,
|
||||
however good the story is. This one discipline invalidates most informal agent
|
||||
comparisons, including ones made in good faith.
|
||||
|
||||
## Report the distribution and the n
|
||||
|
||||
"9 of 40" is honest and comparable. "Significantly better" is neither. Give the
|
||||
count, the denominator and the spread, and say how many runs each side had.
|
||||
|
||||
## Beware the metric becoming the target
|
||||
|
||||
An agent optimised against a judge learns the judge. If the measure is a model's
|
||||
verdict, keep an independent check — a different provider family, or a
|
||||
deterministic assertion the agent cannot talk its way past. A rising score with
|
||||
flat real-world outcomes is the signal that this has happened.
|
||||
|
||||
## A negative result is a result
|
||||
|
||||
"No measurable difference" is worth recording and prevents the change being
|
||||
proposed again in three months. Most changes do not help; a process that only
|
||||
reports wins is not measuring.
|
||||
@@ -0,0 +1,47 @@
|
||||
---
|
||||
name: prior-art-search
|
||||
description: Establishing whether an idea is new, and finding the work that already did it before you build.
|
||||
when_to_use: You are checking novelty before proposing or building something.
|
||||
tags: [research, search]
|
||||
---
|
||||
|
||||
# Assume it has been done, and go looking
|
||||
|
||||
The default hypothesis for any idea is that someone published it. Searching to
|
||||
confirm novelty is a different activity from searching to find prior work, and
|
||||
only the second one is honest.
|
||||
|
||||
## Search the mechanism, not your name for it
|
||||
|
||||
Your framing is unlikely to match the literature's. Decompose into the
|
||||
mechanism and search that:
|
||||
|
||||
> "agent memory that remembers what it already read"
|
||||
> → deduplication, seen-set, incremental corpus, novelty detection,
|
||||
> continual retrieval
|
||||
|
||||
Three or four vocabularies, each searched separately. A single-phrase search
|
||||
returning nothing is evidence about your phrasing, not about the field.
|
||||
|
||||
## Follow citations in both directions
|
||||
|
||||
- **Backwards**: the related-work section of the closest paper you find is a
|
||||
curated survey someone else already did.
|
||||
- **Forwards**: who cites it. This is where the field's response lives — a
|
||||
strong result from two years ago that nobody cites was probably not
|
||||
reproducible, and that is worth knowing before you build on it.
|
||||
|
||||
Two hops in each direction from one well-chosen paper covers a field faster than
|
||||
any keyword sweep.
|
||||
|
||||
## The negative result is the deliverable
|
||||
|
||||
If the search finds it has been done, say so plainly and stop. That is a
|
||||
successful search that saved the build. The failure mode is a search that
|
||||
concludes "novel" because the searcher wanted to build it.
|
||||
|
||||
## Record the search, not just the conclusion
|
||||
|
||||
Which terms, which databases, which date range, and what was found. A novelty
|
||||
claim without its search is unfalsifiable, and six months later nobody can tell
|
||||
whether the field moved or the search was thin.
|
||||
@@ -0,0 +1,54 @@
|
||||
---
|
||||
name: scientific-writing-conventions
|
||||
description: Structure, hedging and claim discipline for a technical write-up that will be read by people who will check it.
|
||||
when_to_use: You are drafting a paper, a technical report, or any document making empirical claims.
|
||||
tags: [research, writing]
|
||||
---
|
||||
|
||||
# Write so a sceptic can check you
|
||||
|
||||
The reader's question is "is this true?" Every convention below exists to let
|
||||
them answer it without asking you.
|
||||
|
||||
## Structure
|
||||
|
||||
```
|
||||
ABSTRACT problem, what you did, headline result WITH its number
|
||||
INTRO why this matters, what was missing, contributions as a list
|
||||
METHOD enough that a competent reader could reimplement it
|
||||
RESULTS what happened, including what did not work
|
||||
DISCUSSION what it means, what it does not, threats to validity
|
||||
```
|
||||
|
||||
Contributions belong as an explicit list. If you cannot write three sentences
|
||||
each starting "we show that...", the work is not finished.
|
||||
|
||||
## Hedge exactly as much as the evidence hedges
|
||||
|
||||
Both directions are failures. "Our method improves retrieval" from one dataset
|
||||
overclaims; "may potentially suggest a possible improvement" for a result with
|
||||
tight confidence intervals underclaims and reads as though you do not trust your
|
||||
own experiment.
|
||||
|
||||
Match the language to the strength:
|
||||
- measured, with intervals, replicated → *shows*, *demonstrates*
|
||||
- measured once, single configuration → *indicates on X*
|
||||
- consistent with, not directly tested → *is consistent with*
|
||||
|
||||
## Every number carries its conditions
|
||||
|
||||
A number without its dataset, configuration and variance is not a result. Put
|
||||
them in the sentence or the table, never in a paragraph three pages away.
|
||||
|
||||
## Threats to validity is not a formality
|
||||
|
||||
State the ways you could be wrong: one seed, one hardware target, a benchmark
|
||||
your method was tuned on, a baseline you implemented yourself. A reviewer will
|
||||
find these; a paper that names them first is far more credible than one that
|
||||
lets them be discovered.
|
||||
|
||||
## Say what you did not do
|
||||
|
||||
Scope limits are part of the contribution. "We do not evaluate on multilingual
|
||||
corpora" prevents a reader assuming you did and finding out later — which costs
|
||||
you far more credibility than the limitation itself.
|
||||
@@ -0,0 +1,51 @@
|
||||
---
|
||||
name: structured-paper-summary
|
||||
description: Reading a paper for what it demonstrates rather than what it claims, and writing the summary in a fixed shape.
|
||||
when_to_use: You are summarising a research paper for a digest, a brief, or a decision.
|
||||
tags: [research, reading]
|
||||
---
|
||||
|
||||
# Summarise the evidence, not the abstract
|
||||
|
||||
An abstract is the authors' pitch. The summary that is worth writing states what
|
||||
was actually demonstrated, on what, against what — and where those differ from
|
||||
the claim, that difference is the most valuable line in your summary.
|
||||
|
||||
## The shape
|
||||
|
||||
```
|
||||
CLAIM what the authors say they show, one sentence
|
||||
METHOD what they actually did — the experiment, not the framing
|
||||
EVIDENCE dataset, baselines, headline numbers, and what was NOT tested
|
||||
STRENGTH strong / suggestive / anecdotal, with the reason
|
||||
BEARING which of our problems this touches, or none
|
||||
```
|
||||
|
||||
`BEARING: none` is a complete and useful entry. Most papers do not apply.
|
||||
|
||||
## Read these four things first
|
||||
|
||||
1. **The baselines.** A method that beats a weak baseline has shown almost
|
||||
nothing. Check whether the comparison is against the current standard or
|
||||
against a version of it the authors implemented.
|
||||
2. **The dataset size and shape.** A result on a million short web snippets may
|
||||
not survive on ten thousand long documents. Note the scale explicitly — it is
|
||||
the most common reason a transferable-looking result does not transfer.
|
||||
3. **The ablations.** A paper with no ablation has not shown which part of its
|
||||
contribution matters. If they claim four components and ablate none, treat
|
||||
the mechanism as unproven even if the number is real.
|
||||
4. **What is missing.** No error bars, one seed, one hardware configuration, no
|
||||
negative results — each weakens the claim, and none of them appear in the
|
||||
abstract.
|
||||
|
||||
## Numbers with their conditions or not at all
|
||||
|
||||
"12% better recall" is unusable. "12% better recall at k=10 on SIFT1M against
|
||||
HNSW with M=16" can be checked, transferred or dismissed. If the conditions are
|
||||
not in the paper, that is itself the finding.
|
||||
|
||||
## Say what would change your mind
|
||||
|
||||
The most useful line for a reader deciding whether to act: what result would
|
||||
make this wrong, and did the authors test it? A summary that a reader can act on
|
||||
tells them where the risk is, not just what the number was.
|
||||
@@ -0,0 +1,52 @@
|
||||
---
|
||||
name: web-search-triage
|
||||
description: Deciding fast which search results are worth reading, and recognising the ones that are restating each other.
|
||||
when_to_use: You are sweeping the open web for signal on a topic rather than reading a known source.
|
||||
tags: [research, search]
|
||||
---
|
||||
|
||||
# Most results restate a smaller number of sources
|
||||
|
||||
The web's response to any technical development is a primary source and then
|
||||
many summaries of it. Triage is mostly about finding the primary source and
|
||||
recognising the rest as one item.
|
||||
|
||||
## Rank by distance from the primary source
|
||||
|
||||
```
|
||||
0 the paper, the spec, the commit, the release notes
|
||||
1 the author's own blog post or thread
|
||||
2 a technical write-up that adds analysis or reproduction
|
||||
3 a summary of (2)
|
||||
4 an aggregator restating (3)
|
||||
```
|
||||
|
||||
Read 0 and 1. Read 2 only when it adds something the primary source did not — a
|
||||
reproduction, a benchmark, a counter-argument. Everything at 3 and below is the
|
||||
same item and should be recorded once, if at all.
|
||||
|
||||
## Signals that a page is worth reading
|
||||
|
||||
- It contains a number, a diff, or a reproduction someone else could run.
|
||||
- It disagrees with the primary source and says why.
|
||||
- It is dated, and the date is recent enough to be about the current version.
|
||||
|
||||
## Signals to skip
|
||||
|
||||
- No date, or a date that is silently the crawl date.
|
||||
- Contains "revolutionary", "game-changing", or a numbered list of tools.
|
||||
- Restates the abstract without adding a measurement.
|
||||
- The claim is entirely in the title and the body never returns to it.
|
||||
|
||||
## Undated is a finding
|
||||
|
||||
For fast-moving topics, a page without a date cannot be triaged at all — the
|
||||
same sentence can be current or two years stale. Treat undated as low priority
|
||||
regardless of quality, and say so, rather than reading it and being unable to
|
||||
place it.
|
||||
|
||||
## Stop deliberately
|
||||
|
||||
Sweeping has no natural end. Decide the budget first — "the top three primary
|
||||
sources per subtopic" — and stop there. An exhaustive sweep that never reports
|
||||
is worth less than a bounded one that does.
|
||||
@@ -0,0 +1,59 @@
|
||||
---
|
||||
name: scene-graph-planning
|
||||
description: Structuring a three.js scene graph so transforms, culling and disposal stay tractable as it grows.
|
||||
when_to_use: You are the scene designer laying out a three.js scene before objects are built.
|
||||
tags: [threejs, architecture]
|
||||
---
|
||||
|
||||
# The graph decides what is cheap later
|
||||
|
||||
Scene-graph shape determines transform cost, culling effectiveness and whether
|
||||
teardown is possible. All three are painful to change once content exists.
|
||||
|
||||
## Group by what moves together, not by what looks similar
|
||||
|
||||
Every `Object3D` with a dirty transform forces a matrix recomputation down its
|
||||
subtree. A graph grouped by *material* or by *asset file* means moving one
|
||||
object dirties unrelated branches. Grouped by motion, a static branch stays
|
||||
clean for the life of the scene.
|
||||
|
||||
```
|
||||
world
|
||||
├── static ← matrixAutoUpdate = false, set once
|
||||
│ ├── terrain
|
||||
│ └── props
|
||||
└── dynamic
|
||||
├── player
|
||||
└── vehicles
|
||||
```
|
||||
|
||||
`matrixAutoUpdate = false` on the static branch removes it from per-frame
|
||||
traversal entirely. This is usually the single largest CPU win in a scene with
|
||||
many objects, and it costs one line.
|
||||
|
||||
## Frustum culling works on bounding volumes, not intent
|
||||
|
||||
Culling is per-`Mesh` against its bounding sphere. Two consequences:
|
||||
|
||||
- **A merged mesh cannot be partially culled.** Merging 500 props into one draw
|
||||
call also means all 500 are drawn whenever any part is on screen. Merge by
|
||||
spatial locality, not by material alone.
|
||||
- **A wrong bounding volume silently misbehaves.** After deforming geometry,
|
||||
call `computeBoundingSphere()`, or the object pops out of view when its
|
||||
stale sphere leaves the frustum.
|
||||
|
||||
## Plan disposal with the graph
|
||||
|
||||
WebGL resources are not garbage collected. Every geometry, material and texture
|
||||
needs an explicit `dispose()`. If ownership is not planned into the graph, a
|
||||
scene swap leaks GPU memory until the tab dies — see `threejs-perf-and-teardown`.
|
||||
|
||||
The rule that makes this tractable: **one owner per resource**, recorded where
|
||||
it is created. A texture shared by twenty materials is disposed once, by the
|
||||
thing that loaded it, not by whichever material is torn down first.
|
||||
|
||||
## Depth is not free
|
||||
|
||||
Deep hierarchies cost traversal on every frame. Prefer a shallow graph with
|
||||
explicit groups over mirroring an asset's exported nesting, which is usually an
|
||||
artefact of how it was modelled rather than how it behaves.
|
||||
@@ -0,0 +1,55 @@
|
||||
---
|
||||
name: shader-authoring-glsl-wgsl
|
||||
description: Writing GLSL and WGSL shaders that compile on both, and the precision and uniform traps that only appear on some hardware.
|
||||
when_to_use: You are the shader author on a three.js/WebGL team, writing or porting a shader.
|
||||
tags: [threejs, shaders]
|
||||
---
|
||||
|
||||
# GLSL and WGSL are the same ideas, spelled differently
|
||||
|
||||
WebGL2 takes GLSL ES 3.0; WebGPU takes WGSL. Porting is mostly mechanical, and
|
||||
the mechanical parts are not where the bugs are.
|
||||
|
||||
| | GLSL ES 3.0 | WGSL |
|
||||
|---|---|---|
|
||||
| entry | `void main()` | `@fragment fn fs_main(...) -> @location(0) vec4f` |
|
||||
| varyings | `in`/`out` at global scope | struct fields with `@location(n)` |
|
||||
| uniforms | `uniform` block | `var<uniform>` in a bind group |
|
||||
| texture | `texture(sampler2D, uv)` | `textureSample(t, s, uv)` — texture and sampler are SEPARATE |
|
||||
| vec | `vec3` | `vec3f` (alias of `vec3<f32>`) |
|
||||
|
||||
The separated texture/sampler split is the one that changes structure: WGSL
|
||||
binds them independently, so a GLSL shader using four samplers becomes four
|
||||
textures plus (often) one shared sampler.
|
||||
|
||||
## Precision is not decoration
|
||||
|
||||
`mediump` in a fragment shader means **at least** 10 bits of mantissa, and on
|
||||
mobile GPUs it means exactly that. A computation that is fine on a desktop
|
||||
where `mediump` is silently promoted to 32-bit will band, banding-clamp or
|
||||
NaN on a phone.
|
||||
|
||||
Rules that avoid the whole class:
|
||||
- World-space positions and time accumulators are `highp`. Always. A `mediump`
|
||||
time uniform visibly stutters within minutes of page load.
|
||||
- Normalise in `highp`, then downcast.
|
||||
- Test on a real mobile device or an emulator that honours precision. Desktop
|
||||
Chrome will not show you this bug.
|
||||
|
||||
## Uniforms are a budget
|
||||
|
||||
Each uniform, varying and texture unit is a hardware-limited slot, and the
|
||||
limits are much lower than desktop defaults suggest (`MAX_VARYING_VECTORS` can
|
||||
be 8). Pack related scalars into a `vec4` rather than declaring four floats, and
|
||||
query the limits rather than assuming.
|
||||
|
||||
## Shader compile errors are silent by default
|
||||
|
||||
three.js logs a compile failure to the console and renders black. That is
|
||||
indistinguishable from a material bug, a camera bug or a culling bug. When
|
||||
something renders black, check the shader log FIRST — it is a two-second check
|
||||
that eliminates a large fraction of the search space.
|
||||
|
||||
Keep a flat-colour fallback: a shader that fails to compile should show
|
||||
magenta, never black. Black is a colour the scene might legitimately be;
|
||||
magenta is not.
|
||||
@@ -0,0 +1,52 @@
|
||||
---
|
||||
name: webgl-frame-profiling
|
||||
description: Finding what actually costs a frame — draw calls, overdraw, shader cost — using Chrome DevTools and Spector.js.
|
||||
when_to_use: You are the performance engineer on a three.js/WebGL team and a scene is dropping frames.
|
||||
tags: [threejs, profiling]
|
||||
---
|
||||
|
||||
# Find the frame's real cost before changing the scene
|
||||
|
||||
A dropped frame has a small number of possible causes and they are
|
||||
distinguishable in minutes. Guessing usually leads to optimising geometry when
|
||||
the cost was overdraw, or the reverse.
|
||||
|
||||
## The order
|
||||
|
||||
1. **CPU or GPU?** Chrome DevTools Performance panel: record 5 seconds of the
|
||||
stall. Long scripting bars mean the CPU is the problem (scene-graph
|
||||
traversal, matrix updates, garbage). A near-idle main thread with dropped
|
||||
frames means the GPU is.
|
||||
2. **How many draw calls?** `renderer.info.render.calls`. Anything in the
|
||||
thousands is the answer on its own — instance, merge, or batch by material.
|
||||
`renderer.info` is free and should be on screen during development.
|
||||
3. **Overdraw?** Spector.js captures a frame and lists every GL call in order.
|
||||
Transparent objects drawn back-to-front over the whole viewport are the
|
||||
usual culprit; the same scene with `transparent: false` running fast
|
||||
confirms it in one test.
|
||||
4. **Shader cost?** Only now. Halve the canvas resolution: if the frame time
|
||||
halves, you are fragment-bound and the shader (or overdraw) is the cost. If
|
||||
it does not move, you are not.
|
||||
|
||||
## The resolution test is the cheapest diagnostic
|
||||
|
||||
`renderer.setPixelRatio(1)` versus `2` changes fragment work 4× and geometry
|
||||
work not at all. That one toggle separates vertex/CPU cost from fragment cost
|
||||
faster than any profiler, and it needs no tooling.
|
||||
|
||||
## What Spector.js is for
|
||||
|
||||
It is a *capture*, not a sampler: one frame, every call, with state at each
|
||||
step. Use it to answer "what is this frame actually doing" — unexpected state
|
||||
changes, redundant binds, a texture uploaded per frame — not to measure time.
|
||||
For time, use the DevTools timeline.
|
||||
|
||||
## Measure the steady state
|
||||
|
||||
The first seconds include shader compilation, texture upload and JIT warmup.
|
||||
three.js compiles a material's program on first use, so a stutter the first time
|
||||
an object becomes visible is a compile, not a leak. Pre-warm with
|
||||
`renderer.compile(scene, camera)` and profile after.
|
||||
|
||||
Record the device and the pixel ratio with any number you report. A frame time
|
||||
without a resolution is not a measurement.
|
||||
@@ -4,13 +4,13 @@ description = "Rust backend teams — Postgres, DuckDB, graph databases, AP
|
||||
stack = ["rust", "postgres", "duckdb", "graph", "api", "middleware"]
|
||||
default_topology = "pipeline"
|
||||
risk_profile = "coding_readwrite"
|
||||
mcp_bundles = ["clawmates_door", "gitea_forge"]
|
||||
mcp_bundles = ["clawmates_door", "clawmates_skills"]
|
||||
version = 1
|
||||
|
||||
[[roles]]
|
||||
slot = "api_designer"
|
||||
order_idx = 0
|
||||
skills = ["decompose-int-items", "openapi_schema", "small-focused-commits", "api-pagination-day-1"]
|
||||
skills = ["decompose-int-items", "openapi-contract-first", "small-focused-commits", "api-pagination-day-1"]
|
||||
system_prompt = """
|
||||
You are the API DESIGNER of a Backend team.
|
||||
|
||||
@@ -31,7 +31,7 @@ brain_seed = """
|
||||
[[roles]]
|
||||
slot = "db_engineer"
|
||||
order_idx = 1
|
||||
skills = ["postgres-migrations-forward-only", "postgres-index-selection", "explain_analyze", "write-rust-current-edition", "workspace-repo-commit-protocol"]
|
||||
skills = ["postgres-migrations-forward-only", "postgres-index-selection", "postgres-explain-analyze", "write-rust-current-edition", "workspace-repo-commit-protocol"]
|
||||
system_prompt = """
|
||||
You are the DB ENGINEER of a Backend team.
|
||||
|
||||
@@ -55,7 +55,7 @@ brain_seed = """
|
||||
[[roles]]
|
||||
slot = "coder"
|
||||
order_idx = 2
|
||||
skills = ["write-rust-current-edition", "cargo_build", "cargo-test-driven-development", "workspace-repo-commit-protocol", "int-xx-marker-protocol", "rust-error-handling", "rust-async-tokio-idioms"]
|
||||
skills = ["write-rust-current-edition", "cargo-test-driven-development", "workspace-repo-commit-protocol", "int-xx-marker-protocol", "rust-error-handling", "rust-async-tokio-idioms"]
|
||||
system_prompt = """
|
||||
You are the CODER of a Backend team.
|
||||
|
||||
@@ -68,7 +68,7 @@ brain_seed = ""
|
||||
[[roles]]
|
||||
slot = "tester"
|
||||
order_idx = 3
|
||||
skills = ["cargo-test-driven-development", "integration_tests_pg", "coverage_report", "tdd-red-green-refactor"]
|
||||
skills = ["cargo-test-driven-development", "postgres-integration-testing", "tdd-red-green-refactor"]
|
||||
system_prompt = """
|
||||
You are the TESTER of a Backend team.
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ stack = ["research", "code-forensics", "obsidian", "documentation"]
|
||||
category = "research"
|
||||
default_topology = "pipeline"
|
||||
risk_profile = "research_readonly"
|
||||
mcp_bundles = ["clawmates_door", "clawmates_skills", "gitea_forge"]
|
||||
mcp_bundles = ["clawmates_door", "clawmates_skills"]
|
||||
version = 1
|
||||
|
||||
[[roles]]
|
||||
@@ -51,7 +51,7 @@ brain_seed = """
|
||||
[[roles]]
|
||||
slot = "architecture_mapper"
|
||||
order_idx = 1
|
||||
skills = ["ast-grep-repo-index", "dependency-graph", "workspace-repo-commit-protocol"]
|
||||
skills = ["ast-grep-repo-index", "workspace-repo-commit-protocol"]
|
||||
system_prompt = """
|
||||
You are the ARCHITECTURE MAPPER of a Codebase Research team.
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ version = 1
|
||||
[[roles]]
|
||||
slot = "brain_inspector"
|
||||
order_idx = 0
|
||||
skills = ["brain-file-reading", "role-purpose-audit", "workspace-repo-commit-protocol"]
|
||||
skills = ["brain-file-reading", "workspace-repo-commit-protocol"]
|
||||
system_prompt = """
|
||||
You are the BRAIN INSPECTOR of a Continuous Improvement team.
|
||||
|
||||
@@ -47,7 +47,7 @@ brain_seed = """
|
||||
[[roles]]
|
||||
slot = "improvement_proposer"
|
||||
order_idx = 1
|
||||
skills = ["level-up-proposal-shape", "brain-consolidation", "workspace-repo-commit-protocol"]
|
||||
skills = ["level-up-proposal-shape", "workspace-repo-commit-protocol"]
|
||||
system_prompt = """
|
||||
You are the IMPROVEMENT PROPOSER of a Continuous Improvement team.
|
||||
|
||||
|
||||
@@ -5,11 +5,16 @@ stack = ["research", "papers", "digest", "obsidian", "podcast"]
|
||||
category = "research"
|
||||
default_topology = "pipeline"
|
||||
risk_profile = "research_readonly"
|
||||
# `web_fetch` was declared here and is INERT: runtime_provision.rs binds every
|
||||
# mission claw to `mcp_bundles = ["clawmates_door"]` and never reads this field.
|
||||
# Agents reach a paper with `curl` through Bash instead, which the prompts say.
|
||||
# Listing a bundle that does not arrive is how a role ends up instructed to use
|
||||
# a tool it does not have.
|
||||
# This field IS now read. `runtime_provision::provision_claw` used to write the
|
||||
# constant `["clawmates_door"]` and ignore it — which is how every skill in the
|
||||
# catalogue became unreachable from a mission, since the `clawmates_skills` MCP
|
||||
# server is their only delivery channel. It now provisions what is listed here,
|
||||
# with the door always added.
|
||||
#
|
||||
# `web_fetch` was removed rather than kept: there is no such bundle to deliver,
|
||||
# and now that this list is honoured, naming one that does not exist is worse
|
||||
# than naming one that was ignored. Agents reach a paper with `curl` through
|
||||
# Bash, which the prompts say.
|
||||
mcp_bundles = ["clawmates_door", "clawmates_skills"]
|
||||
version = 2
|
||||
|
||||
|
||||
@@ -4,13 +4,13 @@ description = "Latest React + TailwindCSS + ShadCN — component authoring,
|
||||
stack = ["typescript", "react", "tailwindcss", "shadcn", "next"]
|
||||
default_topology = "pipeline"
|
||||
risk_profile = "coding_readwrite"
|
||||
mcp_bundles = ["clawmates_door", "gitea_forge"]
|
||||
mcp_bundles = ["clawmates_door", "clawmates_skills"]
|
||||
version = 1
|
||||
|
||||
[[roles]]
|
||||
slot = "designer"
|
||||
order_idx = 0
|
||||
skills = ["decompose-int-items", "design_system_check", "a11y_checklist"]
|
||||
skills = ["decompose-int-items", "a11y-checklist"]
|
||||
system_prompt = """
|
||||
You are the DESIGNER of a Frontend team.
|
||||
|
||||
@@ -34,7 +34,7 @@ brain_seed = """
|
||||
[[roles]]
|
||||
slot = "coder"
|
||||
order_idx = 1
|
||||
skills = ["write_typescript_react", "tailwind-v4-idioms", "workspace-repo-commit-protocol", "int-xx-marker-protocol", "component-4-state-model", "react-19-server-components"]
|
||||
skills = ["tailwind-v4-idioms", "workspace-repo-commit-protocol", "int-xx-marker-protocol", "component-4-state-model", "react-19-server-components"]
|
||||
system_prompt = """
|
||||
You are the CODER of a Frontend team.
|
||||
|
||||
@@ -53,7 +53,7 @@ brain_seed = """
|
||||
[[roles]]
|
||||
slot = "tester"
|
||||
order_idx = 2
|
||||
skills = ["playwright_e2e", "vitest_unit", "a11y_axe", "tdd-red-green-refactor"]
|
||||
skills = ["playwright-e2e-patterns", "a11y-checklist", "tdd-red-green-refactor"]
|
||||
system_prompt = """
|
||||
You are the TESTER of a Frontend team.
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ description = "CUDA, Metal, ROCm from Rust — low-level GPU application de
|
||||
stack = ["rust", "cuda", "metal", "rocm", "gpu"]
|
||||
default_topology = "pipeline"
|
||||
risk_profile = "coding_readwrite"
|
||||
mcp_bundles = ["clawmates_door", "gitea_forge"]
|
||||
mcp_bundles = ["clawmates_door", "clawmates_skills"]
|
||||
version = 1
|
||||
|
||||
[[roles]]
|
||||
@@ -32,7 +32,7 @@ brain_seed = """
|
||||
[[roles]]
|
||||
slot = "kernel_author"
|
||||
order_idx = 1
|
||||
skills = ["write_cuda", "write_metal", "write_rocm", "write_rust_ffi", "workspace-repo-commit-protocol"]
|
||||
skills = ["gpu-kernel-authoring", "workspace-repo-commit-protocol"]
|
||||
system_prompt = """
|
||||
You are the KERNEL AUTHOR of a GPU team.
|
||||
|
||||
@@ -52,7 +52,7 @@ brain_seed = """
|
||||
[[roles]]
|
||||
slot = "bench_engineer"
|
||||
order_idx = 2
|
||||
skills = ["nsight_profile", "metal_frame_capture", "rocprof", "criterion_bench"]
|
||||
skills = ["gpu-profiling-workflow", "criterion-benchmarking"]
|
||||
system_prompt = """
|
||||
You are the BENCH ENGINEER of a GPU team.
|
||||
|
||||
@@ -65,7 +65,7 @@ brain_seed = ""
|
||||
[[roles]]
|
||||
slot = "coder"
|
||||
order_idx = 3
|
||||
skills = ["write-rust-current-edition", "cargo_build", "cargo-test-driven-development", "workspace-repo-commit-protocol", "int-xx-marker-protocol"]
|
||||
skills = ["write-rust-current-edition", "cargo-test-driven-development", "workspace-repo-commit-protocol", "int-xx-marker-protocol"]
|
||||
system_prompt = """
|
||||
You are the RUST-SIDE CODER. Integrate the kernel + FFI into the Rust
|
||||
library, add safe wrappers, and expose ergonomic APIs. Own the
|
||||
|
||||
@@ -5,13 +5,15 @@ stack = ["research", "novelty", "publication", "obsidian", "citation-
|
||||
category = "research"
|
||||
default_topology = "pipeline"
|
||||
risk_profile = "research_readonly"
|
||||
mcp_bundles = ["clawmates_door", "clawmates_skills", "web_fetch"]
|
||||
# `web_fetch` removed: no such bundle is defined, and provision_claw now
|
||||
# honours this list — agents reach a page with `curl` through Bash.
|
||||
mcp_bundles = ["clawmates_door", "clawmates_skills"]
|
||||
version = 1
|
||||
|
||||
[[roles]]
|
||||
slot = "implementation_tracker"
|
||||
order_idx = 0
|
||||
skills = ["git-log-forensics", "paper-citation-parsing", "workspace-repo-commit-protocol"]
|
||||
skills = ["git-log-forensics", "workspace-repo-commit-protocol"]
|
||||
system_prompt = """
|
||||
You are the IMPLEMENTATION TRACKER of an Insight Research team.
|
||||
|
||||
@@ -85,7 +87,7 @@ brain_seed = """
|
||||
[[roles]]
|
||||
slot = "publication_drafter"
|
||||
order_idx = 2
|
||||
skills = ["scientific-writing-conventions", "figure-planning", "workspace-repo-commit-protocol", "small-focused-commits"]
|
||||
skills = ["scientific-writing-conventions", "workspace-repo-commit-protocol", "small-focused-commits"]
|
||||
system_prompt = """
|
||||
You are the PUBLICATION DRAFTER of an Insight Research team.
|
||||
|
||||
|
||||
@@ -4,13 +4,13 @@ description = "Expo + React Native for iOS/Android — camera, comms, netwo
|
||||
stack = ["typescript", "react-native", "expo", "ios", "android"]
|
||||
default_topology = "pipeline"
|
||||
risk_profile = "coding_readwrite"
|
||||
mcp_bundles = ["clawmates_door", "gitea_forge"]
|
||||
mcp_bundles = ["clawmates_door", "clawmates_skills"]
|
||||
version = 1
|
||||
|
||||
[[roles]]
|
||||
slot = "designer"
|
||||
order_idx = 0
|
||||
skills = ["decompose-int-items", "ios_hig_check", "material_you_check"]
|
||||
skills = ["decompose-int-items", "mobile-platform-conventions"]
|
||||
system_prompt = """
|
||||
You are the DESIGNER of a Mobile team.
|
||||
|
||||
@@ -28,7 +28,7 @@ brain_seed = """
|
||||
[[roles]]
|
||||
slot = "coder"
|
||||
order_idx = 1
|
||||
skills = ["write_typescript_react_native", "expo-managed-vs-bare", "workspace-repo-commit-protocol", "int-xx-marker-protocol", "component-4-state-model", "rn-flashlist-perf"]
|
||||
skills = ["expo-managed-vs-bare", "workspace-repo-commit-protocol", "int-xx-marker-protocol", "component-4-state-model", "rn-flashlist-perf"]
|
||||
system_prompt = """
|
||||
You are the CODER of a Mobile team.
|
||||
|
||||
@@ -46,7 +46,7 @@ brain_seed = """
|
||||
[[roles]]
|
||||
slot = "tester"
|
||||
order_idx = 2
|
||||
skills = ["detox_e2e", "jest_unit", "ios_simulator_check", "android_emulator_check", "tdd-red-green-refactor"]
|
||||
skills = ["mobile-e2e-and-simulators", "tdd-red-green-refactor"]
|
||||
system_prompt = """
|
||||
You are the TESTER of a Mobile team.
|
||||
|
||||
|
||||
@@ -5,13 +5,15 @@ stack = ["research", "papers", "arxiv", "obsidian", "library"]
|
||||
category = "research"
|
||||
default_topology = "pipeline"
|
||||
risk_profile = "research_web_readonly"
|
||||
mcp_bundles = ["clawmates_door", "clawmates_skills", "web_fetch"]
|
||||
# `web_fetch` removed: no such bundle is defined, and provision_claw now
|
||||
# honours this list — agents reach a page with `curl` through Bash.
|
||||
mcp_bundles = ["clawmates_door", "clawmates_skills"]
|
||||
version = 1
|
||||
|
||||
[[roles]]
|
||||
slot = "domain_scout"
|
||||
order_idx = 0
|
||||
skills = ["arxiv-query", "semantic-scholar-query", "web-search-triage", "decompose-int-items"]
|
||||
skills = ["arxiv-daily", "web-search-triage", "decompose-int-items"]
|
||||
system_prompt = """
|
||||
You are the DOMAIN SCOUT of a Papers & Online Research team.
|
||||
|
||||
@@ -47,7 +49,7 @@ brain_seed = """
|
||||
[[roles]]
|
||||
slot = "paper_reader"
|
||||
order_idx = 1
|
||||
skills = ["structured-paper-summary", "pdf-text-extraction", "workspace-repo-commit-protocol"]
|
||||
skills = ["structured-paper-summary", "workspace-repo-commit-protocol"]
|
||||
system_prompt = """
|
||||
You are the PAPER READER of a Papers & Online Research team.
|
||||
|
||||
|
||||
@@ -4,13 +4,13 @@ description = "Full software lifecycle for Rust projects — planning, impl
|
||||
stack = ["rust", "systems", "distributed", "backend"]
|
||||
default_topology = "pipeline"
|
||||
risk_profile = "coding_readwrite"
|
||||
mcp_bundles = ["clawmates_door", "gitea_forge"]
|
||||
mcp_bundles = ["clawmates_door", "clawmates_skills"]
|
||||
version = 1
|
||||
|
||||
[[roles]]
|
||||
slot = "planner"
|
||||
order_idx = 0
|
||||
skills = ["read_roadmap", "decompose-int-items", "estimate_effort", "small-focused-commits"]
|
||||
skills = ["decompose-int-items", "small-focused-commits"]
|
||||
system_prompt = """
|
||||
You are the PLANNER of a Rust SDLC team.
|
||||
|
||||
@@ -43,7 +43,7 @@ brain_seed = """
|
||||
[[roles]]
|
||||
slot = "coder"
|
||||
order_idx = 1
|
||||
skills = ["write-rust-current-edition", "cargo_build", "cargo-test-driven-development", "workspace-repo-commit-protocol", "small-focused-commits", "int-xx-marker-protocol", "rust-error-handling", "rust-async-tokio-idioms", "react-19-server-components"]
|
||||
skills = ["write-rust-current-edition", "cargo-test-driven-development", "workspace-repo-commit-protocol", "small-focused-commits", "int-xx-marker-protocol", "rust-error-handling", "rust-async-tokio-idioms", "react-19-server-components"]
|
||||
system_prompt = """
|
||||
You are the CODER of a Rust SDLC team.
|
||||
|
||||
@@ -78,7 +78,7 @@ brain_seed = """
|
||||
[[roles]]
|
||||
slot = "tester"
|
||||
order_idx = 2
|
||||
skills = ["cargo-test-driven-development", "cargo_nextest", "coverage_report", "criterion_bench", "tdd-red-green-refactor"]
|
||||
skills = ["cargo-test-driven-development", "criterion-benchmarking", "tdd-red-green-refactor"]
|
||||
system_prompt = """
|
||||
You are the TESTER of a Rust SDLC team.
|
||||
|
||||
@@ -96,7 +96,7 @@ brain_seed = ""
|
||||
[[roles]]
|
||||
slot = "reviewer"
|
||||
order_idx = 3
|
||||
skills = ["code-review-checklist", "read_diff", "small-focused-commits", "cargo-audit-workflow", "secret-scanning-gitleaks"]
|
||||
skills = ["code-review-checklist", "small-focused-commits", "cargo-audit-workflow", "secret-scanning-gitleaks"]
|
||||
system_prompt = """
|
||||
You are the REVIEWER of a Rust SDLC team.
|
||||
|
||||
|
||||
@@ -4,13 +4,13 @@ description = "Immersive graphics + game dev in the browser — 3D, isometr
|
||||
stack = ["typescript", "threejs", "webgl", "webgpu", "gsap"]
|
||||
default_topology = "pipeline"
|
||||
risk_profile = "coding_readwrite"
|
||||
mcp_bundles = ["clawmates_door", "gitea_forge"]
|
||||
mcp_bundles = ["clawmates_door", "clawmates_skills"]
|
||||
version = 1
|
||||
|
||||
[[roles]]
|
||||
slot = "scene_designer"
|
||||
order_idx = 0
|
||||
skills = ["decompose-int-items", "scene_graph_planning"]
|
||||
skills = ["decompose-int-items", "scene-graph-planning"]
|
||||
system_prompt = """
|
||||
You are the SCENE DESIGNER of a three.js team.
|
||||
|
||||
@@ -28,7 +28,7 @@ brain_seed = """
|
||||
[[roles]]
|
||||
slot = "coder"
|
||||
order_idx = 1
|
||||
skills = ["write_typescript", "threejs-perf-and-teardown", "workspace-repo-commit-protocol", "int-xx-marker-protocol"]
|
||||
skills = ["threejs-perf-and-teardown", "workspace-repo-commit-protocol", "int-xx-marker-protocol"]
|
||||
system_prompt = """
|
||||
You are the CODER of a three.js team.
|
||||
|
||||
@@ -47,7 +47,7 @@ brain_seed = """
|
||||
[[roles]]
|
||||
slot = "shader_author"
|
||||
order_idx = 2
|
||||
skills = ["write_glsl", "write_wgsl", "write_typescript"]
|
||||
skills = ["shader-authoring-glsl-wgsl", "webgl-frame-profiling", "workspace-repo-commit-protocol"]
|
||||
system_prompt = """
|
||||
You are the SHADER AUTHOR. Author vertex/fragment shaders (GLSL for
|
||||
WebGL, WGSL for WebGPU). Comment mathematical steps. Provide a
|
||||
@@ -58,7 +58,7 @@ brain_seed = ""
|
||||
[[roles]]
|
||||
slot = "perf_engineer"
|
||||
order_idx = 3
|
||||
skills = ["chrome_devtools_perf", "spector_js_capture", "webgl_frame_capture"]
|
||||
skills = ["webgl-frame-profiling", "threejs-perf-and-teardown", "shader-authoring-glsl-wgsl"]
|
||||
system_prompt = """
|
||||
You are the PERF ENGINEER. Profile with SpectorJS or Chrome DevTools
|
||||
Performance panel. Report per-frame breakdown (JS / GPU / paint) and
|
||||
|
||||
@@ -5,16 +5,59 @@ requires_repo = true
|
||||
|
||||
default_team_template = "rust_sdlc"
|
||||
|
||||
# ── What this recipe actually does ───────────────────────────────────
|
||||
#
|
||||
# `harness` is REAL and was mislabelled: `benchmark_runner::harness_from_config`
|
||||
# reads it, and `phase_runner`'s benchmark sweep runs a baseline through it
|
||||
# automatically, recording a `benchmark_snapshots` row. It was listed in
|
||||
# `phase_config.rs` as NOT IMPLEMENTED — now corrected there. `mode` genuinely
|
||||
# is inert.
|
||||
#
|
||||
# The real gap was that the phase carried no `task` and no `done_when` — so a
|
||||
# benchmark mission ran
|
||||
# one unjudged phase with a generic directive and reported `completed` whether
|
||||
# it wrote a benchmark or not. Since the entire point of this workflow is to
|
||||
# produce numbers a later refactor is measured against, a run that silently
|
||||
# produced none is worse than no run: the next mission compares against a
|
||||
# baseline that does not exist.
|
||||
#
|
||||
# The instructions therefore live in `task`/`done_when`, which ARE read, rather
|
||||
# than in `mode`/`harness`, which are not. `benchmark` is in PRODUCING_KINDS, so
|
||||
# the empty-delivery rule applies once there is something to deliver.
|
||||
|
||||
[[phases]]
|
||||
kind = "benchmark"
|
||||
order_idx = 0
|
||||
[phases.config]
|
||||
# Slice 7 runs the benchmark_snapshots baseline pass here. This
|
||||
# workflow's job is to AUTHOR the benchmarks + establish the
|
||||
# baseline; subsequent refactor missions consume them.
|
||||
produces = ["md"]
|
||||
# NOTE: `mode` is INERT — no code selects behaviour from it.
|
||||
mode = "author_and_baseline"
|
||||
# Which harness to use; per-stack defaults if unset:
|
||||
# rust → criterion / cargo bench
|
||||
# ts/js → vitest --bench / mitata
|
||||
# py → pytest-benchmark
|
||||
# READ by benchmark_runner. "auto" lets it detect the stack; naming a harness
|
||||
# explicitly ("criterion" + `bench_name`, "cargo_bench", "vitest_bench",
|
||||
# "pytest_bench", or "shell" + `cmd`) pins it. The automatic baseline this
|
||||
# drives is separate from, and a check on, the numbers the agent reports.
|
||||
harness = "auto"
|
||||
task = """
|
||||
Author benchmarks for this repository and record a baseline that a later \
|
||||
refactor can be measured against.
|
||||
|
||||
Pick the harness that matches the stack — criterion for Rust, `vitest --bench` \
|
||||
or mitata for TS/JS, pytest-benchmark for Python — and say in the report which \
|
||||
you chose and why.
|
||||
|
||||
Benchmark what the project's own hot path is, not what is easy to measure. A \
|
||||
microbenchmark of a function nobody calls produces a number that will never \
|
||||
change and teaches a future refactor nothing. Read the code first and name the \
|
||||
operation you are measuring and why it is the one that matters.
|
||||
|
||||
Record the baseline in BENCHMARKS.md: the machine, the command, the numbers, \
|
||||
AND the run-to-run spread from at least three runs. The spread is not optional \
|
||||
detail — without it, nobody can tell whether a later 5% "improvement" is real \
|
||||
or noise, which makes the entire baseline unusable for its only purpose.
|
||||
|
||||
Commit the benchmark code itself, not just the results. The next mission has to \
|
||||
be able to re-run exactly what you ran.
|
||||
"""
|
||||
done_when = "BENCHMARKS.md exists and records, for each benchmark authored, the operation measured, the exact command to re-run it, the baseline numbers from at least three runs including the observed spread, and the machine they were taken on — and the benchmark source is committed to the repository"
|
||||
max_iterations = 2
|
||||
commit_policy = "always"
|
||||
|
||||
@@ -5,29 +5,120 @@ requires_repo = true
|
||||
|
||||
default_team_template = "rust_sdlc"
|
||||
|
||||
# ── What is real here, and what is decoration ────────────────────────
|
||||
#
|
||||
# The scanners themselves are REAL: `gitleaks`, `trivy`, `semgrep` and
|
||||
# `cargo-audit` are installed in the runtime image and `runtime_preflight`
|
||||
# probes for all four at boot, naming the consequence when one is absent. An
|
||||
# agent in a security_scan phase can run them from Bash today.
|
||||
#
|
||||
# `tools` is REAL too: `security_scan::run` reads it to pick which scanners to
|
||||
# run and upserts each finding as a mission_task. What was missing was anything
|
||||
# that FIRED it — it was reachable only from an operator button, so this
|
||||
# workflow's scan phase never scanned. `phase_runner::scan_finished_security_phases`
|
||||
# now runs it when the phase finishes, guarded on a completion marker so a
|
||||
# clean repo is not rescanned forever.
|
||||
#
|
||||
# What remains decoration is annotated inline below. And the part that actually
|
||||
# mattered: no phase carried a `task` or a `done_when`.
|
||||
# A phase with no `done_when` never enters `evaluating`, is never judged, and
|
||||
# reports `completed` whatever it did. So this recipe could run all three
|
||||
# phases, scan nothing, patch nothing, and go green. That is the same defect
|
||||
# `research_and_code.toml` was fixed for, and it is why the tasks below name
|
||||
# the scanners explicitly rather than trusting `tools` to deliver them.
|
||||
|
||||
[[phases]]
|
||||
kind = "security_scan"
|
||||
order_idx = 0
|
||||
[phases.config]
|
||||
# Slice 8 wires these tools as an MCP bundle. Each finding becomes
|
||||
# a mission_task with external_id = CVE/RUSTSEC/gitleaks fingerprint.
|
||||
# READ by `security_scan::run` — this list gates which scanners the platform
|
||||
# runs against the checkout after the phase finishes. The agent ALSO runs them
|
||||
# itself during the phase (see the task): the platform pass is the independent
|
||||
# record, the agent pass is what lets it write a report about what it found.
|
||||
tools = ["cargo_audit", "gitleaks", "trivy_fs", "semgrep"]
|
||||
produces = ["md"]
|
||||
task = """
|
||||
Scan this repository for security problems and write findings down.
|
||||
|
||||
Run the scanners that are installed in your container — `cargo audit`, \
|
||||
`gitleaks detect`, `trivy fs .` and `semgrep --config auto` — from the repo \
|
||||
root. If one is missing or errors, say so explicitly in the report rather than \
|
||||
omitting it: a section absent because a tool failed reads identically to a \
|
||||
section absent because nothing was found, and those are opposite conclusions.
|
||||
|
||||
Write SECURITY-FINDINGS.md with one entry per finding: the identifier \
|
||||
(CVE / RUSTSEC / rule id), the file and line, what an attacker could actually \
|
||||
do with it, and whether it is reachable in our code or sits in an unused \
|
||||
dependency path. Rank by exploitability, not by the scanner's severity field.
|
||||
|
||||
A clean scan is a real and useful result. Say which tools ran, on what, and \
|
||||
that they found nothing — do not manufacture findings to fill the report.
|
||||
"""
|
||||
done_when = "SECURITY-FINDINGS.md exists and states, for each of the four scanners, whether it ran and what it found, with every reported finding carrying an identifier, a location, and a reachability judgement"
|
||||
max_iterations = 2
|
||||
commit_policy = "always"
|
||||
|
||||
[[phases]]
|
||||
kind = "research"
|
||||
order_idx = 1
|
||||
[phases.config]
|
||||
produces = ["md", "pdf"]
|
||||
# `pdf` dropped: PDF rendering was removed from the delivery path (artifacts are
|
||||
# served as Markdown), so asking for it produced a format nothing generates.
|
||||
produces = ["md"]
|
||||
default_topology = "hub_spoke"
|
||||
# The research phase reads the security_scan phase's findings from
|
||||
# mission_tasks and produces a patch strategy per finding.
|
||||
# NOTE: `input_from_phase` is INERT — DECLARED_BUT_UNREAD. Phases do not receive
|
||||
# a structured hand-off from a named predecessor; the next phase reads the
|
||||
# previous phase's committed FILES out of the shared checkout. That is why the
|
||||
# task names SECURITY-FINDINGS.md by path.
|
||||
input_from_phase = "security_scan"
|
||||
task = """
|
||||
Turn the scan findings into a patch strategy.
|
||||
|
||||
Read SECURITY-FINDINGS.md from the repo root — that is the previous phase's \
|
||||
output, committed to this mission's branch. For each finding that is actually \
|
||||
reachable, write into SECURITY-PLAN.md: the fix, the specific file and \
|
||||
function it touches, what could break, and how the fix will be verified.
|
||||
|
||||
Where the fix is a dependency bump, check what the new version changes — a \
|
||||
major bump presented as "update the version" is how a security patch becomes \
|
||||
an outage. Where a finding is not worth fixing, say so and say why; an \
|
||||
unreachable advisory in a dev-dependency is a legitimate "no action".
|
||||
"""
|
||||
done_when = "SECURITY-PLAN.md exists and gives, for every reachable finding in SECURITY-FINDINGS.md, either a named fix with the file it touches and how it will be verified, or an explicit justification for taking no action"
|
||||
max_iterations = 2
|
||||
commit_policy = "always"
|
||||
|
||||
[[phases]]
|
||||
kind = "coding"
|
||||
order_idx = 2
|
||||
[phases.config]
|
||||
# NOTE: `loop` is INERT — DECLARED_BUT_UNREAD ("phase iteration uses
|
||||
# max_iterations + done_when"). Kept so the intent stays visible beside the two
|
||||
# keys that actually drive the loop.
|
||||
loop = "until_all_findings_closed"
|
||||
max_iterations = 3
|
||||
# Security requires reviewer approval on top of green tests.
|
||||
commit_policy = "on_reviewer_approval"
|
||||
mcp_bundles = ["clawmates_door", "clawmates_skills", "gitea_forge", "security_scan"]
|
||||
# NOTE: `mcp_bundles` is INERT AT PHASE LEVEL — bundles come from the TEAM
|
||||
# template (`mission_orchestrator` binds `template.mcp_bundles`). This phase
|
||||
# gets nothing from this line. `gitea_forge` and `security_scan` were removed
|
||||
# from it entirely: neither is defined anywhere, and now that provision_claw
|
||||
# HONOURS the team template's bundle list, naming a bundle that does not exist
|
||||
# stopped being harmlessly inert.
|
||||
mcp_bundles = ["clawmates_door", "clawmates_skills"]
|
||||
task = """
|
||||
Apply the patch strategy and prove it worked.
|
||||
|
||||
Work through SECURITY-PLAN.md. Make the smallest change that closes each \
|
||||
finding, and run the test suite after each one so a regression is attributable \
|
||||
to a single fix rather than to the batch.
|
||||
|
||||
Then RE-RUN the scanner that produced each finding and record the new output in \
|
||||
SECURITY-FINDINGS.md under a "after remediation" heading. A fix that was not \
|
||||
re-scanned is a claim, not a result — and this phase's whole value is the \
|
||||
difference between those two.
|
||||
|
||||
If a fix cannot be made safely, leave the finding open and say why. An open \
|
||||
finding that is documented is worth more than a closed one that is not true.
|
||||
"""
|
||||
done_when = "every finding in SECURITY-PLAN.md marked for fixing is either closed with a re-run of the scanner that found it recorded in SECURITY-FINDINGS.md, or left open with a stated reason, and the test suite passes"
|
||||
|
||||
Reference in New Issue
Block a user