Compare commits
22
Commits
0b4d91889a
...
7525be3791
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7525be3791 | ||
|
|
5220f3bfea | ||
|
|
b47ae7fa6b | ||
|
|
4b160c5a1b | ||
|
|
42e014976d | ||
|
|
02d5f5a8c9 | ||
|
|
3aeee070b8 | ||
|
|
73f5d71c55 | ||
|
|
2668191e30 | ||
|
|
8591585e60 | ||
|
|
3f26dfeaca | ||
|
|
19c4de36e4 | ||
|
|
6af1149e45 | ||
|
|
ceec0423ad | ||
|
|
4f4ce34203 | ||
|
|
6f2b0a8f43 | ||
|
|
9560aaec41 | ||
|
|
c209e654d9 | ||
|
|
4a6d0dfe01 | ||
|
|
d0b657a24b | ||
|
|
1a6fdfc0e6 | ||
|
|
8cb38d1320 |
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO auth_sessions (token_hash, user_id, expires_at, scope)\n VALUES ($1, $2, $3, $4)",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Uuid",
|
||||
"Timestamptz",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "105f8cc147247c69b3c45e2e3eb27fc33b1976accdda66ec3ccc7c57afecc8b9"
|
||||
}
|
||||
+8
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT u.id, u.workspace_id, u.role\n FROM auth_sessions s\n JOIN users u ON u.id = s.user_id\n WHERE s.token_hash = $1 AND s.expires_at > now()",
|
||||
"query": "SELECT u.id, u.workspace_id, u.role, s.scope\n FROM auth_sessions s\n JOIN users u ON u.id = s.user_id\n WHERE s.token_hash = $1 AND s.expires_at > now()",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -17,6 +17,11 @@
|
||||
"ordinal": 2,
|
||||
"name": "role",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "scope",
|
||||
"type_info": "Text"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
@@ -25,10 +30,11 @@
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "900827c5c8c24f4861120e98e3cc8a5b70f22e9f4b4168c9e8eb51c53d68bdae"
|
||||
"hash": "e8f7cb9c34be37fe16c5406e9263159693674dda567b60a1f87c6763ec448951"
|
||||
}
|
||||
@@ -89,6 +89,92 @@ fn build_install_script() -> String {
|
||||
)
|
||||
}
|
||||
|
||||
/// The MCP configuration `claude -p --mcp-config` is pointed at.
|
||||
///
|
||||
/// Under `/root` with the hooks, never under `/mission/repo`: it carries a
|
||||
/// bearer token, and anything written into the checkout arrives in the diff the
|
||||
/// mission delivers.
|
||||
pub const MCP_CONFIG_PATH: &str = "/root/toolhooks/clawmates-mcp.json";
|
||||
|
||||
/// Where the mission container reaches this server.
|
||||
///
|
||||
/// Mission containers join `clawmates_core`, the same network the API is on, so
|
||||
/// the API is reachable by container name. The name differs between
|
||||
/// deployments (`clawmates-server-1` locally, `clawmates_server_1` on gw-04),
|
||||
/// so the default is derived from **our own** hostname — docker's embedded DNS
|
||||
/// resolves a container id on a user-defined network, which makes this
|
||||
/// self-configuring rather than a constant that is right in one place.
|
||||
/// Measured from a sibling container: both the id and the name return 200.
|
||||
pub fn api_origin() -> Option<String> {
|
||||
if let Ok(v) = std::env::var("CLAWMATES_API_ORIGIN") {
|
||||
if !v.trim().is_empty() {
|
||||
return Some(v.trim().trim_end_matches('/').to_string());
|
||||
}
|
||||
}
|
||||
let host = std::env::var("HOSTNAME").ok()?;
|
||||
let host = host.trim();
|
||||
if host.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(format!("http://{host}:8080"))
|
||||
}
|
||||
|
||||
/// The `--mcp-config` document: one HTTP server, carrying its own credential.
|
||||
///
|
||||
/// The token is a `skills:read` session and nothing else. It is written into a
|
||||
/// file the agent can read — it runs `Bash` — so the only thing keeping this
|
||||
/// safe is that the credential authenticates to exactly one route. See
|
||||
/// `cm_auth::authenticate_scoped`.
|
||||
pub fn mcp_document(origin: &str, token: &str) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"mcpServers": {
|
||||
"clawmates_skills": {
|
||||
"type": "http",
|
||||
"url": format!("{origin}/mcp/skills"),
|
||||
"headers": { "Authorization": format!("Bearer {token}") }
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// NOTE on `--allowedTools`. The provider passes it only when the config sets
|
||||
// `tools`, and the seed already does — without it `claude -p` stops mid-turn to
|
||||
// ask for write permission. Whether the MCP tools ALSO need naming there is not
|
||||
// documented anywhere we control, and the daemon exposes no config read to
|
||||
// merge into that list safely: overwriting it would take `Write` and `Bash`
|
||||
// away from every mission agent, and that failure would look like agents that
|
||||
// stopped working rather than a config that was replaced.
|
||||
//
|
||||
// So it is left alone and the question is answered by running a mission with
|
||||
// the door installed. Guessing here is how the last three defects in this file
|
||||
// were introduced.
|
||||
|
||||
/// Write the MCP configuration into a mission container.
|
||||
///
|
||||
/// Returns the path on success. `None` means the mission runs without a door —
|
||||
/// logged, never fatal, exactly like the hooks above. A phase that cannot
|
||||
/// retrieve a skill still delivers; a phase that fails to start because a
|
||||
/// config write failed delivers nothing.
|
||||
pub async fn install_door(docker: &Docker, container: &str, doc: &serde_json::Value) -> Option<String> {
|
||||
// `printf %s` with the JSON single-quoted, not a heredoc: the document is
|
||||
// one line and contains no newline to terminate on.
|
||||
let script = format!(
|
||||
"mkdir -p {HOOK_DIR} && printf '%s' {} > {MCP_CONFIG_PATH} && chmod 600 {MCP_CONFIG_PATH}",
|
||||
crate::vm_tool_tap::shell_quote(&doc.to_string()),
|
||||
);
|
||||
let argv = vec!["sh".to_string(), "-lc".to_string(), script];
|
||||
match crate::container_exec::exec_as_root(docker, container, None, &argv, INSTALL_TIMEOUT).await
|
||||
{
|
||||
Ok(out) if out.exit_code == Some(0) => Some(MCP_CONFIG_PATH.to_string()),
|
||||
other => {
|
||||
eprintln!(
|
||||
"container_tool_hooks: could not write the MCP config in {container} ({other:?}) — this mission runs without the skills door"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The tap file inside the mission container.
|
||||
pub fn tap_file() -> String {
|
||||
format!("{TAP_DIR}/tools.jsonl")
|
||||
|
||||
@@ -60,12 +60,26 @@ fn err(id: Option<Value>, code: i64, message: &str) -> Json<Value> {
|
||||
|
||||
// ── Auth ─────────────────────────────────────────────────────────
|
||||
|
||||
/// This endpoint accepts a **narrow** credential as well as a person's session.
|
||||
///
|
||||
/// It is the one route a mission container is given a token for, and that token
|
||||
/// sits in a file the agent can `cat`. Mission agents run arbitrary `Bash` with
|
||||
/// egress and no read gate, so a full session here would be an owner-privileged
|
||||
/// API key handed to something explicitly untrusted — which is why
|
||||
/// `SCOPE_SKILLS_READ` exists and why this is the only call site that names it.
|
||||
///
|
||||
/// `authenticate_scoped` still accepts `full`, so the UI and any human caller
|
||||
/// are unaffected.
|
||||
async fn authed(state: &AppState, headers: &HeaderMap) -> Option<cm_auth::AuthedUser> {
|
||||
let token = headers
|
||||
.get(AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.strip_prefix("Bearer "))?;
|
||||
state.auth.authenticate(token).await.ok()
|
||||
state
|
||||
.auth
|
||||
.authenticate_scoped(token, cm_auth::SCOPE_SKILLS_READ)
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// Resolve the calling agent via `X-ZeroClaw-Agent` header
|
||||
@@ -100,7 +114,7 @@ fn skill_uri(workspace_id: Option<Uuid>, name: &str) -> String {
|
||||
}
|
||||
|
||||
/// Parse `skill:global/<name>` or `skill:workspace/<ws>/<name>`.
|
||||
fn parse_uri(uri: &str) -> Option<(Option<Uuid>, String)> {
|
||||
pub(crate) fn parse_uri(uri: &str) -> Option<(Option<Uuid>, String)> {
|
||||
if let Some(name) = uri.strip_prefix(URI_PREFIX_GLOBAL) {
|
||||
return Some((None, name.to_string()));
|
||||
}
|
||||
|
||||
@@ -233,6 +233,12 @@ impl<V: PhaseVm> TurnExecutor for MicroVmTurnExecutor<V> {
|
||||
self.phase_id,
|
||||
self.run_id,
|
||||
&outcome.tools,
|
||||
// No turn agents supplied, so nothing is attributed — the same
|
||||
// `agent_id: None` this path has always written. Resolving the
|
||||
// graph node to an agent uuid is the fix, and it cannot be tested
|
||||
// while the fleet is offline; guessing at it here would put one
|
||||
// node's actions on another node's record.
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
|
||||
|
||||
@@ -183,6 +183,75 @@ pub async fn narrative_for_mission(
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// One action an agent took, as a reader gets it back.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ToolEvidence {
|
||||
/// The tool's name, e.g. `Bash`, `Write`.
|
||||
pub tool: String,
|
||||
/// The absolute path inside the sandbox, when the tool named one.
|
||||
///
|
||||
/// Absolute, unlike the sibling `file.touch` row's `target`. See the note
|
||||
/// in `phase_runner::record_vm_tools`: normalising is what destroys the
|
||||
/// only question a path can settle.
|
||||
pub path: Option<String>,
|
||||
/// The tool's arguments, bounded by `vm_tool_tap::bounded_input`.
|
||||
pub input: Value,
|
||||
/// What a command produced, bounded by `vm_tool_tap::bounded_response`.
|
||||
///
|
||||
/// Null for every tool that is not a command. This is where a failing test
|
||||
/// run is visible, and it is the only place it is — the recorded stream has
|
||||
/// no exit codes.
|
||||
pub response: Value,
|
||||
}
|
||||
|
||||
impl ToolEvidence {
|
||||
/// The shell command, for the tools that run one.
|
||||
pub fn command(&self) -> Option<&str> {
|
||||
self.input.get("command").and_then(Value::as_str)
|
||||
}
|
||||
}
|
||||
|
||||
/// Every tool call recorded for a mission, in order.
|
||||
///
|
||||
/// The counterpart to [`narrative_for_mission`], and the reason it exists: the
|
||||
/// narrative is what an agent *said* it did. These rows are what it did. A
|
||||
/// measurement built on the narrative alone scores prose, and prose is written
|
||||
/// by the thing being measured.
|
||||
///
|
||||
/// **Bounded by [`PER_PHASE_CAP`].** A phase that ran more tools than the cap
|
||||
/// returns the first `PER_PHASE_CAP` and no marker saying so, so a check that
|
||||
/// concludes "this never happened" from an empty result is only sound for
|
||||
/// phases under the cap. Every check in `skill_use` is one-sided in the safe
|
||||
/// direction for that reason: it reports a violation it can see, never
|
||||
/// compliance it inferred from silence.
|
||||
pub async fn tool_evidence_for_mission(
|
||||
pool: &PgPool,
|
||||
mission_id: Uuid,
|
||||
) -> Result<Vec<ToolEvidence>, sqlx::Error> {
|
||||
let rows: Vec<(Option<String>, Value)> = sqlx::query_as(
|
||||
"SELECT target, detail
|
||||
FROM mission_events
|
||||
WHERE mission_id = $1 AND kind = $2
|
||||
ORDER BY id",
|
||||
)
|
||||
.bind(mission_id)
|
||||
.bind(TOOL_CALL)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows
|
||||
.into_iter()
|
||||
.map(|(target, detail)| ToolEvidence {
|
||||
tool: target.unwrap_or_default(),
|
||||
path: detail
|
||||
.get("path")
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string),
|
||||
input: detail.get("input").cloned().unwrap_or(Value::Null),
|
||||
response: detail.get("response").cloned().unwrap_or(Value::Null),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub async fn record_all(pool: &PgPool, events: Vec<MissionEvent>) {
|
||||
for e in events {
|
||||
record(pool, e).await;
|
||||
|
||||
@@ -346,6 +346,19 @@ pub async fn on_launch(
|
||||
settings ({e}) — this mission's tool calls run unchecked"
|
||||
);
|
||||
}
|
||||
// Only when this mission got its OWN container — the shared runtime is
|
||||
// not ours to reconfigure, and `mission_gateway` being Some is exactly
|
||||
// the signal that `ensure_container` ran.
|
||||
if mission_gateway.is_some() {
|
||||
install_skills_door(
|
||||
pool,
|
||||
user_id,
|
||||
mission_id,
|
||||
&crate::mission_runtime::container_name(mission_id),
|
||||
p,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
let mut first_team_id: Option<Uuid> = None;
|
||||
let mut provisioned_claws: Vec<cm_domain::AgentId> = Vec::new();
|
||||
@@ -883,3 +896,78 @@ fn default_accent_for(slot: &str) -> &'static str {
|
||||
_ => "#8a8a92",
|
||||
}
|
||||
}
|
||||
|
||||
/// Give this mission's agents a reachable, narrow door to the skills catalogue.
|
||||
///
|
||||
/// Two halves that must both happen: the document goes into the container, and
|
||||
/// the daemon is told to pass it to `claude -p --mcp-config`. Doing one without
|
||||
/// the other leaves a door that is installed and unreachable, which looks
|
||||
/// exactly like a door nobody walked through — the same shape as the hooks that
|
||||
/// were installed and inert.
|
||||
///
|
||||
/// # The credential
|
||||
///
|
||||
/// A `skills:read` session, not a user's. It is written into a file the agent
|
||||
/// can `cat` — it runs `Bash` with egress — so the only thing keeping this safe
|
||||
/// is that the token authenticates to exactly one route and nowhere else. See
|
||||
/// `cm_auth::AuthService::authenticate_scoped`. A full session here would be an
|
||||
/// owner-privileged API key handed to something explicitly untrusted, which is
|
||||
/// why the door went undeployed rather than being deployed the easy way.
|
||||
///
|
||||
/// Every failure degrades to "no door", never to a failed launch. A mission
|
||||
/// that cannot retrieve a skill still delivers.
|
||||
async fn install_skills_door(
|
||||
pool: &PgPool,
|
||||
user_id: cm_domain::UserId,
|
||||
mission_id: Uuid,
|
||||
container: &str,
|
||||
prov: &RuntimeProvisioner,
|
||||
) {
|
||||
let Some(origin) = crate::container_tool_hooks::api_origin() else {
|
||||
eprintln!(
|
||||
"mission_orchestrator: no API origin for the skills door (set \
|
||||
CLAWMATES_API_ORIGIN) — mission {mission_id} runs without it"
|
||||
);
|
||||
return;
|
||||
};
|
||||
// Outlives the longest mission we have seen, and expires on its own so a
|
||||
// leaked container does not leave a live credential behind indefinitely.
|
||||
let auth = cm_auth::AuthService::new(pool.clone());
|
||||
let token = match auth
|
||||
.mint_scoped(user_id, cm_auth::SCOPE_SKILLS_READ, time::Duration::hours(24))
|
||||
.await
|
||||
{
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"mission_orchestrator: could not mint a skills token ({e}) — \
|
||||
mission {mission_id} runs without the door"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let docker = match crate::container_exec::connect() {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
eprintln!("mission_orchestrator: cannot reach docker for the skills door: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let doc = crate::container_tool_hooks::mcp_document(&origin, &token);
|
||||
let Some(path) = crate::container_tool_hooks::install_door(&docker, container, &doc).await
|
||||
else {
|
||||
// `install_door` already said why.
|
||||
return;
|
||||
};
|
||||
if let Err(e) = prov.set_claude_cli_mcp_config(&path).await {
|
||||
eprintln!(
|
||||
"mission_orchestrator: wrote the MCP config but could not point \
|
||||
claude_cli at it ({e}) — the door is installed and unreachable"
|
||||
);
|
||||
return;
|
||||
}
|
||||
eprintln!(
|
||||
"mission_orchestrator: skills door installed for mission {mission_id} \
|
||||
({origin}/mcp/skills)"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1070,6 +1070,103 @@ impl MissionRuntimeProvisioner {
|
||||
/// this to decide whether to KEEP a binding for a retry, and answering
|
||||
/// "still there" when docker cannot be reached would pin the binding open
|
||||
/// on an unreachable daemon rather than on a real container.
|
||||
/// Every `cm-runtime-mission-*` container on this engine, running or not.
|
||||
///
|
||||
/// The piece the row-driven sweep never had. Without it "which containers
|
||||
/// exist" is a question the platform cannot ask, and a container the
|
||||
/// database has forgotten is not merely unreaped — it is unseeable.
|
||||
pub async fn list_mission_containers(&self) -> Result<Vec<(String, Option<i64>)>, String> {
|
||||
let mut filters = std::collections::HashMap::new();
|
||||
filters.insert("name".to_string(), vec!["cm-runtime-mission-".to_string()]);
|
||||
let opts = bollard::query_parameters::ListContainersOptionsBuilder::default()
|
||||
.all(true)
|
||||
.filters(&filters)
|
||||
.build();
|
||||
let list = self
|
||||
.docker
|
||||
.list_containers(Some(opts))
|
||||
.await
|
||||
.map_err(|e| format!("list mission containers: {e}"))?;
|
||||
Ok(list
|
||||
.into_iter()
|
||||
.filter_map(|c| {
|
||||
let name = c
|
||||
.names
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
// Docker returns names with a leading slash.
|
||||
.map(|n| n.trim_start_matches('/').to_string())
|
||||
.find(|n| n.starts_with("cm-runtime-mission-"))?;
|
||||
Some((name, c.created))
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// How long ago docker says this container was created.
|
||||
///
|
||||
/// Taken from the listing rather than a second `inspect`: `created` is
|
||||
/// already a unix timestamp there, so this needs neither a date parser nor
|
||||
/// another round-trip. `None` when docker reported none, and the caller
|
||||
/// treats that as "do not reap" — a container we cannot date is exactly the
|
||||
/// one worth leaving.
|
||||
pub fn container_age(created_epoch: Option<i64>) -> Option<std::time::Duration> {
|
||||
let created = created_epoch?;
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.ok()?
|
||||
.as_secs() as i64;
|
||||
u64::try_from(now - created).ok().map(std::time::Duration::from_secs)
|
||||
}
|
||||
|
||||
/// Does this container's checkout hold commits no remote has?
|
||||
///
|
||||
/// Answered by `git` inside the container, because only it knows which
|
||||
/// refs the remote had. `--not --remotes` lists every commit reachable
|
||||
/// from any local ref and from no remote-tracking ref — which is exactly
|
||||
/// "work that exists only here".
|
||||
///
|
||||
/// Every failure path returns `SomeOrUnknown`. A container we cannot
|
||||
/// question is not a container we may delete.
|
||||
pub async fn unpushed_commits(&self, name: &str) -> UnpushedWork {
|
||||
let script = "cd /mission/repo 2>/dev/null || exit 91; \
|
||||
git rev-list --all --not --remotes 2>/dev/null | wc -l";
|
||||
let argv = vec!["sh".to_string(), "-lc".to_string(), script.to_string()];
|
||||
let out = match crate::container_exec::exec_as_root(
|
||||
&self.docker,
|
||||
name,
|
||||
None,
|
||||
&argv,
|
||||
std::time::Duration::from_secs(30),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(o) => o,
|
||||
Err(e) => {
|
||||
return UnpushedWork::SomeOrUnknown(format!("could not ask git ({e})"));
|
||||
}
|
||||
};
|
||||
if out.exit_code == Some(91) {
|
||||
// No checkout at all — nothing to lose.
|
||||
return UnpushedWork::None;
|
||||
}
|
||||
if out.exit_code != Some(0) {
|
||||
return UnpushedWork::SomeOrUnknown(format!(
|
||||
"git probe exited {:?}",
|
||||
out.exit_code
|
||||
));
|
||||
}
|
||||
match out.stdout.trim().parse::<u64>() {
|
||||
Ok(0) => UnpushedWork::None,
|
||||
Ok(n) => UnpushedWork::SomeOrUnknown(format!(
|
||||
"{n} commit(s) in its checkout are on no remote"
|
||||
)),
|
||||
Err(_) => UnpushedWork::SomeOrUnknown(format!(
|
||||
"unreadable git output {:?}",
|
||||
out.stdout.trim()
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn container_exists(&self, mission_id: Uuid) -> bool {
|
||||
self.docker
|
||||
.inspect_container(&container_name(mission_id), None::<InspectContainerOptions>)
|
||||
@@ -1152,10 +1249,114 @@ pub fn spawn_sweeper(pool: sqlx::PgPool, grace: std::time::Duration) {
|
||||
if let Err(e) = sweep_once(&pool, grace).await {
|
||||
eprintln!("mission_runtime::sweeper: sweep failed: {e}");
|
||||
}
|
||||
if let Err(e) = sweep_orphans(&pool, ORPHAN_GRACE).await {
|
||||
eprintln!("mission_runtime::sweeper: orphan sweep failed: {e}");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// How long a container with no mission row may sit before it is reaped.
|
||||
///
|
||||
/// Long, deliberately. The row-driven sweep above handles every container the
|
||||
/// platform still knows about, so anything reaching this path is already
|
||||
/// unexpected — and the one real orphan we have seen held ten unpushed commits.
|
||||
/// A day of disk is cheaper than being wrong about that.
|
||||
const ORPHAN_GRACE: std::time::Duration = std::time::Duration::from_secs(24 * 3600);
|
||||
|
||||
/// Reap `cm-runtime-mission-*` containers that no `missions` row points at.
|
||||
///
|
||||
/// [`sweep_once`] selects `FROM missions`, and `teardown_container` is only
|
||||
/// ever called with an id that came from that query. So a container whose row
|
||||
/// is gone is invisible to every reaper: nothing enumerates docker, nothing
|
||||
/// errors, and the only symptom is disk.
|
||||
///
|
||||
/// Found on gw-04 2026-08-21 — a container `Up` for nine days holding 2.5G,
|
||||
/// against a `missions` table with **zero rows**.
|
||||
///
|
||||
/// # It refuses to reap work that exists nowhere else
|
||||
///
|
||||
/// That container's checkout held **ten commits on a branch that had never
|
||||
/// been pushed** (+3451/-30 across 30 files). A reaper that deleted on sight
|
||||
/// would have destroyed all of it, silently, as its designed behaviour. So
|
||||
/// before removing anything this asks the checkout whether it holds commits
|
||||
/// that no remote has, and leaves the container alone — loudly, every tick —
|
||||
/// when it does.
|
||||
///
|
||||
/// The check is deliberately one-sided in the safe direction: an inspection
|
||||
/// that fails for any reason counts as "might hold work", never as "safe to
|
||||
/// delete". Losing a day of disk to an unreadable container is recoverable;
|
||||
/// the other way round is not.
|
||||
pub async fn sweep_orphans(pool: &sqlx::PgPool, grace: std::time::Duration) -> Result<(), String> {
|
||||
let Some(prov) = MissionRuntimeProvisioner::from_env() else {
|
||||
return Ok(());
|
||||
};
|
||||
let names = prov.list_mission_containers().await?;
|
||||
if names.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
for (name, created) in names {
|
||||
let Some(id) = mission_id_from_container(&name) else {
|
||||
continue;
|
||||
};
|
||||
// `WHERE id = $1` across every workspace on purpose: the question is
|
||||
// whether ANY row still points at this container, not whether one the
|
||||
// caller can see does.
|
||||
let known: Option<(Uuid,)> = sqlx::query_as("SELECT id FROM missions WHERE id = $1")
|
||||
.bind(id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(|e| format!("look up mission {id}: {e}"))?;
|
||||
if known.is_some() {
|
||||
continue;
|
||||
}
|
||||
match MissionRuntimeProvisioner::container_age(created) {
|
||||
Some(age) if age < grace => continue,
|
||||
None => continue,
|
||||
Some(_) => {}
|
||||
}
|
||||
match prov.unpushed_commits(&name).await {
|
||||
// The safe answer, and the one an error also produces.
|
||||
UnpushedWork::SomeOrUnknown(why) => {
|
||||
eprintln!(
|
||||
"mission_runtime::orphans: {name} has no mission row and is older than \
|
||||
the grace period, but it is NOT safe to reap: {why}. Recover the work \
|
||||
(`git bundle create … origin/main..HEAD`, or push the branch) and then \
|
||||
remove it by hand."
|
||||
);
|
||||
}
|
||||
UnpushedWork::None => {
|
||||
eprintln!(
|
||||
"mission_runtime::orphans: reaping {name} — no mission row, older than \
|
||||
the grace period, and its checkout holds nothing a remote does not"
|
||||
);
|
||||
if let Err(e) = prov.teardown_container(id).await {
|
||||
eprintln!("mission_runtime::orphans: reap {name}: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether an orphan's checkout holds commits no remote has.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum UnpushedWork {
|
||||
/// Every commit is reachable from a remote ref — nothing is lost.
|
||||
None,
|
||||
/// There IS unpushed work, or the question could not be answered. One
|
||||
/// variant for both, because the reaper must treat them identically.
|
||||
SomeOrUnknown(String),
|
||||
}
|
||||
|
||||
/// The mission id encoded in a runtime container's name, if it is one.
|
||||
///
|
||||
/// The inverse of [`container_name`], which formats the uuid `simple` (no
|
||||
/// dashes). Anything that does not parse is not ours and is left alone.
|
||||
pub fn mission_id_from_container(name: &str) -> Option<Uuid> {
|
||||
Uuid::parse_str(name.strip_prefix("cm-runtime-mission-")?).ok()
|
||||
}
|
||||
|
||||
async fn sweep_once(pool: &sqlx::PgPool, grace: std::time::Duration) -> Result<(), String> {
|
||||
use sqlx::Row;
|
||||
let grace_secs = grace.as_secs() as f64;
|
||||
@@ -1569,6 +1770,48 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// The name→id round trip the orphan sweep depends on.
|
||||
///
|
||||
/// If this is wrong the sweep either skips every orphan (harmless) or
|
||||
/// resolves a container to the WRONG mission id and asks the database
|
||||
/// about a mission that does exist — reading "still live, leave it" for a
|
||||
/// container that is not. Cheap to get right, expensive to get wrong.
|
||||
#[test]
|
||||
fn a_container_name_round_trips_to_its_mission() {
|
||||
let id = Uuid::now_v7();
|
||||
assert_eq!(mission_id_from_container(&container_name(id)), Some(id));
|
||||
// Not ours, and not a panic.
|
||||
assert_eq!(mission_id_from_container("clawmates_server_1"), None);
|
||||
assert_eq!(mission_id_from_container("cm-runtime-mission-nonsense"), None);
|
||||
assert_eq!(mission_id_from_container("cm-sandbox-abc"), None);
|
||||
}
|
||||
|
||||
/// A container docker will not date must not be reaped.
|
||||
#[test]
|
||||
fn an_undatable_container_has_no_age() {
|
||||
assert_eq!(MissionRuntimeProvisioner::container_age(None), None);
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs() as i64;
|
||||
let age = MissionRuntimeProvisioner::container_age(Some(now - 3600)).expect("age");
|
||||
assert!(age.as_secs() >= 3500 && age.as_secs() <= 3700, "{age:?}");
|
||||
// A clock skew that puts creation in the future must not underflow into
|
||||
// a colossal age that reads as "long past the grace period".
|
||||
assert_eq!(MissionRuntimeProvisioner::container_age(Some(now + 600)), None);
|
||||
}
|
||||
|
||||
/// The grace period is long, and that is the point.
|
||||
#[test]
|
||||
fn the_orphan_grace_is_generous() {
|
||||
assert!(
|
||||
ORPHAN_GRACE >= std::time::Duration::from_secs(12 * 3600),
|
||||
"the row-driven sweep already handles everything the platform knows \
|
||||
about, so anything reaching the orphan path is unexpected — and the \
|
||||
one real orphan held ten unpushed commits"
|
||||
);
|
||||
}
|
||||
|
||||
const SAMPLE_CONFIG: &str = r#"# top comment
|
||||
[agents.claw_a]
|
||||
model_provider = "anthropic.default"
|
||||
|
||||
@@ -254,18 +254,31 @@ mod tests {
|
||||
"order_idx",
|
||||
"requires_repo",
|
||||
"default_team_template",
|
||||
"default_phase_teams",
|
||||
"default_topology",
|
||||
"phases",
|
||||
"description",
|
||||
];
|
||||
// `[default_phase_teams]` maps a phase PURPOSE to a team template key,
|
||||
// so its keys are not config keys and must not be checked as such.
|
||||
// They are checked against the purposes `phase_runner::purposes_for`
|
||||
// can actually emit instead — a typo'd purpose matches no phase and
|
||||
// that phase silently falls back to the mission-wide team, which is
|
||||
// exactly the kind of quiet wrong staffing this table exists to end.
|
||||
const PURPOSES: &[&str] = &["research", "coding", "security", "mission"];
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.extension().and_then(|e| e.to_str()) != Some("toml") {
|
||||
continue;
|
||||
}
|
||||
let body = std::fs::read_to_string(&path).unwrap();
|
||||
let mut table = String::new();
|
||||
for line in body.lines() {
|
||||
let line = line.trim();
|
||||
if line.starts_with('[') {
|
||||
table = line.trim_matches(['[', ']'].as_slice()).to_string();
|
||||
continue;
|
||||
}
|
||||
if line.starts_with('#') || !line.contains('=') {
|
||||
continue;
|
||||
}
|
||||
@@ -273,6 +286,16 @@ mod tests {
|
||||
if key.is_empty() || key.contains(' ') || key.contains('[') {
|
||||
continue;
|
||||
}
|
||||
if table == "default_phase_teams" {
|
||||
assert!(
|
||||
PURPOSES.contains(&key),
|
||||
"{} staffs purpose `{key}`, which `purposes_for` never emits — \
|
||||
that phase would fall back to the mission-wide team with \
|
||||
nothing reporting it",
|
||||
path.display()
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let accounted = ENVELOPE.contains(&key)
|
||||
|| is_listed(key, KNOWN_KEYS)
|
||||
|| is_listed(key, DECLARED_BUT_UNREAD);
|
||||
|
||||
@@ -309,6 +309,81 @@ mod skill_delivery_wiring_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod attribution_tests {
|
||||
use super::attribute_sessions;
|
||||
use crate::vm_tool_tap::Observed;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn call(session: Option<&str>) -> Observed {
|
||||
Observed {
|
||||
tool: "Bash".into(),
|
||||
path: None,
|
||||
session: session.map(str::to_string),
|
||||
input: serde_json::json!({"command": "ls"}),
|
||||
response: serde_json::Value::Null,
|
||||
}
|
||||
}
|
||||
|
||||
/// The ordinary case: three turns, three sessions, in order.
|
||||
#[test]
|
||||
fn each_session_lands_on_the_turn_that_ran_it() {
|
||||
let (a, b, c) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7());
|
||||
let tools = [
|
||||
call(Some("s1")),
|
||||
call(Some("s1")),
|
||||
call(Some("s2")),
|
||||
call(Some("s3")),
|
||||
call(Some("s2")),
|
||||
];
|
||||
assert_eq!(
|
||||
attribute_sessions(&tools, &[a, b, c]),
|
||||
vec![Some(a), Some(a), Some(b), Some(c), Some(b)],
|
||||
"sessions are ordered by FIRST appearance, so a later call from an \
|
||||
earlier session still belongs to that earlier turn"
|
||||
);
|
||||
}
|
||||
|
||||
/// More sessions than turns — something happened this model does not
|
||||
/// describe, so it must not produce a confident answer.
|
||||
#[test]
|
||||
fn a_count_mismatch_attributes_nothing() {
|
||||
let a = Uuid::now_v7();
|
||||
let tools = [call(Some("s1")), call(Some("s2"))];
|
||||
assert_eq!(
|
||||
attribute_sessions(&tools, &[a]),
|
||||
vec![None, None],
|
||||
"a plausible-looking wrong attribution puts one agent's actions on \
|
||||
another agent's record, and a person later reasons from it"
|
||||
);
|
||||
// And the other direction.
|
||||
assert_eq!(
|
||||
attribute_sessions(&[call(Some("s1"))], &[a, Uuid::now_v7()]),
|
||||
vec![None]
|
||||
);
|
||||
}
|
||||
|
||||
/// One call with no session id poisons the ORDER, not just itself.
|
||||
#[test]
|
||||
fn a_single_missing_session_refuses_the_whole_batch() {
|
||||
let (a, b) = (Uuid::now_v7(), Uuid::now_v7());
|
||||
let tools = [call(Some("s1")), call(None), call(Some("s2"))];
|
||||
assert_eq!(
|
||||
attribute_sessions(&tools, &[a, b]),
|
||||
vec![None, None, None],
|
||||
"a hole shifts every later session onto the wrong turn"
|
||||
);
|
||||
}
|
||||
|
||||
/// The pre-session tap, and the microVM path that supplies no turns.
|
||||
#[test]
|
||||
fn no_turns_and_no_sessions_stay_unattributed() {
|
||||
assert_eq!(attribute_sessions(&[call(Some("s1"))], &[]), vec![None]);
|
||||
assert_eq!(attribute_sessions(&[call(None)], &[]), vec![None]);
|
||||
assert!(attribute_sessions(&[], &[]).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod repo_less_text_tests {
|
||||
use super::*;
|
||||
@@ -567,12 +642,26 @@ async fn drain_finished_container_phases(pool: &PgPool) -> Result<(), String> {
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
// The agent of each turn, in the order the turns ran. `prompt.composed`
|
||||
// is written by the tier as it sends each turn, so this IS the running
|
||||
// order — not a reconstruction of it.
|
||||
let turn_agents: Vec<Uuid> = sqlx::query_scalar(
|
||||
"SELECT agent_id FROM mission_events
|
||||
WHERE phase_id = $1 AND kind = $2 AND agent_id IS NOT NULL
|
||||
ORDER BY id",
|
||||
)
|
||||
.bind(phase_id)
|
||||
.bind(crate::mission_events::PROMPT_COMPOSED)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
record_vm_tools(
|
||||
pool,
|
||||
mission_id,
|
||||
phase_id,
|
||||
run_id.unwrap_or(phase_id),
|
||||
&tools,
|
||||
&turn_agents,
|
||||
)
|
||||
.await;
|
||||
eprintln!(
|
||||
@@ -1598,7 +1687,7 @@ async fn launch_microvm_phase(
|
||||
// still records; recording the same calls twice is what the empty
|
||||
// contract exists to prevent.
|
||||
if let Ok(o) = &outcome {
|
||||
record_vm_tools(&pool2, mission_id, phase_id, run_id, &o.tools).await;
|
||||
record_vm_tools(&pool2, mission_id, phase_id, run_id, &o.tools, &[]).await;
|
||||
}
|
||||
let (status, note) = match outcome {
|
||||
// The gate gave up. It is the ONLY thing that runs a
|
||||
@@ -2052,45 +2141,126 @@ pub(crate) fn vm_tool_recorder(
|
||||
let pool = pool.clone();
|
||||
tokio::spawn(async move {
|
||||
while let Some(batch) = rx.recv().await {
|
||||
record_vm_tools(&pool, mission_id, phase_id, run_id, &batch).await;
|
||||
record_vm_tools(&pool, mission_id, phase_id, run_id, &batch, &[]).await;
|
||||
}
|
||||
});
|
||||
tx
|
||||
}
|
||||
|
||||
/// Which agent each observed call belongs to, by session.
|
||||
///
|
||||
/// The tap is per-CONTAINER and every role in a phase shares one, so a phase's
|
||||
/// calls arrive as one undifferentiated stream and `agent_id` was written
|
||||
/// `None` for all of them. That is why every Skill-Use score is per-mission
|
||||
/// rather than per-role, and why the World's per-agent view gets nothing from
|
||||
/// this tier.
|
||||
///
|
||||
/// One `claude -p` invocation is one turn is one agent, and Claude Code stamps
|
||||
/// each invocation with a `session_id`. So the distinct sessions, in the order
|
||||
/// they first appear, are the phase's turns in the order they ran — and
|
||||
/// `prompt.composed` already records the agent of each turn in that same order.
|
||||
///
|
||||
/// # It attributes nothing rather than guessing
|
||||
///
|
||||
/// Only when the counts match exactly. A phase whose sessions and turns differ
|
||||
/// in number has something this correlation does not model — a retry, a turn
|
||||
/// that called no tool, two agents genuinely concurrent — and a
|
||||
/// plausible-looking wrong attribution is worse here than none: it would put
|
||||
/// one agent's `git push` on another agent's record, which is the sort of thing
|
||||
/// a person later reasons from.
|
||||
pub(crate) fn attribute_sessions(
|
||||
tools: &[crate::vm_tool_tap::Observed],
|
||||
turn_agents: &[Uuid],
|
||||
) -> Vec<Option<Uuid>> {
|
||||
let mut order: Vec<&str> = Vec::new();
|
||||
for t in tools {
|
||||
let Some(sid) = t.session.as_deref() else {
|
||||
// A single unattributable call means the sequence has a hole in it,
|
||||
// and a hole shifts every later session onto the wrong turn.
|
||||
return vec![None; tools.len()];
|
||||
};
|
||||
if !order.contains(&sid) {
|
||||
order.push(sid);
|
||||
}
|
||||
}
|
||||
if order.len() != turn_agents.len() || order.is_empty() {
|
||||
return vec![None; tools.len()];
|
||||
}
|
||||
tools
|
||||
.iter()
|
||||
.map(|t| {
|
||||
let sid = t.session.as_deref()?;
|
||||
let idx = order.iter().position(|s| *s == sid)?;
|
||||
turn_agents.get(idx).copied()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) async fn record_vm_tools(
|
||||
pool: &PgPool,
|
||||
mission_id: Uuid,
|
||||
phase_id: Uuid,
|
||||
run_id: Uuid,
|
||||
tools: &[crate::vm_tool_tap::Observed],
|
||||
turn_agents: &[Uuid],
|
||||
) {
|
||||
if tools.is_empty() {
|
||||
return;
|
||||
}
|
||||
let owners = attribute_sessions(tools, turn_agents);
|
||||
if owners.iter().all(Option::is_none) && !turn_agents.is_empty() {
|
||||
eprintln!(
|
||||
"phase_runner: {} tool call(s) for phase {phase_id} could not be \
|
||||
attributed to an agent ({} session(s) across {} turn(s)) — recorded \
|
||||
unattributed rather than guessed",
|
||||
tools.len(),
|
||||
tools
|
||||
.iter()
|
||||
.filter_map(|t| t.session.as_deref())
|
||||
.collect::<std::collections::HashSet<_>>()
|
||||
.len(),
|
||||
turn_agents.len()
|
||||
);
|
||||
}
|
||||
let mut events = Vec::new();
|
||||
for t in tools {
|
||||
for (t, owner) in tools.iter().zip(owners) {
|
||||
events.push(crate::mission_events::MissionEvent {
|
||||
mission_id,
|
||||
phase_id: Some(phase_id),
|
||||
run_id: Some(run_id),
|
||||
agent_id: None,
|
||||
agent_id: owner,
|
||||
kind: crate::mission_events::TOOL_CALL.to_string(),
|
||||
target: Some(t.tool.clone()),
|
||||
detail: serde_json::Value::Null,
|
||||
// `path` because the World's SSE reads `detail.path` for this kind
|
||||
// and was handed a null on every container-tier call; `input`
|
||||
// because the tool name alone cannot answer a single behavioural
|
||||
// question about the phase.
|
||||
detail: serde_json::json!({
|
||||
"path": t.path,
|
||||
"input": t.input,
|
||||
// Only commands carry one; `bounded_response` returns null for
|
||||
// everything else, and a null key here is noise.
|
||||
"response": t.response,
|
||||
}),
|
||||
});
|
||||
if let Some(path) = &t.path {
|
||||
events.push(crate::mission_events::MissionEvent {
|
||||
mission_id,
|
||||
phase_id: Some(phase_id),
|
||||
run_id: Some(run_id),
|
||||
agent_id: None,
|
||||
agent_id: owner,
|
||||
kind: crate::mission_events::FILE_TOUCH.to_string(),
|
||||
target: Some(crate::mission_events::repo_relative(
|
||||
path,
|
||||
&["/mission/repo", "/workspace"],
|
||||
)),
|
||||
detail: serde_json::json!({ "tool": t.tool }),
|
||||
// The ABSOLUTE path as well as the repo-relative one. `target`
|
||||
// is normalised for the map, where a `mission` → `repo` pair of
|
||||
// directory orbs means nothing to a reader — but normalising is
|
||||
// exactly what destroys the question "did this write land
|
||||
// outside the checkout", which is the one boundary a skill can
|
||||
// be scored on.
|
||||
detail: serde_json::json!({ "tool": t.tool, "abs": path }),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -294,6 +294,54 @@ pub async fn create(
|
||||
.get("phase_teams")
|
||||
.and_then(|v| v.as_object())
|
||||
.is_some_and(|o| o.values().any(|v| v.as_array().is_some_and(|a| !a.is_empty())));
|
||||
// Per-purpose defaults first: a multi-phase recipe does not have one job,
|
||||
// and staffing every phase from one team is what put a coder, a tester and
|
||||
// a committer on a repo-less markdown mission. Only applied when the caller
|
||||
// named no team of any kind, so an explicit choice always wins.
|
||||
let mut config = body.config;
|
||||
if team_template_id.is_none() && body.team_id.is_none() && !has_phase_teams {
|
||||
if let Some(r) = recipe {
|
||||
let mut resolved = serde_json::Map::new();
|
||||
for (purpose, key) in &r.default_phase_teams {
|
||||
match cm_db::repo::team_templates::get_by_key(&state.pool, key).await {
|
||||
Ok(Some(t)) => {
|
||||
resolved.insert(
|
||||
purpose.clone(),
|
||||
serde_json::json!([t.id.to_string()]),
|
||||
);
|
||||
}
|
||||
// Loud, and it does NOT fall back silently: a recipe naming
|
||||
// a template that is not loaded would otherwise stage the
|
||||
// wrong crew and look deliberate.
|
||||
Ok(None) => eprintln!(
|
||||
"missions: recipe {} maps purpose {purpose:?} to team template \
|
||||
{key:?}, which is not loaded — that phase will fall back to the \
|
||||
mission-wide default",
|
||||
body.template_kind.trim()
|
||||
),
|
||||
Err(e) => eprintln!("missions: looking up team template {key:?}: {e}"),
|
||||
}
|
||||
}
|
||||
if !resolved.is_empty() {
|
||||
eprintln!(
|
||||
"missions: {} staffs {} phase purpose(s) from the recipe",
|
||||
body.template_kind.trim(),
|
||||
resolved.len()
|
||||
);
|
||||
if let Some(obj) = config.as_object_mut() {
|
||||
obj.insert("phase_teams".into(), serde_json::Value::Object(resolved));
|
||||
} else {
|
||||
config = serde_json::json!({ "phase_teams": resolved });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let has_phase_teams = has_phase_teams
|
||||
|| config
|
||||
.get("phase_teams")
|
||||
.and_then(|v| v.as_object())
|
||||
.is_some_and(|o| o.values().any(|v| v.as_array().is_some_and(|a| !a.is_empty())));
|
||||
|
||||
if team_template_id.is_none() && body.team_id.is_none() && !has_phase_teams {
|
||||
if let Some(key) = recipe.and_then(|r| r.default_team_template.as_deref()) {
|
||||
match cm_db::repo::team_templates::get_by_key(&state.pool, key).await {
|
||||
@@ -324,7 +372,7 @@ pub async fn create(
|
||||
repo_id: body.repo_id,
|
||||
schedule: body.schedule,
|
||||
description: body.description.as_deref(),
|
||||
config: body.config,
|
||||
config,
|
||||
runtime_kind: Some(runtime_kind),
|
||||
target_node_id: body.target_node_id,
|
||||
backend: body.backend.as_deref(),
|
||||
@@ -1761,6 +1809,12 @@ mod tests {
|
||||
blurb: String::new(),
|
||||
requires_repo: true,
|
||||
default_team_template: Some("rust_sdlc".into()),
|
||||
default_phase_teams: [
|
||||
("research".to_string(), "topic_research".to_string()),
|
||||
("coding".to_string(), "rust_sdlc".to_string()),
|
||||
]
|
||||
.into_iter()
|
||||
.collect(),
|
||||
phases: vec:
|
||||
/// writing the document into the container and telling the daemon about it
|
||||
/// are two halves of one thing, and doing one without the other leaves a
|
||||
/// door that is installed and unreachable — which looks exactly like a door
|
||||
/// nobody walked through.
|
||||
pub async fn set_claude_cli_mcp_config(&self, path: &str) -> Result<(), String> {
|
||||
self.set_prop(
|
||||
"providers.models.claude_cli.default.mcp_config",
|
||||
serde_json::json!(path),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Rebind an existing claw's model without touching its risk_profile
|
||||
/// or mcp_bundles. Used by the "change model" UI on the Agents page
|
||||
/// so we don't accidentally demote a coding_readwrite claw back to
|
||||
|
||||
+1017
-45
File diff suppressed because it is too large
Load Diff
@@ -175,33 +175,56 @@ mod tests {
|
||||
mod contradiction_tests {
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn skills_root() -> PathBuf {
|
||||
fn repo_root(rel: &str) -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("../../skills")
|
||||
.join("../..")
|
||||
.join(rel)
|
||||
.canonicalize()
|
||||
.expect("skills dir")
|
||||
.unwrap_or_else(|e| panic!("{rel}: {e}"))
|
||||
}
|
||||
|
||||
fn all_skills() -> Vec<(String, String)> {
|
||||
fn walk(dir: &std::path::Path, out: &mut Vec<(String, String)>) {
|
||||
for e in std::fs::read_dir(dir).expect("read skills dir") {
|
||||
fn walk_ext(dir: &std::path::Path, ext: &str, out: &mut Vec<(String, String)>) {
|
||||
for e in std::fs::read_dir(dir).expect("read dir") {
|
||||
let p = e.expect("entry").path();
|
||||
if p.is_dir() {
|
||||
walk(&p, out);
|
||||
} else if p.extension().and_then(|x| x.to_str()) == Some("md") {
|
||||
walk_ext(&p, ext, out);
|
||||
} else if p.extension().and_then(|x| x.to_str()) == Some(ext) {
|
||||
out.push((
|
||||
p.file_name().unwrap().to_string_lossy().to_string(),
|
||||
std::fs::read_to_string(&p).expect("read skill"),
|
||||
std::fs::read_to_string(&p).expect("read file"),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Skill bodies alone.
|
||||
fn all_skills() -> Vec<(String, String)> {
|
||||
let mut out = Vec::new();
|
||||
walk(&skills_root(), &mut out);
|
||||
walk_ext(&repo_root("skills"), "md", &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
/// No skill may teach a workspace path the platform does not use.
|
||||
/// **Everything we ship that becomes prompt text an agent reads.**
|
||||
///
|
||||
/// Skills and team-template role prompts, in one corpus, because the rules
|
||||
/// below are properties of *what an agent is told* — not of which file it
|
||||
/// happened to be written in.
|
||||
///
|
||||
/// This function is the finding. The `/workspace/repo` guard was written on
|
||||
/// 2026-08-19 against `skills/` only, and the same wrong path had been
|
||||
/// sitting in **four team templates** the whole time — including
|
||||
/// `rust_sdlc`, the default for five of the six workflow recipes, whose
|
||||
/// coder was told "your working directory is /workspace/repo" and whose
|
||||
/// committer was told to `cd` there. A guard that covers one corpus and not
|
||||
/// the other reads exactly like a guard that covers the problem.
|
||||
fn all_shipped_prompts() -> Vec<(String, String)> {
|
||||
let mut out = all_skills();
|
||||
walk_ext(&repo_root("templates/teams"), "toml", &mut out);
|
||||
walk_ext(&repo_root("templates/workflows"), "toml", &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
/// Nothing we ship may teach a workspace path the platform does not mount.
|
||||
///
|
||||
/// `workspace-repo-commit-protocol` told agents that `/workspace/repo` was
|
||||
/// "the ONLY path where source-modifying edits belong". The platform mounts
|
||||
@@ -210,18 +233,18 @@ mod contradiction_tests {
|
||||
/// was delivered twice in a single measured run, so agents received the
|
||||
/// platform's real path and a skill contradicting it in the SAME prompt.
|
||||
#[test]
|
||||
fn no_skill_teaches_a_repo_path_the_platform_does_not_mount() {
|
||||
fn nothing_we_ship_teaches_a_repo_path_the_platform_does_not_mount() {
|
||||
let mut offenders = Vec::new();
|
||||
for (name, body) in all_skills() {
|
||||
for (name, body) in all_shipped_prompts() {
|
||||
if body.contains("/workspace/repo") {
|
||||
offenders.push(name);
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
offenders.is_empty(),
|
||||
"{} skill(s) name /workspace/repo; the mission checkout is \
|
||||
/mission/repo, so an agent following them writes somewhere that is \
|
||||
never delivered: {}",
|
||||
"{} shipped prompt file(s) name /workspace/repo; the mission \
|
||||
checkout is /mission/repo, so an agent following them writes \
|
||||
somewhere that is never delivered: {}",
|
||||
offenders.len(),
|
||||
offenders.join(", ")
|
||||
);
|
||||
@@ -239,7 +262,7 @@ mod contradiction_tests {
|
||||
/// legitimately DISCUSS these names, as this one now does when warning
|
||||
/// against them.
|
||||
#[test]
|
||||
fn no_skill_instructs_an_agent_to_call_a_zeroclaw_tool() {
|
||||
fn nothing_we_ship_instructs_an_agent_to_call_a_zeroclaw_tool() {
|
||||
const ZEROCLAW_TOOLS: &[&str] = &[
|
||||
"`file_read`",
|
||||
"`file_write`",
|
||||
@@ -248,7 +271,7 @@ mod contradiction_tests {
|
||||
"`glob_search`",
|
||||
];
|
||||
let mut offenders = Vec::new();
|
||||
for (name, body) in all_skills() {
|
||||
for (name, body) in all_shipped_prompts() {
|
||||
// The line has to READ as an instruction. "Do not reach for
|
||||
// `file_read`" is the correction, not the defect.
|
||||
for line in body.lines() {
|
||||
@@ -267,10 +290,146 @@ mod contradiction_tests {
|
||||
}
|
||||
assert!(
|
||||
offenders.is_empty(),
|
||||
"{} skill line(s) tell an agent to use a tool its subprocess does \
|
||||
not expose:\n {}",
|
||||
"{} shipped prompt line(s) tell an agent to use a tool its \
|
||||
subprocess does not expose:\n {}",
|
||||
offenders.len(),
|
||||
offenders.join("\n ")
|
||||
);
|
||||
}
|
||||
|
||||
/// No skill may show a marker the real parser rejects.
|
||||
///
|
||||
/// Checked by running `task_card_parser::parse` itself, never a copy of its
|
||||
/// rules — a second implementation of the contract drifts, and then the
|
||||
/// test passes while the mission loop stalls.
|
||||
///
|
||||
/// This is the third instance of one class: the skills were written
|
||||
/// alongside the platform and then never compared to it again. The first
|
||||
/// was a repo path the platform does not mount; the second a tool the agent
|
||||
/// does not have; this one is `PLAN_COMPLETE: INT-01..05` in
|
||||
/// `decompose-int-items`, which a live planner emitted verbatim. Ids are
|
||||
/// strictly `INT-<digits>`, so the range form parses to nothing — the plan
|
||||
/// pass records no completion at all while every item stays open.
|
||||
///
|
||||
/// Scoped to fenced code blocks, which is where a skill puts the text it
|
||||
/// tells an agent to EMIT. A marker named in a sentence is prose.
|
||||
#[test]
|
||||
fn no_skill_shows_a_marker_the_parser_would_reject() {
|
||||
// The templates. `INT-NN` is a placeholder an agent substitutes, not a
|
||||
// literal it emits, so it is not a contradiction.
|
||||
const PLACEHOLDERS: &[&str] = &["INT-NN", "INT-XX", "INT-N", "INT-nn"];
|
||||
let mut offenders = Vec::new();
|
||||
for (name, body) in all_skills() {
|
||||
let mut fenced = false;
|
||||
for line in body.lines() {
|
||||
if line.trim_start().starts_with("```") {
|
||||
fenced = !fenced;
|
||||
continue;
|
||||
}
|
||||
let t = line.trim();
|
||||
if !fenced || !t.contains("INT-") || !t.contains(':') {
|
||||
continue;
|
||||
}
|
||||
let Some((kind, _)) = t.split_once(':') else {
|
||||
continue;
|
||||
};
|
||||
if !MARKER_KINDS.contains(&kind.trim()) {
|
||||
continue;
|
||||
}
|
||||
if PLACEHOLDERS.iter().any(|p| t.contains(p)) {
|
||||
continue;
|
||||
}
|
||||
if crate::task_card_parser::parse(t).is_empty() {
|
||||
offenders.push(format!("{name}: {t}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
offenders.is_empty(),
|
||||
"{} skill line(s) show a marker the parser rejects — an agent that \
|
||||
follows them exactly is silently ignored:\n {}",
|
||||
offenders.len(),
|
||||
offenders.join("\n ")
|
||||
);
|
||||
}
|
||||
|
||||
/// Every team a recipe names must be a team that exists.
|
||||
///
|
||||
/// `create()` logs and carries on when a recipe names a template that is
|
||||
/// not loaded, because failing mission creation over it would be worse.
|
||||
/// That makes a typo here invisible in exactly the way that matters: the
|
||||
/// mission is staffed by the fallback crew and looks deliberate. `research_only`
|
||||
/// pointed at `rust_sdlc` for months and nothing said a word.
|
||||
#[test]
|
||||
fn every_team_a_recipe_names_exists() {
|
||||
let mut keys = std::collections::HashSet::new();
|
||||
for (_, body) in {
|
||||
let mut v = Vec::new();
|
||||
walk_ext(&repo_root("templates/teams"), "toml", &mut v);
|
||||
v
|
||||
} {
|
||||
for line in body.lines() {
|
||||
if let Some(rest) = line.trim().strip_prefix("key") {
|
||||
if let Some((_, val)) = rest.split_once('=') {
|
||||
keys.insert(val.trim().trim_matches('"').to_string());
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(!keys.is_empty(), "no team templates found at all");
|
||||
|
||||
let mut recipes = Vec::new();
|
||||
walk_ext(&repo_root("templates/workflows"), "toml", &mut recipes);
|
||||
let mut missing = Vec::new();
|
||||
for (name, body) in recipes {
|
||||
let mut table = String::new();
|
||||
for line in body.lines() {
|
||||
let line = line.trim();
|
||||
if line.starts_with('[') {
|
||||
table = line.trim_matches(['[', ']'].as_slice()).to_string();
|
||||
continue;
|
||||
}
|
||||
if line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
let named = if let Some((_, v)) = line.split_once('=') {
|
||||
if line.starts_with("default_team_template")
|
||||
|| table == "default_phase_teams"
|
||||
{
|
||||
Some(v.trim().trim_matches('"').to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(k) = named {
|
||||
if !keys.contains(&k) {
|
||||
missing.push(format!("{name} -> {k}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
missing.is_empty(),
|
||||
"{} recipe(s) name a team template that does not exist, so the mission \
|
||||
is staffed by the fallback crew and looks deliberate: {}",
|
||||
missing.len(),
|
||||
missing.join(", ")
|
||||
);
|
||||
}
|
||||
|
||||
/// The marker kinds, as the parser spells them.
|
||||
const MARKER_KINDS: &[&str] = &[
|
||||
"TASK",
|
||||
"PLAN_COMPLETE",
|
||||
"WORK",
|
||||
"HANDOFF",
|
||||
"TEST_PASS",
|
||||
"TEST_FAIL",
|
||||
"REVIEW_APPROVE",
|
||||
"REVIEW_BLOCK",
|
||||
"COMPLETED",
|
||||
];
|
||||
}
|
||||
|
||||
@@ -71,6 +71,132 @@ pub struct Observed {
|
||||
pub tool: String,
|
||||
/// The path the tool's **input** named, if any. From JSON, never prose.
|
||||
pub path: Option<String>,
|
||||
/// Claude Code's session id for the `claude -p` invocation this call
|
||||
/// happened inside.
|
||||
///
|
||||
/// One invocation is one turn is one agent, so this is the only thing in
|
||||
/// the payload that separates one agent's actions from another's. The tap
|
||||
/// is per-CONTAINER and every role in a phase shares one, so without this
|
||||
/// the whole phase arrives as an undifferentiated stream.
|
||||
pub session: Option<String>,
|
||||
/// What a **command** produced, bounded by [`bounded_response`].
|
||||
///
|
||||
/// Only for tools that run something. `Read`'s response is the file it just
|
||||
/// read and `Write`'s is a restatement of what was written — both are
|
||||
/// already knowable from the arguments and the delivered diff, and storing
|
||||
/// them would double the largest write path in the system for nothing.
|
||||
///
|
||||
/// A command's OUTCOME is different: it is the only place a failing test
|
||||
/// run is visible. Without it "did this phase go red before it went green"
|
||||
/// cannot be answered from anything — not from tool order (in Rust the
|
||||
/// unit test lives in the file under test, so one `Edit` adds both), and
|
||||
/// not from the repository either, because `tdd-red-green-refactor` says
|
||||
/// in so many words to "commit the RED-to-GREEN pair as one commit".
|
||||
pub response: Value,
|
||||
/// The tool's arguments, bounded by [`bounded_input`].
|
||||
///
|
||||
/// Kept because the tool NAME alone answers almost nothing. A phase that
|
||||
/// recorded `Bash × 6` is indistinguishable from one that ran the test
|
||||
/// suite six times, one that pushed to a branch it was told not to, and
|
||||
/// one that queried an API a skill forbids. The argument is where the
|
||||
/// behaviour is, and until now this parser read it, took the path out of
|
||||
/// it, and dropped the rest on the floor.
|
||||
pub input: Value,
|
||||
}
|
||||
|
||||
/// How much of one argument string is worth keeping.
|
||||
///
|
||||
/// A shell command longer than this is a heredoc or a generated payload; its
|
||||
/// first half still carries the verb, which is what any check reads.
|
||||
const MAX_ARG_LEN: usize = 512;
|
||||
|
||||
/// Argument keys whose value is a file BODY rather than a description of an
|
||||
/// action.
|
||||
///
|
||||
/// Dropped to a byte count rather than truncated. These carry whole source
|
||||
/// files — `mission_events` is already the largest write path on a coding
|
||||
/// phase, and storing every `Write` twice (once in the event, once in the
|
||||
/// delivered diff) buys nothing: no check reads the body, and the diff is the
|
||||
/// authority on what was written anyway.
|
||||
const BODY_KEYS: [&str; 4] = ["content", "new_string", "old_string", "edits"];
|
||||
|
||||
/// Tools whose response is an outcome rather than a restatement.
|
||||
const RESPONSE_TOOLS: [&str; 1] = ["Bash"];
|
||||
|
||||
/// How much of a command's output to keep.
|
||||
const MAX_OUTPUT_LEN: usize = 600;
|
||||
|
||||
/// Shrink a command's response, keeping the **end** of its output.
|
||||
///
|
||||
/// The opposite of [`bounded_input`], and deliberately so. An argument's
|
||||
/// meaning is at the start — the verb of the command. A command's meaning is at
|
||||
/// the END: `cargo test` prints hundreds of lines and then `test result: ok` or
|
||||
/// `test result: FAILED`, and a head-biased truncation would keep the noise and
|
||||
/// throw away the verdict, which is the one thing being stored for.
|
||||
pub fn bounded_response(tool: &str, response: &Value) -> Value {
|
||||
if !RESPONSE_TOOLS.contains(&tool) {
|
||||
return Value::Null;
|
||||
}
|
||||
let Some(obj) = response.as_object() else {
|
||||
return Value::Null;
|
||||
};
|
||||
let mut out = serde_json::Map::new();
|
||||
for key in ["stdout", "stderr", "interrupted"] {
|
||||
match obj.get(key) {
|
||||
Some(Value::String(s)) if s.len() > MAX_OUTPUT_LEN => {
|
||||
let start = s
|
||||
.char_indices()
|
||||
.map(|(i, _)| i)
|
||||
.find(|i| *i >= s.len().saturating_sub(MAX_OUTPUT_LEN))
|
||||
.unwrap_or(0);
|
||||
out.insert(key.into(), Value::String(format!("[truncated]…{}", &s[start..])));
|
||||
}
|
||||
Some(v) => {
|
||||
out.insert(key.into(), v.clone());
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
}
|
||||
Value::Object(out)
|
||||
}
|
||||
|
||||
/// Shrink a tool's arguments to something safe to store on every call.
|
||||
///
|
||||
/// Bounded rather than whitelisted on purpose. A whitelist of "interesting"
|
||||
/// keys silently drops the one argument that matters the first time a tool
|
||||
/// grows a new field, and the loss is invisible — the event still looks
|
||||
/// complete. Bounding keeps every key and says, in the record itself, where it
|
||||
/// stopped.
|
||||
pub fn bounded_input(input: &Value) -> Value {
|
||||
let Some(obj) = input.as_object() else {
|
||||
return Value::Null;
|
||||
};
|
||||
let mut out = serde_json::Map::new();
|
||||
for (k, v) in obj {
|
||||
if BODY_KEYS.contains(&k.as_str()) {
|
||||
let bytes = match v {
|
||||
Value::String(s) => s.len(),
|
||||
other => other.to_string().len(),
|
||||
};
|
||||
out.insert(k.clone(), json!({ "omitted_bytes": bytes }));
|
||||
continue;
|
||||
}
|
||||
match v {
|
||||
Value::String(s) if s.len() > MAX_ARG_LEN => {
|
||||
let cut = s
|
||||
.char_indices()
|
||||
.map(|(i, _)| i)
|
||||
.take_while(|i| *i <= MAX_ARG_LEN)
|
||||
.last()
|
||||
.unwrap_or(0);
|
||||
out.insert(k.clone(), Value::String(format!("{}…[truncated]", &s[..cut])));
|
||||
}
|
||||
other => {
|
||||
out.insert(k.clone(), other.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
Value::Object(out)
|
||||
}
|
||||
|
||||
/// The hook script. Copies stdin verbatim to the tap file and gets out of the
|
||||
@@ -140,13 +266,13 @@ pub fn install_command(dir: &str) -> String {
|
||||
"mkdir -p {dir} && rm -f {dir}/tools.jsonl \
|
||||
&& printf '%s' {script} > {dir}/tap.sh && chmod +x {dir}/tap.sh",
|
||||
dir = dir,
|
||||
script = q(&hook_script(dir)),
|
||||
script = shell_quote(&hook_script(dir)),
|
||||
)
|
||||
}
|
||||
|
||||
/// Write the composed settings document.
|
||||
pub fn settings_command(path: &str, settings: &Value) -> String {
|
||||
format!("printf '%s' {} > {path}", q(&settings.to_string()))
|
||||
format!("printf '%s' {} > {path}", shell_quote(&settings.to_string()))
|
||||
}
|
||||
|
||||
/// Parse a drained tap.
|
||||
@@ -178,8 +304,20 @@ pub fn parse(raw: &str) -> Vec<Observed> {
|
||||
.or_else(|| v.get("toolInput"))
|
||||
.cloned()
|
||||
.unwrap_or(Value::Null);
|
||||
let response = v
|
||||
.get("tool_response")
|
||||
.or_else(|| v.get("toolResponse"))
|
||||
.cloned()
|
||||
.unwrap_or(Value::Null);
|
||||
Some(Observed {
|
||||
path: crate::mission_events::tool_path(&input),
|
||||
input: bounded_input(&input),
|
||||
response: bounded_response(&tool, &response),
|
||||
session: v
|
||||
.get("session_id")
|
||||
.or_else(|| v.get("sessionId"))
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string),
|
||||
tool,
|
||||
})
|
||||
})
|
||||
@@ -188,7 +326,7 @@ pub fn parse(raw: &str) -> Vec<Observed> {
|
||||
|
||||
/// Single-quote for `sh`. Local copy, same rule as the stop gate's — these two
|
||||
/// modules deliberately share no code, so neither can break the other.
|
||||
fn q(s: &str) -> String {
|
||||
pub fn shell_quote(s: &str) -> String {
|
||||
format!("'{}'", s.replace('\'', r"'\''"))
|
||||
}
|
||||
|
||||
@@ -268,12 +406,78 @@ mod tests {
|
||||
assert_eq!(
|
||||
parse(raw),
|
||||
vec![
|
||||
Observed { tool: "Edit".into(), path: Some("/mission/repo/src/a.rs".into()) },
|
||||
Observed { tool: "Bash".into(), path: None },
|
||||
Observed {
|
||||
tool: "Edit".into(),
|
||||
path: Some("/mission/repo/src/a.rs".into()),
|
||||
input: json!({"file_path": "/mission/repo/src/a.rs"}),
|
||||
session: None,
|
||||
response: Value::Null,
|
||||
},
|
||||
Observed {
|
||||
tool: "Bash".into(),
|
||||
path: None,
|
||||
input: json!({"command": "ls"}),
|
||||
session: None,
|
||||
response: Value::Null,
|
||||
},
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// The command survives the parse.
|
||||
///
|
||||
/// The regression this guards is the one that made the first container-tier
|
||||
/// measurement unusable: six `Bash` calls were recorded and not one of them
|
||||
/// said what it ran, so every behavioural question — did it run the tests,
|
||||
/// did it commit, did it call the API a skill forbids — was unanswerable
|
||||
/// from a record that looked complete.
|
||||
#[test]
|
||||
fn the_argument_is_what_carries_the_behaviour() {
|
||||
let raw = concat!(
|
||||
r#"{"tool_name":"Bash","tool_input":{"command":"cargo nextest run -p cm-api"}}"#,
|
||||
"\n",
|
||||
);
|
||||
let got = parse(raw);
|
||||
assert_eq!(got[0].input["command"], json!("cargo nextest run -p cm-api"));
|
||||
}
|
||||
|
||||
/// A file body is counted, not stored; everything else survives bounded.
|
||||
#[test]
|
||||
fn bodies_are_dropped_and_long_arguments_are_marked() {
|
||||
let long = "x".repeat(MAX_ARG_LEN + 50);
|
||||
let got = bounded_input(&json!({
|
||||
"file_path": "/mission/repo/src/a.rs",
|
||||
"content": "fn main() {}",
|
||||
"command": long,
|
||||
}));
|
||||
assert_eq!(got["file_path"], json!("/mission/repo/src/a.rs"));
|
||||
assert_eq!(
|
||||
got["content"],
|
||||
json!({"omitted_bytes": 12}),
|
||||
"a file body is stored in the delivered diff already; the event only \
|
||||
needs to say how big it was"
|
||||
);
|
||||
let cmd = got["command"].as_str().expect("command kept");
|
||||
assert!(cmd.ends_with("…[truncated]"), "{cmd}");
|
||||
assert!(
|
||||
cmd.len() < MAX_ARG_LEN + 40,
|
||||
"a bounded argument must actually be bounded: {}",
|
||||
cmd.len()
|
||||
);
|
||||
}
|
||||
|
||||
/// Truncation must not split a multi-byte character.
|
||||
///
|
||||
/// `&s[..cut]` on a byte index inside a UTF-8 sequence panics, and the
|
||||
/// panic would land in the drain — losing a whole phase's tap to a command
|
||||
/// that happened to contain an emoji or an em dash.
|
||||
#[test]
|
||||
fn truncation_respects_character_boundaries() {
|
||||
let long = "é".repeat(MAX_ARG_LEN);
|
||||
let got = bounded_input(&json!({ "command": long }));
|
||||
assert!(got["command"].as_str().unwrap().ends_with("…[truncated]"));
|
||||
}
|
||||
|
||||
/// Exactly one place in the tree writes the guest settings document.
|
||||
///
|
||||
/// The unit test above proves `guest_settings` composes correctly; it says
|
||||
|
||||
@@ -34,6 +34,20 @@ pub struct WorkflowRecipe {
|
||||
pub phases: Vec<WorkflowPhase>,
|
||||
#[serde(default)]
|
||||
pub default_team_template: Option<String>,
|
||||
/// Default team **per phase purpose**, by template key:
|
||||
/// `{ research = "topic_research", coding = "rust_sdlc" }`.
|
||||
///
|
||||
/// `default_team_template` names ONE team for a whole mission, and a
|
||||
/// multi-phase recipe does not have one job. `research_and_code` staffs a
|
||||
/// research phase and a coding phase from the same `rust_sdlc` crew, which
|
||||
/// is why its research phase has to spend a paragraph of `task` telling
|
||||
/// coders not to code — a workaround for staffing, written into the prompt.
|
||||
///
|
||||
/// Resolved to `config.phase_teams` at mission-create, which the
|
||||
/// orchestrator and `composed_graph` already read. Purposes come from
|
||||
/// `phase_runner::purposes_for`.
|
||||
#[serde(default)]
|
||||
pub default_phase_teams: std::collections::BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
|
||||
@@ -0,0 +1,352 @@
|
||||
//! The orphan sweep's two Docker-touching seams, against real containers.
|
||||
//!
|
||||
//! `sweep_orphans` force-removes containers. Its decision logic is pure and
|
||||
//! unit-tested in `mission_runtime`, but the two calls that talk to Docker —
|
||||
//! "which containers exist" and "does this checkout hold work no remote has" —
|
||||
//! had never run against a daemon. Those are exactly the ones worth exercising
|
||||
//! for real: the first decides what is considered at all, and the second is the
|
||||
//! only thing standing between a reaper and ten unpushed commits.
|
||||
//!
|
||||
//! That is not hypothetical. The orphan that motivated this sweep held
|
||||
//! +3451/-30 across 30 files on a branch that existed nowhere else.
|
||||
//!
|
||||
//! Skips cleanly when there is no Docker, so a machine or CI runner without one
|
||||
//! reports "not run" rather than failing.
|
||||
|
||||
use cm_api::mission_runtime::{container_name, MissionRuntimeProvisioner, UnpushedWork};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// The image is already local on any machine that runs missions, and it has
|
||||
/// `git`, which the probe needs.
|
||||
const FIXTURE_IMAGE: &str = "clawmates-runtime:hooks";
|
||||
|
||||
/// These tests must not run at the same time as each other.
|
||||
///
|
||||
/// `sweep_orphans` is global: it reaps EVERY orphaned `cm-runtime-mission-*`
|
||||
/// container on the daemon, which on a parallel test runner includes the
|
||||
/// fixtures another test in this file just started. That is not a flaw in the
|
||||
/// sweep — it is what a sweep is — but it means anything here that creates a
|
||||
/// mission-shaped container has to hold this lock.
|
||||
///
|
||||
/// Found the honest way: the reap test deleted the listing test's fixture
|
||||
/// mid-run and the listing test reported a container it could not see.
|
||||
static FIXTURES: std::sync::LazyLock<tokio::sync::Mutex<()>> =
|
||||
std::sync::LazyLock::new(|| tokio::sync::Mutex::new(()));
|
||||
|
||||
fn docker_available() -> bool {
|
||||
std::process::Command::new("docker")
|
||||
.args(["image", "inspect", FIXTURE_IMAGE])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Start a fixture container named like a mission runtime, running a shell
|
||||
/// script that leaves `/mission/repo` in a known state.
|
||||
fn start_fixture(id: Uuid, setup: &str) -> String {
|
||||
let name = container_name(id);
|
||||
let _ = std::process::Command::new("docker")
|
||||
.args(["rm", "-f", &name])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status();
|
||||
let script = format!("{setup}\nsleep 3600");
|
||||
let out = std::process::Command::new("docker")
|
||||
.args([
|
||||
"run", "-d", "--name", &name, "--entrypoint", "sh", FIXTURE_IMAGE, "-c", &script,
|
||||
])
|
||||
.output()
|
||||
.expect("docker run");
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"could not start fixture {name}: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
// The script has to have finished its git work before the probe runs.
|
||||
std::thread::sleep(std::time::Duration::from_secs(3));
|
||||
name
|
||||
}
|
||||
|
||||
fn remove(name: &str) {
|
||||
let _ = std::process::Command::new("docker")
|
||||
.args(["rm", "-f", name])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status();
|
||||
}
|
||||
|
||||
const GIT_INIT: &str = "set -e
|
||||
mkdir -p /mission/repo && cd /mission/repo
|
||||
git init -q .
|
||||
git config user.email t@t && git config user.name t
|
||||
echo hello > a.txt && git add a.txt && git commit -qm 'work nobody else has'";
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_sweep_can_see_containers_and_refuses_the_ones_holding_work() {
|
||||
if !docker_available() {
|
||||
eprintln!("orphan_sweep: no docker or no {FIXTURE_IMAGE} — not run");
|
||||
return;
|
||||
}
|
||||
let Some(prov) = MissionRuntimeProvisioner::from_env() else {
|
||||
eprintln!("orphan_sweep: no docker connection — not run");
|
||||
return;
|
||||
};
|
||||
let _serial = FIXTURES.lock().await;
|
||||
|
||||
let dirty_id = Uuid::now_v7();
|
||||
let clean_id = Uuid::now_v7();
|
||||
let empty_id = Uuid::now_v7();
|
||||
|
||||
// Commits, and no remote ref anywhere: this is the container that must
|
||||
// survive. It is the one the real orphan looked like.
|
||||
let dirty = start_fixture(dirty_id, GIT_INIT);
|
||||
// The same repo, but every commit is reachable from a remote-tracking ref,
|
||||
// which is what "already pushed" looks like to `git rev-list --not
|
||||
// --remotes`.
|
||||
let clean = start_fixture(
|
||||
clean_id,
|
||||
&format!("{GIT_INIT}\ngit update-ref refs/remotes/origin/main HEAD"),
|
||||
);
|
||||
// No checkout at all — nothing to lose.
|
||||
let empty = start_fixture(empty_id, "set -e\nmkdir -p /root");
|
||||
|
||||
let result = async {
|
||||
let names = prov.list_mission_containers().await?;
|
||||
let found: Vec<&String> = names.iter().map(|(n, _)| n).collect();
|
||||
for expected in [&dirty, &clean, &empty] {
|
||||
assert!(
|
||||
found.iter().any(|n| *n == expected),
|
||||
"the sweep cannot see {expected}; a container it cannot list is \
|
||||
one it can never reap, which is the whole defect this closes. \
|
||||
saw: {found:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// Docker's own creation timestamp must come back, or the sweep declines
|
||||
// to reap for want of an age.
|
||||
let (_, created) = names
|
||||
.iter()
|
||||
.find(|(n, _)| n == &dirty)
|
||||
.expect("dirty in listing");
|
||||
assert!(
|
||||
MissionRuntimeProvisioner::container_age(*created).is_some(),
|
||||
"a container docker will not date is never reaped, so an absent \
|
||||
timestamp here would silently disable the sweep"
|
||||
);
|
||||
|
||||
match prov.unpushed_commits(&dirty).await {
|
||||
UnpushedWork::SomeOrUnknown(why) => {
|
||||
assert!(why.contains("no remote"), "{why}");
|
||||
}
|
||||
UnpushedWork::None => panic!(
|
||||
"the probe said a checkout with an unpushed commit holds nothing — \
|
||||
this is the exact answer that destroys work"
|
||||
),
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
prov.unpushed_commits(&clean).await,
|
||||
UnpushedWork::None,
|
||||
"every commit is reachable from a remote ref, so there is nothing to lose"
|
||||
);
|
||||
assert_eq!(
|
||||
prov.unpushed_commits(&empty).await,
|
||||
UnpushedWork::None,
|
||||
"no /mission/repo at all means nothing to lose"
|
||||
);
|
||||
Ok::<(), String>(())
|
||||
}
|
||||
.await;
|
||||
|
||||
remove(&dirty);
|
||||
remove(&clean);
|
||||
remove(&empty);
|
||||
result.expect("orphan sweep probes");
|
||||
}
|
||||
|
||||
/// A container we cannot question is not a container we may delete.
|
||||
#[tokio::test]
|
||||
async fn a_container_that_is_gone_reads_as_holding_work() {
|
||||
if !docker_available() {
|
||||
eprintln!("orphan_sweep: no docker — not run");
|
||||
return;
|
||||
}
|
||||
let Some(prov) = MissionRuntimeProvisioner::from_env() else {
|
||||
return;
|
||||
};
|
||||
match prov
|
||||
.unpushed_commits("cm-runtime-mission-does-not-exist-at-all")
|
||||
.await
|
||||
{
|
||||
UnpushedWork::SomeOrUnknown(_) => {}
|
||||
UnpushedWork::None => panic!(
|
||||
"an unanswerable probe must never read as 'safe to delete' — every \
|
||||
failure path in this check is one-sided for that reason"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set to run the destructive sweep test.
|
||||
///
|
||||
/// The other tests in this file only create fixtures and read them. This one
|
||||
/// calls `sweep_orphans`, which REMOVES containers — and CI runs
|
||||
/// `cargo test --workspace` inside a container with `/var/run/docker.sock`
|
||||
/// mounted, on gw04, which is the host that runs production missions.
|
||||
///
|
||||
/// `adopt_existing` protects everything already present, but it cannot protect
|
||||
/// a mission container created in the seconds between that call and the sweep.
|
||||
/// On a developer machine that race is nothing; on the production host it is a
|
||||
/// mission. So the destructive test is opt-in, and CI simply does not run it.
|
||||
const RUN_DESTRUCTIVE: &str = "CM_TEST_ORPHAN_SWEEP";
|
||||
|
||||
/// The reap decision itself, against real containers.
|
||||
///
|
||||
/// The probes above are the inputs; this is the act. A clean orphan past its
|
||||
/// grace must go, an orphan holding unpushed work must stay, and a young one
|
||||
/// must stay regardless — and all three have to be true of the same sweep, in
|
||||
/// one pass, because that is how it runs.
|
||||
#[tokio::test]
|
||||
async fn the_sweep_reaps_the_clean_orphan_and_spares_the_others() {
|
||||
if !docker_available() {
|
||||
eprintln!("orphan_sweep: no docker — not run");
|
||||
return;
|
||||
}
|
||||
if std::env::var(RUN_DESTRUCTIVE).is_err() {
|
||||
eprintln!(
|
||||
"orphan_sweep: not run — this test removes containers, and the CI \
|
||||
runner shares a docker daemon with production. Set \
|
||||
{RUN_DESTRUCTIVE}=1 to run it."
|
||||
);
|
||||
return;
|
||||
}
|
||||
if MissionRuntimeProvisioner::from_env().is_none() {
|
||||
return;
|
||||
}
|
||||
let _serial = FIXTURES.lock().await;
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
|
||||
// Adopt every mission container that already exists on this daemon.
|
||||
//
|
||||
// The sweep asks the DATABASE whether a container is known, and a fresh
|
||||
// test database knows nothing — so on a developer machine the sweep
|
||||
// classifies the live local stack's mission containers as orphans and
|
||||
// reaps them. It did exactly that on the first run of this test, deleting
|
||||
// two real mission containers.
|
||||
//
|
||||
// Giving each a row makes the test safe AND covers the case the other
|
||||
// assertions do not: a container the platform still knows about is never
|
||||
// touched, whatever its checkout looks like.
|
||||
let adopted = adopt_existing(&pool).await;
|
||||
|
||||
let clean_id = Uuid::now_v7();
|
||||
let dirty_id = Uuid::now_v7();
|
||||
let young_id = Uuid::now_v7();
|
||||
|
||||
let alive = |name: &str| {
|
||||
std::process::Command::new("docker")
|
||||
.args(["inspect", name])
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false)
|
||||
};
|
||||
|
||||
let clean = start_fixture(
|
||||
clean_id,
|
||||
&format!("{GIT_INIT}\ngit update-ref refs/remotes/origin/main HEAD"),
|
||||
);
|
||||
let dirty = start_fixture(dirty_id, GIT_INIT);
|
||||
|
||||
// Pass one, no grace: everything present is past its window, so the
|
||||
// decision is made purely on whether the checkout holds work.
|
||||
let swept = cm_api::mission_runtime::sweep_orphans(&pool, std::time::Duration::ZERO).await;
|
||||
let clean_gone = !alive(&clean);
|
||||
let dirty_alive = alive(&dirty);
|
||||
|
||||
// Pass two, a real grace, on a container minted seconds ago. It is clean
|
||||
// and orphaned — reapable on every axis except its age — so if the grace is
|
||||
// decorative this is where that shows.
|
||||
//
|
||||
// Started AFTER the first pass on purpose: a grace applies to every
|
||||
// container in the sweep, so a fixture created before a zero-grace pass is
|
||||
// reaped by that pass and proves nothing about the window. The first
|
||||
// version of this test made exactly that mistake and failed itself.
|
||||
let young = start_fixture(
|
||||
young_id,
|
||||
&format!("{GIT_INIT}\ngit update-ref refs/remotes/origin/main HEAD"),
|
||||
);
|
||||
let swept2 =
|
||||
cm_api::mission_runtime::sweep_orphans(&pool, std::time::Duration::from_secs(3600)).await;
|
||||
let young_alive = alive(&young);
|
||||
|
||||
remove(&clean);
|
||||
remove(&dirty);
|
||||
remove(&young);
|
||||
|
||||
for name in &adopted {
|
||||
assert!(
|
||||
alive(name),
|
||||
"the sweep reaped {name}, which HAS a mission row — a container the \
|
||||
platform still knows about must never be touched"
|
||||
);
|
||||
}
|
||||
|
||||
swept.expect("first sweep");
|
||||
swept2.expect("second sweep");
|
||||
assert!(
|
||||
clean_gone,
|
||||
"a clean orphan past its grace is exactly what this sweep exists to \
|
||||
reclaim; leaving it means the disk leak is still open"
|
||||
);
|
||||
assert!(
|
||||
dirty_alive,
|
||||
"an orphan holding commits no remote has MUST survive — the container \
|
||||
that motivated this held ten of them"
|
||||
);
|
||||
assert!(
|
||||
young_alive,
|
||||
"a container inside the grace window must be left alone even when it is \
|
||||
otherwise reapable, or the grace is decorative"
|
||||
);
|
||||
}
|
||||
|
||||
/// Give every mission container already on this daemon a row, so the sweep
|
||||
/// treats it as known and leaves it alone.
|
||||
///
|
||||
/// Returns the names, which then double as an assertion: none of them may be
|
||||
/// reaped.
|
||||
async fn adopt_existing(pool: &sqlx::PgPool) -> Vec<String> {
|
||||
let Some(prov) = MissionRuntimeProvisioner::from_env() else {
|
||||
return Vec::new();
|
||||
};
|
||||
let Ok(existing) = prov.list_mission_containers().await else {
|
||||
return Vec::new();
|
||||
};
|
||||
let ws = Uuid::now_v7();
|
||||
sqlx::query("INSERT INTO workspaces (id, name, plan) VALUES ($1, 'orphan-test', 'free')")
|
||||
.bind(ws)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("seed workspace");
|
||||
let mut names = Vec::new();
|
||||
for (name, _) in existing {
|
||||
let Some(id) = cm_api::mission_runtime::mission_id_from_container(&name) else {
|
||||
continue;
|
||||
};
|
||||
sqlx::query(
|
||||
"INSERT INTO missions (id, workspace_id, title, template_kind)
|
||||
VALUES ($1, $2, 'adopted by orphan_sweep test', 'research_only')
|
||||
ON CONFLICT (id) DO NOTHING",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(ws)
|
||||
.execute(pool)
|
||||
.await
|
||||
.expect("adopt container");
|
||||
names.push(name);
|
||||
}
|
||||
names
|
||||
}
|
||||
@@ -12,5 +12,7 @@ mod token;
|
||||
|
||||
pub use bootstrap::bootstrap_owner;
|
||||
pub use jwt::{ExternalClaims, JwtError, JwtVerifier};
|
||||
pub use service::{AuthError, AuthService, AuthedUser, SESSION_TTL};
|
||||
pub use service::{
|
||||
AuthError, AuthService, AuthedUser, SCOPE_FULL, SCOPE_SKILLS_READ, SESSION_TTL,
|
||||
};
|
||||
pub use token::SessionToken;
|
||||
|
||||
@@ -10,6 +10,17 @@ use crate::token::{hash_token, SessionToken};
|
||||
/// How long a login session stays valid.
|
||||
pub const SESSION_TTL: Duration = Duration::days(7);
|
||||
|
||||
/// A person's session. Accepted by every route.
|
||||
pub const SCOPE_FULL: &str = "full";
|
||||
|
||||
/// Read the skills catalogue over MCP, and nothing else.
|
||||
///
|
||||
/// The credential a mission container is given so its agent can retrieve skill
|
||||
/// bodies on demand. Deliberately its own constant rather than a string
|
||||
/// literal at the two call sites: a typo in one of them would produce a token
|
||||
/// that authenticates nowhere, which fails safely but silently.
|
||||
pub const SCOPE_SKILLS_READ: &str = "skills:read";
|
||||
|
||||
/// The authenticated caller attached to every API request: everything RBAC
|
||||
/// decisions need, nothing more.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -263,13 +274,44 @@ impl AuthService {
|
||||
/// JWTs (three dot-separated segments) take the hosted-identity path;
|
||||
/// everything else is a local opaque session token.
|
||||
pub async fn authenticate(&self, token_secret: &str) -> Result<AuthedUser, AuthError> {
|
||||
self.authenticate_scoped(token_secret, SCOPE_FULL).await
|
||||
}
|
||||
|
||||
/// Resolve a bearer token that is allowed to be narrow.
|
||||
///
|
||||
/// `required` is the scope this call site accepts *in addition to*
|
||||
/// [`SCOPE_FULL`], which is a person's session and is accepted everywhere.
|
||||
///
|
||||
/// # Fail closed
|
||||
///
|
||||
/// [`authenticate`](Self::authenticate) delegates here with `SCOPE_FULL`,
|
||||
/// so a narrow token is **rejected by every existing caller** and a route
|
||||
/// has to opt in by naming the scope it accepts. That direction matters:
|
||||
/// the likely mistake is adding a scope and forgetting to wire a check, and
|
||||
/// this way that mistake grants nothing instead of granting everything.
|
||||
///
|
||||
/// A narrow credential exists because the alternative is worse. Reaching
|
||||
/// `/mcp/skills` from a mission container means putting a bearer token in a
|
||||
/// file inside it, and mission agents run arbitrary `Bash` with egress and
|
||||
/// no read gate — so a full session there is an owner-privileged API key
|
||||
/// handed to something explicitly untrusted.
|
||||
pub async fn authenticate_scoped(
|
||||
&self,
|
||||
token_secret: &str,
|
||||
required: &str,
|
||||
) -> Result<AuthedUser, AuthError> {
|
||||
if let Some(verifier) = self.verifier.clone() {
|
||||
if token_secret.matches('.').count() == 2 {
|
||||
// An external issuer JWT is always a person. There is no
|
||||
// narrow form of it, so it satisfies only `full`.
|
||||
if required != SCOPE_FULL {
|
||||
return Err(AuthError::Unauthenticated);
|
||||
}
|
||||
return self.authenticate_external(token_secret, &verifier).await;
|
||||
}
|
||||
}
|
||||
let row = sqlx::query!(
|
||||
"SELECT u.id, u.workspace_id, u.role
|
||||
"SELECT u.id, u.workspace_id, u.role, s.scope
|
||||
FROM auth_sessions s
|
||||
JOIN users u ON u.id = s.user_id
|
||||
WHERE s.token_hash = $1 AND s.expires_at > now()",
|
||||
@@ -278,6 +320,9 @@ impl AuthService {
|
||||
.fetch_optional(&self.pool)
|
||||
.await?
|
||||
.ok_or(AuthError::Unauthenticated)?;
|
||||
if row.scope != SCOPE_FULL && row.scope != required {
|
||||
return Err(AuthError::Unauthenticated);
|
||||
}
|
||||
Ok(AuthedUser {
|
||||
user_id: UserId::from(row.id),
|
||||
workspace_id: WorkspaceId::from(row.workspace_id),
|
||||
@@ -289,6 +334,35 @@ impl AuthService {
|
||||
})
|
||||
}
|
||||
|
||||
/// Mint a narrow, short-lived credential for something that is not a person.
|
||||
///
|
||||
/// Returns the secret, which is the only time it exists in plaintext here.
|
||||
pub async fn mint_scoped(
|
||||
&self,
|
||||
user_id: UserId,
|
||||
scope: &str,
|
||||
ttl: Duration,
|
||||
) -> Result<String, AuthError> {
|
||||
if scope == SCOPE_FULL {
|
||||
// A caller reaching for this wants a narrow token; handing back a
|
||||
// full one because the argument was wrong is the failure this
|
||||
// whole change exists to prevent.
|
||||
return Err(AuthError::Unauthenticated);
|
||||
}
|
||||
let token = SessionToken::generate();
|
||||
sqlx::query!(
|
||||
"INSERT INTO auth_sessions (token_hash, user_id, expires_at, scope)
|
||||
VALUES ($1, $2, $3, $4)",
|
||||
hash_token(token.secret()),
|
||||
user_id.as_uuid(),
|
||||
OffsetDateTime::now_utc() + ttl,
|
||||
scope,
|
||||
)
|
||||
.execute(&self.pool)
|
||||
.await?;
|
||||
Ok(token.secret().to_string())
|
||||
}
|
||||
|
||||
/// Mint a long-lived opaque session for an internal service caller
|
||||
/// (e.g. the per-team ZeroClaw runtime calling back into the MCP door).
|
||||
/// Returns the plaintext token — the caller is responsible for handing
|
||||
|
||||
@@ -125,3 +125,81 @@ async fn tokens_are_unique_per_login() {
|
||||
auth.authenticate(a.secret()).await.unwrap();
|
||||
auth.authenticate(b.secret()).await.unwrap();
|
||||
}
|
||||
|
||||
/// A narrow credential must be refused everywhere it was not explicitly
|
||||
/// allowed.
|
||||
///
|
||||
/// This is the whole security property. The skills token lives in a file
|
||||
/// inside a mission container, where an agent running arbitrary `Bash` can
|
||||
/// read it — so what matters is not that `/mcp/skills` accepts it but that
|
||||
/// **nothing else does**.
|
||||
#[tokio::test]
|
||||
async fn a_scoped_token_is_refused_by_every_unscoped_caller() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (_ws, user) = seeded(&pool).await;
|
||||
let auth = AuthService::new(pool);
|
||||
|
||||
let narrow = auth
|
||||
.mint_scoped(user.id, cm_auth::SCOPE_SKILLS_READ, time::Duration::hours(1))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// `authenticate` is what every ordinary route calls.
|
||||
assert!(
|
||||
matches!(
|
||||
auth.authenticate(&narrow).await,
|
||||
Err(AuthError::Unauthenticated)
|
||||
),
|
||||
"a skills token must not authenticate a normal API call — the token is \
|
||||
readable by the agent it is given to"
|
||||
);
|
||||
|
||||
// And it is refused for a DIFFERENT narrow scope, not just for `full`.
|
||||
assert!(matches!(
|
||||
auth.authenticate_scoped(&narrow, "some:other").await,
|
||||
Err(AuthError::Unauthenticated)
|
||||
));
|
||||
|
||||
// It does work for the one thing it is for.
|
||||
let ok = auth
|
||||
.authenticate_scoped(&narrow, cm_auth::SCOPE_SKILLS_READ)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(ok.user_id, user.id);
|
||||
}
|
||||
|
||||
/// A person's session keeps working everywhere, including the scoped route.
|
||||
#[tokio::test]
|
||||
async fn a_full_session_still_satisfies_a_scoped_route() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (_ws, user) = seeded(&pool).await;
|
||||
let auth = AuthService::new(pool);
|
||||
auth.set_password(user.id, "correct horse battery staple")
|
||||
.await
|
||||
.unwrap();
|
||||
let token = auth
|
||||
.login_local("[email protected]", "correct horse battery staple")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(auth
|
||||
.authenticate_scoped(token.secret(), cm_auth::SCOPE_SKILLS_READ)
|
||||
.await
|
||||
.is_ok());
|
||||
}
|
||||
|
||||
/// `mint_scoped` must refuse to mint a full token.
|
||||
///
|
||||
/// A caller reaching for this wants a narrow credential; handing back a full
|
||||
/// one because an argument was wrong is precisely the failure the scope column
|
||||
/// exists to prevent, and it would be invisible — the token would work.
|
||||
#[tokio::test]
|
||||
async fn mint_scoped_refuses_to_mint_a_full_token() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (_ws, user) = seeded(&pool).await;
|
||||
let auth = AuthService::new(pool);
|
||||
assert!(auth
|
||||
.mint_scoped(user.id, cm_auth::SCOPE_FULL, time::Duration::hours(1))
|
||||
.await
|
||||
.is_err());
|
||||
}
|
||||
|
||||
+165
-123
@@ -1,177 +1,219 @@
|
||||
# Where this left off — 2026-08-21
|
||||
# Where this left off — 2026-08-21 (second pass)
|
||||
|
||||
Read `CAPABILITY-REVIEW.md` for the system picture,
|
||||
`TOOL-CALL-ARCHITECTURE.md` for how mission tools actually work (and the two
|
||||
wrong theories that preceded it), and `SKILL-USE-BASELINE.md` for the
|
||||
measurement — noting that its Trigger column is now out of date in a good way
|
||||
(see "Next, in order" §2).
|
||||
`TOOL-CALL-ARCHITECTURE.md` for how mission tools actually work, and
|
||||
`SKILL-USE-BASELINE.md` for the measurement — which now scores behaviour rather
|
||||
than the agent's own account of it.
|
||||
|
||||
## State of the tree
|
||||
|
||||
Everything is pushed. `main` is at `5a11fae`; the fork's
|
||||
`merge/upstream-v0.8.4` is at `be9c34b1c`. Local suite green: **107 test
|
||||
binaries, 412 lib tests**, frontend builds.
|
||||
Local suite green: **107 test binaries, 796 tests** (`cargo test --workspace`), and the workspace builds with `--all-targets`.
|
||||
Five measurement missions ran on the local stack (`scripts/skill-use-run.sh`);
|
||||
all five are held 90 days and re-scorable with `--score <id>`.
|
||||
10 commits on `main` this pass, **not pushed** — a push to `main` auto-deploys
|
||||
to gw-04, and the container-tier work already deployed is unexercised there
|
||||
(see below).
|
||||
|
||||
**CI is green and production is current.** Run 507 passed `test` and `build`,
|
||||
and gw-04 rolled to it. Production also runs `clawmates-runtime:hooks` for
|
||||
per-mission containers (see below).
|
||||
## The premise of the last handoff's item 1 was wrong
|
||||
|
||||
## Container-tier tool gate + telemetry — SHIPPED 2026-08-21
|
||||
It said the container-tier gate and tap were "proven locally and unproven in
|
||||
prod", and told you to watch for the first production mission.
|
||||
|
||||
The tier that actually runs missions now has both, verified end to end on a
|
||||
real mission (locally first, then deployed):
|
||||
**Production has never run a mission.**
|
||||
|
||||
```
|
||||
tool.call 10 Bash 6, Read 3, Write 1
|
||||
file.touch 4 research/tapproof.md
|
||||
gw-04$ select count(*) from missions; -> 0
|
||||
gw-04$ select count(*) from mission_events; -> 0
|
||||
```
|
||||
|
||||
That is the first time the container tier has ever been observable.
|
||||
Prod is armed correctly — server restarted with
|
||||
`CLAWMATES_RUNTIME_IMAGE=clawmates-runtime:hooks`, image present. There is
|
||||
simply nothing to watch. Prod auth is Clerk, so a mission cannot be launched
|
||||
from a terminal; someone has to click. Generalise the lesson: before debugging
|
||||
why a deployed thing shows no evidence, check whether anything ran.
|
||||
|
||||
**How, after `stream-json` failed.** Claude Code runs its tools inside its own
|
||||
subprocess, so they never reach ZeroClaw's executor and never become a
|
||||
`TurnEvent::ToolCall`. Hooks bypass that entirely: `claude -p --settings <doc>`
|
||||
honours `PreToolUse` and `PostToolUse`, so the gate blocks and the tap records
|
||||
without ZeroClaw being involved at all.
|
||||
### A leaked mission container nothing can reap
|
||||
|
||||
Pieces: `--settings` on `claude_cli` (fork `be9c34b1c`),
|
||||
`container_tool_hooks` writes both hook scripts and one settings document into
|
||||
the mission container, `set_claude_cli_settings` points the provider at it, and
|
||||
`phase_runner::drain_finished_container_phases` collects the tap into
|
||||
`mission_events` (idempotent by truncation — no cursor column).
|
||||
`cm-runtime-mission-019ff5b157ce77028f308ebd3dc92748` has been `Up` since
|
||||
**2026-08-12** on `clawmates-runtime:sync`, with no `missions` row behind it.
|
||||
|
||||
**Production state:** server on `f6e6037`;
|
||||
`CLAWMATES_RUNTIME_IMAGE=clawmates-runtime:hooks` in `/opt/clawmates/.env`
|
||||
(backup at `.env.bak.prehooks`; rollback = restore it and recreate). Note that
|
||||
host uses **legacy `docker-compose`**, not the v2 plugin.
|
||||
`mission_runtime`'s terminal sweeper selects `FROM missions WHERE status IN
|
||||
(…)`, and `teardown_container(mission_id)` is only ever called with an id from
|
||||
that query. Nothing enumerates Docker for `cm-runtime-mission-*` containers with
|
||||
no matching row, so a container whose row is gone is invisible to every reaper.
|
||||
Same shape as the earlier agent-container reap drift, different table.
|
||||
|
||||
**Not yet observed in production** — no prod mission has run since the flip.
|
||||
Prod auth is Clerk, so a mission cannot be launched from here by password. The
|
||||
check when one runs:
|
||||
**It was not idle.** Its checkout held **ten commits on a branch that had never
|
||||
been pushed** — +3451/-30 across 30 files, eighteen INT items on `clawhdf5`
|
||||
including AES-256-GCM, Ed25519 signing and HNSW batch insert. The remote had
|
||||
eight other `clawmates/*` branches and not this one.
|
||||
|
||||
```
|
||||
ssh gw-04 'docker logs clawmates_server_1 2>&1 | grep -E "per-mission runtime image|drained .* tool call"'
|
||||
```
|
||||
Handled: bundled and verified, branch pushed to git.redclaw.dev, confirmed on
|
||||
the remote at the tip (`87039e9`), container removed. 57G → 59G free. The
|
||||
bundle is kept at `/opt/clawmates/rescued/rescue-019ff5b1.bundle`.
|
||||
|
||||
### Three bugs the live test found, all the same shape
|
||||
`mission_runtime::sweep_orphans` now closes the gap, and **that container is
|
||||
why it refuses to reap a checkout holding commits no remote has.** A reaper
|
||||
that deleted on sight would have destroyed all of it silently, as its designed
|
||||
behaviour. Every unanswerable case — docker will not date it, git will not
|
||||
answer, the clock skewed — resolves to *do not reap*.
|
||||
|
||||
Each left every other link looking correct:
|
||||
## What shipped this pass
|
||||
|
||||
1. The settings document pointed `PostToolUse` at a path the installer never
|
||||
wrote. Claude Code does not complain about a missing hook command — it
|
||||
records nothing. A test now compares the document's commands against the
|
||||
files the installer creates.
|
||||
2. The mission container uses `CLAWMATES_RUNTIME_IMAGE`, not the shared
|
||||
`clawmates-runtime` container — it was on an older image whose daemon schema
|
||||
had no `settings` field, so the prop write returned `404 path_not_found`.
|
||||
3. The drain used `connect_with_local_defaults()`; the server reaches Docker
|
||||
through a **socket proxy**, so it failed — and returned `Ok(())` silently.
|
||||
### The tool tap kept the name and discarded the argument
|
||||
|
||||
### CI: build failures were disk, not code
|
||||
The container tier's first measured mission recorded `Bash × 6` and not one of
|
||||
them said what it ran. `vm_tool_tap::parse` read `tool_input` to pull the path
|
||||
out of it and dropped the rest, so every behavioural question about a phase was
|
||||
unanswerable from a record that looked complete.
|
||||
|
||||
Runs 503–506 failed at `build` with an unreadable log, and the first casualty
|
||||
was a **docs-only** commit. Cause: building runtime images by hand on gw-04
|
||||
competes with CI for the same 150G volume; the frontend image build lost.
|
||||
`docker builder prune` reclaimed 34GB (22G → 57G free) and the next run went
|
||||
green. The build job now writes breadcrumbs, a `df -h` snapshot, and which
|
||||
services actually pushed — the failing runs had pushed `server`, aborted on
|
||||
`frontend`, and left `:latest` unmoved, which surfaced three steps later as
|
||||
"the deploy did not happen".
|
||||
`Observed.input` now keeps it, bounded: file bodies become a byte count, other
|
||||
long strings truncate with a marker. **Host-side only, no image rebuild** — the
|
||||
arguments were always in the tap file. `tool.call` also gained `detail.path`
|
||||
(the World's SSE reads it and had been getting null on every container-tier
|
||||
call) and `file.touch` gained `detail.abs`.
|
||||
|
||||
**Operational note:** do not build images by hand on gw-04 while CI may run.
|
||||
### Skill-Use is scored from actions
|
||||
|
||||
`skill_use::Evidence` carries `tool.call` rows alongside the narrative, and
|
||||
every check prefers them. `workspace-repo-commit-protocol`'s boundary was a
|
||||
substring search for `/workspace/repo` in prose — an agent that wrote to the
|
||||
wrong root **without narrating it scored a clean pass**. Two verdicts changed
|
||||
for honesty: silence is `NotObservable` rather than `Pass`, and a test that ran
|
||||
after the first write is undecidable rather than a failure.
|
||||
|
||||
**Trigger is still `NotObservable`, and half of its old reason is now wrong.**
|
||||
"`claude_cli` cannot surface a tool call" is false. What still holds is that we
|
||||
**inline** skill bodies, so there is no retrieval to observe. The blocker moved
|
||||
from the transport to the delivery model, and the door (§3) closes it with no
|
||||
scorer change at all.
|
||||
|
||||
### Research phases are staffed by a research team
|
||||
|
||||
`research_only` — repo-less, one research phase — defaulted to `rust_sdlc`, so
|
||||
it was staffed with a planner, coder, tester, reviewer and committer, four of
|
||||
whom had nothing to do. New `topic_research` team, plus `default_phase_teams`
|
||||
so a recipe can staff each phase *purpose* separately. Measured: 5 roles → 3,
|
||||
14 skill deliveries → 4, 50KB of prompt → 24KB, and **1 of 9 delivered skills
|
||||
applicable → 4 of 4**.
|
||||
|
||||
The scores barely moved, and that is the honest reading: what changed is that
|
||||
`not_applicable` now means "no machine-checkable consequence" rather than "this
|
||||
skill had nothing to do with this phase".
|
||||
|
||||
Three existing research templates were also wrong in ways nothing checked.
|
||||
`papers_research` bound **`arxiv-daily`** — a skill whose content is "do not
|
||||
search arXiv yourself" — to the DOMAIN SCOUT, the role whose job is searching.
|
||||
Its PAPER READER was told to "fetch the PDF, extract text"; the runtime image
|
||||
has no pdftotext, no mutool and no pypdf, so every paper would have hit the
|
||||
`[read: abstract only]` fallback, which reads exactly like the fallback working.
|
||||
`insight_research` cross-referenced "our repos'" history when a mission binds
|
||||
one. `codebase_research` wrote to a vault that is not mounted.
|
||||
|
||||
### The wrong repo path was in the team templates too
|
||||
|
||||
The `/workspace/repo` guard was written against `skills/` only. The same path
|
||||
was in four team templates — including `rust_sdlc`, default for five of six
|
||||
recipes, whose coder was told "your working directory is /workspace/repo". The
|
||||
guards now walk one corpus: skills, team templates and recipes together.
|
||||
|
||||
### Two more skills contradicted the platform
|
||||
|
||||
Both found by reading the source of truth before writing a check against it —
|
||||
which is the only reason they were found.
|
||||
|
||||
- `decompose-int-items` taught `PLAN_COMPLETE: INT-01..05`. Ids are strictly
|
||||
`INT-<digits>`, so the range form is rejected and the plan pass records
|
||||
nothing while every item stays open.
|
||||
- `workspace-repo-commit-protocol` claimed the task-card parser advances mission
|
||||
state on the INT id in your commit subject. **Nothing in the platform reads
|
||||
commit messages** — `apply_for_run` reads `run_events`, the turn output.
|
||||
|
||||
`no_skill_shows_a_marker_the_parser_would_reject` guards the class, running the
|
||||
real parser over every marker in every skill's fenced blocks.
|
||||
|
||||
## Next, in order
|
||||
|
||||
1. **Watch the first production mission.** Nothing has run since the runtime
|
||||
flip, so the container-tier gate and tap are proven locally and unproven in
|
||||
prod. Prod auth is Clerk, so a mission cannot be launched from here.
|
||||
```
|
||||
ssh gw-04 'docker logs clawmates_server_1 2>&1 | grep -E "per-mission runtime image|drained .* tool call"'
|
||||
ssh gw-04 'docker exec <cm-runtime-mission-…> cat /root/toolhooks/tap/tools.jsonl | head'
|
||||
```
|
||||
If tools appear in `mission_events`, the loop is closed. If not, check the
|
||||
three failure shapes listed above — each looked correct from every other
|
||||
angle.
|
||||
1. **The TDD check cannot confirm red-first, and that is structural.** Run 4
|
||||
(`research_and_code`, real repo) edited `src/lib.rs` once — implementation
|
||||
*and* `#[cfg(test)] mod tests` in the same write — then ran `cargo test`
|
||||
five times. In Rust the unit test lives in the file under test, so that
|
||||
ordering is what following the skill precisely looks like from outside. The
|
||||
check detects "wrote source, never ran a test" and nothing more. If you want
|
||||
red-first, it needs the diff (did the test exist before the impl?), not the
|
||||
tool order.
|
||||
|
||||
2. **Now that tool calls are observable, redo the Skill-Use measurement.**
|
||||
`docs/SKILL-USE-BASELINE.md` reports Trigger as `not_observable` on the
|
||||
container tier because there was no tool evidence. There is now.
|
||||
`mission_events` carries `tool.call` and `file.touch` per phase, so
|
||||
Trigger can be scored from behaviour instead of prose — which is what the
|
||||
paper actually measures. This is the single highest-value follow-up: it
|
||||
turns the baseline from "the first honest number" into a real one.
|
||||
2. **Attribute tool calls to agents.** `record_vm_tools` writes
|
||||
`agent_id: None`, because the container tap is per-container and all roles
|
||||
share one. Every Skill-Use score is therefore per-**mission**, not per-role,
|
||||
and the World's per-agent view gets nothing from the container tier. The
|
||||
hook payload carries `session_id`; mapping it back to a turn is the fix.
|
||||
|
||||
3. **Give the microVM tier the same treatment, or retire the difference.** It
|
||||
has the gate and a tap already, but by a different route (`vm_tool_tap`
|
||||
installs into the guest, `microvm_executor` drains inside the turn). Two
|
||||
mechanisms for one job is how they drift. Worth folding onto
|
||||
`container_tool_hooks` once the fleet is back — tank and morpheus have been
|
||||
offline for over a week, so the microVM tier cannot be tested at all today.
|
||||
3. **Deploy the door** (`TOOL-CALL-ARCHITECTURE.md` §3). Config, not code:
|
||||
`/zeroclaw-data/clawmates-mcp.json` plus a door-shaped provider alias. It is
|
||||
now the single change that makes **Trigger** a real measurement, and the
|
||||
precondition for skills moving from inlined bodies to progressive
|
||||
disclosure — which would also cut the prompt cost in item 1.
|
||||
|
||||
4. **Deploy the door** (`docs/TOOL-CALL-ARCHITECTURE.md` §3). Config, not code:
|
||||
`/zeroclaw-data/clawmates-mcp.json` plus a door-shaped provider alias. Now
|
||||
less urgent than it looked — the gate no longer depends on it — but it is
|
||||
still the precondition for `clawmates_skills` being reachable, and therefore
|
||||
for skills moving from inlined bodies to progressive disclosure.
|
||||
4. **Fold the microVM tier onto `container_tool_hooks`.** It has a gate and a
|
||||
tap by a different route (`vm_tool_tap` installs into the guest,
|
||||
`microvm_executor` drains inside the turn). Two mechanisms for one job is how
|
||||
they drift — and the argument-discarding bug above lived in the shared parser
|
||||
precisely because nobody looked at it from the container side. The fleet has
|
||||
been offline for over a week, so this cannot be tested today.
|
||||
|
||||
5. **Pull upstream's egress policy** — `0db7d999a feat(plugins): add shared
|
||||
egress policy foundation (#9137)`. We are ~220 commits behind; this is the
|
||||
one item identified as worth taking, and it is defence for a problem we have
|
||||
not solved.
|
||||
|
||||
**Dropped from this list:** "give the direct-session tier a tap". That tier is
|
||||
dormant — `CLAWMATES_MISSION_EXECUTOR` is unset in production, so it never
|
||||
runs. Checking that before building for it saved the work.
|
||||
one item worth taking, and it is defence for a problem we have not solved.
|
||||
|
||||
## Open decisions that are yours
|
||||
|
||||
- **Self-authoring scope.** Agents now apply their own `skill_candidate` items
|
||||
with no human click (`CLAWMATES_SKILL_SELF_AUTHORING=0` restores the gate).
|
||||
- **Push.** 10 commits are local. Pushing `main` triggers CI → auto-deploy to
|
||||
gw-04.
|
||||
- **Self-authoring scope.** Agents apply their own `skill_candidate` items with
|
||||
no human click (`CLAWMATES_SKILL_SELF_AUTHORING=0` restores the gate).
|
||||
`identity_refinement` and `brain_consolidation` still wait for a human,
|
||||
because they change what an agent IS rather than adding a procedure it can
|
||||
consult. Say if you want those autonomous too.
|
||||
- **Skill-Use Compliance coverage.** Most skills still score `not_applicable` —
|
||||
we cannot tell whether they changed anything. `small-focused-commits` and
|
||||
`tdd-red-green-refactor` are the next candidates and both need the repository
|
||||
diff rather than the turn text.
|
||||
consult.
|
||||
|
||||
## Deliberately not done
|
||||
|
||||
- **The mission executor swap** (running turns through `ProviderExecutor` or the
|
||||
chat `Runtime`). The blockers are structural, not wiring: `cm-runtime`'s
|
||||
`files` tool rejects absolute paths *by construction*, `shell` runs in a
|
||||
per-agent sandbox with no mission mount, `ToolContext` carries no path or VM
|
||||
handle, and approvals key on `(session_id, message_id)`. The cheap fixes
|
||||
deliver what it was wanted for.
|
||||
- **`cm-brain` offline tests** — 6 of 9 need live `clawbrainhub.com`. Stubbing
|
||||
means reproducing an external registry protocol we have no spec for.
|
||||
- **The mission executor swap.** Blockers are structural: `cm-runtime`'s `files`
|
||||
tool rejects absolute paths by construction, `shell` runs in a per-agent
|
||||
sandbox with no mission mount, `ToolContext` carries no path or VM handle, and
|
||||
approvals key on `(session_id, message_id)`.
|
||||
- **A tap for the direct-session tier.** Dormant —
|
||||
`CLAWMATES_MISSION_EXECUTOR` is unset in production, so it never runs.
|
||||
- **`cm-brain` offline tests** — 6 of 9 need live `clawbrainhub.com`.
|
||||
- **Graph memory / `clawhdf5-agent`** — in the workspace manifest, used by no
|
||||
crate. Measure against a baseline before migrating.
|
||||
crate.
|
||||
|
||||
## Operational facts that cost time to learn
|
||||
|
||||
- The Gitea **actions-log API returns 403** for the token in
|
||||
`deploy/compose/.env`. Every CI failure this week was debugged blind because
|
||||
of it. Steps now write to `/tmp/ci-logs` on the runner host as a workaround;
|
||||
**a token with the `actions` scope** remains the highest-value thing to
|
||||
obtain.
|
||||
- **gw-04 uses legacy `docker-compose`**, not the v2 plugin. `docker compose`
|
||||
fails there.
|
||||
`deploy/compose/.env`. A token with the `actions` scope remains the
|
||||
highest-value thing to obtain.
|
||||
- **gw-04 uses legacy `docker-compose`**, not the v2 plugin.
|
||||
- **Do not build images by hand on gw-04 while CI may run** — same 150G volume,
|
||||
and the frontend image build is what loses.
|
||||
- The server reaches Docker through a **socket proxy** (`DOCKER_HOST`). Use
|
||||
`container_exec::connect()`, never `connect_with_local_defaults()`.
|
||||
- Prod auth is **Clerk**; the bootstrap password in `deploy/compose/.env` works
|
||||
only against the local stack.
|
||||
- Rebuilding the local server image is a **full Rust compile inside Docker**
|
||||
(~8 min); the layer cache does not preserve `target/`. Budget for it before
|
||||
any measurement that needs new server code.
|
||||
- macOS has no `timeout(1)`.
|
||||
|
||||
## Two corrections made this session, worth remembering
|
||||
## The recurring shape, now seven times over
|
||||
|
||||
- **"Missions can't call tools at all" was wrong.** They call `Bash` and `Write`
|
||||
with permissions pre-accepted. The gap was observing and gating, not having.
|
||||
- **Raw test counts are a bad coverage metric.** They pointed at `cm-safety`,
|
||||
whose seven tests already covered its critical paths, and missed a Slack
|
||||
replay hole that let one captured request authenticate forever.
|
||||
**A claim in a comment or a doc, believed and never checked.** Every significant
|
||||
finding this pass came from reading the source of truth — the parser, the
|
||||
recipe, the production table — rather than the text describing it. The two new
|
||||
skill contradictions were found *while writing checks against those skills*,
|
||||
which is the cheapest place to catch them and the reason to always read first.
|
||||
|
||||
The recurring shape, now seven times over: **a claim in a comment or a doc,
|
||||
believed and never checked.** Every significant finding this session came from
|
||||
running the thing rather than reading about it.
|
||||
The corollary the measurement itself demonstrated: **its own first verdict was
|
||||
wrong**, and scoring a research phase as a TDD failure would have buried the
|
||||
real finding (item 1). A check that reports a system defect as an agent defect
|
||||
is worse than no check.
|
||||
|
||||
+261
-112
@@ -1,7 +1,7 @@
|
||||
# Skill-Use baseline
|
||||
|
||||
*First measurement of whether ClawMates' skills change what agents do.
|
||||
2026-08-19.*
|
||||
*Whether ClawMates' skills change what agents do. First measured 2026-08-19;
|
||||
re-measured 2026-08-21 against tool evidence rather than agent prose.*
|
||||
|
||||
Scored on the three axes from `Skill-Use` (arXiv, 2026-08-05): **Trigger** (did
|
||||
the agent reach for the skill), **Compliance** (did it follow the procedure),
|
||||
@@ -10,165 +10,314 @@ the agent reach for the skill), **Compliance** (did it follow the procedure),
|
||||
Read the method before the numbers. A measurement whose limits are not stated
|
||||
is worse than none, because it gets quoted without them.
|
||||
|
||||
## Why there was no baseline before today
|
||||
## What changed on 2026-08-21
|
||||
|
||||
Not because nobody ran it. Because **it could not have returned anything but
|
||||
zero**, for two structural reasons that had nothing to do with agent behaviour:
|
||||
The 2026-08-19 measurement scored Compliance and Boundary from the
|
||||
`reasoning` events — the agent's own account of its turn. Since then the
|
||||
container tier records what agents actually **do**
|
||||
(`container_tool_hooks`, `PostToolUse`), and `vm_tool_tap` stopped throwing the
|
||||
tool's arguments away, so `Bash` commands and `Write` paths are on the record.
|
||||
|
||||
1. 55 of 85 role skill bindings resolved to skills that were never authored.
|
||||
2. Even resolved skills had no delivery channel to a mission agent — the
|
||||
catalogue's only route was an MCP server that mission claws cannot reach.
|
||||
`skill_use` now reads those. The difference is not cosmetic:
|
||||
|
||||
Both were fixed in the two commits preceding this document. Anyone who had run
|
||||
this measurement in July would have concluded "our agents ignore their skills",
|
||||
which would have been false and expensive.
|
||||
- `workspace-repo-commit-protocol`'s Boundary was a substring search for
|
||||
`/workspace/repo` in the narrative. **An agent that wrote to the wrong root
|
||||
without narrating it scored a clean pass.** It now reads the write paths.
|
||||
- `arxiv-daily`'s Boundary read a URL in prose, which may be the agent
|
||||
explaining that it did *not* fetch it. It now reads the `curl` that ran.
|
||||
- `tdd-red-green-refactor`, `cargo-test-driven-development` and
|
||||
`small-focused-commits` gained their first checks at all.
|
||||
|
||||
Two verdicts changed for honesty rather than coverage. **Silence used to score
|
||||
`Pass`** — a mission with no evidence scored identically to one checked and
|
||||
found clean; it is now `NotObservable`. And a test that ran *after* the first
|
||||
write is `NotObservable`, not a failure, because a Rust unit test lives in the
|
||||
file under test.
|
||||
|
||||
### Trigger: the reason changed, and only half of it went away
|
||||
|
||||
The 2026-08-19 document said Trigger was unobservable because `claude_cli`
|
||||
"cannot surface a tool call — there is nothing to retrieve *with*."
|
||||
|
||||
**That half is now false.** Mission tool calls are recorded on both tiers; a
|
||||
retrieval would be as visible as any other call.
|
||||
|
||||
The other half still holds and is the one that decides the verdict: **we still
|
||||
inline**. `pinned_skills_text` puts full skill bodies in the prompt, so the
|
||||
agent never reaches for anything — it is simply holding one. Trigger is now
|
||||
*instrumentable* and still not *observable*, and the blocker has moved from the
|
||||
transport to the delivery model.
|
||||
|
||||
Making it real is one change and no scorer work: serve skills through the door
|
||||
(`TOOL-CALL-ARCHITECTURE.md` §3) so retrieval becomes a tool call.
|
||||
|
||||
## Method, and what it cannot see
|
||||
|
||||
Scored by `cm_api::skill_use` from what the platform records: the
|
||||
`prompt.composed` event (the exact bytes an agent received) and the `reasoning`
|
||||
events (what it said it did). No re-derivation from the catalogue — the
|
||||
catalogue changes, and now that agents author their own skills it changes by
|
||||
itself.
|
||||
Scored by `cm_api::skill_use` from what the platform records: `prompt.composed`
|
||||
(the exact bytes an agent received), `reasoning` (what it said it did), and
|
||||
`tool.call` (what it did). No re-derivation from the catalogue — the catalogue
|
||||
changes, and now that agents author their own skills it changes by itself.
|
||||
|
||||
### Trigger is not observable here, and that is a finding
|
||||
|
||||
The paper measures agents under **progressive disclosure**: the agent sees a
|
||||
name and description and must decide to retrieve the body. That retrieval is a
|
||||
tool call, which makes Trigger observable.
|
||||
|
||||
We do not deliver skills that way. `pinned_skills_text` inlines full bodies into
|
||||
the prompt, because mission claws run on `claude_cli`, which cannot surface a
|
||||
tool call — there is nothing to retrieve *with*. The agent never reaches for a
|
||||
skill; it is simply holding one.
|
||||
|
||||
So Trigger is reported as `not_observable` with the reason attached, **never as
|
||||
zero**. Scoring it zero would report a delivery-model property as an agent
|
||||
failure — the same confusion that kept 55 empty bindings invisible.
|
||||
|
||||
### Compliance and Boundary are checked mechanically, or not at all
|
||||
**Every tool-backed check is one-sided.** It reports a violation it can see and
|
||||
never infers compliance from silence: the recorded stream is capped per phase
|
||||
(`PER_PHASE_CAP = 400`), so an absent call is not proof of an absent action.
|
||||
|
||||
Only skills whose procedure has a machine-checkable consequence are scored.
|
||||
Everything else returns `not_applicable` rather than a guess: a heuristic that
|
||||
Everything else returns `not_applicable` rather than a guess — a heuristic that
|
||||
scores prose by keyword overlap produces a number that looks like a measurement
|
||||
and is not one.
|
||||
|
||||
Compliance for `int-xx-marker-protocol` is checked by running the **real**
|
||||
`task_card_parser`, not a copy of its rules — a second implementation would
|
||||
`task_card_parser`, not a copy of its rules; a second implementation would
|
||||
drift, and then the score would pass while the mission loop still stalled.
|
||||
|
||||
## The runs
|
||||
|
||||
Two missions on the container/ZeroClaw tier, local stack, `research_only`.
|
||||
Five missions on the container/ZeroClaw tier, local stack. Runs 1–3 are
|
||||
`research_only`; run 3 uses the **same task text as run 2**, so the only
|
||||
variable is the scorer. Run 4 is `research_and_code` against a real repository,
|
||||
because a research mission writes no code and makes no commits — the TDD and
|
||||
commit checks could never fire on one.
|
||||
|
||||
| | run 1 | run 2 |
|
||||
|---|---|---|
|
||||
| distinct skills delivered | 3 | **9** |
|
||||
| total deliveries (per role prompt) | 3 | 14 |
|
||||
| phantom "skills" scored | **2** | 0 |
|
||||
| | run 1 | run 2 | run 3 | run 4 |
|
||||
|---|---|---|---|---|
|
||||
| date | 08-19 | 08-19 | 08-21 | 08-21 |
|
||||
| workflow | research | research | research | **code** |
|
||||
| distinct skills delivered | 3 | 9 | 9 | 9 |
|
||||
| total deliveries (per role prompt) | 3 | 14 | 14 | 28 |
|
||||
| phantom "skills" scored | **2** | 0 | 0 | 0 |
|
||||
| tool calls recorded | 0 | 0 | **49** | **33** |
|
||||
| writes outside `/mission/repo` | ? | ? | 0 | 0 |
|
||||
|
||||
Run 1's phantom entries are the finding of the run, described below.
|
||||
|
||||
Run 2, per skill (all `source_kind=builtin`; no agent-authored skill has been
|
||||
Run 3, per skill (all `source_kind=builtin`; no agent-authored skill has been
|
||||
delivered yet):
|
||||
|
||||
| skill | deliveries | compliance | boundary |
|
||||
|---|---|---|---|
|
||||
| `int-xx-marker-protocol` | 1 | **pass** | n/a |
|
||||
| `small-focused-commits` | 4 | n/a | n/a |
|
||||
| `workspace-repo-commit-protocol` | 2 | n/a | **pass** *(from 12 write paths)* |
|
||||
| `small-focused-commits` | 4 | n/a | n/a *(no commit ran)* |
|
||||
| `cargo-test-driven-development` | 2 | n/a | n/a |
|
||||
| `workspace-repo-commit-protocol` | 2 | n/a | n/a |
|
||||
| `tdd-red-green-refactor` | 1 | n/a | n/a |
|
||||
| `decompose-int-items` | 1 | n/a | n/a |
|
||||
| `write-rust-current-edition` | 1 | n/a | n/a |
|
||||
| `code-review-checklist` | 1 | n/a | n/a |
|
||||
| `criterion-benchmarking` | 1 | n/a | n/a |
|
||||
| `tdd-red-green-refactor` | 1 | n/a | n/a |
|
||||
|
||||
**n = 2 runs. No spread is reported because two runs cannot establish one.**
|
||||
This is a baseline in the sense of "the first honest number", not in the sense
|
||||
of `metrics-baseline-comparison.md`, which requires enough runs to see the noise
|
||||
floor before any change is judged against it. Do not compare a future number to
|
||||
this one without first establishing that floor.
|
||||
**n = 5 runs. No spread is reported because five cannot establish one.** This
|
||||
is a baseline in the sense of "the first honest number", not in the sense of
|
||||
`metrics-baseline-comparison.md`, which requires enough runs to see the noise
|
||||
floor before any change is judged against it.
|
||||
|
||||
`workspace-repo-commit-protocol`'s pass is the one score that materially
|
||||
improved: it now rests on twelve recorded `Write`/`Edit` paths, every one under
|
||||
`/mission/repo`, instead of on the absence of a string in prose.
|
||||
|
||||
### Run 4 — the first run that could have violated the new checks
|
||||
|
||||
`research_and_code` against `clawmates-delivery-scratch`, task: add a `slugify`
|
||||
utility and commit it. The task says nothing about testing; priming it would
|
||||
have measured the prompt rather than the skill.
|
||||
|
||||
| skill | deliveries | compliance | boundary |
|
||||
|---|---|---|---|
|
||||
| `int-xx-marker-protocol` | 2 | **pass** | n/a |
|
||||
| `workspace-repo-commit-protocol` | 4 | n/a | **pass** |
|
||||
| `cargo-test-driven-development` | 4 | **not observable** | n/a |
|
||||
| `tdd-red-green-refactor` | 2 | **not observable** | n/a |
|
||||
| `small-focused-commits` | 8 | n/a | n/a |
|
||||
| the other four | 2 each | n/a | n/a |
|
||||
|
||||
The agents edited `src/lib.rs` once, ran `cargo test` five times, and committed
|
||||
with `INT-01` on the subject. Every one of 33 tool calls stayed inside
|
||||
`/mission/repo`.
|
||||
|
||||
**The TDD verdict is `not_observable`, and that is the honest answer rather
|
||||
than a gap in the run.** The single `Edit` to `src/lib.rs` added the
|
||||
implementation *and* a `#[cfg(test)] mod tests` block, then the tests ran. In
|
||||
Rust the unit test lives in the file under test, so "wrote the file, then ran
|
||||
the test" is exactly what writing the failing test first looks like from the
|
||||
outside. The check therefore detects one thing only — **a phase that wrote
|
||||
source and never ran a test at all** — and cannot confirm red-first. That is a
|
||||
real limit of scoring TDD from tool ordering, and it applies to the most common
|
||||
Rust shape, not an edge case.
|
||||
|
||||
#### The parsing bug run 4 found
|
||||
|
||||
Claude Code writes a multi-line commit message as
|
||||
|
||||
```
|
||||
git commit -m "$(cat <<'EOF'
|
||||
INT-01 Add slugify function to src/lib.rs
|
||||
…
|
||||
EOF
|
||||
)"
|
||||
```
|
||||
|
||||
and `commit_subjects` read the first line of the `-m` value — which is the
|
||||
heredoc *opener*, `$(cat <<'EOF'`. Every commit check was scoring a string the
|
||||
agent never wrote. It happened to score no violation, because `$(cat <<'EOF'`
|
||||
is not one of the never-merge messages; that is luck, not a check. Fixed, with
|
||||
a regression test built from the exact command in `mission_events`.
|
||||
|
||||
The verdicts in the table above are unchanged by the fix — the real subject,
|
||||
`INT-01 Add slugify function to src/lib.rs`, is not a never-merge message
|
||||
either — so the table reproduces against the shipped scorer.
|
||||
|
||||
## What the measurement found
|
||||
|
||||
Four defects, none of which any test or log would have surfaced.
|
||||
### 1–4: the 2026-08-19 findings
|
||||
|
||||
### 1. The prompt format made its own record unparseable
|
||||
Four defects, none of which any test or log would have surfaced: the prompt
|
||||
format made its own record unparseable (`## <name>` against markdown bodies);
|
||||
a prompt was recorded that was never sent; a pinned skill taught
|
||||
`/workspace/repo`, a path the platform does not mount; and
|
||||
`int-xx-marker-protocol` documented a `PLAN_COMPLETE` marker the parser had
|
||||
never implemented. All four are fixed, with guards in
|
||||
`skills_loader::contradiction_tests` and `topology_exec`. The detail is in this
|
||||
file's git history.
|
||||
|
||||
Skills were introduced with `## <name>`, and skill bodies are markdown full of
|
||||
`##` headings. Run 1 duly scored **"Sizing heuristic"** and **"The output
|
||||
shape"** — both subheadings inside `decompose-int-items` — as skills with no
|
||||
catalogue row.
|
||||
### 5. The tool tap recorded the name and discarded the argument
|
||||
|
||||
Fixed with an unambiguous `--- SKILL: <name> ---` marker, and both writers now
|
||||
share one renderer so the reader cannot drift from the writer.
|
||||
The first container-tier mission with telemetry recorded `Bash × 6` and not one
|
||||
of them said what it ran. `vm_tool_tap::parse` read `tool_input` to pull the
|
||||
path out of it and dropped the rest.
|
||||
|
||||
### 2. A prompt was recorded that was never sent
|
||||
Every behavioural question was therefore unanswerable from a record that looked
|
||||
complete — which is the recurring shape, not a new one. Fixed host-side: the
|
||||
arguments were always in the tap file.
|
||||
|
||||
The phase prompt was recorded at the dispatch fork, before the tier was chosen.
|
||||
The container tier does not send that text — it sends the bare task and appends
|
||||
skills per turn. So every container mission logged a `solo` prompt that reached
|
||||
no agent.
|
||||
### 6. Two more skills contradicted the platform
|
||||
|
||||
A provenance record of something that did not happen is worse than no record: it
|
||||
is the wrong answer, delivered confidently. Recording now happens inside each
|
||||
tier, and a test asserts every launcher records the prompt it actually sends.
|
||||
Same class as finding 3, and both found by reading the source of truth before
|
||||
writing a check against it.
|
||||
|
||||
### 3. A pinned skill contradicted the platform in the same prompt
|
||||
- **`decompose-int-items` taught `PLAN_COMPLETE: INT-01..05`.** An id is
|
||||
strictly `INT-` plus digits, so the range form is rejected outright: the plan
|
||||
pass records nothing while every item stays open. A live planner emitted
|
||||
exactly that line.
|
||||
- **`workspace-repo-commit-protocol` claimed the task-card parser advances
|
||||
mission state on the INT id in your commit subject.** Nothing in the platform
|
||||
reads commit messages. `task_card_parser::apply_for_run` reads `run_events` —
|
||||
the agent's turn output. An agent that believed this would commit with the id,
|
||||
never emit `COMPLETED: INT-NN`, and leave the mission open on an item it had
|
||||
already finished.
|
||||
|
||||
`workspace-repo-commit-protocol` told agents that **`/workspace/repo`** was "the
|
||||
ONLY path where source-modifying edits belong". The platform mounts and
|
||||
advertises **`/mission/repo`** — 26 references in the code; `/workspace/repo`
|
||||
appears in none. The skill is bound on **29 role bindings** and was delivered
|
||||
twice in run 2, so agents received the real path in the tool preamble and a
|
||||
skill contradicting it a few hundred tokens later.
|
||||
`no_skill_shows_a_marker_the_parser_would_reject` now runs the real parser over
|
||||
every marker in every skill's fenced blocks, negative-controlled against the
|
||||
range form.
|
||||
|
||||
It also instructed `file_read` / `file_write` / `shell` — ZeroClaw's tool names,
|
||||
the exact ones `phase_task_text` was fixed to stop advertising after five agents
|
||||
on one mission spent 7.4k tokens describing the mismatch instead of working.
|
||||
### 7. A repo-less research mission is staffed with a Rust SDLC crew
|
||||
|
||||
An agent that obeyed this skill wrote source into a directory nothing collects,
|
||||
and reached for tools its subprocess does not expose. Rewritten against what the
|
||||
code actually does, with two guards in `skills_loader::contradiction_tests`: no
|
||||
skill may name a repo path the platform does not mount, and none may instruct a
|
||||
tool the agent does not have. Both negative-controlled.
|
||||
This is the finding of run 3, and it explains most of the `not_applicable`
|
||||
column above.
|
||||
|
||||
This is the same shape as the finding below and it is worth stating as a class:
|
||||
**the skills were never checked against the platform they describe.** Nothing
|
||||
compared them, so a skill could contradict the prompt it ships inside and stay
|
||||
that way indefinitely.
|
||||
`templates/workflows/research_only.toml` declares `requires_repo = false` and a
|
||||
single `research` phase — and `default_team_template = "rust_sdlc"`. So the
|
||||
mission was staffed with **planner, coder, tester, reviewer, committer**, and
|
||||
each received the skills its role is bound to:
|
||||
|
||||
### 4. The skill documents a marker the platform never implemented
|
||||
```
|
||||
coder :: write-rust-current-edition, cargo-test-driven-development,
|
||||
workspace-repo-commit-protocol, small-focused-commits,
|
||||
int-xx-marker-protocol
|
||||
tester :: cargo-test-driven-development, criterion-benchmarking,
|
||||
tdd-red-green-refactor
|
||||
committer :: workspace-repo-commit-protocol, small-focused-commits
|
||||
reviewer :: code-review-checklist, small-focused-commits
|
||||
planner :: decompose-int-items, small-focused-commits
|
||||
```
|
||||
|
||||
`int-xx-marker-protocol` lists `PLAN_COMPLETE: INT-NN` in its ladder.
|
||||
`task_card_parser` has **no such kind** and never has. An agent following the
|
||||
skill exactly emits a marker that is silently ignored.
|
||||
There is no repository, nothing to test, nothing to review and nothing to
|
||||
commit. Four of the five roles have no work, and 50KB of prompt (~12.6k tokens)
|
||||
is spent staffing them.
|
||||
|
||||
Observed live: run 2's planner emitted `PLAN_COMPLETE: INT-01..02`, which is
|
||||
also the range form — on the kinds that *are* parsed, that yields the id
|
||||
`INT-01..02`, a task card for an item that does not exist while the two real
|
||||
items stay open.
|
||||
**The skills are correctly bound to the roles. The roles are wrong for the
|
||||
workflow.** That distinction matters: a reader who saw only "7 of 9 skills
|
||||
scored not_applicable" would conclude the skills are useless, when what the
|
||||
number actually measures is a staffing default.
|
||||
|
||||
**This is a skill/implementation mismatch, not an agent failure**, and it is
|
||||
precisely what this measurement exists to find: the agent did what it was told,
|
||||
and what it was told was wrong. Both shapes are now scored as failures; the
|
||||
underlying reconciliation — implement `PLAN_COMPLETE` or remove it from the
|
||||
skill — is deliberately left as a decision rather than guessed at here.
|
||||
**Fixed the same day, and measured again as run 5.** `research_only` now
|
||||
defaults to a new `topic_research` team, and `default_phase_teams` lets a
|
||||
recipe staff each phase *purpose* separately — `research_and_code` and
|
||||
`security_hardening` give their research phases the research team and keep
|
||||
`rust_sdlc` for coding. `benchmark` and `refactor` were checked and left alone:
|
||||
one coding-purpose phase each, correctly staffed already.
|
||||
|
||||
Run 5 is run 3's task, re-run against the new staffing:
|
||||
|
||||
| | run 3 | run 5 |
|
||||
|---|---|---|
|
||||
| roles staffed | 5 | **3** |
|
||||
| distinct skills delivered | 9 | **4** |
|
||||
| total deliveries | 14 | **4** |
|
||||
| prompt bytes across roles | 50,449 | **24,065** |
|
||||
| skills applicable to the phase | 1 of 9 | **4 of 4** |
|
||||
|
||||
**Read the last row carefully, and not the ones above it.** The *scores* barely
|
||||
moved: run 5 has one `pass` and three `not_applicable`. What changed is what
|
||||
`not_applicable` now means. In run 3 it mostly meant "this skill had nothing to
|
||||
do with what this phase was doing"; in run 5 it means "this skill's procedure
|
||||
has no machine-checkable consequence" — which is the honest, permanent reason,
|
||||
and the one this measurement was designed to report.
|
||||
|
||||
Cutting the prompt in half is real but incidental. The finding is that the
|
||||
denominator was wrong: seven of the nine skills in run 3 were never applicable,
|
||||
so any ratio computed over them measured staffing, not skill use.
|
||||
|
||||
The agents also produced exactly the structure the new team's task specifies —
|
||||
`research/questions.md`, `research/evidence.md`, `research/REPORT.md` — with
|
||||
zero writes outside `/mission/repo`.
|
||||
|
||||
### 8. The check that got it wrong first
|
||||
|
||||
Run 3's first scoring reported `cargo-test-driven-development` and
|
||||
`tdd-red-green-refactor` as **compliance = fail**: files were written and no
|
||||
test ever ran.
|
||||
|
||||
That verdict was wrong, and wrong in the way this whole document exists to
|
||||
prevent. The phase wrote fifteen markdown notes and a helper script. There was
|
||||
no code to test-drive. Reporting it as an agent failure would have been a system
|
||||
defect wearing an agent's name — and it would have buried the real finding,
|
||||
which is finding 7 above.
|
||||
|
||||
The check is now scoped to files with a source extension in the languages the
|
||||
skill itself names. It is recorded here rather than quietly corrected, because
|
||||
a measurement that hides its own false positives cannot be trusted about
|
||||
anyone else's.
|
||||
|
||||
## Honest limits
|
||||
|
||||
- **Two runs, one tier, one workflow.** Nothing here generalises to the microVM
|
||||
or session tiers yet, and this document should not be read as if it does.
|
||||
- **Most skills score `not_applicable`** on both observable axes. That is not a
|
||||
pass. It means we cannot currently tell whether those skills changed anything.
|
||||
`workspace-repo-commit-protocol` now has a Boundary check (writing outside
|
||||
`/mission/repo`); `small-focused-commits` and `tdd-red-green-refactor` remain
|
||||
candidates, and both need the repository diff rather than the turn text.
|
||||
- **No agent-authored skill has been measured.** Self-authoring shipped in the
|
||||
same pass; `source_kind` is carried through the scorer specifically so a
|
||||
rising score on agent-authored skills is visible rather than averaged in.
|
||||
- **Five runs, one tier, two workflows.** Nothing here generalises to the
|
||||
microVM or session tiers.
|
||||
- **The TDD check is one-sided and the common Rust case is undecidable.** It
|
||||
catches "wrote source, never ran a test". It cannot confirm red-first,
|
||||
because a Rust unit test lives in the file under test — see run 4.
|
||||
- **Four runs, and run 4 is the only coding one.** The commit checks have been
|
||||
*reached* live exactly once.
|
||||
- **Tool calls carry no agent attribution.** `record_vm_tools` writes
|
||||
`agent_id: None` — the container tier's tap is per-container, and all five
|
||||
roles share one container. Every score above is therefore per-**mission**,
|
||||
not per-role, and the World's per-agent view gets nothing from it. Mapping
|
||||
the hook payload's `session_id` back to a turn would fix it.
|
||||
- **Most skills still score `not_applicable`** on both observable axes. That is
|
||||
not a pass. See finding 7 for why the number is what it is.
|
||||
- **No agent-authored skill has been measured.** `source_kind` is carried
|
||||
through the scorer specifically so a rising score on agent-authored skills is
|
||||
visible rather than averaged in.
|
||||
- **Evidence expires.** Mission events are reaped after 7 days unless
|
||||
`retain_events_until` is set. Both runs here are held for 90 days. An empty
|
||||
score means "no evidence", never "no compliance", and the API says so in its
|
||||
payload rather than leaving the caller to infer it.
|
||||
`retain_events_until` is set; `scripts/skill-use-run.sh` holds every run for
|
||||
90 days so it stays re-scorable when the scorer changes again — which is
|
||||
exactly what happened to run 3. An empty score means "no evidence", never "no
|
||||
compliance", and the API says so in its payload.
|
||||
|
||||
## Reproducing
|
||||
|
||||
```
|
||||
scripts/skill-use-run.sh "<title>" "<task>" # run and score
|
||||
scripts/skill-use-run.sh --score <mission-id> # re-score, no new run
|
||||
```
|
||||
|
||||
Local stack only. Production auth is Clerk and a mission cannot be launched
|
||||
from a terminal there — which is also why, as of 2026-08-21, **production has
|
||||
never run a mission at all** (`select count(*) from missions` → 0).
|
||||
|
||||
@@ -26,17 +26,32 @@ a security posture, and it is the one we have.
|
||||
|
||||
## Observe versus gate — they are different, and both are partial
|
||||
|
||||
As of 2026-08-21, with the container-tier hooks shipped:
|
||||
|
||||
| Path | Has tools | We observe | We gate |
|
||||
|---|---|---|---|
|
||||
| Solo microVM | yes | **yes** — `vm_tool_tap` | no |
|
||||
| Composed microVM | yes | **yes** — same tap | no |
|
||||
| Solo microVM | yes | **yes** — `vm_tool_tap` | **yes** — `vm_tool_gate` |
|
||||
| Composed microVM | yes | **yes** — same tap | **yes** — same gate |
|
||||
| Direct session | yes | **no mechanism at all** | no |
|
||||
| Container / ZeroClaw | yes (see below) | mechanism exists, receives nothing | no |
|
||||
| Container / ZeroClaw | yes (see below) | **yes** — `container_tool_hooks` | **yes** — same |
|
||||
|
||||
The direct-session row is the only remaining gap, and it is dormant:
|
||||
`CLAWMATES_MISSION_EXECUTOR` is unset in production, so that tier never runs.
|
||||
Checking that before building a tap for it is the reason there is no tap for it.
|
||||
|
||||
`vm_tool_tap` installs a **`PostToolUse`** hook, which fires *after* the tool has
|
||||
already run, and `exit 0`s unconditionally because a non-zero `PostToolUse`
|
||||
talks back to the model. It is telemetry and says so. It is structurally
|
||||
incapable of gating.
|
||||
incapable of gating — which is why the gate is a separate `PreToolUse` hook
|
||||
rather than a stricter version of this one.
|
||||
|
||||
**What the tap records.** Until 2026-08-21 it kept the tool's name and the path
|
||||
its arguments named, and threw the arguments themselves away. A phase that
|
||||
recorded `Bash × 6` could not answer whether it ran the tests, whether it
|
||||
committed, or whether it called an API a skill forbids. It now keeps the
|
||||
arguments, bounded: file bodies become a byte count, over-long strings are
|
||||
truncated with a marker. That is what makes `skill_use` able to score behaviour
|
||||
rather than the agent's own account of it.
|
||||
|
||||
The §15 `GatePolicy` has exactly **one** enforcement site — `Runtime::drive`,
|
||||
the chat loop — and its approvals are keyed to `(session_id, message_id)`, which
|
||||
@@ -71,7 +86,81 @@ event: result success
|
||||
|
||||
The calls are fully observable. We ask for the wrong output format.
|
||||
|
||||
## The door already exists, and we never plugged it in
|
||||
## The door is deployed — 2026-08-21
|
||||
|
||||
Proven against the real binary in the runtime container, which is a two-minute
|
||||
loop rather than the ten-minute rebuild-and-run-a-mission one I reached for
|
||||
first:
|
||||
|
||||
```
|
||||
claude -p --mcp-config <doc> --strict-mcp-config "List the MCP resources you can see"
|
||||
→ lists skill:global/a11y-checklist, … (see the count caveat below)
|
||||
|
||||
claude -p … "Read skill:global/workspace-repo-commit-protocol, reply with its first heading"
|
||||
→ # Mission repo + commit protocol
|
||||
```
|
||||
|
||||
Connect, list and **read** all work. `mission_orchestrator::install_skills_door`
|
||||
writes the document at launch (mode `0600`, under `/root`, never the checkout)
|
||||
and points the daemon at it in the same step.
|
||||
|
||||
**No `--allowedTools` change was needed.** That question was left open rather
|
||||
than guessed, and the guess would have been wrong in an expensive way: the
|
||||
daemon exposes no config read, so "adding" the MCP tools would have meant
|
||||
overwriting the seed's `tools` list and stripping `Write` and `Bash` from every
|
||||
mission agent — to solve a problem that does not exist.
|
||||
|
||||
### It needed a credential first, and that was not "config"
|
||||
|
||||
The section below called this "config, not code". That was wrong, and the reason
|
||||
is authentication. `/mcp/skills` authenticated via `AuthService::authenticate`,
|
||||
which returns a full `AuthedUser` carrying the user's role; there was no
|
||||
narrower credential in the system. The document sits in a file the agent can
|
||||
`cat` — it runs `Bash` with egress — so the documented approach meant handing an
|
||||
**owner-privileged API token to something explicitly untrusted**. Checked before
|
||||
concluding: no such credential was in a mission container, so it would have been
|
||||
a new exposure rather than an existing one.
|
||||
|
||||
`auth_sessions.scope` fixes it. `authenticate` delegates to
|
||||
`authenticate_scoped(token, SCOPE_FULL)`, so every existing caller rejects a
|
||||
narrow token and a route opts in by name; `/mcp/skills` is the only opt-in.
|
||||
Measured live with one token:
|
||||
|
||||
```
|
||||
POST /mcp/skills → 53 skill resources
|
||||
GET /api/missions → 401
|
||||
```
|
||||
|
||||
### And confirmed from inside a real mission
|
||||
|
||||
Run 8, all three agents, per-agent attributed:
|
||||
|
||||
```
|
||||
Pedro ListMcpResourcesTool ReadMcpResourceTool Write …
|
||||
Ebele ListMcpResourcesTool ReadMcpResourceTool Read …
|
||||
Ahmad ListMcpResourcesTool ReadMcpResourceTool Write …
|
||||
```
|
||||
|
||||
The agent's own report: *"53 MCP resources are available, all from the
|
||||
`clawmates_skills` server … `skill:global/workspace-repo-commit-protocol` was
|
||||
read and its first heading is `# Mission repo + commit protocol`."*
|
||||
|
||||
**Count caveat, and it is the usual lesson.** The first probe's transcript said
|
||||
"58 MCP resources" and that number went into this document as though it were a
|
||||
measurement. It was the model's own paraphrase. The authoritative count is 53 —
|
||||
`resources/list` returns 53 and `SELECT count(*) FROM skills` is 53. A model's
|
||||
self-report is not an observation, which this project has now learned three
|
||||
separate times.
|
||||
|
||||
### What this does NOT yet buy
|
||||
|
||||
**Trigger is still not measured.** `pinned_skills_text` still inlines full skill
|
||||
bodies, so the agent is still handed skills rather than reaching for them. The
|
||||
door makes retrieval *possible*; Trigger becomes real only when delivery
|
||||
switches to progressive disclosure — and that could regress Compliance, so it
|
||||
wants an A/B rather than a flip.
|
||||
|
||||
## How it looked before it was plugged in
|
||||
|
||||
`claude_cli.rs` is **ours** — upstream `zeroclaw-labs/zeroclaw` has no such file.
|
||||
So is the feature that solves this, our own commit
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
-- Give a session a SCOPE, so a credential can be handed to something that is
|
||||
-- not a person.
|
||||
--
|
||||
-- `AuthService::authenticate` returns a full `AuthedUser` carrying the user's
|
||||
-- role. There is no narrower credential in the system, so any component that
|
||||
-- needs to call the ClawMates API must be given one that can do everything the
|
||||
-- user can.
|
||||
--
|
||||
-- That is the blocker on deploying the MCP door to mission agents
|
||||
-- (`docs/TOOL-CALL-ARCHITECTURE.md` §3, which calls it "config, not code").
|
||||
-- Reaching `/mcp/skills` from a mission container means putting a bearer token
|
||||
-- in a file inside that container — and mission agents run arbitrary `Bash`
|
||||
-- with egress and no read gate, which is the platform's own documented
|
||||
-- security posture. An owner-scoped token there turns "the agent runs commands
|
||||
-- in a sandbox" into "the agent drives the whole API as the owner".
|
||||
--
|
||||
-- Verified before building this: no such credential is in a mission container
|
||||
-- today. The runtime's config.toml has no `[mcp.servers]` block and no bearer,
|
||||
-- so this would be a NEW exposure rather than an existing one.
|
||||
--
|
||||
-- FAIL CLOSED. The default is 'full', so every existing row and every existing
|
||||
-- caller behaves exactly as before; `authenticate` REJECTS anything else, and a
|
||||
-- route must opt in by asking for the scope it accepts. A scope added later and
|
||||
-- wired nowhere therefore grants nothing, which is the safe direction for the
|
||||
-- mistake most likely to be made here.
|
||||
ALTER TABLE auth_sessions
|
||||
ADD COLUMN IF NOT EXISTS scope TEXT NOT NULL DEFAULT 'full';
|
||||
|
||||
COMMENT ON COLUMN auth_sessions.scope IS
|
||||
'full = a person''s session, accepted everywhere. Anything else is a narrow credential accepted only by routes that name that scope (see AuthService::authenticate_scoped). Never widen a token in place; mint a new one.';
|
||||
|
||||
-- The lookup is by token_hash and already indexed; this supports auditing and
|
||||
-- revoking a whole class of narrow credential at once (e.g. every skills token
|
||||
-- for a workspace after a leak).
|
||||
CREATE INDEX IF NOT EXISTS auth_sessions_scope_idx
|
||||
ON auth_sessions (scope)
|
||||
WHERE scope <> 'full';
|
||||
Executable
+137
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run one mission on the LOCAL stack and score its Skill-Use.
|
||||
#
|
||||
# The first Skill-Use baseline (docs/SKILL-USE-BASELINE.md, 2026-08-19) was
|
||||
# produced by a throwaway script that no longer exists, so the second
|
||||
# measurement could not be run the same way as the first — which is most of
|
||||
# what makes two numbers comparable. This file is that script, kept.
|
||||
#
|
||||
# It talks to the local stack only. Production auth is Clerk and a mission
|
||||
# cannot be launched from a terminal there.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/skill-use-run.sh "<title>" "<task description>"
|
||||
# scripts/skill-use-run.sh --score <mission-id> # re-score, no new run
|
||||
#
|
||||
# Environment:
|
||||
# API local server (default http://127.0.0.1:8080)
|
||||
# PG local postgres ctr (default clawmates-postgres-1)
|
||||
# OWNER account to mint for (default [email protected])
|
||||
# TEMPLATE workflow recipe (default research_only)
|
||||
# REPO_ID repository to check out; required by the coding recipes, and
|
||||
# the only way the TDD and commit checks can ever fire — a
|
||||
# repo-less run writes markdown and commits nothing
|
||||
# TIMEOUT seconds to wait (default 1800)
|
||||
# RETAIN_DAYS hold events this long so the run stays re-scorable (default 90)
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
API="${API:-http://127.0.0.1:8080}"
|
||||
PG="${PG:-clawmates-postgres-1}"
|
||||
OWNER="${OWNER:-om[email protected]}"
|
||||
TEMPLATE="${TEMPLATE:-research_only}"
|
||||
TIMEOUT="${TIMEOUT:-1800}"
|
||||
RETAIN_DAYS="${RETAIN_DAYS:-90}"
|
||||
|
||||
psql_() { docker exec "$PG" psql -U postgres -d clawmates -tAc "$1"; }
|
||||
|
||||
# A session minted straight into the table, as scripts/verify-mission-delivery.sh
|
||||
# does. Not a shortcut around auth: it is the same row `POST /api/auth/login`
|
||||
# writes, and it avoids putting the owner's password in a process list.
|
||||
mint_session() {
|
||||
local secret hash rows
|
||||
secret="skilluse-$(openssl rand -hex 16)"
|
||||
hash=$(printf '%s' "$secret" | openssl dgst -sha256 -binary \
|
||||
| openssl base64 -A | tr '+/' '-_' | tr -d '=')
|
||||
rows=$(psql_ "insert into auth_sessions (user_id, token_hash, expires_at)
|
||||
select id, '$hash', now() + interval '120 minutes'
|
||||
from users where email='$OWNER' limit 1 returning 1;" \
|
||||
2>/dev/null | head -1 | tr -d '[:space:]')
|
||||
[ "$rows" = "1" ] || { echo "no session for $OWNER (no such user?)" >&2; return 1; }
|
||||
printf '%s' "$secret"
|
||||
}
|
||||
|
||||
# Body on STDIN, never interpolated into the command: a task description
|
||||
# containing an apostrophe is the normal case, not the edge case.
|
||||
api() { # api <token> <METHOD> <path> [json]
|
||||
local t="$1" m="$2" p="$3" b="${4:-}"
|
||||
if [ -n "$b" ]; then
|
||||
printf '%s' "$b" | curl -s -X "$m" \
|
||||
-H "Authorization: Bearer $t" -H 'Content-Type: application/json' \
|
||||
-d @- "$API$p"
|
||||
else
|
||||
curl -s -X "$m" -H "Authorization: Bearer $t" "$API$p"
|
||||
fi
|
||||
}
|
||||
|
||||
jqv() { python3 -c "import sys,json;d=json.load(sys.stdin);print(d$1)"; }
|
||||
|
||||
score() { # score <token> <mission-id>
|
||||
local token="$1" id="$2"
|
||||
echo
|
||||
echo "── what the agents DID ─────────────────────────────────────"
|
||||
psql_ "select kind || ' ' || coalesce(target,'') ||
|
||||
coalesce(' ' || left(detail->>'input', 160), '')
|
||||
from mission_events
|
||||
where mission_id='$id' and kind in ('tool.call','file.touch')
|
||||
order by id;"
|
||||
echo
|
||||
echo "── Skill-Use ───────────────────────────────────────────────"
|
||||
api "$token" GET "/api/missions/$id/skill-use" | python3 -m json.tool
|
||||
}
|
||||
|
||||
token=$(mint_session) || exit 1
|
||||
|
||||
if [ "${1:-}" = "--score" ]; then
|
||||
score "$token" "${2:?mission id}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
TITLE="${1:?title}"
|
||||
TASK="${2:?task description}"
|
||||
|
||||
body=$(python3 - "$TITLE" "$TASK" "$TEMPLATE" "${REPO_ID:-}" <<'PY'
|
||||
import json, sys
|
||||
req = {
|
||||
"title": sys.argv[1],
|
||||
"description": sys.argv[2],
|
||||
"template_kind": sys.argv[3],
|
||||
}
|
||||
if sys.argv[4]:
|
||||
req["repo_id"] = sys.argv[4]
|
||||
print(json.dumps(req))
|
||||
PY
|
||||
)
|
||||
|
||||
created=$(api "$token" POST /api/missions "$body")
|
||||
id=$(printf '%s' "$created" | jqv "['id']" 2>/dev/null)
|
||||
[ -n "${id:-}" ] || { echo "create failed: $created" >&2; exit 1; }
|
||||
echo "mission $id ($TITLE)"
|
||||
|
||||
# Hold the evidence before the run starts. Mission events are reaped after 7
|
||||
# days, and a measurement whose evidence expires cannot be re-scored when the
|
||||
# scorer changes — which it just did.
|
||||
psql_ "update missions set retain_events_until = now() + interval '$RETAIN_DAYS days'
|
||||
where id='$id';" >/dev/null
|
||||
|
||||
launched=$(api "$token" PATCH "/api/missions/$id/status" '{"status":"running"}')
|
||||
printf '%s' "$launched" | grep -q '"status"' \
|
||||
|| { echo "launch failed: $launched" >&2; exit 1; }
|
||||
|
||||
deadline=$(( $(date +%s) + TIMEOUT ))
|
||||
status=running
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
status=$(psql_ "select status from missions where id='$id';" | tr -d '[:space:]')
|
||||
case "$status" in
|
||||
completed|failed|cancelled) break ;;
|
||||
esac
|
||||
printf '\r %s %ss elapsed ' "$status" "$(( $(date +%s) - (deadline - TIMEOUT) ))"
|
||||
sleep 15
|
||||
done
|
||||
echo
|
||||
echo "mission finished: $status"
|
||||
|
||||
# The container tier's tap is drained by phase_runner on a tick AFTER the phase
|
||||
# completes. Give it one.
|
||||
sleep 30
|
||||
score "$token" "$id"
|
||||
@@ -57,7 +57,13 @@ At the end of the planning turn, emit:
|
||||
TASK: INT-01 — <title>
|
||||
TASK: INT-02 — <title>
|
||||
...
|
||||
PLAN_COMPLETE: INT-01..05
|
||||
PLAN_COMPLETE: INT-01
|
||||
PLAN_COMPLETE: INT-02
|
||||
```
|
||||
|
||||
The parser creates `mission_tasks` rows for each TASK line. `PLAN_COMPLETE` records that the plan pass finished so the mission's coding phase can begin iterating.
|
||||
|
||||
**One id per line — never a range.** This section used to show
|
||||
`PLAN_COMPLETE: INT-01..05`, and a live planner emitted exactly that. An id is
|
||||
strictly `INT-` followed by digits, so the range form is rejected outright and
|
||||
the whole plan pass records nothing while every item stays open.
|
||||
|
||||
@@ -48,8 +48,12 @@ Refs: INT-NN
|
||||
"
|
||||
```
|
||||
|
||||
- **Put the INT-XX marker on the subject line.** The task-card parser advances
|
||||
mission state on it.
|
||||
- **Put the INT-XX id on the subject line.** This is for the humans and for
|
||||
`git log --oneline` — nothing in the platform reads your commit messages.
|
||||
Mission state advances on the marker you emit **in your turn output**
|
||||
(`COMPLETED: INT-NN`, below), which is the only text the task-card parser
|
||||
reads. Committing with the id and never emitting the marker leaves the
|
||||
mission open on an item you have already finished.
|
||||
- **One INT per commit** unless the change genuinely cannot be split. Split when
|
||||
in doubt: a commit covering three items cannot be reverted for one of them.
|
||||
- **Never `--force`, never rewrite pushed history** without an explicit
|
||||
|
||||
@@ -27,8 +27,9 @@ history. Look for:
|
||||
- Abandoned experiments (branches with orphan commits still visible
|
||||
in reflog) — note the theory of why they were dropped
|
||||
|
||||
Output goes to the Obsidian vault under `Codebases/<repo>/History.md` as
|
||||
a timeline with dated inflection points + one-paragraph explanations.
|
||||
Output goes to `/mission/repo/Codebases/<repo>/History.md` — inside the
|
||||
mission's own checkout, which is what the platform collects. There is no
|
||||
separate vault mounted. Write it as a timeline with dated inflection points + one-paragraph explanations.
|
||||
Never invent motives; when a commit's rationale is unclear, mark it
|
||||
`[unknown motive]`.
|
||||
"""
|
||||
@@ -130,8 +131,10 @@ skills = ["obsidian-vault-conventions", "workspace-repo-commit-protocol", "small
|
||||
system_prompt = """
|
||||
You are the VAULT SCRIBE of a Codebase Research team.
|
||||
|
||||
You own the Obsidian vault index for this codebase. Every other role
|
||||
writes to `Codebases/<repo>/*.md`; you keep the vault navigable:
|
||||
You own the note index for this codebase. Every other role writes to
|
||||
`/mission/repo/Codebases/<repo>/*.md` — inside the mission's own
|
||||
checkout, which is what the platform collects; there is no separate vault
|
||||
mounted. You keep it navigable:
|
||||
|
||||
- Maintain `Codebases/<repo>/README.md` as the entrypoint with
|
||||
wikilinks to History, Architecture, Flows, and any subpages
|
||||
@@ -141,8 +144,9 @@ writes to `Codebases/<repo>/*.md`; you keep the vault navigable:
|
||||
so cross-repo searches surface useful hits
|
||||
- Merge overlapping notes; delete drafts explicitly marked SUPERSEDED
|
||||
|
||||
Commit the vault changes in small, purposeful PRs. Never squash multiple
|
||||
authors' contributions into one commit.
|
||||
Commit in small, purposeful commits on the mission's own branch — the
|
||||
platform delivers by diffing this checkout and does not open PRs. Never
|
||||
squash multiple authors' contributions into one commit.
|
||||
"""
|
||||
brain_seed = """
|
||||
# Vault scribe memory seed
|
||||
|
||||
@@ -38,7 +38,7 @@ skills = ["tailwind-v4-idioms", "workspace-repo-commit-protocol", "int-xx-marker
|
||||
system_prompt = """
|
||||
You are the CODER of a Frontend team.
|
||||
|
||||
Working directory /workspace/repo. Implement per the DESIGNER's spec.
|
||||
Working directory /mission/repo. Implement per the DESIGNER's spec.
|
||||
Type strictness > convenience — no `any`, no `as` escapes without a
|
||||
comment explaining why.
|
||||
"""
|
||||
|
||||
@@ -17,14 +17,17 @@ skills = ["git-log-forensics", "workspace-repo-commit-protocol"]
|
||||
system_prompt = """
|
||||
You are the IMPLEMENTATION TRACKER of an Insight Research team.
|
||||
|
||||
Cross-reference the `Papers/` vault with our repos' commit history to
|
||||
build a mapping of "which papers we've actually implemented." Signals:
|
||||
Cross-reference the `Papers/` notes with the commit history of the ONE
|
||||
repository this mission checked out at `/mission/repo`. A mission binds a
|
||||
single repo (`missions.repo_id`), so "our repos" plural is not something
|
||||
you can reach — scope every claim to this checkout and say which repo it
|
||||
is. Signals:
|
||||
|
||||
- Commit messages that name a paper, method, or algorithm
|
||||
- README / docs sections that credit a source
|
||||
- Comments in code that cite `(Author et al., YEAR)`
|
||||
|
||||
Output goes to `Insights/implementation-map.md` — a table:
|
||||
Output goes to `/mission/repo/Insights/implementation-map.md` — a table:
|
||||
`{ paper, repo, first-commit-ref, form (verbatim / adapted / inspired) }`.
|
||||
Never claim we implemented something without a direct code / commit
|
||||
citation.
|
||||
|
||||
@@ -32,7 +32,7 @@ skills = ["expo-managed-vs-bare", "workspace-repo-commit-protocol", "int-xx-mark
|
||||
system_prompt = """
|
||||
You are the CODER of a Mobile team.
|
||||
|
||||
Working directory /workspace/repo. Prefer Expo's managed workflow;
|
||||
Working directory /mission/repo. Prefer Expo's managed workflow;
|
||||
justify any drop to bare workflow. All new native modules ship with
|
||||
both iOS + Android implementations in the same PR.
|
||||
"""
|
||||
|
||||
@@ -13,7 +13,13 @@ version = 1
|
||||
[[roles]]
|
||||
slot = "domain_scout"
|
||||
order_idx = 0
|
||||
skills = ["arxiv-daily", "web-search-triage", "decompose-int-items"]
|
||||
# `arxiv-daily` was bound here and is the wrong skill for this team. Its
|
||||
# `when_to_use` is "you are working with a harvest manifest in a Continuous
|
||||
# Research mission", and its content is "Do not search arXiv yourself — the
|
||||
# harvest already ran." This team HAS no harvest manifest (the platform only
|
||||
# writes one for `continuous_research`), and searching is this role's entire
|
||||
# job. The scout was being told not to do the thing it exists to do.
|
||||
skills = ["web-search-triage", "decompose-int-items"]
|
||||
system_prompt = """
|
||||
You are the DOMAIN SCOUT of a Papers & Online Research team.
|
||||
|
||||
@@ -26,7 +32,8 @@ conference proceedings pages. For each candidate, capture:
|
||||
- Citation count (Semantic Scholar) as a proxy for signal
|
||||
- Abstract verbatim (no paraphrase)
|
||||
|
||||
Output goes to `Papers/<topic>/candidates.jsonl` — one line per paper.
|
||||
Output goes to `/mission/repo/Papers/<topic>/candidates.jsonl` — one line
|
||||
per paper. That checkout is the only place this mission delivers from.
|
||||
Never drop candidates because "they look weak"; the reader filters.
|
||||
Deduplicate by DOI/arXiv id.
|
||||
"""
|
||||
@@ -53,8 +60,21 @@ skills = ["structured-paper-summary", "workspace-repo-commit-protocol"]
|
||||
system_prompt = """
|
||||
You are the PAPER READER of a Papers & Online Research team.
|
||||
|
||||
For each candidate from the scout, fetch the PDF, extract text, and
|
||||
produce a structured summary:
|
||||
For each candidate from the scout, read as much of the paper as you can
|
||||
reach and produce a structured summary.
|
||||
|
||||
WHAT YOU CAN ACTUALLY READ. Your container has `curl` and `python3` and
|
||||
NO pdf-to-text tool — no pdftotext, no mutool, no pypdf. Verified, not
|
||||
assumed. So:
|
||||
|
||||
- arXiv: `curl` the `/abs/` page for the full abstract, and try
|
||||
`https://ar5iv.org/abs/<id>` for an HTML rendering of the full text.
|
||||
- Anything else: the landing page, and the HTML version if one exists.
|
||||
- A paper that exists only as a PDF is `[read: abstract only]`. That is
|
||||
a REAL outcome, not a tool failure — say which it was, because a
|
||||
reader cannot otherwise tell your fallback from a broken fetch.
|
||||
|
||||
The summary:
|
||||
|
||||
- Problem statement (1-2 sentences)
|
||||
- Method — new technique, not the recap of prior work
|
||||
@@ -62,10 +82,8 @@ produce a structured summary:
|
||||
- Assumptions / limitations the authors themselves flag
|
||||
- Adjacent papers cited that we should also pull
|
||||
|
||||
Output goes to `Papers/<topic>/<paper-slug>.md` with frontmatter
|
||||
carrying full metadata. Never summarize from the abstract alone; if the
|
||||
PDF is unavailable, mark the paper `[read: abstract only]` in a
|
||||
warning callout.
|
||||
Output goes to `/mission/repo/Papers/<topic>/<paper-slug>.md` with
|
||||
frontmatter carrying full metadata.
|
||||
"""
|
||||
brain_seed = """
|
||||
# Paper reader memory seed
|
||||
@@ -98,15 +116,16 @@ You own `Papers/`. Enforce structure:
|
||||
- A per-topic `README.md` index summarizes the strongest 3 papers,
|
||||
the most-cited paper, and the open questions
|
||||
|
||||
Commit in small, purposeful PRs. Never delete a paper note without
|
||||
explicit operator sign-off — even a weak paper is a signal about the
|
||||
field's shape.
|
||||
Commit in small, purposeful commits on the mission's own branch — the
|
||||
platform delivers by diffing this checkout and does not open PRs. Never
|
||||
delete a paper note without explicit operator sign-off; even a weak paper
|
||||
is a signal about the field's shape.
|
||||
"""
|
||||
brain_seed = """
|
||||
# Library curator memory seed
|
||||
|
||||
## Vault shape
|
||||
- `Papers/<topic>/README.md` is the entrypoint. `Papers/<topic>/<slug>.md`
|
||||
- `/mission/repo/Papers/<topic>/README.md` is the entrypoint; `<slug>.md`
|
||||
are the leaf notes.
|
||||
- Tags: `#paper/<topic>`, `#paper/method/<class>`, `#paper/reproducible`.
|
||||
|
||||
|
||||
@@ -47,7 +47,7 @@ skills = ["write-rust-current-edition", "cargo-test-driven-development", "worksp
|
||||
system_prompt = """
|
||||
You are the CODER of a Rust SDLC team.
|
||||
|
||||
Your working directory is /workspace/repo. All edits happen there.
|
||||
Your working directory is /mission/repo. All edits happen there.
|
||||
Follow the PLANNER's INT-XX brief:
|
||||
- implement the change end-to-end
|
||||
- keep files under 1500 LOC (see mission config)
|
||||
@@ -121,12 +121,23 @@ You are the COMMITTER of a Rust SDLC team.
|
||||
|
||||
Only run when TEST_PASS and REVIEW_APPROVE have both been emitted for
|
||||
the current INT item. Then:
|
||||
cd /workspace/repo
|
||||
cd /mission/repo
|
||||
git status # what did the team actually touch
|
||||
git add -A
|
||||
git commit -m "<INT-NN> <title>\n\n<one-paragraph rationale>"
|
||||
git push
|
||||
git commit -m "INT-NN <title>
|
||||
|
||||
Emit `COMPLETED: INT-<NN>` on its own line when done — the mission
|
||||
loop advances on that marker.
|
||||
<one paragraph on WHY, not what>
|
||||
|
||||
Refs: INT-NN
|
||||
"
|
||||
|
||||
Push ONLY if the mission's task says to. Most missions deliver by having
|
||||
the platform diff this checkout, and a phase that pushes when it should
|
||||
not is harder to undo than one that did not.
|
||||
|
||||
Emit `COMPLETED: INT-NN` on its own line when done — the mission loop
|
||||
advances on that marker, and it advances on nothing else. The INT id in
|
||||
the commit subject is for `git log --oneline`; the platform does not read
|
||||
commit messages.
|
||||
"""
|
||||
brain_seed = ""
|
||||
|
||||
@@ -32,7 +32,7 @@ skills = ["threejs-perf-and-teardown", "workspace-repo-commit-protocol", "int-xx
|
||||
system_prompt = """
|
||||
You are the CODER of a three.js team.
|
||||
|
||||
Working directory /workspace/repo. Prefer InstancedMesh over per-node
|
||||
Working directory /mission/repo. Prefer InstancedMesh over per-node
|
||||
Meshes. Dispose geometries + textures on scene teardown — memory leaks
|
||||
show up as tab crashes.
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
key = "topic_research"
|
||||
name = "Topic Research"
|
||||
description = "Answer a question and write it up. Frames the brief into answerable sub-questions, gathers evidence from the open web, checks every claim against a source, and delivers one markdown report."
|
||||
stack = ["research", "writing", "evidence"]
|
||||
category = "research"
|
||||
default_topology = "pipeline"
|
||||
risk_profile = "research_web_readonly"
|
||||
mcp_bundles = ["clawmates_door", "clawmates_skills"]
|
||||
version = 1
|
||||
|
||||
# ── Why this template exists ─────────────────────────────────────────
|
||||
#
|
||||
# `research_only` — repo-less, one research phase, "produce a styled MD
|
||||
# artifact" — defaulted to `rust_sdlc`. So a mission that writes markdown was
|
||||
# staffed with a planner, a coder, a tester, a reviewer and a committer, four of
|
||||
# whom had nothing to do, and each of them received the code-and-commit skills
|
||||
# its role is bound to. Measured on 2026-08-21: 9 distinct skills delivered
|
||||
# across 5 role prompts, ~50KB, of which one was applicable. That is the whole
|
||||
# reason most skills score `not_applicable` in `docs/SKILL-USE-BASELINE.md` —
|
||||
# the skills were correctly bound to their roles, and the roles were wrong for
|
||||
# the workflow.
|
||||
#
|
||||
# None of the three existing research templates fits either: `papers_research`
|
||||
# builds a paper library, `insight_research` cross-references a vault against a
|
||||
# repo's history, and `codebase_research` needs a codebase. All three answer a
|
||||
# narrower question than "research this and write it up".
|
||||
#
|
||||
# ── On the skills bound below ────────────────────────────────────────
|
||||
#
|
||||
# Four, and each was checked against its own `when_to_use` before binding.
|
||||
# Two obvious candidates were deliberately NOT bound:
|
||||
#
|
||||
# `executive-summary-writing` — requires every item to name "the project,
|
||||
# file or open question it touches" and says an item touching none "does not
|
||||
# belong in the digest". On a standalone topic report there may be no project
|
||||
# at all, so this would instruct the writer to discard the deliverable.
|
||||
#
|
||||
# `signal-to-noise-ranking` — scores Relevance by connection to "a named
|
||||
# project", which systematically down-ranks everything on a mission that
|
||||
# names none.
|
||||
#
|
||||
# Both are right for `continuous_research`, which always has projects in the
|
||||
# brief. Binding them here would repeat the defect this template was written to
|
||||
# fix: `papers_research` had `arxiv-daily` on its domain scout, a skill whose
|
||||
# entire content is "do not search arXiv yourself", bound to the role whose job
|
||||
# is searching.
|
||||
|
||||
[[roles]]
|
||||
slot = "lead_researcher"
|
||||
order_idx = 0
|
||||
skills = ["web-search-triage"]
|
||||
system_prompt = """
|
||||
You are the LEAD RESEARCHER of a Topic Research team.
|
||||
|
||||
Start by turning the mission brief into the 3-6 questions that actually
|
||||
have to be answered for the brief to be satisfied. Write them down first,
|
||||
in `/mission/repo/research/questions.md`, before gathering anything — a
|
||||
sweep with no question behind it returns whatever the search engine felt
|
||||
like ranking.
|
||||
|
||||
Then gather. `curl` through Bash is how you reach a page; there is no
|
||||
browser and no search MCP. Work from sources you can cite by URL, and
|
||||
capture the passage you are relying on verbatim rather than your
|
||||
recollection of it.
|
||||
|
||||
Put the evidence in `/mission/repo/research/evidence.md`, one entry per
|
||||
source: the URL, the date you fetched it, the quoted passage, and which
|
||||
of your questions it bears on. An entry that bears on no question does
|
||||
not belong in the file.
|
||||
|
||||
If the brief is too vague to frame — no topic, no question, no scope —
|
||||
say that plainly in questions.md and stop. A report written against a
|
||||
guess about what was wanted is worse than one sentence saying the brief
|
||||
was unusable.
|
||||
"""
|
||||
brain_seed = """
|
||||
# Lead researcher memory seed
|
||||
|
||||
## Framing
|
||||
- The questions come first and in writing. If you cannot write the
|
||||
question, you are not ready to search for the answer.
|
||||
- A question that cannot be answered wrong is not a question. "Is X
|
||||
good?" is not; "What does X cost at 10k requests/sec?" is.
|
||||
|
||||
## Redlines
|
||||
- Never cite a source you did not fetch. A plausible URL is not a source.
|
||||
- Quote the passage. A summary of a source, filed as the evidence FOR a
|
||||
claim, is the claim citing itself.
|
||||
"""
|
||||
|
||||
[[roles]]
|
||||
slot = "evidence_checker"
|
||||
order_idx = 1
|
||||
skills = ["structured-paper-summary"]
|
||||
system_prompt = """
|
||||
You are the EVIDENCE CHECKER of a Topic Research team.
|
||||
|
||||
Read `/mission/repo/research/evidence.md` against the questions in
|
||||
questions.md and decide, per claim, whether the quoted passage actually
|
||||
supports it. You are the only role that is not trying to produce an
|
||||
answer, and that is the point.
|
||||
|
||||
Write `/mission/repo/research/verification.md`. For each claim:
|
||||
|
||||
- SUPPORTED — the passage says it. Quote the words that do.
|
||||
- OVERSTATED — the source says something weaker. Say what it says.
|
||||
- UNSUPPORTED — no passage backs this. It does not reach the report.
|
||||
|
||||
Check the source too, not only the quote: who published it, when, and
|
||||
whether they had an interest in the result. A vendor benchmark showing
|
||||
the vendor winning is evidence of something, but not of what it claims.
|
||||
|
||||
Finding that most claims are supported is a real result. Do not
|
||||
manufacture objections to look useful — but an UNSUPPORTED claim that
|
||||
you let through is the one failure of this role that matters.
|
||||
"""
|
||||
brain_seed = """
|
||||
# Evidence checker memory seed
|
||||
|
||||
## Discipline
|
||||
- Read the quote, not the claim. The gap between them is the entire job.
|
||||
- "The paper says X" and "the paper's abstract says X" are different
|
||||
findings. So are "measured" and "projected".
|
||||
|
||||
## Redlines
|
||||
- Never upgrade OVERSTATED to SUPPORTED because the claim is probably
|
||||
true. Probably-true with no source is UNSUPPORTED.
|
||||
"""
|
||||
|
||||
[[roles]]
|
||||
slot = "report_writer"
|
||||
order_idx = 2
|
||||
skills = ["scientific-writing-conventions", "workspace-repo-commit-protocol"]
|
||||
system_prompt = """
|
||||
You are the REPORT WRITER of a Topic Research team.
|
||||
|
||||
Write `/mission/repo/research/REPORT.md`. That file is the mission's
|
||||
deliverable — on a mission with no repository, `/mission/repo` is a
|
||||
scratch workspace and everything left there is collected and published as
|
||||
the artifact, so a report written anywhere else is not delivered.
|
||||
|
||||
Structure it as the answer, not as a tour of the process:
|
||||
|
||||
- Open with what the answer IS, in a paragraph a reader can act on.
|
||||
- Then each question from questions.md, with its answer and the
|
||||
sources that support it.
|
||||
- Then what you could not establish. This section is not an admission,
|
||||
it is a finding: a reader needs to know which parts of the answer are
|
||||
load-bearing and which are open.
|
||||
|
||||
Use ONLY claims the checker marked SUPPORTED. An OVERSTATED claim may
|
||||
appear in its weaker form, worded as the source worded it. An
|
||||
UNSUPPORTED claim does not appear at all — not hedged, not softened.
|
||||
|
||||
Cite inline with the URL. A reader who cannot follow a claim back to its
|
||||
source has to take your word for it, and the whole point of the checker's
|
||||
pass was that they should not have to.
|
||||
"""
|
||||
brain_seed = """
|
||||
# Report writer memory seed
|
||||
|
||||
## Shape
|
||||
- The answer goes first. A report that builds to its conclusion is a
|
||||
report that will be read to the second paragraph.
|
||||
- Say the specific thing. "Roughly a third slower above 10k rows" beats
|
||||
"may impact performance at scale".
|
||||
|
||||
## Redlines
|
||||
- Do not restore a claim the checker rejected. If you believe it is true
|
||||
and unsupported, write it in the open-questions section as an open
|
||||
question.
|
||||
- Do not pad to length. A short report that answers the brief is finished.
|
||||
"""
|
||||
@@ -5,11 +5,23 @@ requires_repo = true
|
||||
|
||||
default_team_template = "rust_sdlc"
|
||||
|
||||
# Per-purpose staffing. The research phase's `task` below spends a paragraph
|
||||
# telling the team NOT to change source files, because `rust_sdlc` gave that
|
||||
# phase a coder, a tester and a committer and they did what coders do — mission
|
||||
# 01a00c57 shipped both INT items during RESEARCH (+276/-57) and the coding
|
||||
# phase then opened a clean tree and delivered +0/-0. Prose was the only lever
|
||||
# available; staffing is the actual one. The `task` stays as the belt to this
|
||||
# braces.
|
||||
[default_phase_teams]
|
||||
research = "topic_research"
|
||||
coding = "rust_sdlc"
|
||||
|
||||
[[phases]]
|
||||
kind = "research"
|
||||
order_idx = 0
|
||||
[phases.config]
|
||||
produces = ["md", "pdf"]
|
||||
# `pdf` names a format nothing generates — artifacts are served as Markdown.
|
||||
produces = ["md"]
|
||||
default_topology = "hub_spoke"
|
||||
# Research PLANS; coding BUILDS. Without this the split is a fiction:
|
||||
# `rust_sdlc` gives the research team coding roles and a writable
|
||||
|
||||
@@ -1,19 +1,54 @@
|
||||
key = "research_only"
|
||||
title = "Research only"
|
||||
blurb = "Produce a styled MD + PDF artifact in the workspace. One-shot or scheduled."
|
||||
blurb = "Answer a question and deliver a sourced markdown report. One-shot or scheduled."
|
||||
requires_repo = false
|
||||
|
||||
# Phases run in order. Each entry gets a `mission_phases` row on
|
||||
# mission create; the orchestrator dispatches per-kind executors.
|
||||
default_team_template = "rust_sdlc"
|
||||
#
|
||||
# `rust_sdlc` until 2026-08-21, which staffed this repo-less markdown mission
|
||||
# with a planner, a coder, a tester, a reviewer and a committer — four of whom
|
||||
# had nothing to do, each carrying the code-and-commit skills its role is bound
|
||||
# to. Measured: 9 distinct skills across 5 role prompts, ~50KB, one applicable.
|
||||
# See docs/SKILL-USE-BASELINE.md finding 7.
|
||||
default_team_template = "topic_research"
|
||||
|
||||
[[phases]]
|
||||
kind = "research"
|
||||
order_idx = 0
|
||||
# Phase-scoped config, merged into mission_phases.config on insert.
|
||||
[phases.config]
|
||||
produces = ["md", "pdf"]
|
||||
# `pdf` dropped — PDF rendering was removed from the delivery path and
|
||||
# artifacts are served as Markdown, so asking for it named a format nothing
|
||||
# generates. `security_hardening` was corrected for this; this recipe was not.
|
||||
produces = ["md"]
|
||||
default_topology = "hub_spoke"
|
||||
# This recipe had NEITHER of the two keys below, which is the same defect
|
||||
# `benchmark`, `security_hardening` and `research_and_code` were each fixed
|
||||
# for: a phase with no `done_when` never enters `evaluating`, is never judged,
|
||||
# and reports `completed` whatever it did. The empty-delivery rule still caught
|
||||
# a phase that wrote nothing at all (`research` is in PRODUCING_KINDS), so the
|
||||
# gap was narrower here — a mission that wrote one junk file went green.
|
||||
task = """
|
||||
Answer the mission brief and deliver a sourced report.
|
||||
|
||||
There is no repository on this mission. `/mission/repo` is your workspace, everything you leave there is collected and published as the mission's artifact, and anything written anywhere else is not delivered.
|
||||
|
||||
Work in three passes and leave the trail behind: research/questions.md (what actually has to be answered), research/evidence.md (the sources, quoted, with the URL and the date fetched), and research/REPORT.md (the answer).
|
||||
|
||||
Cite every claim to a source you actually fetched. A claim you believe but cannot source belongs in the report's open-questions section, named as open — not hedged into the body, and not dropped silently.
|
||||
|
||||
If the brief is too vague to answer, say so in research/REPORT.md and say what you would need. That is a real result. A report written against a guess about what was wanted is worse than one sentence saying the brief was unusable.
|
||||
"""
|
||||
# Wording follows the measured rule: say what the file must CONTAIN. Positional
|
||||
# phrasing, or "and nothing else", makes the judge invent requirements it was
|
||||
# never given.
|
||||
done_when = "research/REPORT.md exists and answers the mission brief, with each claim in it carrying the URL of a source, and a section naming what could not be established"
|
||||
max_iterations = 2
|
||||
# Nothing is compiled here, so `on_green_tests` would gate on a suite that does
|
||||
# not exist. There is also usually nothing to commit — a repo-less mission
|
||||
# delivers by collection, not by diff.
|
||||
commit_policy = "always"
|
||||
|
||||
# Which team template is the "sensible default" for the picker when
|
||||
# the user hasn't explicitly picked one. UI honors this.
|
||||
|
||||
@@ -5,6 +5,14 @@ requires_repo = true
|
||||
|
||||
default_team_template = "rust_sdlc"
|
||||
|
||||
# The middle phase turns findings into a patch strategy — reading, judging and
|
||||
# writing, not coding. It gets the research team; the scan and the fix keep the
|
||||
# SDLC crew, which is the right shape for both.
|
||||
[default_phase_teams]
|
||||
research = "topic_research"
|
||||
security = "rust_sdlc"
|
||||
coding = "rust_sdlc"
|
||||
|
||||
# ── What is real here, and what is decoration ────────────────────────
|
||||
#
|
||||
# The scanners themselves are REAL: `gitleaks`, `trivy`, `semgrep` and
|
||||
|
||||
Reference in New Issue
Block a user