Compare commits
5
Commits
7525be3791
...
accae7fa94
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
accae7fa94 | ||
|
|
22eeaa6f15 | ||
|
|
f52cff3e04 | ||
|
|
b58f0347e6 | ||
|
|
72eda8b3d2 |
@@ -54,6 +54,7 @@ pub mod security_scan;
|
|||||||
pub mod session_executor;
|
pub mod session_executor;
|
||||||
pub mod container_tool_hooks;
|
pub mod container_tool_hooks;
|
||||||
pub mod gateway_preflight;
|
pub mod gateway_preflight;
|
||||||
|
pub mod skill_delivery;
|
||||||
pub mod skill_self_authoring;
|
pub mod skill_self_authoring;
|
||||||
pub mod skill_use;
|
pub mod skill_use;
|
||||||
pub mod skills_loader;
|
pub mod skills_loader;
|
||||||
|
|||||||
@@ -237,12 +237,21 @@ async fn mint_grant(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Authenticate the bearer header → workspace/user. `None` if missing/invalid.
|
/// Authenticate the bearer header → workspace/user. `None` if missing/invalid.
|
||||||
|
///
|
||||||
|
/// Accepts [`cm_auth::SCOPE_AGENT_DOOR`] as well as a person's session. This
|
||||||
|
/// route is the one that can `delegate`, and the thing that will eventually
|
||||||
|
/// hold a token for it is an agent runtime — so the narrow credential has to
|
||||||
|
/// exist before something reaches for the only one that does.
|
||||||
async fn authed(state: &AppState, headers: &HeaderMap) -> Option<cm_auth::AuthedUser> {
|
async fn authed(state: &AppState, headers: &HeaderMap) -> Option<cm_auth::AuthedUser> {
|
||||||
let token = headers
|
let token = headers
|
||||||
.get(AUTHORIZATION)
|
.get(AUTHORIZATION)
|
||||||
.and_then(|v| v.to_str().ok())
|
.and_then(|v| v.to_str().ok())
|
||||||
.and_then(|v| v.strip_prefix("Bearer "))?;
|
.and_then(|v| v.strip_prefix("Bearer "))?;
|
||||||
state.auth.authenticate(token).await.ok()
|
state
|
||||||
|
.auth
|
||||||
|
.authenticate_scoped(token, cm_auth::SCOPE_AGENT_DOOR)
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resolve the specific claw making the call. Our ZeroClaw fork stamps the
|
/// Resolve the specific claw making the call. Our ZeroClaw fork stamps the
|
||||||
|
|||||||
@@ -106,7 +106,7 @@ async fn caller_agent(
|
|||||||
|
|
||||||
// ── URI helpers ──────────────────────────────────────────────────
|
// ── URI helpers ──────────────────────────────────────────────────
|
||||||
|
|
||||||
fn skill_uri(workspace_id: Option<Uuid>, name: &str) -> String {
|
pub(crate) fn skill_uri(workspace_id: Option<Uuid>, name: &str) -> String {
|
||||||
match workspace_id {
|
match workspace_id {
|
||||||
Some(ws) => format!("{URI_PREFIX_WORKSPACE}{ws}/{name}"),
|
Some(ws) => format!("{URI_PREFIX_WORKSPACE}{ws}/{name}"),
|
||||||
None => format!("{URI_PREFIX_GLOBAL}{name}"),
|
None => format!("{URI_PREFIX_GLOBAL}{name}"),
|
||||||
|
|||||||
@@ -349,7 +349,7 @@ pub async fn on_launch(
|
|||||||
// Only when this mission got its OWN container — the shared runtime is
|
// Only when this mission got its OWN container — the shared runtime is
|
||||||
// not ours to reconfigure, and `mission_gateway` being Some is exactly
|
// not ours to reconfigure, and `mission_gateway` being Some is exactly
|
||||||
// the signal that `ensure_container` ran.
|
// the signal that `ensure_container` ran.
|
||||||
if mission_gateway.is_some() {
|
let door = if mission_gateway.is_some() {
|
||||||
install_skills_door(
|
install_skills_door(
|
||||||
pool,
|
pool,
|
||||||
user_id,
|
user_id,
|
||||||
@@ -357,8 +357,22 @@ pub async fn on_launch(
|
|||||||
&crate::mission_runtime::container_name(mission_id),
|
&crate::mission_runtime::container_name(mission_id),
|
||||||
p,
|
p,
|
||||||
)
|
)
|
||||||
.await;
|
.await
|
||||||
}
|
} else {
|
||||||
|
false
|
||||||
|
};
|
||||||
|
// Decided here and recorded, not re-derived per turn: this is the only
|
||||||
|
// point that knows whether the door actually installed, and an arm that
|
||||||
|
// could change mid-mission would make the run unattributable.
|
||||||
|
record_skill_delivery(
|
||||||
|
pool,
|
||||||
|
mission_id,
|
||||||
|
crate::skill_delivery::resolve(
|
||||||
|
crate::skill_delivery::requested_for(&mission.config),
|
||||||
|
door,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
let mut first_team_id: Option<Uuid> = None;
|
let mut first_team_id: Option<Uuid> = None;
|
||||||
let mut provisioned_claws: Vec<cm_domain::AgentId> = Vec::new();
|
let mut provisioned_claws: Vec<cm_domain::AgentId> = Vec::new();
|
||||||
@@ -916,19 +930,23 @@ fn default_accent_for(slot: &str) -> &'static str {
|
|||||||
///
|
///
|
||||||
/// Every failure degrades to "no door", never to a failed launch. A mission
|
/// Every failure degrades to "no door", never to a failed launch. A mission
|
||||||
/// that cannot retrieve a skill still delivers.
|
/// that cannot retrieve a skill still delivers.
|
||||||
|
/// Returns whether the door is installed AND reachable. The caller needs the
|
||||||
|
/// answer, not just the log line: the `index` delivery arm hands agents a list
|
||||||
|
/// of uris to fetch, and without a door every one of them is a dead end that
|
||||||
|
/// reads as an agent ignoring its skills.
|
||||||
async fn install_skills_door(
|
async fn install_skills_door(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
user_id: cm_domain::UserId,
|
user_id: cm_domain::UserId,
|
||||||
mission_id: Uuid,
|
mission_id: Uuid,
|
||||||
container: &str,
|
container: &str,
|
||||||
prov: &RuntimeProvisioner,
|
prov: &RuntimeProvisioner,
|
||||||
) {
|
) -> bool {
|
||||||
let Some(origin) = crate::container_tool_hooks::api_origin() else {
|
let Some(origin) = crate::container_tool_hooks::api_origin() else {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"mission_orchestrator: no API origin for the skills door (set \
|
"mission_orchestrator: no API origin for the skills door (set \
|
||||||
CLAWMATES_API_ORIGIN) — mission {mission_id} runs without it"
|
CLAWMATES_API_ORIGIN) — mission {mission_id} runs without it"
|
||||||
);
|
);
|
||||||
return;
|
return false;
|
||||||
};
|
};
|
||||||
// Outlives the longest mission we have seen, and expires on its own so a
|
// Outlives the longest mission we have seen, and expires on its own so a
|
||||||
// leaked container does not leave a live credential behind indefinitely.
|
// leaked container does not leave a live credential behind indefinitely.
|
||||||
@@ -943,31 +961,54 @@ async fn install_skills_door(
|
|||||||
"mission_orchestrator: could not mint a skills token ({e}) — \
|
"mission_orchestrator: could not mint a skills token ({e}) — \
|
||||||
mission {mission_id} runs without the door"
|
mission {mission_id} runs without the door"
|
||||||
);
|
);
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let docker = match crate::container_exec::connect() {
|
let docker = match crate::container_exec::connect() {
|
||||||
Ok(d) => d,
|
Ok(d) => d,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("mission_orchestrator: cannot reach docker for the skills door: {e}");
|
eprintln!("mission_orchestrator: cannot reach docker for the skills door: {e}");
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let doc = crate::container_tool_hooks::mcp_document(&origin, &token);
|
let doc = crate::container_tool_hooks::mcp_document(&origin, &token);
|
||||||
let Some(path) = crate::container_tool_hooks::install_door(&docker, container, &doc).await
|
let Some(path) = crate::container_tool_hooks::install_door(&docker, container, &doc).await
|
||||||
else {
|
else {
|
||||||
// `install_door` already said why.
|
// `install_door` already said why.
|
||||||
return;
|
return false;
|
||||||
};
|
};
|
||||||
if let Err(e) = prov.set_claude_cli_mcp_config(&path).await {
|
if let Err(e) = prov.set_claude_cli_mcp_config(&path).await {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"mission_orchestrator: wrote the MCP config but could not point \
|
"mission_orchestrator: wrote the MCP config but could not point \
|
||||||
claude_cli at it ({e}) — the door is installed and unreachable"
|
claude_cli at it ({e}) — the door is installed and unreachable"
|
||||||
);
|
);
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"mission_orchestrator: skills door installed for mission {mission_id} \
|
"mission_orchestrator: skills door installed for mission {mission_id} \
|
||||||
({origin}/mcp/skills)"
|
({origin}/mcp/skills)"
|
||||||
);
|
);
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record which arm this mission runs, so every turn composes the same one and
|
||||||
|
/// the score can be attributed to it afterwards.
|
||||||
|
///
|
||||||
|
/// A write failure is not fatal: `skill_delivery_mode` reads NULL as `inline`,
|
||||||
|
/// which is the arm that needs nothing installed. A mission that quietly ran
|
||||||
|
/// the control arm is a lost data point; a mission that failed to launch over
|
||||||
|
/// a telemetry column is a lost mission.
|
||||||
|
async fn record_skill_delivery(pool: &PgPool, mission_id: Uuid, mode: crate::skill_delivery::Mode) {
|
||||||
|
if let Err(e) = sqlx::query("UPDATE missions SET skill_delivery = $2 WHERE id = $1")
|
||||||
|
.bind(mission_id)
|
||||||
|
.bind(mode.as_str())
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
eprintln!(
|
||||||
|
"mission_orchestrator: could not record skill_delivery={} for mission \
|
||||||
|
{mission_id} ({e}) — its turns will compose skills inline",
|
||||||
|
mode.as_str()
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1141,7 +1141,16 @@ async fn launch_phase(
|
|||||||
// `topology_exec`, with the running node's own role, and appending here too
|
// `topology_exec`, with the running node's own role, and appending here too
|
||||||
// would put every crew member's skills in every turn twice.
|
// would put every crew member's skills in every turn twice.
|
||||||
let task_with_skills = match phase_skills_text(pool, mission_id).await {
|
let task_with_skills = match phase_skills_text(pool, mission_id).await {
|
||||||
Some(skills) => crate::topology_exec::compose_turn_prompt(&task, Some(&skills)),
|
// Always `Inline` here, and not because it is the default: the solo
|
||||||
|
// tiers get no skills door (`install_skills_door` runs only for a
|
||||||
|
// mission with its own container), so an index would list uris nothing
|
||||||
|
// in the VM can fetch. When the microVM tier folds onto
|
||||||
|
// `container_tool_hooks` this becomes a real choice; today it is a fact.
|
||||||
|
Some(skills) => crate::topology_exec::compose_turn_prompt(
|
||||||
|
&task,
|
||||||
|
Some(&skills),
|
||||||
|
crate::skill_delivery::Mode::Inline,
|
||||||
|
),
|
||||||
None => task.clone(),
|
None => task.clone(),
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,255 @@
|
|||||||
|
//! How a mission agent receives the skills bound to it.
|
||||||
|
//!
|
||||||
|
//! Two arms, and this module exists to hold them side by side rather than to
|
||||||
|
//! replace one with the other:
|
||||||
|
//!
|
||||||
|
//! - [`Mode::Inline`] — every pinned skill's full body is appended to the turn
|
||||||
|
//! prompt. What production has always done.
|
||||||
|
//! - [`Mode::Index`] — the prompt carries each skill's name, description and
|
||||||
|
//! `when_to_use` plus the URI that returns its body, and the agent fetches
|
||||||
|
//! the ones it judges relevant.
|
||||||
|
//!
|
||||||
|
//! # Why this is an A/B and not a switch
|
||||||
|
//!
|
||||||
|
//! Trigger — did the agent reach for the skill when it applied? — is
|
||||||
|
//! unmeasurable under `Inline` by construction. Nothing was reached for; the
|
||||||
|
//! text was handed over. `skill_use` reports `NotObservable` for exactly that
|
||||||
|
//! reason, and it is right to.
|
||||||
|
//!
|
||||||
|
//! `Index` makes Trigger observable, because retrieval is a recorded
|
||||||
|
//! `ReadMcpResourceTool` call. But it can only *cost* Compliance: under
|
||||||
|
//! `Inline` the procedure is in front of the model whether or not it noticed
|
||||||
|
//! it applied, and under `Index` a missed judgement means the body is never
|
||||||
|
//! read at all. Trading a measured axis for an unmeasured regression in
|
||||||
|
//! another is not an improvement, so the arm is selected per mission and
|
||||||
|
//! recorded on the mission row, and both arms stay runnable.
|
||||||
|
//!
|
||||||
|
//! # `Index` requires the door, and degrades rather than lying
|
||||||
|
//!
|
||||||
|
//! An index names a body and tells the agent how to fetch it. If the
|
||||||
|
//! `clawmates_skills` MCP server is not reachable from the container, that is
|
||||||
|
//! an index of procedures the agent cannot obtain — strictly worse than
|
||||||
|
//! `Inline`, and it fails as an agent that ignored its skills rather than as a
|
||||||
|
//! missing config. [`resolve`] therefore takes the door's install result and
|
||||||
|
//! refuses `Index` without it. This is the same failure the old
|
||||||
|
//! `pinned_skills_text` doc comment warned about; what changed is that the
|
||||||
|
//! door now exists, not that the warning stopped applying.
|
||||||
|
|
||||||
|
/// Where the index tells agents to fetch a skill body from.
|
||||||
|
///
|
||||||
|
/// Must match the server name in
|
||||||
|
/// [`crate::container_tool_hooks::mcp_document`] — the agent passes it
|
||||||
|
/// straight to `ReadMcpResourceTool`.
|
||||||
|
pub const MCP_SERVER: &str = "clawmates_skills";
|
||||||
|
|
||||||
|
/// Selects the arm. Unset or unrecognised means [`Mode::Inline`].
|
||||||
|
pub const ENV_VAR: &str = "CLAWMATES_SKILL_DELIVERY";
|
||||||
|
|
||||||
|
/// The `# Your skills` preamble under [`Mode::Inline`].
|
||||||
|
///
|
||||||
|
/// **Byte-identical to what production has always sent.** The A arm of an A/B
|
||||||
|
/// has to be the thing already running, or the comparison measures this edit
|
||||||
|
/// as well as the change under test.
|
||||||
|
pub const INLINE_PREAMBLE: &str = "These are procedures you are expected to follow for \
|
||||||
|
this kind of work. Where one applies to what you are about to do, follow it.";
|
||||||
|
|
||||||
|
/// The `# Your skills` preamble under [`Mode::Index`].
|
||||||
|
///
|
||||||
|
/// Written and matched in one place ([`mode_in_prompt`]) so the reader cannot
|
||||||
|
/// drift from the writer — the same rule `SKILL_MARKER` is under, and for the
|
||||||
|
/// same reason: a scorer that misreads the arm reports the wrong axis.
|
||||||
|
pub const INDEX_PREAMBLE: &str = "These procedures are AVAILABLE to you; their bodies are \
|
||||||
|
not included below. Each entry names one, says when it applies, and gives the uri that \
|
||||||
|
returns it. Where an entry applies to what you are about to do, read it FIRST and then \
|
||||||
|
follow it.";
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum Mode {
|
||||||
|
Inline,
|
||||||
|
Index,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Mode {
|
||||||
|
pub fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Mode::Inline => "inline",
|
||||||
|
Mode::Index => "index",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse a recorded or configured arm. Unrecognised input is `None`, and every
|
||||||
|
/// caller resolves that to `Inline` — an unreadable value must not silently
|
||||||
|
/// select the arm that needs a door.
|
||||||
|
pub fn parse(s: &str) -> Option<Mode> {
|
||||||
|
match s.trim().to_ascii_lowercase().as_str() {
|
||||||
|
"inline" => Some(Mode::Inline),
|
||||||
|
"index" | "progressive" => Some(Mode::Index),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The arm this deployment asks for, before the door is taken into account.
|
||||||
|
pub fn requested() -> Mode {
|
||||||
|
let Ok(raw) = std::env::var(ENV_VAR) else {
|
||||||
|
return Mode::Inline;
|
||||||
|
};
|
||||||
|
if raw.trim().is_empty() {
|
||||||
|
return Mode::Inline;
|
||||||
|
}
|
||||||
|
match parse(&raw) {
|
||||||
|
Some(m) => m,
|
||||||
|
None => {
|
||||||
|
eprintln!(
|
||||||
|
"skill_delivery: {ENV_VAR}={raw:?} is not `inline` or `index` — \
|
||||||
|
delivering skills inline"
|
||||||
|
);
|
||||||
|
Mode::Inline
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The arm for one mission: `config.skill_delivery` if it names one, otherwise
|
||||||
|
/// the deployment default.
|
||||||
|
///
|
||||||
|
/// Per-mission and not only per-deployment because the alternative is
|
||||||
|
/// restarting the server between arms, and an A/B whose two halves ran against
|
||||||
|
/// different server processes has a confound in it that nothing in the numbers
|
||||||
|
/// will show. This way both arms run against one binary, interleaved.
|
||||||
|
pub fn requested_for(config: &serde_json::Value) -> Mode {
|
||||||
|
let Some(raw) = config.get("skill_delivery").and_then(|v| v.as_str()) else {
|
||||||
|
return requested();
|
||||||
|
};
|
||||||
|
match parse(raw) {
|
||||||
|
Some(m) => m,
|
||||||
|
None => {
|
||||||
|
eprintln!(
|
||||||
|
"skill_delivery: config.skill_delivery={raw:?} is not `inline` or \
|
||||||
|
`index` — falling back to the deployment default"
|
||||||
|
);
|
||||||
|
requested()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The arm a mission will actually run, given whether its skills door installed.
|
||||||
|
pub fn resolve(requested: Mode, door_installed: bool) -> Mode {
|
||||||
|
match (requested, door_installed) {
|
||||||
|
(Mode::Index, true) => Mode::Index,
|
||||||
|
(Mode::Index, false) => {
|
||||||
|
eprintln!(
|
||||||
|
"skill_delivery: {ENV_VAR} asked for `index` but this mission has no \
|
||||||
|
skills door — falling back to `inline`, because an index the agent \
|
||||||
|
cannot fetch from is worse than no index"
|
||||||
|
);
|
||||||
|
Mode::Inline
|
||||||
|
}
|
||||||
|
(Mode::Inline, _) => Mode::Inline,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The `# Your skills` section heading for an arm.
|
||||||
|
pub fn preamble(mode: Mode) -> &'static str {
|
||||||
|
match mode {
|
||||||
|
Mode::Inline => INLINE_PREAMBLE,
|
||||||
|
Mode::Index => INDEX_PREAMBLE,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Which arm produced a recorded prompt.
|
||||||
|
///
|
||||||
|
/// Read back from the prompt rather than from the mission row on purpose: the
|
||||||
|
/// row says what the mission was configured to do *now*, and a score is being
|
||||||
|
/// computed against a prompt that was composed then. The recorded prompt is
|
||||||
|
/// the only artefact that cannot have changed since the turn ran.
|
||||||
|
///
|
||||||
|
/// Matched as a whole line. A skill body that quotes the preamble mid-sentence
|
||||||
|
/// is prose; this is the same rule `skill_names_in` learned the hard way.
|
||||||
|
pub fn mode_in_prompt(prompt: &str) -> Mode {
|
||||||
|
if prompt.lines().any(|l| l.trim() == INDEX_PREAMBLE) {
|
||||||
|
Mode::Index
|
||||||
|
} else {
|
||||||
|
Mode::Inline
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One index entry's text — everything under the `--- SKILL: <name> ---`
|
||||||
|
/// marker, which [`crate::topology_exec::render_pinned_skill`] writes.
|
||||||
|
///
|
||||||
|
/// `when_to_use` is the load-bearing field: it is the only thing the agent has
|
||||||
|
/// to judge relevance from, so a skill with none says so rather than omitting
|
||||||
|
/// the line and leaving the model to infer from the description alone.
|
||||||
|
pub fn index_entry(description: &str, when_to_use: Option<&str>, uri: &str) -> String {
|
||||||
|
let when = when_to_use
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|w| !w.is_empty())
|
||||||
|
.unwrap_or("not stated — judge from the description");
|
||||||
|
format!(
|
||||||
|
"{}\nWhen to use: {}\nRead it: ReadMcpResourceTool(server=\"{}\", uri=\"{}\")",
|
||||||
|
description.trim(),
|
||||||
|
when,
|
||||||
|
MCP_SERVER,
|
||||||
|
uri,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_unreadable_arm_never_selects_the_one_that_needs_a_door() {
|
||||||
|
assert_eq!(parse("nonsense"), None);
|
||||||
|
assert_eq!(parse("INDEX"), Some(Mode::Index));
|
||||||
|
assert_eq!(parse(" inline "), Some(Mode::Inline));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_mission_can_name_its_own_arm() {
|
||||||
|
assert_eq!(
|
||||||
|
requested_for(&serde_json::json!({ "skill_delivery": "index" })),
|
||||||
|
Mode::Index
|
||||||
|
);
|
||||||
|
// Unreadable values and absent ones both defer to the deployment
|
||||||
|
// default, which is `Inline` unless the environment says otherwise.
|
||||||
|
assert_eq!(
|
||||||
|
requested_for(&serde_json::json!({ "skill_delivery": "sideways" })),
|
||||||
|
requested()
|
||||||
|
);
|
||||||
|
assert_eq!(requested_for(&serde_json::json!({})), requested());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn index_without_a_door_falls_back() {
|
||||||
|
assert_eq!(resolve(Mode::Index, false), Mode::Inline);
|
||||||
|
assert_eq!(resolve(Mode::Index, true), Mode::Index);
|
||||||
|
assert_eq!(resolve(Mode::Inline, true), Mode::Inline);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The scorer reads the arm off the prompt, so the writer and this reader
|
||||||
|
/// have to agree for every arm — including the one that writes no marker.
|
||||||
|
#[test]
|
||||||
|
fn the_arm_is_recoverable_from_the_prompt_that_was_sent() {
|
||||||
|
let inline = format!("Task: x\n\n# Your skills\n\n{INLINE_PREAMBLE}\n\nbody");
|
||||||
|
let index = format!("Task: x\n\n# Your skills\n\n{INDEX_PREAMBLE}\n\nentry");
|
||||||
|
assert_eq!(mode_in_prompt(&inline), Mode::Inline);
|
||||||
|
assert_eq!(mode_in_prompt(&index), Mode::Index);
|
||||||
|
assert_eq!(mode_in_prompt("Task: x"), Mode::Inline);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A body quoting the preamble must not re-label the arm — the same
|
||||||
|
/// failure `SKILL_MARKER` had when a heading inside a body counted.
|
||||||
|
#[test]
|
||||||
|
fn a_body_quoting_the_preamble_does_not_change_the_arm() {
|
||||||
|
let body = format!("The index arm opens with \"{INDEX_PREAMBLE}\" and then lists.");
|
||||||
|
let prompt = format!("Task: x\n\n# Your skills\n\n{INLINE_PREAMBLE}\n\n{body}");
|
||||||
|
assert_eq!(mode_in_prompt(&prompt), Mode::Inline);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_entry_states_a_missing_when_to_use_rather_than_dropping_the_line() {
|
||||||
|
let e = index_entry("Summarise a paper.", None, "skill:global/x");
|
||||||
|
assert!(e.contains("When to use: not stated"), "{e}");
|
||||||
|
assert!(e.contains("ReadMcpResourceTool(server=\"clawmates_skills\""), "{e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
+175
-14
@@ -189,6 +189,55 @@ pub fn retrieved_skills(ev: &Evidence<'_>) -> Vec<String> {
|
|||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Did the agent reach for this skill?
|
||||||
|
///
|
||||||
|
/// The answer depends on whether it was ever given the chance, which is what
|
||||||
|
/// the delivery arm decides — so this takes the arm rather than assuming one.
|
||||||
|
/// Getting that wrong is not a rounding error: under `Index` the old text
|
||||||
|
/// would have said "this skill was inlined into the prompt" about a skill that
|
||||||
|
/// was not, and scored a real miss as a structural blind spot.
|
||||||
|
fn trigger_verdict(
|
||||||
|
mode: crate::skill_delivery::Mode,
|
||||||
|
retrieved: bool,
|
||||||
|
compliance: &Verdict,
|
||||||
|
boundary: &Verdict,
|
||||||
|
) -> Verdict {
|
||||||
|
if retrieved {
|
||||||
|
// The agent reached for it. That is the paper's Trigger, and it is a
|
||||||
|
// recorded tool call like any other.
|
||||||
|
return Verdict::Pass;
|
||||||
|
}
|
||||||
|
match mode {
|
||||||
|
// Handed over, so there was no reaching-for to observe. Not a failure
|
||||||
|
// and not a pass — the axis simply does not exist in this arm.
|
||||||
|
crate::skill_delivery::Mode::Inline => Verdict::NotObservable(
|
||||||
|
"this skill was inlined into the prompt, not retrieved — the agent \
|
||||||
|
was handed it, so there is no reaching-for to observe. Serve it \
|
||||||
|
through the door instead and this becomes a tool call"
|
||||||
|
.into(),
|
||||||
|
),
|
||||||
|
crate::skill_delivery::Mode::Index => {
|
||||||
|
// Offered by name and `when_to_use`, and never opened. Whether that
|
||||||
|
// is a miss depends on whether the skill had anything to say about
|
||||||
|
// this phase at all: a skill with no machine-checkable consequence
|
||||||
|
// here is one an agent is right to pass over, and scoring that as a
|
||||||
|
// failure would punish correct triage.
|
||||||
|
if matches!(compliance, Verdict::NotApplicable)
|
||||||
|
&& matches!(boundary, Verdict::NotApplicable)
|
||||||
|
{
|
||||||
|
Verdict::NotApplicable
|
||||||
|
} else {
|
||||||
|
Verdict::Fail(
|
||||||
|
"offered in the index with its `when_to_use`, and never read — \
|
||||||
|
the agent had the entry in front of it and did not fetch the \
|
||||||
|
procedure"
|
||||||
|
.into(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Score every skill a phase's prompt delivered, against what the agent produced.
|
/// Score every skill a phase's prompt delivered, against what the agent produced.
|
||||||
pub fn score(
|
pub fn score(
|
||||||
prompt: &str,
|
prompt: &str,
|
||||||
@@ -196,6 +245,10 @@ pub fn score(
|
|||||||
source_kinds: &dyn Fn(&str) -> String,
|
source_kinds: &dyn Fn(&str) -> String,
|
||||||
) -> Vec<SkillUse> {
|
) -> Vec<SkillUse> {
|
||||||
let retrieved = retrieved_skills(evidence);
|
let retrieved = retrieved_skills(evidence);
|
||||||
|
// Read off the prompt that was actually sent, not off the mission row: the
|
||||||
|
// row says what the mission is configured to do now, and this is scoring a
|
||||||
|
// turn that ran then.
|
||||||
|
let mode = crate::skill_delivery::mode_in_prompt(prompt);
|
||||||
// Delivered by either route. A skill that was retrieved and never inlined
|
// Delivered by either route. A skill that was retrieved and never inlined
|
||||||
// is invisible to `skills_in_prompt`, and under progressive disclosure that
|
// is invisible to `skills_in_prompt`, and under progressive disclosure that
|
||||||
// is EVERY skill — so scoring only the prompt would report zero for the
|
// is EVERY skill — so scoring only the prompt would report zero for the
|
||||||
@@ -212,19 +265,12 @@ pub fn score(
|
|||||||
let (compliance, boundary) = check(&skill, evidence);
|
let (compliance, boundary) = check(&skill, evidence);
|
||||||
SkillUse {
|
SkillUse {
|
||||||
source_kind: source_kinds(&skill),
|
source_kind: source_kinds(&skill),
|
||||||
trigger: if retrieved.contains(&skill) {
|
trigger: trigger_verdict(
|
||||||
// The agent reached for it. That is the paper's Trigger,
|
mode,
|
||||||
// and it is now a recorded tool call like any other.
|
retrieved.contains(&skill),
|
||||||
Verdict::Pass
|
&compliance,
|
||||||
} else {
|
&boundary,
|
||||||
Verdict::NotObservable(
|
),
|
||||||
"this skill was inlined into the prompt, not retrieved — \
|
|
||||||
the agent was handed it, so there is no reaching-for to \
|
|
||||||
observe. Serve it through the door instead and this \
|
|
||||||
becomes a tool call"
|
|
||||||
.into(),
|
|
||||||
)
|
|
||||||
},
|
|
||||||
compliance,
|
compliance,
|
||||||
boundary,
|
boundary,
|
||||||
skill,
|
skill,
|
||||||
@@ -855,7 +901,122 @@ mod tests {
|
|||||||
.iter()
|
.iter()
|
||||||
.map(|(n, b)| crate::topology_exec::render_pinned_skill(n, b))
|
.map(|(n, b)| crate::topology_exec::render_pinned_skill(n, b))
|
||||||
.collect();
|
.collect();
|
||||||
crate::topology_exec::compose_turn_prompt("Task: do the thing", Some(&body))
|
crate::topology_exec::compose_turn_prompt(
|
||||||
|
"Task: do the thing",
|
||||||
|
Some(&body),
|
||||||
|
crate::skill_delivery::Mode::Inline,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A prompt as the delivery layer renders it under the INDEX arm.
|
||||||
|
fn rendered_index(skills: &[(&str, &str)]) -> String {
|
||||||
|
let body: String = skills
|
||||||
|
.iter()
|
||||||
|
.map(|(name, when)| {
|
||||||
|
crate::topology_exec::render_pinned_skill(
|
||||||
|
name,
|
||||||
|
&crate::skill_delivery::index_entry(
|
||||||
|
"a procedure",
|
||||||
|
Some(when),
|
||||||
|
&format!("skill:global/{name}"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
crate::topology_exec::compose_turn_prompt(
|
||||||
|
"Task: do the thing",
|
||||||
|
Some(&body),
|
||||||
|
crate::skill_delivery::Mode::Index,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_skill(uri: &str) -> ToolEvidence {
|
||||||
|
ToolEvidence {
|
||||||
|
tool: "ReadMcpResourceTool".into(),
|
||||||
|
path: None,
|
||||||
|
input: json!({ "server": "clawmates_skills", "uri": uri }),
|
||||||
|
response: serde_json::Value::Null,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The whole loop, end to end: the index writes a uri, the agent reads that
|
||||||
|
/// exact uri back, and the scorer recovers the skill's name from it.
|
||||||
|
///
|
||||||
|
/// Three components have to agree on one string — `mcp_skills::skill_uri`
|
||||||
|
/// writes it, `skill_delivery::index_entry` puts it in the prompt, and
|
||||||
|
/// `parse_uri` reads it. Asserting them separately would let any pair drift
|
||||||
|
/// while each one's own test stayed green.
|
||||||
|
#[test]
|
||||||
|
fn the_uri_the_index_advertises_is_the_one_the_scorer_recovers() {
|
||||||
|
let prompt = rendered_index(&[("workspace-repo-commit-protocol", "before committing")]);
|
||||||
|
assert!(
|
||||||
|
prompt.contains("uri=\"skill:global/workspace-repo-commit-protocol\""),
|
||||||
|
"the entry must name the uri to fetch:\n{prompt}"
|
||||||
|
);
|
||||||
|
let tools = vec![read_skill("skill:global/workspace-repo-commit-protocol")];
|
||||||
|
let ev = Evidence::new("", &tools);
|
||||||
|
assert_eq!(
|
||||||
|
retrieved_skills(&ev),
|
||||||
|
vec!["workspace-repo-commit-protocol"],
|
||||||
|
"the uri the prompt advertised must parse back to the skill's name"
|
||||||
|
);
|
||||||
|
let scored = score(&prompt, &ev, &builtin);
|
||||||
|
assert_eq!(scored.len(), 1);
|
||||||
|
assert!(
|
||||||
|
matches!(scored[0].trigger, Verdict::Pass),
|
||||||
|
"reaching for an indexed skill IS the Trigger axis: {:?}",
|
||||||
|
scored[0].trigger
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The index arm must not report a miss as a blind spot.
|
||||||
|
///
|
||||||
|
/// Under `Inline` "never retrieved" is `NotObservable`, and that is honest
|
||||||
|
/// there. Reusing it here would say "this skill was inlined into the
|
||||||
|
/// prompt" about a skill whose body was never sent — the exact shape of a
|
||||||
|
/// check reporting a system defect where an agent behaviour belongs.
|
||||||
|
#[test]
|
||||||
|
fn an_indexed_skill_that_was_never_read_is_a_miss_not_a_blind_spot() {
|
||||||
|
let prompt = rendered_index(&[("workspace-repo-commit-protocol", "before committing")]);
|
||||||
|
// It wrote outside the mission checkout, so the boundary check applies
|
||||||
|
// — this skill had something to say about this phase and went unread.
|
||||||
|
let tools = acted(&[("Write", Some("/tmp/scratch.rs"), json!({}))]);
|
||||||
|
let scored = score(&prompt, &Evidence::new("", &tools), &builtin);
|
||||||
|
assert!(
|
||||||
|
matches!(scored[0].trigger, Verdict::Fail(_)),
|
||||||
|
"offered by name and when_to_use, never fetched: {:?}",
|
||||||
|
scored[0].trigger
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// ...but only when the skill had a consequence to check.
|
||||||
|
///
|
||||||
|
/// A skill with nothing machine-checkable in this phase is one an agent is
|
||||||
|
/// right to pass over, and scoring that as a Trigger failure would punish
|
||||||
|
/// correct triage — which is the behaviour progressive disclosure is
|
||||||
|
/// supposed to reward.
|
||||||
|
#[test]
|
||||||
|
fn passing_over_a_skill_with_nothing_to_check_is_not_a_trigger_failure() {
|
||||||
|
let prompt = rendered_index(&[("some-unchecked-skill", "when writing prose")]);
|
||||||
|
let scored = score(&prompt, &narrative("wrote the report"), &builtin);
|
||||||
|
assert!(
|
||||||
|
matches!(scored[0].trigger, Verdict::NotApplicable),
|
||||||
|
"{:?}",
|
||||||
|
scored[0].trigger
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The control arm has to be unchanged, or the A/B measures this edit too.
|
||||||
|
#[test]
|
||||||
|
fn the_inline_arm_still_reports_trigger_as_unobservable() {
|
||||||
|
let prompt = rendered(&[("workspace-repo-commit-protocol", "body")]);
|
||||||
|
let tools = acted(&[("Write", Some("/tmp/scratch.rs"), json!({}))]);
|
||||||
|
let scored = score(&prompt, &Evidence::new("", &tools), &builtin);
|
||||||
|
assert!(
|
||||||
|
matches!(scored[0].trigger, Verdict::NotObservable(_)),
|
||||||
|
"{:?}",
|
||||||
|
scored[0].trigger
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The prompt is the record of what was delivered, so parsing it must match
|
/// The prompt is the record of what was delivered, so parsing it must match
|
||||||
|
|||||||
@@ -337,12 +337,50 @@ impl ZeroClawDriveExecutor {
|
|||||||
/// for a mission role was unreachable prose, and no measurement of whether
|
/// for a mission role was unreachable prose, and no measurement of whether
|
||||||
/// skills fire could have returned anything but zero.
|
/// skills fire could have returned anything but zero.
|
||||||
///
|
///
|
||||||
/// Bodies, not an index. The chat path lists names and lets the claw call
|
/// Bodies or an index, depending on the mission's arm — see
|
||||||
/// `skills.read`; there is no such tool here, so an index would advertise a
|
/// [`crate::skill_delivery`]. Bodies were once the only honest option:
|
||||||
/// capability that does not exist — the exact failure this whole change is
|
/// there was no tool on the mission path that could fetch one, so an index
|
||||||
/// about. Pinned only (`pin_in_context`), because everything else would go
|
/// would have advertised a capability that did not exist. The skills door
|
||||||
/// in unbounded and unread.
|
/// changed that, and the arm is now recorded per mission so both can run.
|
||||||
|
///
|
||||||
|
/// Pinned only (`pin_in_context`) in either arm, because everything else
|
||||||
|
/// would go in unbounded and unread.
|
||||||
pub async fn pinned_skills_text(&self, alias: &str) -> Option<String> {
|
pub async fn pinned_skills_text(&self, alias: &str) -> Option<String> {
|
||||||
|
let mode = self.skill_delivery_mode().await;
|
||||||
|
self.pinned_skills_in_mode(alias, mode).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The arm this mission was launched with.
|
||||||
|
///
|
||||||
|
/// Read per turn rather than cached on the executor: the executor is
|
||||||
|
/// constructed from the environment by `topology_worker`, which knows
|
||||||
|
/// nothing about a mission, and the arm is decided at launch by the code
|
||||||
|
/// that also learns whether the door installed.
|
||||||
|
///
|
||||||
|
/// Anything unreadable — no tap, no row, an unrecognised value — resolves
|
||||||
|
/// to `Inline`, which is the arm that needs nothing to be true.
|
||||||
|
pub(crate) async fn skill_delivery_mode(&self) -> crate::skill_delivery::Mode {
|
||||||
|
let Some(tap) = self.tap.as_ref() else {
|
||||||
|
return crate::skill_delivery::Mode::Inline;
|
||||||
|
};
|
||||||
|
sqlx::query_scalar::<_, Option<String>>(
|
||||||
|
"SELECT skill_delivery FROM missions WHERE id = $1",
|
||||||
|
)
|
||||||
|
.bind(tap.mission_id)
|
||||||
|
.fetch_optional(&tap.pool)
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.flatten()
|
||||||
|
.flatten()
|
||||||
|
.and_then(|s| crate::skill_delivery::parse(&s))
|
||||||
|
.unwrap_or(crate::skill_delivery::Mode::Inline)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn pinned_skills_in_mode(
|
||||||
|
&self,
|
||||||
|
alias: &str,
|
||||||
|
mode: crate::skill_delivery::Mode,
|
||||||
|
) -> Option<String> {
|
||||||
let tap = self.tap.as_ref()?;
|
let tap = self.tap.as_ref()?;
|
||||||
let agent_id = crate::runtime_provision::claw_from_alias(alias)?;
|
let agent_id = crate::runtime_provision::claw_from_alias(alias)?;
|
||||||
let link = cm_db::repo::agent_template_link::get(&tap.pool, agent_id)
|
let link = cm_db::repo::agent_template_link::get(&tap.pool, agent_id)
|
||||||
@@ -361,17 +399,29 @@ impl ZeroClawDriveExecutor {
|
|||||||
let mut out = String::new();
|
let mut out = String::new();
|
||||||
let mut n = 0usize;
|
let mut n = 0usize;
|
||||||
for b in bindings.iter().filter(|b| b.pin_in_context) {
|
for b in bindings.iter().filter(|b| b.pin_in_context) {
|
||||||
|
let text = match mode {
|
||||||
|
crate::skill_delivery::Mode::Inline => b.skill.body.clone(),
|
||||||
|
// An entry is a few hundred bytes whatever the body weighs, so
|
||||||
|
// the index arm cannot hit the cap that follows. That is the
|
||||||
|
// point of it, and the reason the cap is checked against the
|
||||||
|
// rendered text rather than against the body.
|
||||||
|
crate::skill_delivery::Mode::Index => crate::skill_delivery::index_entry(
|
||||||
|
&b.skill.description,
|
||||||
|
b.skill.when_to_use.as_deref(),
|
||||||
|
&crate::mcp_skills::skill_uri(b.skill.workspace_id, &b.skill.name),
|
||||||
|
),
|
||||||
|
};
|
||||||
// Bounded, and truncation is STATED. A silently clipped procedure
|
// Bounded, and truncation is STATED. A silently clipped procedure
|
||||||
// is worse than an absent one: the agent follows the half it can
|
// is worse than an absent one: the agent follows the half it can
|
||||||
// see and reports success against a rule it never read.
|
// see and reports success against a rule it never read.
|
||||||
if out.len() + b.skill.body.len() > MAX_PINNED_SKILL_BYTES {
|
if out.len() + text.len() > MAX_PINNED_SKILL_BYTES {
|
||||||
out.push_str(&format!(
|
out.push_str(&format!(
|
||||||
"\n[skill \"{}\" omitted — the pinned set exceeded {} bytes]\n",
|
"\n[skill \"{}\" omitted — the pinned set exceeded {} bytes]\n",
|
||||||
b.skill.name, MAX_PINNED_SKILL_BYTES
|
b.skill.name, MAX_PINNED_SKILL_BYTES
|
||||||
));
|
));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
out.push_str(&render_pinned_skill(&b.skill.name, &b.skill.body));
|
out.push_str(&render_pinned_skill(&b.skill.name, &text));
|
||||||
n += 1;
|
n += 1;
|
||||||
}
|
}
|
||||||
if n == 0 {
|
if n == 0 {
|
||||||
@@ -778,9 +828,14 @@ impl TurnExecutor for ZeroClawDriveExecutor {
|
|||||||
);
|
);
|
||||||
fallback
|
fallback
|
||||||
});
|
});
|
||||||
|
// One lookup, used for the section, its preamble and the record.
|
||||||
|
// Deriving it three times would let a mission compose an index under
|
||||||
|
// an inline heading if the row changed mid-run.
|
||||||
|
let mode = self.skill_delivery_mode().await;
|
||||||
let prompt = compose_turn_prompt(
|
let prompt = compose_turn_prompt(
|
||||||
&Self::build_prompt(&req),
|
&Self::build_prompt(&req),
|
||||||
self.pinned_skills_text(&alias).await.as_deref(),
|
self.pinned_skills_in_mode(&alias, mode).await.as_deref(),
|
||||||
|
mode,
|
||||||
);
|
);
|
||||||
// Record what this agent is ACTUALLY about to receive, before driving.
|
// Record what this agent is ACTUALLY about to receive, before driving.
|
||||||
// Re-deriving it later would re-run the skill lookup against a
|
// Re-deriving it later would re-run the skill lookup against a
|
||||||
@@ -795,7 +850,14 @@ impl TurnExecutor for ZeroClawDriveExecutor {
|
|||||||
ev.run_id = tap.run_id;
|
ev.run_id = tap.run_id;
|
||||||
ev.agent_id = crate::runtime_provision::claw_from_alias(&alias);
|
ev.agent_id = crate::runtime_provision::claw_from_alias(&alias);
|
||||||
ev.target = Some(req.role.clone());
|
ev.target = Some(req.role.clone());
|
||||||
ev.detail = serde_json::json!({ "text": prompt, "tier": "container" });
|
ev.detail = serde_json::json!({
|
||||||
|
"text": prompt,
|
||||||
|
"tier": "container",
|
||||||
|
// The A/B arm, alongside the prompt it produced. `skill_use`
|
||||||
|
// recovers this from the prompt text itself, so this field is
|
||||||
|
// for reporting and for catching the two disagreeing.
|
||||||
|
"skill_delivery": mode.as_str(),
|
||||||
|
});
|
||||||
crate::mission_events::record(&tap.pool, ev).await;
|
crate::mission_events::record(&tap.pool, ev).await;
|
||||||
}
|
}
|
||||||
self.drive(&alias, &prompt).await
|
self.drive(&alias, &prompt).await
|
||||||
@@ -807,17 +869,22 @@ impl TurnExecutor for ZeroClawDriveExecutor {
|
|||||||
/// Split out from `run_turn` so the wiring is testable: `pinned_skills_text`
|
/// Split out from `run_turn` so the wiring is testable: `pinned_skills_text`
|
||||||
/// working and `run_turn` actually calling it are different claims, and the
|
/// working and `run_turn` actually calling it are different claims, and the
|
||||||
/// second is the one that was false for every skill in the catalogue.
|
/// second is the one that was false for every skill in the catalogue.
|
||||||
pub fn compose_turn_prompt(base: &str, skills: Option<&str>) -> String {
|
pub fn compose_turn_prompt(
|
||||||
|
base: &str,
|
||||||
|
skills: Option<&str>,
|
||||||
|
mode: crate::skill_delivery::Mode,
|
||||||
|
) -> String {
|
||||||
let Some(skills) = skills.map(str::trim).filter(|s| !s.is_empty()) else {
|
let Some(skills) = skills.map(str::trim).filter(|s| !s.is_empty()) else {
|
||||||
// No heading when there is nothing under it. An empty "Your skills"
|
// No heading when there is nothing under it. An empty "Your skills"
|
||||||
// section tells the model it has skills and then shows it none, which
|
// section tells the model it has skills and then shows it none, which
|
||||||
// is worse than silence.
|
// is worse than silence.
|
||||||
return base.to_string();
|
return base.to_string();
|
||||||
};
|
};
|
||||||
format!(
|
// The preamble differs per arm and lives in `skill_delivery`, because it
|
||||||
"{base}\n\n# Your skills\n\nThese are procedures you are expected to follow for \
|
// is also what the scorer reads the arm back from. Two copies of this
|
||||||
this kind of work. Where one applies to what you are about to do, follow it.\n\n{skills}"
|
// sentence is two chances for the reader to stop recognising the writer.
|
||||||
)
|
let preamble = crate::skill_delivery::preamble(mode);
|
||||||
|
format!("{base}\n\n# Your skills\n\n{preamble}\n\n{skills}")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parse `role=alias,role=alias` into a map (blank/malformed entries skipped).
|
/// Parse `role=alias,role=alias` into a map (blank/malformed entries skipped).
|
||||||
|
|||||||
@@ -10,6 +10,7 @@
|
|||||||
//! structurally unreadable. That is why this is a test and not a comment: the
|
//! structurally unreadable. That is why this is a test and not a comment: the
|
||||||
//! failure produced no error anywhere, and every layer reported success.
|
//! failure produced no error anywhere, and every layer reported success.
|
||||||
|
|
||||||
|
use cm_api::skill_delivery::Mode;
|
||||||
use cm_api::topology_exec::{MissionTap, ZeroClawDriveExecutor};
|
use cm_api::topology_exec::{MissionTap, ZeroClawDriveExecutor};
|
||||||
use cm_domain::{Agent, AgentId, AgentStatus, Role, User, UserId, Workspace, WorkspaceId};
|
use cm_domain::{Agent, AgentId, AgentStatus, Role, User, UserId, Workspace, WorkspaceId};
|
||||||
|
|
||||||
@@ -121,6 +122,14 @@ async fn seed_claw_with_pinned_skill(pool: &sqlx::PgPool, body: &str) -> (String
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn executor(pool: &sqlx::PgPool, workspace_id: WorkspaceId) -> ZeroClawDriveExecutor {
|
fn executor(pool: &sqlx::PgPool, workspace_id: WorkspaceId) -> ZeroClawDriveExecutor {
|
||||||
|
executor_for(pool, workspace_id, Uuid::now_v7())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn executor_for(
|
||||||
|
pool: &sqlx::PgPool,
|
||||||
|
workspace_id: WorkspaceId,
|
||||||
|
mission_id: Uuid,
|
||||||
|
) -> ZeroClawDriveExecutor {
|
||||||
ZeroClawDriveExecutor::new(
|
ZeroClawDriveExecutor::new(
|
||||||
"http://127.0.0.1:1".into(),
|
"http://127.0.0.1:1".into(),
|
||||||
"unused".into(),
|
"unused".into(),
|
||||||
@@ -130,12 +139,93 @@ fn executor(pool: &sqlx::PgPool, workspace_id: WorkspaceId) -> ZeroClawDriveExec
|
|||||||
.with_tap(MissionTap {
|
.with_tap(MissionTap {
|
||||||
pool: pool.clone(),
|
pool: pool.clone(),
|
||||||
workspace_id: workspace_id.as_uuid(),
|
workspace_id: workspace_id.as_uuid(),
|
||||||
mission_id: Uuid::now_v7(),
|
mission_id,
|
||||||
phase_id: None,
|
phase_id: None,
|
||||||
run_id: None,
|
run_id: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A mission row carrying an explicit delivery arm.
|
||||||
|
async fn seed_mission_on_arm(pool: &sqlx::PgPool, ws: WorkspaceId, arm: Option<&str>) -> Uuid {
|
||||||
|
let mission = Uuid::now_v7();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO missions (id, workspace_id, title, template_kind, status, skill_delivery)
|
||||||
|
VALUES ($1, $2, 'arm', 'research_only', 'running', $3)",
|
||||||
|
)
|
||||||
|
.bind(mission)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.bind(arm)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
mission
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── The index arm ───────────────────────────────────────────────────
|
||||||
|
//
|
||||||
|
// Trigger — did the agent reach for the skill? — cannot exist while every body
|
||||||
|
// is handed over unasked. These tests cover the arm that makes it a question,
|
||||||
|
// and the one guarantee the control arm needs: that it did not change.
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn the_index_arm_sends_the_uri_and_withholds_the_body() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
const MARKER: &str = "Never review a paper from its title alone.";
|
||||||
|
let (alias, ws) = seed_claw_with_pinned_skill(&pool, MARKER).await;
|
||||||
|
let mission = seed_mission_on_arm(&pool, ws, Some("index")).await;
|
||||||
|
|
||||||
|
let text = executor_for(&pool, ws, mission)
|
||||||
|
.pinned_skills_text(&alias)
|
||||||
|
.await
|
||||||
|
.expect("an indexed skill is still delivered — as an entry, not a body");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
!text.contains(MARKER),
|
||||||
|
"the BODY is what the index withholds; leaving it in delivers both \
|
||||||
|
arms at once and measures neither. Got:\n{text}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
text.contains("uri=\"skill:global/delivery-test-skill-"),
|
||||||
|
"an entry without a fetchable uri names a procedure the agent cannot \
|
||||||
|
obtain — worse than inlining it. Got:\n{text}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
text.contains("When to use: always"),
|
||||||
|
"`when_to_use` is the only thing the agent can judge relevance from, \
|
||||||
|
and judging relevance is the entire axis. Got:\n{text}"
|
||||||
|
);
|
||||||
|
// The scorer counts delivered skills by the marker, in both arms.
|
||||||
|
assert_eq!(
|
||||||
|
cm_api::topology_exec::skill_names_in(&text).len(),
|
||||||
|
1,
|
||||||
|
"an indexed skill must still count as delivered:\n{text}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn an_unrecorded_arm_delivers_bodies() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
const MARKER: &str = "Never review a paper from its title alone.";
|
||||||
|
let (alias, ws) = seed_claw_with_pinned_skill(&pool, MARKER).await;
|
||||||
|
|
||||||
|
// Every mission that ran before the column existed, plus any row whose
|
||||||
|
// value is unreadable, plus a turn with no mission row at all. All three
|
||||||
|
// resolve to the arm that needs nothing installed to work.
|
||||||
|
for arm in [None, Some("nonsense")] {
|
||||||
|
let mission = seed_mission_on_arm(&pool, ws, arm).await;
|
||||||
|
let text = executor_for(&pool, ws, mission)
|
||||||
|
.pinned_skills_text(&alias)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
text.contains(MARKER),
|
||||||
|
"skill_delivery={arm:?} must deliver the body — an index arm \
|
||||||
|
selected by accident hands out uris behind a door that may not \
|
||||||
|
be installed. Got:\n{text}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn a_pinned_skill_body_reaches_the_turn_prompt() {
|
async fn a_pinned_skill_body_reaches_the_turn_prompt() {
|
||||||
let pool = cm_testkit::test_pool().await;
|
let pool = cm_testkit::test_pool().await;
|
||||||
@@ -149,9 +239,9 @@ async fn a_pinned_skill_body_reaches_the_turn_prompt() {
|
|||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
text.contains(MARKER),
|
text.contains(MARKER),
|
||||||
"the skill BODY must be present, not just its name — there is no \
|
"the default arm delivers the BODY. It is the control in the delivery \
|
||||||
`skills.read` tool on the mission path, so an index would name a \
|
A/B, so it has to stay what production has always sent; the index arm \
|
||||||
procedure the agent has no way to fetch. Got:\n{text}"
|
is selected per mission and tested separately. Got:\n{text}"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -197,13 +287,13 @@ async fn an_agent_with_no_pinned_skills_adds_nothing() {
|
|||||||
fn the_composed_prompt_carries_the_skill_and_omits_the_heading_when_empty() {
|
fn the_composed_prompt_carries_the_skill_and_omits_the_heading_when_empty() {
|
||||||
let base = "You are the \"researcher\" agent.\n\nTask: read the papers";
|
let base = "You are the \"researcher\" agent.\n\nTask: read the papers";
|
||||||
|
|
||||||
let with = cm_api::topology_exec::compose_turn_prompt(base, Some("## arxiv-daily\nDo not re-search."));
|
let with = cm_api::topology_exec::compose_turn_prompt(base, Some("## arxiv-daily\nDo not re-search."), Mode::Inline);
|
||||||
assert!(with.contains("Task: read the papers"), "the base turn must survive");
|
assert!(with.contains("Task: read the papers"), "the base turn must survive");
|
||||||
assert!(with.contains("Do not re-search."), "the skill body must be in the prompt");
|
assert!(with.contains("Do not re-search."), "the skill body must be in the prompt");
|
||||||
assert!(with.contains("# Your skills"), "the section needs a heading");
|
assert!(with.contains("# Your skills"), "the section needs a heading");
|
||||||
|
|
||||||
for empty in [None, Some(""), Some(" \n ")] {
|
for empty in [None, Some(""), Some(" \n ")] {
|
||||||
let without = cm_api::topology_exec::compose_turn_prompt(base, empty);
|
let without = cm_api::topology_exec::compose_turn_prompt(base, empty, Mode::Inline);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
without, base,
|
without, base,
|
||||||
"with no skills the prompt must be byte-identical to the base — an \
|
"with no skills the prompt must be byte-identical to the base — an \
|
||||||
@@ -277,7 +367,7 @@ async fn the_microvm_and_session_tiers_get_the_skill_in_their_task_text() {
|
|||||||
);
|
);
|
||||||
|
|
||||||
// All three tiers share this composition, so testing it once covers them.
|
// All three tiers share this composition, so testing it once covers them.
|
||||||
let composed = cm_api::topology_exec::compose_turn_prompt("Task: read the papers", Some(&skills));
|
let composed = cm_api::topology_exec::compose_turn_prompt("Task: read the papers", Some(&skills), Mode::Inline);
|
||||||
assert!(composed.contains(MARKER));
|
assert!(composed.contains(MARKER));
|
||||||
assert!(composed.contains("Task: read the papers"));
|
assert!(composed.contains("Task: read the papers"));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ mod token;
|
|||||||
pub use bootstrap::bootstrap_owner;
|
pub use bootstrap::bootstrap_owner;
|
||||||
pub use jwt::{ExternalClaims, JwtError, JwtVerifier};
|
pub use jwt::{ExternalClaims, JwtError, JwtVerifier};
|
||||||
pub use service::{
|
pub use service::{
|
||||||
AuthError, AuthService, AuthedUser, SCOPE_FULL, SCOPE_SKILLS_READ, SESSION_TTL,
|
AuthError, AuthService, AuthedUser, SCOPE_AGENT_DOOR, SCOPE_FULL, SCOPE_SKILLS_READ,
|
||||||
|
SESSION_TTL,
|
||||||
};
|
};
|
||||||
pub use token::SessionToken;
|
pub use token::SessionToken;
|
||||||
|
|||||||
@@ -21,6 +21,19 @@ pub const SCOPE_FULL: &str = "full";
|
|||||||
/// that authenticates nowhere, which fails safely but silently.
|
/// that authenticates nowhere, which fails safely but silently.
|
||||||
pub const SCOPE_SKILLS_READ: &str = "skills:read";
|
pub const SCOPE_SKILLS_READ: &str = "skills:read";
|
||||||
|
|
||||||
|
/// Act through the §15 MCP door (`/mcp`), and nothing else.
|
||||||
|
///
|
||||||
|
/// The door is the actuator: `email_send`, `slack_post`, `delegate`. Reaching
|
||||||
|
/// it means a token an agent's runtime holds, and the same reasoning as
|
||||||
|
/// [`SCOPE_SKILLS_READ`] applies — a full session there is an owner-privileged
|
||||||
|
/// API key handed to a process whose whole purpose is to act on instructions
|
||||||
|
/// from a model.
|
||||||
|
///
|
||||||
|
/// Nothing mints one yet; the route accepts it so that whatever does will not
|
||||||
|
/// have to reach for a person's session to be understood. `full` still works,
|
||||||
|
/// so the UI and any human caller are unaffected.
|
||||||
|
pub const SCOPE_AGENT_DOOR: &str = "agent:door";
|
||||||
|
|
||||||
/// The authenticated caller attached to every API request: everything RBAC
|
/// The authenticated caller attached to every API request: everything RBAC
|
||||||
/// decisions need, nothing more.
|
/// decisions need, nothing more.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
|||||||
@@ -168,6 +168,54 @@ async fn a_scoped_token_is_refused_by_every_unscoped_caller() {
|
|||||||
assert_eq!(ok.user_id, user.id);
|
assert_eq!(ok.user_id, user.id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The two narrow scopes must not substitute for each other.
|
||||||
|
///
|
||||||
|
/// They protect different things — one reads the skills catalogue, the other
|
||||||
|
/// operates the §15 door that can `delegate`. Both tokens live where an agent
|
||||||
|
/// can read them, so the whole value of having two constants is that holding
|
||||||
|
/// one grants nothing the other has.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn one_narrow_scope_does_not_open_the_other() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let (_ws, user) = seeded(&pool).await;
|
||||||
|
let auth = AuthService::new(pool);
|
||||||
|
|
||||||
|
let skills = auth
|
||||||
|
.mint_scoped(user.id, cm_auth::SCOPE_SKILLS_READ, time::Duration::hours(1))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let door = auth
|
||||||
|
.mint_scoped(user.id, cm_auth::SCOPE_AGENT_DOOR, time::Duration::hours(1))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
matches!(
|
||||||
|
auth.authenticate_scoped(&skills, cm_auth::SCOPE_AGENT_DOOR).await,
|
||||||
|
Err(AuthError::Unauthenticated)
|
||||||
|
),
|
||||||
|
"a skills token must not reach the door — the door can `delegate`"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
matches!(
|
||||||
|
auth.authenticate_scoped(&door, cm_auth::SCOPE_SKILLS_READ).await,
|
||||||
|
Err(AuthError::Unauthenticated)
|
||||||
|
),
|
||||||
|
"and a door token must not read the catalogue"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
matches!(
|
||||||
|
auth.authenticate(&door).await,
|
||||||
|
Err(AuthError::Unauthenticated)
|
||||||
|
),
|
||||||
|
"nor authenticate an ordinary API call"
|
||||||
|
);
|
||||||
|
assert!(auth
|
||||||
|
.authenticate_scoped(&door, cm_auth::SCOPE_AGENT_DOOR)
|
||||||
|
.await
|
||||||
|
.is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
/// A person's session keeps working everywhere, including the scoped route.
|
/// A person's session keeps working everywhere, including the scoped route.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn a_full_session_still_satisfies_a_scoped_route() {
|
async fn a_full_session_still_satisfies_a_scoped_route() {
|
||||||
|
|||||||
@@ -30,6 +30,10 @@ async fn server() -> &'static PgServer {
|
|||||||
SERVER
|
SERVER
|
||||||
.get_or_init(|| async {
|
.get_or_init(|| async {
|
||||||
if let Ok(url) = std::env::var("CM_TEST_DATABASE_URL") {
|
if let Ok(url) = std::env::var("CM_TEST_DATABASE_URL") {
|
||||||
|
// Only on the shared-server path. The testcontainer below is
|
||||||
|
// torn down with the process, so it has nothing to reap and a
|
||||||
|
// sweep there would be pure cost.
|
||||||
|
reap_stale_databases(&url).await;
|
||||||
return PgServer {
|
return PgServer {
|
||||||
admin_url: url,
|
admin_url: url,
|
||||||
_container: None,
|
_container: None,
|
||||||
@@ -57,6 +61,86 @@ async fn server() -> &'static PgServer {
|
|||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// How long a test database may sit before another test process reaps it.
|
||||||
|
///
|
||||||
|
/// Comfortably longer than any test run, so a database in use by a
|
||||||
|
/// concurrently-running binary is never a candidate. Nothing here needs to be
|
||||||
|
/// prompt — the point is that the set stays bounded, not that it stays empty.
|
||||||
|
const STALE_AFTER_MS: u64 = 2 * 60 * 60 * 1000;
|
||||||
|
|
||||||
|
/// Drop test databases left behind by earlier runs.
|
||||||
|
///
|
||||||
|
/// `test_pool` creates a database per test and nothing ever dropped it. On the
|
||||||
|
/// testcontainer path that is invisible: the container dies with the process
|
||||||
|
/// and takes them with it. But `CM_TEST_DATABASE_URL` points at a SHARED
|
||||||
|
/// server that outlives the run — which is the path CI uses and the path
|
||||||
|
/// `.cargo/config.toml` sets for local development — so on both of those every
|
||||||
|
/// database ever created is still there.
|
||||||
|
///
|
||||||
|
/// Measured before writing this: **3,546 databases, 38 GB** on one developer
|
||||||
|
/// machine. It grows with every `cargo test`.
|
||||||
|
///
|
||||||
|
/// Age comes from the name, not the catalogue. Postgres records no creation
|
||||||
|
/// time for a database, but the names are `test_<uuid-v7>` and UUIDv7 puts the
|
||||||
|
/// millisecond timestamp in its first 48 bits — the same property
|
||||||
|
/// `mission_runtime::container_name` relies on.
|
||||||
|
///
|
||||||
|
/// Best-effort throughout: a test must never fail because housekeeping could
|
||||||
|
/// not run.
|
||||||
|
async fn reap_stale_databases(admin_url: &str) {
|
||||||
|
let Ok(admin) = PgPoolOptions::new()
|
||||||
|
.max_connections(1)
|
||||||
|
.connect(admin_url)
|
||||||
|
.await
|
||||||
|
else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let names: Vec<String> = sqlx::query_scalar(
|
||||||
|
"SELECT datname FROM pg_database WHERE datname LIKE 'test\\_%'",
|
||||||
|
)
|
||||||
|
.fetch_all(&admin)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let now_ms = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_millis() as u64)
|
||||||
|
.unwrap_or(0);
|
||||||
|
let mut dropped = 0usize;
|
||||||
|
for name in names {
|
||||||
|
let Some(created) = uuid_v7_millis(&name) else {
|
||||||
|
// Not a name we minted; leave it entirely alone.
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if now_ms.saturating_sub(created) < STALE_AFTER_MS {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// FORCE terminates any leftover connection; without it a single stale
|
||||||
|
// session pins the database and the reap silently does nothing.
|
||||||
|
if sqlx::query(&format!("DROP DATABASE IF EXISTS {name} WITH (FORCE)"))
|
||||||
|
.execute(&admin)
|
||||||
|
.await
|
||||||
|
.is_ok()
|
||||||
|
{
|
||||||
|
dropped += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if dropped > 0 {
|
||||||
|
eprintln!("cm-testkit: reaped {dropped} stale test database(s)");
|
||||||
|
}
|
||||||
|
admin.close().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The millisecond timestamp encoded in the leading 48 bits of a
|
||||||
|
/// `test_<uuid-v7-simple>` name.
|
||||||
|
fn uuid_v7_millis(db_name: &str) -> Option<u64> {
|
||||||
|
let hex = db_name.strip_prefix("test_")?;
|
||||||
|
if hex.len() != 32 || !hex.chars().all(|c| c.is_ascii_hexdigit()) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
u64::from_str_radix(&hex[..12], 16).ok()
|
||||||
|
}
|
||||||
|
|
||||||
/// Creates a unique database, runs all migrations, and returns a pool
|
/// Creates a unique database, runs all migrations, and returns a pool
|
||||||
/// connected to it.
|
/// connected to it.
|
||||||
pub async fn test_pool() -> PgPool {
|
pub async fn test_pool() -> PgPool {
|
||||||
@@ -93,3 +177,46 @@ fn swap_database(url: &str, db_name: &str) -> String {
|
|||||||
None => format!("{head}/{db_name}"),
|
None => format!("{head}/{db_name}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::{uuid_v7_millis, STALE_AFTER_MS};
|
||||||
|
|
||||||
|
/// Age comes from the NAME, because Postgres records no creation time for
|
||||||
|
/// a database. UUIDv7 puts the millisecond timestamp in its first 48 bits.
|
||||||
|
#[test]
|
||||||
|
fn a_test_database_name_carries_its_own_age() {
|
||||||
|
let id = uuid::Uuid::now_v7();
|
||||||
|
let name = format!("test_{}", id.simple());
|
||||||
|
let ms = uuid_v7_millis(&name).expect("a name we minted parses");
|
||||||
|
let now = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.unwrap()
|
||||||
|
.as_millis() as u64;
|
||||||
|
assert!(
|
||||||
|
now.saturating_sub(ms) < 5_000,
|
||||||
|
"a database created just now must read as new, or the reaper drops \
|
||||||
|
one another test process is still using"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Anything we did not mint is left alone.
|
||||||
|
#[test]
|
||||||
|
fn only_our_own_names_are_reapable() {
|
||||||
|
assert!(uuid_v7_millis("clawmates").is_none());
|
||||||
|
assert!(uuid_v7_millis("postgres").is_none());
|
||||||
|
assert!(uuid_v7_millis("template1").is_none());
|
||||||
|
// Right prefix, wrong shape — a human-made `test_scratch` survives.
|
||||||
|
assert!(uuid_v7_millis("test_scratch").is_none());
|
||||||
|
assert!(uuid_v7_millis("test_").is_none());
|
||||||
|
// Right length, not hex.
|
||||||
|
assert!(uuid_v7_millis(&format!("test_{}", "z".repeat(32))).is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The window has to be longer than a test run, or the reaper deletes a
|
||||||
|
/// database out from under a binary running in parallel.
|
||||||
|
#[test]
|
||||||
|
fn the_stale_window_outlasts_any_test_run() {
|
||||||
|
assert!(STALE_AFTER_MS >= 60 * 60 * 1000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+106
-35
@@ -7,12 +7,12 @@ than the agent's own account of it.
|
|||||||
|
|
||||||
## State of the tree
|
## State of the tree
|
||||||
|
|
||||||
Local suite green: **107 test binaries, 796 tests** (`cargo test --workspace`), and the workspace builds with `--all-targets`.
|
Local suite green: **108 test binaries, 815 tests** (`cargo test --workspace`).
|
||||||
Five measurement missions ran on the local stack (`scripts/skill-use-run.sh`);
|
**Pushed** — 22 commits, CI green on gw-04, deployed.
|
||||||
all five are held 90 days and re-scorable with `--score <id>`.
|
|
||||||
10 commits on `main` this pass, **not pushed** — a push to `main` auto-deploys
|
Eight measurement missions ran on the local stack
|
||||||
to gw-04, and the container-tier work already deployed is unexercised there
|
(`scripts/skill-use-run.sh`); all are held 90 days and re-scorable with
|
||||||
(see below).
|
`--score <id>`.
|
||||||
|
|
||||||
## The premise of the last handoff's item 1 was wrong
|
## The premise of the last handoff's item 1 was wrong
|
||||||
|
|
||||||
@@ -132,44 +132,115 @@ which is the only reason they were found.
|
|||||||
`no_skill_shows_a_marker_the_parser_would_reject` guards the class, running the
|
`no_skill_shows_a_marker_the_parser_would_reject` guards the class, running the
|
||||||
real parser over every marker in every skill's fenced blocks.
|
real parser over every marker in every skill's fenced blocks.
|
||||||
|
|
||||||
|
### Trigger is measured, and the door is what made it possible
|
||||||
|
|
||||||
|
`skill_use` scores Trigger from `ReadMcpResourceTool` calls, parsed through
|
||||||
|
`mcp_skills::parse_uri` — the function that wrote the URI. Run 8, one mission,
|
||||||
|
one clean A/B:
|
||||||
|
|
||||||
|
| skill | delivered by | Trigger |
|
||||||
|
|---|---|---|
|
||||||
|
| `workspace-repo-commit-protocol` | retrieval | **pass** |
|
||||||
|
| `web-search-triage` | inlined | not observable |
|
||||||
|
| `structured-paper-summary` | inlined | not observable |
|
||||||
|
| `scientific-writing-conventions` | inlined | not observable |
|
||||||
|
|
||||||
|
Careful with what that proves: the retrieval was *instructed* by the task. It
|
||||||
|
demonstrates the instrument, not a spontaneous relevance judgement.
|
||||||
|
|
||||||
|
### Red-first is observable, and it needed neither ordering nor the diff
|
||||||
|
|
||||||
|
The previous version of this list said the TDD check needed the repository
|
||||||
|
diff. That was wrong, and `tdd-red-green-refactor` says why: *"Commit the
|
||||||
|
RED-to-GREEN pair as one commit."* The failing test and its fix land together
|
||||||
|
by instruction, so the diff and the commit history are as blind as tool
|
||||||
|
ordering already was.
|
||||||
|
|
||||||
|
The witness is what each run PRINTED, and the tap was discarding it. Claude
|
||||||
|
Code's `PostToolUse` payload carries `tool_response` (stdout/stderr/
|
||||||
|
interrupted) — verified against the binary. `bounded_response` keeps the END of
|
||||||
|
that output, the opposite of `bounded_input`, because a command's meaning is
|
||||||
|
its verdict and `cargo test` prints it last.
|
||||||
|
|
||||||
|
failing run then a passing one → Pass
|
||||||
|
every run failed → Fail
|
||||||
|
every run passed → NotObservable, because a test that never
|
||||||
|
failed is equally what a correct
|
||||||
|
implementation written first looks like
|
||||||
|
|
||||||
|
## The four days after that pass went into infrastructure
|
||||||
|
|
||||||
|
No product commits landed between 2026-08-21 and 2026-08-25. What did happen,
|
||||||
|
re-verified live on the 25th:
|
||||||
|
|
||||||
|
- `PasswordAuthentication no` is effective on gw-01/gw-02/gw-04 (checked with
|
||||||
|
`sudo sshd -T`, not by reading a config a drop-in can override), and
|
||||||
|
`fail2ban` is active on all three — **95 / 74 / 77 total bans**, so it is
|
||||||
|
catching real traffic. `ignoreip` must keep `100.64.0.0/10`: every operator
|
||||||
|
and every fleet node reaches these hosts from CGNAT space.
|
||||||
|
- gw-02's three Postgres containers were published on `0.0.0.0` and are now
|
||||||
|
bound to `127.0.0.1` (5434 mindbridge, 5435 clawbridge, 5437 smartclaw).
|
||||||
|
- Roughly 93G reclaimed; the largest single piece was a source-level leak of one
|
||||||
|
test database per test, fixed in `b58f034`.
|
||||||
|
|
||||||
|
Two failure shapes worth keeping:
|
||||||
|
|
||||||
|
- Recreating a DB container leaves the owning app holding **dead pool
|
||||||
|
connections**. The port binding reads perfect and the app still returns its
|
||||||
|
baseline status code, so a shallow check calls it done while every pool is
|
||||||
|
stale. Restart the owning unit and assert on the established-connection count.
|
||||||
|
- A `grep` for the service name matched `smartclaw-email` before `smartclaw`,
|
||||||
|
so the wrong unit was restarted. Only the connection count being `0` caught it.
|
||||||
|
|
||||||
|
**Open, and not safe to assume otherwise:** gw-02 has *no host firewall* —
|
||||||
|
`INPUT` policy `ACCEPT`, `DOCKER-USER` empty. Ports 18789/3000/3010/8082 are
|
||||||
|
bound `0.0.0.0` by native processes (a compose edit will not move them), and the
|
||||||
|
only thing keeping them off the internet is an upstream cloud firewall the host
|
||||||
|
cannot see. gw-05 and web-01 have no fail2ban and `sudo` there needs a password.
|
||||||
|
|
||||||
## Next, in order
|
## Next, in order
|
||||||
|
|
||||||
1. **The TDD check cannot confirm red-first, and that is structural.** Run 4
|
1. **Watch the first production mission.** Everything below the fold in this
|
||||||
(`research_and_code`, real repo) edited `src/lib.rs` once — implementation
|
document is now deployed and unexercised in prod, because prod has still
|
||||||
*and* `#[cfg(test)] mod tests` in the same write — then ran `cargo test`
|
never run a mission. The door, the staffing, the attribution and the orphan
|
||||||
five times. In Rust the unit test lives in the file under test, so that
|
sweep all engage on the next launch. Prod auth is Clerk, so somebody has to
|
||||||
ordering is what following the skill precisely looks like from outside. The
|
click.
|
||||||
check detects "wrote source, never ran a test" and nothing more. If you want
|
|
||||||
red-first, it needs the diff (did the test exist before the impl?), not the
|
|
||||||
tool order.
|
|
||||||
|
|
||||||
2. **Attribute tool calls to agents.** `record_vm_tools` writes
|
```
|
||||||
`agent_id: None`, because the container tap is per-container and all roles
|
ssh gw-04 'docker logs clawmates_server_1 2>&1 | grep -E "skills door installed|drained .* tool call|orphans:"'
|
||||||
share one. Every Skill-Use score is therefore per-**mission**, not per-role,
|
```
|
||||||
and the World's per-agent view gets nothing from the container tier. The
|
|
||||||
hook payload carries `session_id`; mapping it back to a turn is the fix.
|
|
||||||
|
|
||||||
3. **Deploy the door** (`TOOL-CALL-ARCHITECTURE.md` §3). Config, not code:
|
2. ~~**Measure Trigger properly.**~~ **Done — 2026-08-25.** `skill_delivery`
|
||||||
`/zeroclaw-data/clawmates-mcp.json` plus a door-shaped provider alias. It is
|
ships both arms; `index` sends name + `when_to_use` + a uri and the agent
|
||||||
now the single change that makes **Trigger** a real measurement, and the
|
fetches bodies through the door. Runs 9/10 in `SKILL-USE-BASELINE.md` are the
|
||||||
precondition for skills moving from inlined bodies to progressive
|
A/B: identical task, one server process, and **the first retrieval this
|
||||||
disclosure — which would also cut the prompt cost in item 1.
|
project has seen that the task did not ask for**. Each agent fetched the
|
||||||
|
skill bound to its own role and neither fetched another's. No Compliance
|
||||||
|
regression appeared — 34% fewer tokens, both arms judged met, deliverables
|
||||||
|
slightly larger. n=1 per arm, so that is a signal and not a rate.
|
||||||
|
|
||||||
4. **Fold the microVM tier onto `container_tool_hooks`.** It has a gate and a
|
What is left here is volume: run more pairs before believing any number, and
|
||||||
tap by a different route (`vm_tool_tap` installs into the guest,
|
decide whether `index` becomes the default. It is currently opt-in per
|
||||||
`microvm_executor` drains inside the turn). Two mechanisms for one job is how
|
mission (`config.skill_delivery`) or per deployment
|
||||||
they drift — and the argument-discarding bug above lived in the shared parser
|
(`CLAWMATES_SKILL_DELIVERY`).
|
||||||
precisely because nobody looked at it from the container side. The fleet has
|
|
||||||
been offline for over a week, so this cannot be tested today.
|
|
||||||
|
|
||||||
5. **Pull upstream's egress policy** — `0db7d999a feat(plugins): add shared
|
3. **Fold the microVM tier onto `container_tool_hooks`.** Two mechanisms for
|
||||||
egress policy foundation (#9137)`. We are ~220 commits behind; this is the
|
one job is how they drift — the argument-discarding bug lived in the shared
|
||||||
one item worth taking, and it is defence for a problem we have not solved.
|
parser precisely because nobody looked at it from the container side. The
|
||||||
|
microVM path also still passes no turn agents, so its tool calls stay
|
||||||
|
unattributed. Blocked: the fleet has been offline for over a week.
|
||||||
|
|
||||||
|
4. **Pull upstream's egress policy** — `0db7d999a feat(plugins): add shared
|
||||||
|
egress policy foundation (#9137)`. ~220 commits behind; this is the one item
|
||||||
|
worth taking, and it is defence for a problem we have not solved.
|
||||||
|
|
||||||
|
5. **A narrow credential for the rest of the door.** `SCOPE_SKILLS_READ` covers
|
||||||
|
`/mcp/skills`. The `/mcp` door proper (`mcp_door.rs`) still authenticates
|
||||||
|
with a full session, and it is the one that can `delegate`. Nothing hands it
|
||||||
|
a token today; anything that does should not hand it a person's.
|
||||||
|
|
||||||
## Open decisions that are yours
|
## Open decisions that are yours
|
||||||
|
|
||||||
- **Push.** 10 commits are local. Pushing `main` triggers CI → auto-deploy to
|
|
||||||
gw-04.
|
|
||||||
- **Self-authoring scope.** Agents apply their own `skill_candidate` items with
|
- **Self-authoring scope.** Agents apply their own `skill_candidate` items with
|
||||||
no human click (`CLAWMATES_SKILL_SELF_AUTHORING=0` restores the gate).
|
no human click (`CLAWMATES_SKILL_SELF_AUTHORING=0` restores the gate).
|
||||||
`identity_refinement` and `brain_consolidation` still wait for a human,
|
`identity_refinement` and `brain_consolidation` still wait for a human,
|
||||||
|
|||||||
@@ -286,6 +286,90 @@ skill itself names. It is recorded here rather than quietly corrected, because
|
|||||||
a measurement that hides its own false positives cannot be trusted about
|
a measurement that hides its own false positives cannot be trusted about
|
||||||
anyone else's.
|
anyone else's.
|
||||||
|
|
||||||
|
## Runs 9 and 10 — the delivery A/B, and the first unprompted Trigger
|
||||||
|
|
||||||
|
Two `research_only` missions, **identical task text**, launched minutes apart
|
||||||
|
against one server process. The task never mentions skills, MCP or retrieval —
|
||||||
|
which is the whole point. Run 8 demonstrated the instrument, but its retrieval
|
||||||
|
was *instructed by the task*; nothing there showed an agent judging relevance.
|
||||||
|
|
||||||
|
| | run 9 (`inline`) | run 10 (`index`) |
|
||||||
|
|---|---|---|
|
||||||
|
| skills delivered | 4 | 4 |
|
||||||
|
| prompt bytes (3 turns) | 8642 / 7689 / 10165 | 7329 / 5524 / 5762 |
|
||||||
|
| tool calls recorded | 89 | **59** |
|
||||||
|
| tokens (3 steps) | 52,191 | **35,090** |
|
||||||
|
| `ReadMcpResourceTool` | 0 | **2** |
|
||||||
|
| judge verdict | met | met |
|
||||||
|
|
||||||
|
Per skill:
|
||||||
|
|
||||||
|
| skill | run 9 trigger | run 10 trigger | boundary (both) |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `web-search-triage` | not observable | **pass** | n/a |
|
||||||
|
| `scientific-writing-conventions` | not observable | **pass** | n/a |
|
||||||
|
| `structured-paper-summary` | not observable | n/a | n/a |
|
||||||
|
| `workspace-repo-commit-protocol` | not observable | **FAIL** | pass |
|
||||||
|
|
||||||
|
### What the retrievals actually show
|
||||||
|
|
||||||
|
Attribution is the load-bearing detail, and it is only available because
|
||||||
|
`attribute_sessions` maps a phase's tool calls back to the agent that made
|
||||||
|
them:
|
||||||
|
|
||||||
|
Solveig (lead_researcher) -> skill:global/web-search-triage
|
||||||
|
Olamide (report_writer) -> skill:global/scientific-writing-conventions
|
||||||
|
|
||||||
|
**Each agent reached for the skill bound to its own role, and neither reached
|
||||||
|
for another role's.** That is a relevance judgement, not a sweep — an agent
|
||||||
|
that fetched all four would have shown nothing except that it could.
|
||||||
|
|
||||||
|
Two skills went unread. `structured-paper-summary` scores `NotApplicable`: its
|
||||||
|
holder passed over it and nothing in that phase could check whether it should
|
||||||
|
have, so calling that a miss would punish correct triage.
|
||||||
|
`workspace-repo-commit-protocol` scores **Fail**, and that verdict is the one
|
||||||
|
to argue with: the agent never fetched the procedure, yet every write it made
|
||||||
|
landed inside the checkout, so `boundary` passes. It behaved correctly without
|
||||||
|
reading the rule. Under the rule as written — an applicable skill offered and
|
||||||
|
not read is a Trigger failure — that is a fail, and the pass beside it is the
|
||||||
|
honest counterweight rather than a contradiction.
|
||||||
|
|
||||||
|
### The regression the A/B existed to catch did not appear
|
||||||
|
|
||||||
|
Progressive disclosure can only cost Compliance: under `inline` the procedure
|
||||||
|
sits in front of the model whether or not it noticed it applied. It did not
|
||||||
|
cost anything measurable here. The `index` arm used **34% fewer tokens and 30
|
||||||
|
fewer tool calls**, both arms passed the independent judge, and the deliverables
|
||||||
|
came out slightly larger, not thinner:
|
||||||
|
|
||||||
|
research/speculative-decoding.md 4870 -> 6182
|
||||||
|
research/evidence.md 10823 -> 11727
|
||||||
|
research/evidence-check.md 3340 -> 5336
|
||||||
|
research/questions.md 1894 -> 2402
|
||||||
|
research/REPORT.md 9867 -> 7901
|
||||||
|
|
||||||
|
The `Edit` count is where the arms differ most (31 -> 5). That is a behavioural
|
||||||
|
difference the measurement did not predict and cannot explain from one run
|
||||||
|
each.
|
||||||
|
|
||||||
|
### Two readings this data does not support
|
||||||
|
|
||||||
|
**The prompt saving is small.** Skill bodies are a minority of a turn prompt,
|
||||||
|
so withholding them cut 15-43% of the bytes, not the order of magnitude the
|
||||||
|
framing suggests. Progressive disclosure is worth doing for Trigger, not for
|
||||||
|
context economy.
|
||||||
|
|
||||||
|
**Every `tool.call` in a phase carries the same `created_at`** — the drain
|
||||||
|
timestamp, not the call time. All 59 rows in run 10 read `12:48:12`. Ordering
|
||||||
|
tool calls by that column produces a confident, entirely fabricated narrative;
|
||||||
|
the first draft of this section said the report writer had fetched both skills,
|
||||||
|
because both retrievals appeared inside its turn window. Attribution by
|
||||||
|
`agent_id` is the real answer and it says something different.
|
||||||
|
|
||||||
|
**n = 1 per arm.** Two missions do not establish a rate. What they establish is
|
||||||
|
that the axis now produces a signal at all, and that the control arm's
|
||||||
|
structural blind spot is gone rather than papered over.
|
||||||
|
|
||||||
## Honest limits
|
## Honest limits
|
||||||
|
|
||||||
- **Five runs, one tier, two workflows.** Nothing here generalises to the
|
- **Five runs, one tier, two workflows.** Nothing here generalises to the
|
||||||
|
|||||||
@@ -0,0 +1,16 @@
|
|||||||
|
-- Which arm of the skill-delivery A/B a mission ran.
|
||||||
|
--
|
||||||
|
-- 'inline' — every pinned skill's body is appended to the turn prompt.
|
||||||
|
-- 'index' — the prompt carries name + when_to_use + uri; the agent fetches
|
||||||
|
-- the bodies it judges relevant through the skills door.
|
||||||
|
--
|
||||||
|
-- Nullable, and NULL means 'inline'. Every mission that ran before this column
|
||||||
|
-- existed ran inline, so a default would be a claim about history that a NULL
|
||||||
|
-- correctly declines to make. `topology_exec::skill_delivery_mode` resolves
|
||||||
|
-- NULL, an unrecognised value and a missing row identically, to the arm that
|
||||||
|
-- requires nothing to be true.
|
||||||
|
--
|
||||||
|
-- Written once at launch by `mission_orchestrator`, which is the only place
|
||||||
|
-- that knows whether the skills door installed — 'index' is refused without
|
||||||
|
-- one, because an index the agent cannot fetch from is worse than no index.
|
||||||
|
ALTER TABLE missions ADD COLUMN IF NOT EXISTS skill_delivery TEXT;
|
||||||
@@ -21,6 +21,13 @@
|
|||||||
# REPO_ID repository to check out; required by the coding recipes, and
|
# REPO_ID repository to check out; required by the coding recipes, and
|
||||||
# the only way the TDD and commit checks can ever fire — a
|
# the only way the TDD and commit checks can ever fire — a
|
||||||
# repo-less run writes markdown and commits nothing
|
# repo-less run writes markdown and commits nothing
|
||||||
|
# DELIVERY skill delivery arm `inline` (default) or `index`
|
||||||
|
# `index` sends name + when_to_use + a uri and makes the agent
|
||||||
|
# fetch bodies through the skills door, which is the only arm
|
||||||
|
# where Trigger is a question at all. Set per mission, so both
|
||||||
|
# arms run against ONE server process — restarting between arms
|
||||||
|
# would put a confound in the comparison that the numbers do
|
||||||
|
# not show.
|
||||||
# TIMEOUT seconds to wait (default 1800)
|
# TIMEOUT seconds to wait (default 1800)
|
||||||
# RETAIN_DAYS hold events this long so the run stays re-scorable (default 90)
|
# RETAIN_DAYS hold events this long so the run stays re-scorable (default 90)
|
||||||
|
|
||||||
@@ -30,6 +37,7 @@ API="${API:-http://127.0.0.1:8080}"
|
|||||||
PG="${PG:-clawmates-postgres-1}"
|
PG="${PG:-clawmates-postgres-1}"
|
||||||
OWNER="${OWNER:-om[email protected]}"
|
OWNER="${OWNER:-om[email protected]}"
|
||||||
TEMPLATE="${TEMPLATE:-research_only}"
|
TEMPLATE="${TEMPLATE:-research_only}"
|
||||||
|
DELIVERY="${DELIVERY:-}"
|
||||||
TIMEOUT="${TIMEOUT:-1800}"
|
TIMEOUT="${TIMEOUT:-1800}"
|
||||||
RETAIN_DAYS="${RETAIN_DAYS:-90}"
|
RETAIN_DAYS="${RETAIN_DAYS:-90}"
|
||||||
|
|
||||||
@@ -67,7 +75,7 @@ api() { # api <token> <METHOD> <path> [json]
|
|||||||
jqv() { python3 -c "import sys,json;d=json.load(sys.stdin);print(d$1)"; }
|
jqv() { python3 -c "import sys,json;d=json.load(sys.stdin);print(d$1)"; }
|
||||||
|
|
||||||
score() { # score <token> <mission-id>
|
score() { # score <token> <mission-id>
|
||||||
local token="$1" id="$2"
|
local token="$1" id="$2" arm
|
||||||
echo
|
echo
|
||||||
echo "── what the agents DID ─────────────────────────────────────"
|
echo "── what the agents DID ─────────────────────────────────────"
|
||||||
psql_ "select kind || ' ' || coalesce(target,'') ||
|
psql_ "select kind || ' ' || coalesce(target,'') ||
|
||||||
@@ -77,6 +85,11 @@ score() { # score <token> <mission-id>
|
|||||||
order by id;"
|
order by id;"
|
||||||
echo
|
echo
|
||||||
echo "── Skill-Use ───────────────────────────────────────────────"
|
echo "── Skill-Use ───────────────────────────────────────────────"
|
||||||
|
# Which arm ran, printed next to the scores it produced. A Trigger column of
|
||||||
|
# "not observable" means something different under each arm, and a report
|
||||||
|
# that does not say which one is not comparable to the next one.
|
||||||
|
arm=$(psql_ "select coalesce(skill_delivery, 'inline (unrecorded)') from missions where id='$id';" | tr -d '[:space:]')
|
||||||
|
echo "delivery arm: $arm"
|
||||||
api "$token" GET "/api/missions/$id/skill-use" | python3 -m json.tool
|
api "$token" GET "/api/missions/$id/skill-use" | python3 -m json.tool
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -90,7 +103,7 @@ fi
|
|||||||
TITLE="${1:?title}"
|
TITLE="${1:?title}"
|
||||||
TASK="${2:?task description}"
|
TASK="${2:?task description}"
|
||||||
|
|
||||||
body=$(python3 - "$TITLE" "$TASK" "$TEMPLATE" "${REPO_ID:-}" <<'PY'
|
body=$(python3 - "$TITLE" "$TASK" "$TEMPLATE" "${REPO_ID:-}" "$DELIVERY" <<'PY'
|
||||||
import json, sys
|
import json, sys
|
||||||
req = {
|
req = {
|
||||||
"title": sys.argv[1],
|
"title": sys.argv[1],
|
||||||
@@ -99,6 +112,10 @@ req = {
|
|||||||
}
|
}
|
||||||
if sys.argv[4]:
|
if sys.argv[4]:
|
||||||
req["repo_id"] = sys.argv[4]
|
req["repo_id"] = sys.argv[4]
|
||||||
|
if sys.argv[5]:
|
||||||
|
# The recipe merges `phase_teams` INTO this object rather than replacing
|
||||||
|
# it, so an arm set here survives staffing.
|
||||||
|
req["config"] = {"skill_delivery": sys.argv[5]}
|
||||||
print(json.dumps(req))
|
print(json.dumps(req))
|
||||||
PY
|
PY
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user