Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
accae7fa94 | ||
|
|
22eeaa6f15 | ||
|
|
f52cff3e04 | ||
|
|
b58f0347e6 | ||
|
|
72eda8b3d2 |
@@ -54,6 +54,7 @@ pub mod security_scan;
|
||||
pub mod session_executor;
|
||||
pub mod container_tool_hooks;
|
||||
pub mod gateway_preflight;
|
||||
pub mod skill_delivery;
|
||||
pub mod skill_self_authoring;
|
||||
pub mod skill_use;
|
||||
pub mod skills_loader;
|
||||
|
||||
@@ -237,12 +237,21 @@ async fn mint_grant(
|
||||
}
|
||||
|
||||
/// Authenticate the bearer header → workspace/user. `None` if missing/invalid.
|
||||
///
|
||||
/// Accepts [`cm_auth::SCOPE_AGENT_DOOR`] as well as a person's session. This
|
||||
/// route is the one that can `delegate`, and the thing that will eventually
|
||||
/// hold a token for it is an agent runtime — so the narrow credential has to
|
||||
/// exist before something reaches for the only one that does.
|
||||
async fn authed(state: &AppState, headers: &HeaderMap) -> Option<cm_auth::AuthedUser> {
|
||||
let token = headers
|
||||
.get(AUTHORIZATION)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.strip_prefix("Bearer "))?;
|
||||
state.auth.authenticate(token).await.ok()
|
||||
state
|
||||
.auth
|
||||
.authenticate_scoped(token, cm_auth::SCOPE_AGENT_DOOR)
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// Resolve the specific claw making the call. Our ZeroClaw fork stamps the
|
||||
|
||||
@@ -106,7 +106,7 @@ async fn caller_agent(
|
||||
|
||||
// ── URI helpers ──────────────────────────────────────────────────
|
||||
|
||||
fn skill_uri(workspace_id: Option<Uuid>, name: &str) -> String {
|
||||
pub(crate) fn skill_uri(workspace_id: Option<Uuid>, name: &str) -> String {
|
||||
match workspace_id {
|
||||
Some(ws) => format!("{URI_PREFIX_WORKSPACE}{ws}/{name}"),
|
||||
None => format!("{URI_PREFIX_GLOBAL}{name}"),
|
||||
|
||||
@@ -349,7 +349,7 @@ pub async fn on_launch(
|
||||
// Only when this mission got its OWN container — the shared runtime is
|
||||
// not ours to reconfigure, and `mission_gateway` being Some is exactly
|
||||
// the signal that `ensure_container` ran.
|
||||
if mission_gateway.is_some() {
|
||||
let door = if mission_gateway.is_some() {
|
||||
install_skills_door(
|
||||
pool,
|
||||
user_id,
|
||||
@@ -357,9 +357,23 @@ pub async fn on_launch(
|
||||
&crate::mission_runtime::container_name(mission_id),
|
||||
p,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
false
|
||||
};
|
||||
// Decided here and recorded, not re-derived per turn: this is the only
|
||||
// point that knows whether the door actually installed, and an arm that
|
||||
// could change mid-mission would make the run unattributable.
|
||||
record_skill_delivery(
|
||||
pool,
|
||||
mission_id,
|
||||
crate::skill_delivery::resolve(
|
||||
crate::skill_delivery::requested_for(&mission.config),
|
||||
door,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
let mut first_team_id: Option<Uuid> = None;
|
||||
let mut provisioned_claws: Vec<cm_domain::AgentId> = Vec::new();
|
||||
for (purpose, template_id) in &picks {
|
||||
@@ -916,19 +930,23 @@ fn default_accent_for(slot: &str) -> &'static str {
|
||||
///
|
||||
/// Every failure degrades to "no door", never to a failed launch. A mission
|
||||
/// that cannot retrieve a skill still delivers.
|
||||
/// Returns whether the door is installed AND reachable. The caller needs the
|
||||
/// answer, not just the log line: the `index` delivery arm hands agents a list
|
||||
/// of uris to fetch, and without a door every one of them is a dead end that
|
||||
/// reads as an agent ignoring its skills.
|
||||
async fn install_skills_door(
|
||||
pool: &PgPool,
|
||||
user_id: cm_domain::UserId,
|
||||
mission_id: Uuid,
|
||||
container: &str,
|
||||
prov: &RuntimeProvisioner,
|
||||
) {
|
||||
) -> bool {
|
||||
let Some(origin) = crate::container_tool_hooks::api_origin() else {
|
||||
eprintln!(
|
||||
"mission_orchestrator: no API origin for the skills door (set \
|
||||
CLAWMATES_API_ORIGIN) — mission {mission_id} runs without it"
|
||||
);
|
||||
return;
|
||||
return false;
|
||||
};
|
||||
// Outlives the longest mission we have seen, and expires on its own so a
|
||||
// leaked container does not leave a live credential behind indefinitely.
|
||||
@@ -943,31 +961,54 @@ async fn install_skills_door(
|
||||
"mission_orchestrator: could not mint a skills token ({e}) — \
|
||||
mission {mission_id} runs without the door"
|
||||
);
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let docker = match crate::container_exec::connect() {
|
||||
Ok(d) => d,
|
||||
Err(e) => {
|
||||
eprintln!("mission_orchestrator: cannot reach docker for the skills door: {e}");
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
};
|
||||
let doc = crate::container_tool_hooks::mcp_document(&origin, &token);
|
||||
let Some(path) = crate::container_tool_hooks::install_door(&docker, container, &doc).await
|
||||
else {
|
||||
// `install_door` already said why.
|
||||
return;
|
||||
return false;
|
||||
};
|
||||
if let Err(e) = prov.set_claude_cli_mcp_config(&path).await {
|
||||
eprintln!(
|
||||
"mission_orchestrator: wrote the MCP config but could not point \
|
||||
claude_cli at it ({e}) — the door is installed and unreachable"
|
||||
);
|
||||
return;
|
||||
return false;
|
||||
}
|
||||
eprintln!(
|
||||
"mission_orchestrator: skills door installed for mission {mission_id} \
|
||||
({origin}/mcp/skills)"
|
||||
);
|
||||
true
|
||||
}
|
||||
|
||||
/// Record which arm this mission runs, so every turn composes the same one and
|
||||
/// the score can be attributed to it afterwards.
|
||||
///
|
||||
/// A write failure is not fatal: `skill_delivery_mode` reads NULL as `inline`,
|
||||
/// which is the arm that needs nothing installed. A mission that quietly ran
|
||||
/// the control arm is a lost data point; a mission that failed to launch over
|
||||
/// a telemetry column is a lost mission.
|
||||
async fn record_skill_delivery(pool: &PgPool, mission_id: Uuid, mode: crate::skill_delivery::Mode) {
|
||||
if let Err(e) = sqlx::query("UPDATE missions SET skill_delivery = $2 WHERE id = $1")
|
||||
.bind(mission_id)
|
||||
.bind(mode.as_str())
|
||||
.execute(pool)
|
||||
.await
|
||||
{
|
||||
eprintln!(
|
||||
"mission_orchestrator: could not record skill_delivery={} for mission \
|
||||
{mission_id} ({e}) — its turns will compose skills inline",
|
||||
mode.as_str()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1141,7 +1141,16 @@ async fn launch_phase(
|
||||
// `topology_exec`, with the running node's own role, and appending here too
|
||||
// would put every crew member's skills in every turn twice.
|
||||
let task_with_skills = match phase_skills_text(pool, mission_id).await {
|
||||
Some(skills) => crate::topology_exec::compose_turn_prompt(&task, Some(&skills)),
|
||||
// Always `Inline` here, and not because it is the default: the solo
|
||||
// tiers get no skills door (`install_skills_door` runs only for a
|
||||
// mission with its own container), so an index would list uris nothing
|
||||
// in the VM can fetch. When the microVM tier folds onto
|
||||
// `container_tool_hooks` this becomes a real choice; today it is a fact.
|
||||
Some(skills) => crate::topology_exec::compose_turn_prompt(
|
||||
&task,
|
||||
Some(&skills),
|
||||
crate::skill_delivery::Mode::Inline,
|
||||
),
|
||||
None => task.clone(),
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,255 @@
|
||||
//! How a mission agent receives the skills bound to it.
|
||||
//!
|
||||
//! Two arms, and this module exists to hold them side by side rather than to
|
||||
//! replace one with the other:
|
||||
//!
|
||||
//! - [`Mode::Inline`] — every pinned skill's full body is appended to the turn
|
||||
//! prompt. What production has always done.
|
||||
//! - [`Mode::Index`] — the prompt carries each skill's name, description and
|
||||
//! `when_to_use` plus the URI that returns its body, and the agent fetches
|
||||
//! the ones it judges relevant.
|
||||
//!
|
||||
//! # Why this is an A/B and not a switch
|
||||
//!
|
||||
//! Trigger — did the agent reach for the skill when it applied? — is
|
||||
//! unmeasurable under `Inline` by construction. Nothing was reached for; the
|
||||
//! text was handed over. `skill_use` reports `NotObservable` for exactly that
|
||||
//! reason, and it is right to.
|
||||
//!
|
||||
//! `Index` makes Trigger observable, because retrieval is a recorded
|
||||
//! `ReadMcpResourceTool` call. But it can only *cost* Compliance: under
|
||||
//! `Inline` the procedure is in front of the model whether or not it noticed
|
||||
//! it applied, and under `Index` a missed judgement means the body is never
|
||||
//! read at all. Trading a measured axis for an unmeasured regression in
|
||||
//! another is not an improvement, so the arm is selected per mission and
|
||||
//! recorded on the mission row, and both arms stay runnable.
|
||||
//!
|
||||
//! # `Index` requires the door, and degrades rather than lying
|
||||
//!
|
||||
//! An index names a body and tells the agent how to fetch it. If the
|
||||
//! `clawmates_skills` MCP server is not reachable from the container, that is
|
||||
//! an index of procedures the agent cannot obtain — strictly worse than
|
||||
//! `Inline`, and it fails as an agent that ignored its skills rather than as a
|
||||
//! missing config. [`resolve`] therefore takes the door's install result and
|
||||
//! refuses `Index` without it. This is the same failure the old
|
||||
//! `pinned_skills_text` doc comment warned about; what changed is that the
|
||||
//! door now exists, not that the warning stopped applying.
|
||||
|
||||
/// Where the index tells agents to fetch a skill body from.
|
||||
///
|
||||
/// Must match the server name in
|
||||
/// [`crate::container_tool_hooks::mcp_document`] — the agent passes it
|
||||
/// straight to `ReadMcpResourceTool`.
|
||||
pub const MCP_SERVER: &str = "clawmates_skills";
|
||||
|
||||
/// Selects the arm. Unset or unrecognised means [`Mode::Inline`].
|
||||
pub const ENV_VAR: &str = "CLAWMATES_SKILL_DELIVERY";
|
||||
|
||||
/// The `# Your skills` preamble under [`Mode::Inline`].
|
||||
///
|
||||
/// **Byte-identical to what production has always sent.** The A arm of an A/B
|
||||
/// has to be the thing already running, or the comparison measures this edit
|
||||
/// as well as the change under test.
|
||||
pub const INLINE_PREAMBLE: &str = "These are procedures you are expected to follow for \
|
||||
this kind of work. Where one applies to what you are about to do, follow it.";
|
||||
|
||||
/// The `# Your skills` preamble under [`Mode::Index`].
|
||||
///
|
||||
/// Written and matched in one place ([`mode_in_prompt`]) so the reader cannot
|
||||
/// drift from the writer — the same rule `SKILL_MARKER` is under, and for the
|
||||
/// same reason: a scorer that misreads the arm reports the wrong axis.
|
||||
pub const INDEX_PREAMBLE: &str = "These procedures are AVAILABLE to you; their bodies are \
|
||||
not included below. Each entry names one, says when it applies, and gives the uri that \
|
||||
returns it. Where an entry applies to what you are about to do, read it FIRST and then \
|
||||
follow it.";
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Mode {
|
||||
Inline,
|
||||
Index,
|
||||
}
|
||||
|
||||
impl Mode {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Mode::Inline => "inline",
|
||||
Mode::Index => "index",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a recorded or configured arm. Unrecognised input is `None`, and every
|
||||
/// caller resolves that to `Inline` — an unreadable value must not silently
|
||||
/// select the arm that needs a door.
|
||||
pub fn parse(s: &str) -> Option<Mode> {
|
||||
match s.trim().to_ascii_lowercase().as_str() {
|
||||
"inline" => Some(Mode::Inline),
|
||||
"index" | "progressive" => Some(Mode::Index),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The arm this deployment asks for, before the door is taken into account.
|
||||
pub fn requested() -> Mode {
|
||||
let Ok(raw) = std::env::var(ENV_VAR) else {
|
||||
return Mode::Inline;
|
||||
};
|
||||
if raw.trim().is_empty() {
|
||||
return Mode::Inline;
|
||||
}
|
||||
match parse(&raw) {
|
||||
Some(m) => m,
|
||||
None => {
|
||||
eprintln!(
|
||||
"skill_delivery: {ENV_VAR}={raw:?} is not `inline` or `index` — \
|
||||
delivering skills inline"
|
||||
);
|
||||
Mode::Inline
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The arm for one mission: `config.skill_delivery` if it names one, otherwise
|
||||
/// the deployment default.
|
||||
///
|
||||
/// Per-mission and not only per-deployment because the alternative is
|
||||
/// restarting the server between arms, and an A/B whose two halves ran against
|
||||
/// different server processes has a confound in it that nothing in the numbers
|
||||
/// will show. This way both arms run against one binary, interleaved.
|
||||
pub fn requested_for(config: &serde_json::Value) -> Mode {
|
||||
let Some(raw) = config.get("skill_delivery").and_then(|v| v.as_str()) else {
|
||||
return requested();
|
||||
};
|
||||
match parse(raw) {
|
||||
Some(m) => m,
|
||||
None => {
|
||||
eprintln!(
|
||||
"skill_delivery: config.skill_delivery={raw:?} is not `inline` or \
|
||||
`index` — falling back to the deployment default"
|
||||
);
|
||||
requested()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The arm a mission will actually run, given whether its skills door installed.
|
||||
pub fn resolve(requested: Mode, door_installed: bool) -> Mode {
|
||||
match (requested, door_installed) {
|
||||
(Mode::Index, true) => Mode::Index,
|
||||
(Mode::Index, false) => {
|
||||
eprintln!(
|
||||
"skill_delivery: {ENV_VAR} asked for `index` but this mission has no \
|
||||
skills door — falling back to `inline`, because an index the agent \
|
||||
cannot fetch from is worse than no index"
|
||||
);
|
||||
Mode::Inline
|
||||
}
|
||||
(Mode::Inline, _) => Mode::Inline,
|
||||
}
|
||||
}
|
||||
|
||||
/// The `# Your skills` section heading for an arm.
|
||||
pub fn preamble(mode: Mode) -> &'static str {
|
||||
match mode {
|
||||
Mode::Inline => INLINE_PREAMBLE,
|
||||
Mode::Index => INDEX_PREAMBLE,
|
||||
}
|
||||
}
|
||||
|
||||
/// Which arm produced a recorded prompt.
|
||||
///
|
||||
/// Read back from the prompt rather than from the mission row on purpose: the
|
||||
/// row says what the mission was configured to do *now*, and a score is being
|
||||
/// computed against a prompt that was composed then. The recorded prompt is
|
||||
/// the only artefact that cannot have changed since the turn ran.
|
||||
///
|
||||
/// Matched as a whole line. A skill body that quotes the preamble mid-sentence
|
||||
/// is prose; this is the same rule `skill_names_in` learned the hard way.
|
||||
pub fn mode_in_prompt(prompt: &str) -> Mode {
|
||||
if prompt.lines().any(|l| l.trim() == INDEX_PREAMBLE) {
|
||||
Mode::Index
|
||||
} else {
|
||||
Mode::Inline
|
||||
}
|
||||
}
|
||||
|
||||
/// One index entry's text — everything under the `--- SKILL: <name> ---`
|
||||
/// marker, which [`crate::topology_exec::render_pinned_skill`] writes.
|
||||
///
|
||||
/// `when_to_use` is the load-bearing field: it is the only thing the agent has
|
||||
/// to judge relevance from, so a skill with none says so rather than omitting
|
||||
/// the line and leaving the model to infer from the description alone.
|
||||
pub fn index_entry(description: &str, when_to_use: Option<&str>, uri: &str) -> String {
|
||||
let when = when_to_use
|
||||
.map(str::trim)
|
||||
.filter(|w| !w.is_empty())
|
||||
.unwrap_or("not stated — judge from the description");
|
||||
format!(
|
||||
"{}\nWhen to use: {}\nRead it: ReadMcpResourceTool(server=\"{}\", uri=\"{}\")",
|
||||
description.trim(),
|
||||
when,
|
||||
MCP_SERVER,
|
||||
uri,
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn an_unreadable_arm_never_selects_the_one_that_needs_a_door() {
|
||||
assert_eq!(parse("nonsense"), None);
|
||||
assert_eq!(parse("INDEX"), Some(Mode::Index));
|
||||
assert_eq!(parse(" inline "), Some(Mode::Inline));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_mission_can_name_its_own_arm() {
|
||||
assert_eq!(
|
||||
requested_for(&serde_json::json!({ "skill_delivery": "index" })),
|
||||
Mode::Index
|
||||
);
|
||||
// Unreadable values and absent ones both defer to the deployment
|
||||
// default, which is `Inline` unless the environment says otherwise.
|
||||
assert_eq!(
|
||||
requested_for(&serde_json::json!({ "skill_delivery": "sideways" })),
|
||||
requested()
|
||||
);
|
||||
assert_eq!(requested_for(&serde_json::json!({})), requested());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn index_without_a_door_falls_back() {
|
||||
assert_eq!(resolve(Mode::Index, false), Mode::Inline);
|
||||
assert_eq!(resolve(Mode::Index, true), Mode::Index);
|
||||
assert_eq!(resolve(Mode::Inline, true), Mode::Inline);
|
||||
}
|
||||
|
||||
/// The scorer reads the arm off the prompt, so the writer and this reader
|
||||
/// have to agree for every arm — including the one that writes no marker.
|
||||
#[test]
|
||||
fn the_arm_is_recoverable_from_the_prompt_that_was_sent() {
|
||||
let inline = format!("Task: x\n\n# Your skills\n\n{INLINE_PREAMBLE}\n\nbody");
|
||||
let index = format!("Task: x\n\n# Your skills\n\n{INDEX_PREAMBLE}\n\nentry");
|
||||
assert_eq!(mode_in_prompt(&inline), Mode::Inline);
|
||||
assert_eq!(mode_in_prompt(&index), Mode::Index);
|
||||
assert_eq!(mode_in_prompt("Task: x"), Mode::Inline);
|
||||
}
|
||||
|
||||
/// A body quoting the preamble must not re-label the arm — the same
|
||||
/// failure `SKILL_MARKER` had when a heading inside a body counted.
|
||||
#[test]
|
||||
fn a_body_quoting_the_preamble_does_not_change_the_arm() {
|
||||
let body = format!("The index arm opens with \"{INDEX_PREAMBLE}\" and then lists.");
|
||||
let prompt = format!("Task: x\n\n# Your skills\n\n{INLINE_PREAMBLE}\n\n{body}");
|
||||
assert_eq!(mode_in_prompt(&prompt), Mode::Inline);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_entry_states_a_missing_when_to_use_rather_than_dropping_the_line() {
|
||||
let e = index_entry("Summarise a paper.", None, "skill:global/x");
|
||||
assert!(e.contains("When to use: not stated"), "{e}");
|
||||
assert!(e.contains("ReadMcpResourceTool(server=\"clawmates_skills\""), "{e}");
|
||||
}
|
||||
}
|
||||
+175
-14
@@ -189,6 +189,55 @@ pub fn retrieved_skills(ev: &Evidence<'_>) -> Vec<String> {
|
||||
out
|
||||
}
|
||||
|
||||
/// Did the agent reach for this skill?
|
||||
///
|
||||
/// The answer depends on whether it was ever given the chance, which is what
|
||||
/// the delivery arm decides — so this takes the arm rather than assuming one.
|
||||
/// Getting that wrong is not a rounding error: under `Index` the old text
|
||||
/// would have said "this skill was inlined into the prompt" about a skill that
|
||||
/// was not, and scored a real miss as a structural blind spot.
|
||||
fn trigger_verdict(
|
||||
mode: crate::skill_delivery::Mode,
|
||||
retrieved: bool,
|
||||
compliance: &Verdict,
|
||||
boundary: &Verdict,
|
||||
) -> Verdict {
|
||||
if retrieved {
|
||||
// The agent reached for it. That is the paper's Trigger, and it is a
|
||||
// recorded tool call like any other.
|
||||
return Verdict::Pass;
|
||||
}
|
||||
match mode {
|
||||
// Handed over, so there was no reaching-for to observe. Not a failure
|
||||
// and not a pass — the axis simply does not exist in this arm.
|
||||
crate::skill_delivery::Mode::Inline => Verdict::NotObservable(
|
||||
"this skill was inlined into the prompt, not retrieved — the agent \
|
||||
was handed it, so there is no reaching-for to observe. Serve it \
|
||||
through the door instead and this becomes a tool call"
|
||||
.into(),
|
||||
),
|
||||
crate::skill_delivery::Mode::Index => {
|
||||
// Offered by name and `when_to_use`, and never opened. Whether that
|
||||
// is a miss depends on whether the skill had anything to say about
|
||||
// this phase at all: a skill with no machine-checkable consequence
|
||||
// here is one an agent is right to pass over, and scoring that as a
|
||||
// failure would punish correct triage.
|
||||
if matches!(compliance, Verdict::NotApplicable)
|
||||
&& matches!(boundary, Verdict::NotApplicable)
|
||||
{
|
||||
Verdict::NotApplicable
|
||||
} else {
|
||||
Verdict::Fail(
|
||||
"offered in the index with its `when_to_use`, and never read — \
|
||||
the agent had the entry in front of it and did not fetch the \
|
||||
procedure"
|
||||
.into(),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Score every skill a phase's prompt delivered, against what the agent produced.
|
||||
pub fn score(
|
||||
prompt: &str,
|
||||
@@ -196,6 +245,10 @@ pub fn score(
|
||||
source_kinds: &dyn Fn(&str) -> String,
|
||||
) -> Vec<SkillUse> {
|
||||
let retrieved = retrieved_skills(evidence);
|
||||
// Read off the prompt that was actually sent, not off the mission row: the
|
||||
// row says what the mission is configured to do now, and this is scoring a
|
||||
// turn that ran then.
|
||||
let mode = crate::skill_delivery::mode_in_prompt(prompt);
|
||||
// Delivered by either route. A skill that was retrieved and never inlined
|
||||
// is invisible to `skills_in_prompt`, and under progressive disclosure that
|
||||
// is EVERY skill — so scoring only the prompt would report zero for the
|
||||
@@ -212,19 +265,12 @@ pub fn score(
|
||||
let (compliance, boundary) = check(&skill, evidence);
|
||||
SkillUse {
|
||||
source_kind: source_kinds(&skill),
|
||||
trigger: if retrieved.contains(&skill) {
|
||||
// The agent reached for it. That is the paper's Trigger,
|
||||
// and it is now a recorded tool call like any other.
|
||||
Verdict::Pass
|
||||
} else {
|
||||
Verdict::NotObservable(
|
||||
"this skill was inlined into the prompt, not retrieved — \
|
||||
the agent was handed it, so there is no reaching-for to \
|
||||
observe. Serve it through the door instead and this \
|
||||
becomes a tool call"
|
||||
.into(),
|
||||
)
|
||||
},
|
||||
trigger: trigger_verdict(
|
||||
mode,
|
||||
retrieved.contains(&skill),
|
||||
&compliance,
|
||||
&boundary,
|
||||
),
|
||||
compliance,
|
||||
boundary,
|
||||
skill,
|
||||
@@ -855,7 +901,122 @@ mod tests {
|
||||
.iter()
|
||||
.map(|(n, b)| crate::topology_exec::render_pinned_skill(n, b))
|
||||
.collect();
|
||||
crate::topology_exec::compose_turn_prompt("Task: do the thing", Some(&body))
|
||||
crate::topology_exec::compose_turn_prompt(
|
||||
"Task: do the thing",
|
||||
Some(&body),
|
||||
crate::skill_delivery::Mode::Inline,
|
||||
)
|
||||
}
|
||||
|
||||
/// A prompt as the delivery layer renders it under the INDEX arm.
|
||||
fn rendered_index(skills: &[(&str, &str)]) -> String {
|
||||
let body: String = skills
|
||||
.iter()
|
||||
.map(|(name, when)| {
|
||||
crate::topology_exec::render_pinned_skill(
|
||||
name,
|
||||
&crate::skill_delivery::index_entry(
|
||||
"a procedure",
|
||||
Some(when),
|
||||
&format!("skill:global/{name}"),
|
||||
),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
crate::topology_exec::compose_turn_prompt(
|
||||
"Task: do the thing",
|
||||
Some(&body),
|
||||
crate::skill_delivery::Mode::Index,
|
||||
)
|
||||
}
|
||||
|
||||
fn read_skill(uri: &str) -> ToolEvidence {
|
||||
ToolEvidence {
|
||||
tool: "ReadMcpResourceTool".into(),
|
||||
path: None,
|
||||
input: json!({ "server": "clawmates_skills", "uri": uri }),
|
||||
response: serde_json::Value::Null,
|
||||
}
|
||||
}
|
||||
|
||||
/// The whole loop, end to end: the index writes a uri, the agent reads that
|
||||
/// exact uri back, and the scorer recovers the skill's name from it.
|
||||
///
|
||||
/// Three components have to agree on one string — `mcp_skills::skill_uri`
|
||||
/// writes it, `skill_delivery::index_entry` puts it in the prompt, and
|
||||
/// `parse_uri` reads it. Asserting them separately would let any pair drift
|
||||
/// while each one's own test stayed green.
|
||||
#[test]
|
||||
fn the_uri_the_index_advertises_is_the_one_the_scorer_recovers() {
|
||||
let prompt = rendered_index(&[("workspace-repo-commit-protocol", "before committing")]);
|
||||
assert!(
|
||||
prompt.contains("uri=\"skill:global/workspace-repo-commit-protocol\""),
|
||||
"the entry must name the uri to fetch:\n{prompt}"
|
||||
);
|
||||
let tools = vec![read_skill("skill:global/workspace-repo-commit-protocol")];
|
||||
let ev = Evidence::new("", &tools);
|
||||
assert_eq!(
|
||||
retrieved_skills(&ev),
|
||||
vec!["workspace-repo-commit-protocol"],
|
||||
"the uri the prompt advertised must parse back to the skill's name"
|
||||
);
|
||||
let scored = score(&prompt, &ev, &builtin);
|
||||
assert_eq!(scored.len(), 1);
|
||||
assert!(
|
||||
matches!(scored[0].trigger, Verdict::Pass),
|
||||
"reaching for an indexed skill IS the Trigger axis: {:?}",
|
||||
scored[0].trigger
|
||||
);
|
||||
}
|
||||
|
||||
/// The index arm must not report a miss as a blind spot.
|
||||
///
|
||||
/// Under `Inline` "never retrieved" is `NotObservable`, and that is honest
|
||||
/// there. Reusing it here would say "this skill was inlined into the
|
||||
/// prompt" about a skill whose body was never sent — the exact shape of a
|
||||
/// check reporting a system defect where an agent behaviour belongs.
|
||||
#[test]
|
||||
fn an_indexed_skill_that_was_never_read_is_a_miss_not_a_blind_spot() {
|
||||
let prompt = rendered_index(&[("workspace-repo-commit-protocol", "before committing")]);
|
||||
// It wrote outside the mission checkout, so the boundary check applies
|
||||
// — this skill had something to say about this phase and went unread.
|
||||
let tools = acted(&[("Write", Some("/tmp/scratch.rs"), json!({}))]);
|
||||
let scored = score(&prompt, &Evidence::new("", &tools), &builtin);
|
||||
assert!(
|
||||
matches!(scored[0].trigger, Verdict::Fail(_)),
|
||||
"offered by name and when_to_use, never fetched: {:?}",
|
||||
scored[0].trigger
|
||||
);
|
||||
}
|
||||
|
||||
/// ...but only when the skill had a consequence to check.
|
||||
///
|
||||
/// A skill with nothing machine-checkable in this phase is one an agent is
|
||||
/// right to pass over, and scoring that as a Trigger failure would punish
|
||||
/// correct triage — which is the behaviour progressive disclosure is
|
||||
/// supposed to reward.
|
||||
#[test]
|
||||
fn passing_over_a_skill_with_nothing_to_check_is_not_a_trigger_failure() {
|
||||
let prompt = rendered_index(&[("some-unchecked-skill", "when writing prose")]);
|
||||
let scored = score(&prompt, &narrative("wrote the report"), &builtin);
|
||||
assert!(
|
||||
matches!(scored[0].trigger, Verdict::NotApplicable),
|
||||
"{:?}",
|
||||
scored[0].trigger
|
||||
);
|
||||
}
|
||||
|
||||
/// The control arm has to be unchanged, or the A/B measures this edit too.
|
||||
#[test]
|
||||
fn the_inline_arm_still_reports_trigger_as_unobservable() {
|
||||
let prompt = rendered(&[("workspace-repo-commit-protocol", "body")]);
|
||||
let tools = acted(&[("Write", Some("/tmp/scratch.rs"), json!({}))]);
|
||||
let scored = score(&prompt, &Evidence::new("", &tools), &builtin);
|
||||
assert!(
|
||||
matches!(scored[0].trigger, Verdict::NotObservable(_)),
|
||||
"{:?}",
|
||||
scored[0].trigger
|
||||
);
|
||||
}
|
||||
|
||||
/// The prompt is the record of what was delivered, so parsing it must match
|
||||
|
||||
@@ -337,12 +337,50 @@ impl ZeroClawDriveExecutor {
|
||||
/// for a mission role was unreachable prose, and no measurement of whether
|
||||
/// skills fire could have returned anything but zero.
|
||||
///
|
||||
/// Bodies, not an index. The chat path lists names and lets the claw call
|
||||
/// `skills.read`; there is no such tool here, so an index would advertise a
|
||||
/// capability that does not exist — the exact failure this whole change is
|
||||
/// about. Pinned only (`pin_in_context`), because everything else would go
|
||||
/// in unbounded and unread.
|
||||
/// Bodies or an index, depending on the mission's arm — see
|
||||
/// [`crate::skill_delivery`]. Bodies were once the only honest option:
|
||||
/// there was no tool on the mission path that could fetch one, so an index
|
||||
/// would have advertised a capability that did not exist. The skills door
|
||||
/// changed that, and the arm is now recorded per mission so both can run.
|
||||
///
|
||||
/// Pinned only (`pin_in_context`) in either arm, because everything else
|
||||
/// would go in unbounded and unread.
|
||||
pub async fn pinned_skills_text(&self, alias: &str) -> Option<String> {
|
||||
let mode = self.skill_delivery_mode().await;
|
||||
self.pinned_skills_in_mode(alias, mode).await
|
||||
}
|
||||
|
||||
/// The arm this mission was launched with.
|
||||
///
|
||||
/// Read per turn rather than cached on the executor: the executor is
|
||||
/// constructed from the environment by `topology_worker`, which knows
|
||||
/// nothing about a mission, and the arm is decided at launch by the code
|
||||
/// that also learns whether the door installed.
|
||||
///
|
||||
/// Anything unreadable — no tap, no row, an unrecognised value — resolves
|
||||
/// to `Inline`, which is the arm that needs nothing to be true.
|
||||
pub(crate) async fn skill_delivery_mode(&self) -> crate::skill_delivery::Mode {
|
||||
let Some(tap) = self.tap.as_ref() else {
|
||||
return crate::skill_delivery::Mode::Inline;
|
||||
};
|
||||
sqlx::query_scalar::<_, Option<String>>(
|
||||
"SELECT skill_delivery FROM missions WHERE id = $1",
|
||||
)
|
||||
.bind(tap.mission_id)
|
||||
.fetch_optional(&tap.pool)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.flatten()
|
||||
.and_then(|s| crate::skill_delivery::parse(&s))
|
||||
.unwrap_or(crate::skill_delivery::Mode::Inline)
|
||||
}
|
||||
|
||||
pub(crate) async fn pinned_skills_in_mode(
|
||||
&self,
|
||||
alias: &str,
|
||||
mode: crate::skill_delivery::Mode,
|
||||
) -> Option<String> {
|
||||
let tap = self.tap.as_ref()?;
|
||||
let agent_id = crate::runtime_provision::claw_from_alias(alias)?;
|
||||
let link = cm_db::repo::agent_template_link::get(&tap.pool, agent_id)
|
||||
@@ -361,17 +399,29 @@ impl ZeroClawDriveExecutor {
|
||||
let mut out = String::new();
|
||||
let mut n = 0usize;
|
||||
for b in bindings.iter().filter(|b| b.pin_in_context) {
|
||||
let text = match mode {
|
||||
crate::skill_delivery::Mode::Inline => b.skill.body.clone(),
|
||||
// An entry is a few hundred bytes whatever the body weighs, so
|
||||
// the index arm cannot hit the cap that follows. That is the
|
||||
// point of it, and the reason the cap is checked against the
|
||||
// rendered text rather than against the body.
|
||||
crate::skill_delivery::Mode::Index => crate::skill_delivery::index_entry(
|
||||
&b.skill.description,
|
||||
b.skill.when_to_use.as_deref(),
|
||||
&crate::mcp_skills::skill_uri(b.skill.workspace_id, &b.skill.name),
|
||||
),
|
||||
};
|
||||
// Bounded, and truncation is STATED. A silently clipped procedure
|
||||
// is worse than an absent one: the agent follows the half it can
|
||||
// see and reports success against a rule it never read.
|
||||
if out.len() + b.skill.body.len() > MAX_PINNED_SKILL_BYTES {
|
||||
if out.len() + text.len() > MAX_PINNED_SKILL_BYTES {
|
||||
out.push_str(&format!(
|
||||
"\n[skill \"{}\" omitted — the pinned set exceeded {} bytes]\n",
|
||||
b.skill.name, MAX_PINNED_SKILL_BYTES
|
||||
));
|
||||
continue;
|
||||
}
|
||||
out.push_str(&render_pinned_skill(&b.skill.name, &b.skill.body));
|
||||
out.push_str(&render_pinned_skill(&b.skill.name, &text));
|
||||
n += 1;
|
||||
}
|
||||
if n == 0 {
|
||||
@@ -778,9 +828,14 @@ impl TurnExecutor for ZeroClawDriveExecutor {
|
||||
);
|
||||
fallback
|
||||
});
|
||||
// One lookup, used for the section, its preamble and the record.
|
||||
// Deriving it three times would let a mission compose an index under
|
||||
// an inline heading if the row changed mid-run.
|
||||
let mode = self.skill_delivery_mode().await;
|
||||
let prompt = compose_turn_prompt(
|
||||
&Self::build_prompt(&req),
|
||||
self.pinned_skills_text(&alias).await.as_deref(),
|
||||
self.pinned_skills_in_mode(&alias, mode).await.as_deref(),
|
||||
mode,
|
||||
);
|
||||
// Record what this agent is ACTUALLY about to receive, before driving.
|
||||
// Re-deriving it later would re-run the skill lookup against a
|
||||
@@ -795,7 +850,14 @@ impl TurnExecutor for ZeroClawDriveExecutor {
|
||||
ev.run_id = tap.run_id;
|
||||
ev.agent_id = crate::runtime_provision::claw_from_alias(&alias);
|
||||
ev.target = Some(req.role.clone());
|
||||
ev.detail = serde_json::json!({ "text": prompt, "tier": "container" });
|
||||
ev.detail = serde_json::json!({
|
||||
"text": prompt,
|
||||
"tier": "container",
|
||||
// The A/B arm, alongside the prompt it produced. `skill_use`
|
||||
// recovers this from the prompt text itself, so this field is
|
||||
// for reporting and for catching the two disagreeing.
|
||||
"skill_delivery": mode.as_str(),
|
||||
});
|
||||
crate::mission_events::record(&tap.pool, ev).await;
|
||||
}
|
||||
self.drive(&alias, &prompt).await
|
||||
@@ -807,17 +869,22 @@ impl TurnExecutor for ZeroClawDriveExecutor {
|
||||
/// Split out from `run_turn` so the wiring is testable: `pinned_skills_text`
|
||||
/// working and `run_turn` actually calling it are different claims, and the
|
||||
/// second is the one that was false for every skill in the catalogue.
|
||||
pub fn compose_turn_prompt(base: &str, skills: Option<&str>) -> String {
|
||||
pub fn compose_turn_prompt(
|
||||
base: &str,
|
||||
skills: Option<&str>,
|
||||
mode: crate::skill_delivery::Mode,
|
||||
) -> String {
|
||||
let Some(skills) = skills.map(str::trim).filter(|s| !s.is_empty()) else {
|
||||
// No heading when there is nothing under it. An empty "Your skills"
|
||||
// section tells the model it has skills and then shows it none, which
|
||||
// is worse than silence.
|
||||
return base.to_string();
|
||||
};
|
||||
format!(
|
||||
"{base}\n\n# Your skills\n\nThese are procedures you are expected to follow for \
|
||||
this kind of work. Where one applies to what you are about to do, follow it.\n\n{skills}"
|
||||
)
|
||||
// The preamble differs per arm and lives in `skill_delivery`, because it
|
||||
// is also what the scorer reads the arm back from. Two copies of this
|
||||
// sentence is two chances for the reader to stop recognising the writer.
|
||||
let preamble = crate::skill_delivery::preamble(mode);
|
||||
format!("{base}\n\n# Your skills\n\n{preamble}\n\n{skills}")
|
||||
}
|
||||
|
||||
/// Parse `role=alias,role=alias` into a map (blank/malformed entries skipped).
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
//! structurally unreadable. That is why this is a test and not a comment: the
|
||||
//! failure produced no error anywhere, and every layer reported success.
|
||||
|
||||
use cm_api::skill_delivery::Mode;
|
||||
use cm_api::topology_exec::{MissionTap, ZeroClawDriveExecutor};
|
||||
use cm_domain::{Agent, AgentId, AgentStatus, Role, User, UserId, Workspace, WorkspaceId};
|
||||
|
||||
@@ -121,6 +122,14 @@ async fn seed_claw_with_pinned_skill(pool: &sqlx::PgPool, body: &str) -> (String
|
||||
}
|
||||
|
||||
fn executor(pool: &sqlx::PgPool, workspace_id: WorkspaceId) -> ZeroClawDriveExecutor {
|
||||
executor_for(pool, workspace_id, Uuid::now_v7())
|
||||
}
|
||||
|
||||
fn executor_for(
|
||||
pool: &sqlx::PgPool,
|
||||
workspace_id: WorkspaceId,
|
||||
mission_id: Uuid,
|
||||
) -> ZeroClawDriveExecutor {
|
||||
ZeroClawDriveExecutor::new(
|
||||
"http://127.0.0.1:1".into(),
|
||||
"unused".into(),
|
||||
@@ -130,12 +139,93 @@ fn executor(pool: &sqlx::PgPool, workspace_id: WorkspaceId) -> ZeroClawDriveExec
|
||||
.with_tap(MissionTap {
|
||||
pool: pool.clone(),
|
||||
workspace_id: workspace_id.as_uuid(),
|
||||
mission_id: Uuid::now_v7(),
|
||||
mission_id,
|
||||
phase_id: None,
|
||||
run_id: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// A mission row carrying an explicit delivery arm.
|
||||
async fn seed_mission_on_arm(pool: &sqlx::PgPool, ws: WorkspaceId, arm: Option<&str>) -> Uuid {
|
||||
let mission = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO missions (id, workspace_id, title, template_kind, status, skill_delivery)
|
||||
VALUES ($1, $2, 'arm', 'research_only', 'running', $3)",
|
||||
)
|
||||
.bind(mission)
|
||||
.bind(ws.as_uuid())
|
||||
.bind(arm)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
mission
|
||||
}
|
||||
|
||||
// ── The index arm ───────────────────────────────────────────────────
|
||||
//
|
||||
// Trigger — did the agent reach for the skill? — cannot exist while every body
|
||||
// is handed over unasked. These tests cover the arm that makes it a question,
|
||||
// and the one guarantee the control arm needs: that it did not change.
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_index_arm_sends_the_uri_and_withholds_the_body() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
const MARKER: &str = "Never review a paper from its title alone.";
|
||||
let (alias, ws) = seed_claw_with_pinned_skill(&pool, MARKER).await;
|
||||
let mission = seed_mission_on_arm(&pool, ws, Some("index")).await;
|
||||
|
||||
let text = executor_for(&pool, ws, mission)
|
||||
.pinned_skills_text(&alias)
|
||||
.await
|
||||
.expect("an indexed skill is still delivered — as an entry, not a body");
|
||||
|
||||
assert!(
|
||||
!text.contains(MARKER),
|
||||
"the BODY is what the index withholds; leaving it in delivers both \
|
||||
arms at once and measures neither. Got:\n{text}"
|
||||
);
|
||||
assert!(
|
||||
text.contains("uri=\"skill:global/delivery-test-skill-"),
|
||||
"an entry without a fetchable uri names a procedure the agent cannot \
|
||||
obtain — worse than inlining it. Got:\n{text}"
|
||||
);
|
||||
assert!(
|
||||
text.contains("When to use: always"),
|
||||
"`when_to_use` is the only thing the agent can judge relevance from, \
|
||||
and judging relevance is the entire axis. Got:\n{text}"
|
||||
);
|
||||
// The scorer counts delivered skills by the marker, in both arms.
|
||||
assert_eq!(
|
||||
cm_api::topology_exec::skill_names_in(&text).len(),
|
||||
1,
|
||||
"an indexed skill must still count as delivered:\n{text}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn an_unrecorded_arm_delivers_bodies() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
const MARKER: &str = "Never review a paper from its title alone.";
|
||||
let (alias, ws) = seed_claw_with_pinned_skill(&pool, MARKER).await;
|
||||
|
||||
// Every mission that ran before the column existed, plus any row whose
|
||||
// value is unreadable, plus a turn with no mission row at all. All three
|
||||
// resolve to the arm that needs nothing installed to work.
|
||||
for arm in [None, Some("nonsense")] {
|
||||
let mission = seed_mission_on_arm(&pool, ws, arm).await;
|
||||
let text = executor_for(&pool, ws, mission)
|
||||
.pinned_skills_text(&alias)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
text.contains(MARKER),
|
||||
"skill_delivery={arm:?} must deliver the body — an index arm \
|
||||
selected by accident hands out uris behind a door that may not \
|
||||
be installed. Got:\n{text}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_pinned_skill_body_reaches_the_turn_prompt() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
@@ -149,9 +239,9 @@ async fn a_pinned_skill_body_reaches_the_turn_prompt() {
|
||||
|
||||
assert!(
|
||||
text.contains(MARKER),
|
||||
"the skill BODY must be present, not just its name — there is no \
|
||||
`skills.read` tool on the mission path, so an index would name a \
|
||||
procedure the agent has no way to fetch. Got:\n{text}"
|
||||
"the default arm delivers the BODY. It is the control in the delivery \
|
||||
A/B, so it has to stay what production has always sent; the index arm \
|
||||
is selected per mission and tested separately. Got:\n{text}"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -197,13 +287,13 @@ async fn an_agent_with_no_pinned_skills_adds_nothing() {
|
||||
fn the_composed_prompt_carries_the_skill_and_omits_the_heading_when_empty() {
|
||||
let base = "You are the \"researcher\" agent.\n\nTask: read the papers";
|
||||
|
||||
let with = cm_api::topology_exec::compose_turn_prompt(base, Some("## arxiv-daily\nDo not re-search."));
|
||||
let with = cm_api::topology_exec::compose_turn_prompt(base, Some("## arxiv-daily\nDo not re-search."), Mode::Inline);
|
||||
assert!(with.contains("Task: read the papers"), "the base turn must survive");
|
||||
assert!(with.contains("Do not re-search."), "the skill body must be in the prompt");
|
||||
assert!(with.contains("# Your skills"), "the section needs a heading");
|
||||
|
||||
for empty in [None, Some(""), Some(" \n ")] {
|
||||
let without = cm_api::topology_exec::compose_turn_prompt(base, empty);
|
||||
let without = cm_api::topology_exec::compose_turn_prompt(base, empty, Mode::Inline);
|
||||
assert_eq!(
|
||||
without, base,
|
||||
"with no skills the prompt must be byte-identical to the base — an \
|
||||
@@ -277,7 +367,7 @@ async fn the_microvm_and_session_tiers_get_the_skill_in_their_task_text() {
|
||||
);
|
||||
|
||||
// All three tiers share this composition, so testing it once covers them.
|
||||
let composed = cm_api::topology_exec::compose_turn_prompt("Task: read the papers", Some(&skills));
|
||||
let composed = cm_api::topology_exec::compose_turn_prompt("Task: read the papers", Some(&skills), Mode::Inline);
|
||||
assert!(composed.contains(MARKER));
|
||||
assert!(composed.contains("Task: read the papers"));
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ mod token;
|
||||
pub use bootstrap::bootstrap_owner;
|
||||
pub use jwt::{ExternalClaims, JwtError, JwtVerifier};
|
||||
pub use service::{
|
||||
AuthError, AuthService, AuthedUser, SCOPE_FULL, SCOPE_SKILLS_READ, SESSION_TTL,
|
||||
AuthError, AuthService, AuthedUser, SCOPE_AGENT_DOOR, SCOPE_FULL, SCOPE_SKILLS_READ,
|
||||
SESSION_TTL,
|
||||
};
|
||||
pub use token::SessionToken;
|
||||
|
||||
@@ -21,6 +21,19 @@ pub const SCOPE_FULL: &str = "full";
|
||||
/// that authenticates nowhere, which fails safely but silently.
|
||||
pub const SCOPE_SKILLS_READ: &str = "skills:read";
|
||||
|
||||
/// Act through the §15 MCP door (`/mcp`), and nothing else.
|
||||
///
|
||||
/// The door is the actuator: `email_send`, `slack_post`, `delegate`. Reaching
|
||||
/// it means a token an agent's runtime holds, and the same reasoning as
|
||||
/// [`SCOPE_SKILLS_READ`] applies — a full session there is an owner-privileged
|
||||
/// API key handed to a process whose whole purpose is to act on instructions
|
||||
/// from a model.
|
||||
///
|
||||
/// Nothing mints one yet; the route accepts it so that whatever does will not
|
||||
/// have to reach for a person's session to be understood. `full` still works,
|
||||
/// so the UI and any human caller are unaffected.
|
||||
pub const SCOPE_AGENT_DOOR: &str = "agent:door";
|
||||
|
||||
/// The authenticated caller attached to every API request: everything RBAC
|
||||
/// decisions need, nothing more.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
|
||||
@@ -168,6 +168,54 @@ async fn a_scoped_token_is_refused_by_every_unscoped_caller() {
|
||||
assert_eq!(ok.user_id, user.id);
|
||||
}
|
||||
|
||||
/// The two narrow scopes must not substitute for each other.
|
||||
///
|
||||
/// They protect different things — one reads the skills catalogue, the other
|
||||
/// operates the §15 door that can `delegate`. Both tokens live where an agent
|
||||
/// can read them, so the whole value of having two constants is that holding
|
||||
/// one grants nothing the other has.
|
||||
#[tokio::test]
|
||||
async fn one_narrow_scope_does_not_open_the_other() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let (_ws, user) = seeded(&pool).await;
|
||||
let auth = AuthService::new(pool);
|
||||
|
||||
let skills = auth
|
||||
.mint_scoped(user.id, cm_auth::SCOPE_SKILLS_READ, time::Duration::hours(1))
|
||||
.await
|
||||
.unwrap();
|
||||
let door = auth
|
||||
.mint_scoped(user.id, cm_auth::SCOPE_AGENT_DOOR, time::Duration::hours(1))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
auth.authenticate_scoped(&skills, cm_auth::SCOPE_AGENT_DOOR).await,
|
||||
Err(AuthError::Unauthenticated)
|
||||
),
|
||||
"a skills token must not reach the door — the door can `delegate`"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
auth.authenticate_scoped(&door, cm_auth::SCOPE_SKILLS_READ).await,
|
||||
Err(AuthError::Unauthenticated)
|
||||
),
|
||||
"and a door token must not read the catalogue"
|
||||
);
|
||||
assert!(
|
||||
matches!(
|
||||
auth.authenticate(&door).await,
|
||||
Err(AuthError::Unauthenticated)
|
||||
),
|
||||
"nor authenticate an ordinary API call"
|
||||
);
|
||||
assert!(auth
|
||||
.authenticate_scoped(&door, cm_auth::SCOPE_AGENT_DOOR)
|
||||
.await
|
||||
.is_ok());
|
||||
}
|
||||
|
||||
/// A person's session keeps working everywhere, including the scoped route.
|
||||
#[tokio::test]
|
||||
async fn a_full_session_still_satisfies_a_scoped_route() {
|
||||
|
||||
@@ -30,6 +30,10 @@ async fn server() -> &'static PgServer {
|
||||
SERVER
|
||||
.get_or_init(|| async {
|
||||
if let Ok(url) = std::env::var("CM_TEST_DATABASE_URL") {
|
||||
// Only on the shared-server path. The testcontainer below is
|
||||
// torn down with the process, so it has nothing to reap and a
|
||||
// sweep there would be pure cost.
|
||||
reap_stale_databases(&url).await;
|
||||
return PgServer {
|
||||
admin_url: url,
|
||||
_container: None,
|
||||
@@ -57,6 +61,86 @@ async fn server() -> &'static PgServer {
|
||||
.await
|
||||
}
|
||||
|
||||
/// How long a test database may sit before another test process reaps it.
|
||||
///
|
||||
/// Comfortably longer than any test run, so a database in use by a
|
||||
/// concurrently-running binary is never a candidate. Nothing here needs to be
|
||||
/// prompt — the point is that the set stays bounded, not that it stays empty.
|
||||
const STALE_AFTER_MS: u64 = 2 * 60 * 60 * 1000;
|
||||
|
||||
/// Drop test databases left behind by earlier runs.
|
||||
///
|
||||
/// `test_pool` creates a database per test and nothing ever dropped it. On the
|
||||
/// testcontainer path that is invisible: the container dies with the process
|
||||
/// and takes them with it. But `CM_TEST_DATABASE_URL` points at a SHARED
|
||||
/// server that outlives the run — which is the path CI uses and the path
|
||||
/// `.cargo/config.toml` sets for local development — so on both of those every
|
||||
/// database ever created is still there.
|
||||
///
|
||||
/// Measured before writing this: **3,546 databases, 38 GB** on one developer
|
||||
/// machine. It grows with every `cargo test`.
|
||||
///
|
||||
/// Age comes from the name, not the catalogue. Postgres records no creation
|
||||
/// time for a database, but the names are `test_<uuid-v7>` and UUIDv7 puts the
|
||||
/// millisecond timestamp in its first 48 bits — the same property
|
||||
/// `mission_runtime::container_name` relies on.
|
||||
///
|
||||
/// Best-effort throughout: a test must never fail because housekeeping could
|
||||
/// not run.
|
||||
async fn reap_stale_databases(admin_url: &str) {
|
||||
let Ok(admin) = PgPoolOptions::new()
|
||||
.max_connections(1)
|
||||
.connect(admin_url)
|
||||
.await
|
||||
else {
|
||||
return;
|
||||
};
|
||||
let names: Vec<String> = sqlx::query_scalar(
|
||||
"SELECT datname FROM pg_database WHERE datname LIKE 'test\\_%'",
|
||||
)
|
||||
.fetch_all(&admin)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
let now_ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0);
|
||||
let mut dropped = 0usize;
|
||||
for name in names {
|
||||
let Some(created) = uuid_v7_millis(&name) else {
|
||||
// Not a name we minted; leave it entirely alone.
|
||||
continue;
|
||||
};
|
||||
if now_ms.saturating_sub(created) < STALE_AFTER_MS {
|
||||
continue;
|
||||
}
|
||||
// FORCE terminates any leftover connection; without it a single stale
|
||||
// session pins the database and the reap silently does nothing.
|
||||
if sqlx::query(&format!("DROP DATABASE IF EXISTS {name} WITH (FORCE)"))
|
||||
.execute(&admin)
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
dropped += 1;
|
||||
}
|
||||
}
|
||||
if dropped > 0 {
|
||||
eprintln!("cm-testkit: reaped {dropped} stale test database(s)");
|
||||
}
|
||||
admin.close().await;
|
||||
}
|
||||
|
||||
/// The millisecond timestamp encoded in the leading 48 bits of a
|
||||
/// `test_<uuid-v7-simple>` name.
|
||||
fn uuid_v7_millis(db_name: &str) -> Option<u64> {
|
||||
let hex = db_name.strip_prefix("test_")?;
|
||||
if hex.len() != 32 || !hex.chars().all(|c| c.is_ascii_hexdigit()) {
|
||||
return None;
|
||||
}
|
||||
u64::from_str_radix(&hex[..12], 16).ok()
|
||||
}
|
||||
|
||||
/// Creates a unique database, runs all migrations, and returns a pool
|
||||
/// connected to it.
|
||||
pub async fn test_pool() -> PgPool {
|
||||
@@ -93,3 +177,46 @@ fn swap_database(url: &str, db_name: &str) -> String {
|
||||
None => format!("{head}/{db_name}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{uuid_v7_millis, STALE_AFTER_MS};
|
||||
|
||||
/// Age comes from the NAME, because Postgres records no creation time for
|
||||
/// a database. UUIDv7 puts the millisecond timestamp in its first 48 bits.
|
||||
#[test]
|
||||
fn a_test_database_name_carries_its_own_age() {
|
||||
let id = uuid::Uuid::now_v7();
|
||||
let name = format!("test_{}", id.simple());
|
||||
let ms = uuid_v7_millis(&name).expect("a name we minted parses");
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64;
|
||||
assert!(
|
||||
now.saturating_sub(ms) < 5_000,
|
||||
"a database created just now must read as new, or the reaper drops \
|
||||
one another test process is still using"
|
||||
);
|
||||
}
|
||||
|
||||
/// Anything we did not mint is left alone.
|
||||
#[test]
|
||||
fn only_our_own_names_are_reapable() {
|
||||
assert!(uuid_v7_millis("clawmates").is_none());
|
||||
assert!(uuid_v7_millis("postgres").is_none());
|
||||
assert!(uuid_v7_millis("template1").is_none());
|
||||
// Right prefix, wrong shape — a human-made `test_scratch` survives.
|
||||
assert!(uuid_v7_millis("test_scratch").is_none());
|
||||
assert!(uuid_v7_millis("test_").is_none());
|
||||
// Right length, not hex.
|
||||
assert!(uuid_v7_millis(&format!("test_{}", "z".repeat(32))).is_none());
|
||||
}
|
||||
|
||||
/// The window has to be longer than a test run, or the reaper deletes a
|
||||
/// database out from under a binary running in parallel.
|
||||
#[test]
|
||||
fn the_stale_window_outlasts_any_test_run() {
|
||||
assert!(STALE_AFTER_MS >= 60 * 60 * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
+106
-35
@@ -7,12 +7,12 @@ than the agent's own account of it.
|
||||
|
||||
## State of the tree
|
||||
|
||||
Local suite green: **107 test binaries, 796 tests** (`cargo test --workspace`), and the workspace builds with `--all-targets`.
|
||||
Five measurement missions ran on the local stack (`scripts/skill-use-run.sh`);
|
||||
all five are held 90 days and re-scorable with `--score <id>`.
|
||||
10 commits on `main` this pass, **not pushed** — a push to `main` auto-deploys
|
||||
to gw-04, and the container-tier work already deployed is unexercised there
|
||||
(see below).
|
||||
Local suite green: **108 test binaries, 815 tests** (`cargo test --workspace`).
|
||||
**Pushed** — 22 commits, CI green on gw-04, deployed.
|
||||
|
||||
Eight measurement missions ran on the local stack
|
||||
(`scripts/skill-use-run.sh`); all are held 90 days and re-scorable with
|
||||
`--score <id>`.
|
||||
|
||||
## The premise of the last handoff's item 1 was wrong
|
||||
|
||||
@@ -132,44 +132,115 @@ which is the only reason they were found.
|
||||
`no_skill_shows_a_marker_the_parser_would_reject` guards the class, running the
|
||||
real parser over every marker in every skill's fenced blocks.
|
||||
|
||||
### Trigger is measured, and the door is what made it possible
|
||||
|
||||
`skill_use` scores Trigger from `ReadMcpResourceTool` calls, parsed through
|
||||
`mcp_skills::parse_uri` — the function that wrote the URI. Run 8, one mission,
|
||||
one clean A/B:
|
||||
|
||||
| skill | delivered by | Trigger |
|
||||
|---|---|---|
|
||||
| `workspace-repo-commit-protocol` | retrieval | **pass** |
|
||||
| `web-search-triage` | inlined | not observable |
|
||||
| `structured-paper-summary` | inlined | not observable |
|
||||
| `scientific-writing-conventions` | inlined | not observable |
|
||||
|
||||
Careful with what that proves: the retrieval was *instructed* by the task. It
|
||||
demonstrates the instrument, not a spontaneous relevance judgement.
|
||||
|
||||
### Red-first is observable, and it needed neither ordering nor the diff
|
||||
|
||||
The previous version of this list said the TDD check needed the repository
|
||||
diff. That was wrong, and `tdd-red-green-refactor` says why: *"Commit the
|
||||
RED-to-GREEN pair as one commit."* The failing test and its fix land together
|
||||
by instruction, so the diff and the commit history are as blind as tool
|
||||
ordering already was.
|
||||
|
||||
The witness is what each run PRINTED, and the tap was discarding it. Claude
|
||||
Code's `PostToolUse` payload carries `tool_response` (stdout/stderr/
|
||||
interrupted) — verified against the binary. `bounded_response` keeps the END of
|
||||
that output, the opposite of `bounded_input`, because a command's meaning is
|
||||
its verdict and `cargo test` prints it last.
|
||||
|
||||
failing run then a passing one → Pass
|
||||
every run failed → Fail
|
||||
every run passed → NotObservable, because a test that never
|
||||
failed is equally what a correct
|
||||
implementation written first looks like
|
||||
|
||||
## The four days after that pass went into infrastructure
|
||||
|
||||
No product commits landed between 2026-08-21 and 2026-08-25. What did happen,
|
||||
re-verified live on the 25th:
|
||||
|
||||
- `PasswordAuthentication no` is effective on gw-01/gw-02/gw-04 (checked with
|
||||
`sudo sshd -T`, not by reading a config a drop-in can override), and
|
||||
`fail2ban` is active on all three — **95 / 74 / 77 total bans**, so it is
|
||||
catching real traffic. `ignoreip` must keep `100.64.0.0/10`: every operator
|
||||
and every fleet node reaches these hosts from CGNAT space.
|
||||
- gw-02's three Postgres containers were published on `0.0.0.0` and are now
|
||||
bound to `127.0.0.1` (5434 mindbridge, 5435 clawbridge, 5437 smartclaw).
|
||||
- Roughly 93G reclaimed; the largest single piece was a source-level leak of one
|
||||
test database per test, fixed in `b58f034`.
|
||||
|
||||
Two failure shapes worth keeping:
|
||||
|
||||
- Recreating a DB container leaves the owning app holding **dead pool
|
||||
connections**. The port binding reads perfect and the app still returns its
|
||||
baseline status code, so a shallow check calls it done while every pool is
|
||||
stale. Restart the owning unit and assert on the established-connection count.
|
||||
- A `grep` for the service name matched `smartclaw-email` before `smartclaw`,
|
||||
so the wrong unit was restarted. Only the connection count being `0` caught it.
|
||||
|
||||
**Open, and not safe to assume otherwise:** gw-02 has *no host firewall* —
|
||||
`INPUT` policy `ACCEPT`, `DOCKER-USER` empty. Ports 18789/3000/3010/8082 are
|
||||
bound `0.0.0.0` by native processes (a compose edit will not move them), and the
|
||||
only thing keeping them off the internet is an upstream cloud firewall the host
|
||||
cannot see. gw-05 and web-01 have no fail2ban and `sudo` there needs a password.
|
||||
|
||||
## Next, in order
|
||||
|
||||
1. **The TDD check cannot confirm red-first, and that is structural.** Run 4
|
||||
(`research_and_code`, real repo) edited `src/lib.rs` once — implementation
|
||||
*and* `#[cfg(test)] mod tests` in the same write — then ran `cargo test`
|
||||
five times. In Rust the unit test lives in the file under test, so that
|
||||
ordering is what following the skill precisely looks like from outside. The
|
||||
check detects "wrote source, never ran a test" and nothing more. If you want
|
||||
red-first, it needs the diff (did the test exist before the impl?), not the
|
||||
tool order.
|
||||
1. **Watch the first production mission.** Everything below the fold in this
|
||||
document is now deployed and unexercised in prod, because prod has still
|
||||
never run a mission. The door, the staffing, the attribution and the orphan
|
||||
sweep all engage on the next launch. Prod auth is Clerk, so somebody has to
|
||||
click.
|
||||
|
||||
2. **Attribute tool calls to agents.** `record_vm_tools` writes
|
||||
`agent_id: None`, because the container tap is per-container and all roles
|
||||
share one. Every Skill-Use score is therefore per-**mission**, not per-role,
|
||||
and the World's per-agent view gets nothing from the container tier. The
|
||||
hook payload carries `session_id`; mapping it back to a turn is the fix.
|
||||
```
|
||||
ssh gw-04 'docker logs clawmates_server_1 2>&1 | grep -E "skills door installed|drained .* tool call|orphans:"'
|
||||
```
|
||||
|
||||
3. **Deploy the door** (`TOOL-CALL-ARCHITECTURE.md` §3). Config, not code:
|
||||
`/zeroclaw-data/clawmates-mcp.json` plus a door-shaped provider alias. It is
|
||||
now the single change that makes **Trigger** a real measurement, and the
|
||||
precondition for skills moving from inlined bodies to progressive
|
||||
disclosure — which would also cut the prompt cost in item 1.
|
||||
2. ~~**Measure Trigger properly.**~~ **Done — 2026-08-25.** `skill_delivery`
|
||||
ships both arms; `index` sends name + `when_to_use` + a uri and the agent
|
||||
fetches bodies through the door. Runs 9/10 in `SKILL-USE-BASELINE.md` are the
|
||||
A/B: identical task, one server process, and **the first retrieval this
|
||||
project has seen that the task did not ask for**. Each agent fetched the
|
||||
skill bound to its own role and neither fetched another's. No Compliance
|
||||
regression appeared — 34% fewer tokens, both arms judged met, deliverables
|
||||
slightly larger. n=1 per arm, so that is a signal and not a rate.
|
||||
|
||||
4. **Fold the microVM tier onto `container_tool_hooks`.** It has a gate and a
|
||||
tap by a different route (`vm_tool_tap` installs into the guest,
|
||||
`microvm_executor` drains inside the turn). Two mechanisms for one job is how
|
||||
they drift — and the argument-discarding bug above lived in the shared parser
|
||||
precisely because nobody looked at it from the container side. The fleet has
|
||||
been offline for over a week, so this cannot be tested today.
|
||||
What is left here is volume: run more pairs before believing any number, and
|
||||
decide whether `index` becomes the default. It is currently opt-in per
|
||||
mission (`config.skill_delivery`) or per deployment
|
||||
(`CLAWMATES_SKILL_DELIVERY`).
|
||||
|
||||
5. **Pull upstream's egress policy** — `0db7d999a feat(plugins): add shared
|
||||
egress policy foundation (#9137)`. We are ~220 commits behind; this is the
|
||||
one item worth taking, and it is defence for a problem we have not solved.
|
||||
3. **Fold the microVM tier onto `container_tool_hooks`.** Two mechanisms for
|
||||
one job is how they drift — the argument-discarding bug lived in the shared
|
||||
parser precisely because nobody looked at it from the container side. The
|
||||
microVM path also still passes no turn agents, so its tool calls stay
|
||||
unattributed. Blocked: the fleet has been offline for over a week.
|
||||
|
||||
4. **Pull upstream's egress policy** — `0db7d999a feat(plugins): add shared
|
||||
egress policy foundation (#9137)`. ~220 commits behind; this is the one item
|
||||
worth taking, and it is defence for a problem we have not solved.
|
||||
|
||||
5. **A narrow credential for the rest of the door.** `SCOPE_SKILLS_READ` covers
|
||||
`/mcp/skills`. The `/mcp` door proper (`mcp_door.rs`) still authenticates
|
||||
with a full session, and it is the one that can `delegate`. Nothing hands it
|
||||
a token today; anything that does should not hand it a person's.
|
||||
|
||||
## Open decisions that are yours
|
||||
|
||||
- **Push.** 10 commits are local. Pushing `main` triggers CI → auto-deploy to
|
||||
gw-04.
|
||||
- **Self-authoring scope.** Agents apply their own `skill_candidate` items with
|
||||
no human click (`CLAWMATES_SKILL_SELF_AUTHORING=0` restores the gate).
|
||||
`identity_refinement` and `brain_consolidation` still wait for a human,
|
||||
|
||||
@@ -286,6 +286,90 @@ skill itself names. It is recorded here rather than quietly corrected, because
|
||||
a measurement that hides its own false positives cannot be trusted about
|
||||
anyone else's.
|
||||
|
||||
## Runs 9 and 10 — the delivery A/B, and the first unprompted Trigger
|
||||
|
||||
Two `research_only` missions, **identical task text**, launched minutes apart
|
||||
against one server process. The task never mentions skills, MCP or retrieval —
|
||||
which is the whole point. Run 8 demonstrated the instrument, but its retrieval
|
||||
was *instructed by the task*; nothing there showed an agent judging relevance.
|
||||
|
||||
| | run 9 (`inline`) | run 10 (`index`) |
|
||||
|---|---|---|
|
||||
| skills delivered | 4 | 4 |
|
||||
| prompt bytes (3 turns) | 8642 / 7689 / 10165 | 7329 / 5524 / 5762 |
|
||||
| tool calls recorded | 89 | **59** |
|
||||
| tokens (3 steps) | 52,191 | **35,090** |
|
||||
| `ReadMcpResourceTool` | 0 | **2** |
|
||||
| judge verdict | met | met |
|
||||
|
||||
Per skill:
|
||||
|
||||
| skill | run 9 trigger | run 10 trigger | boundary (both) |
|
||||
|---|---|---|---|
|
||||
| `web-search-triage` | not observable | **pass** | n/a |
|
||||
| `scientific-writing-conventions` | not observable | **pass** | n/a |
|
||||
| `structured-paper-summary` | not observable | n/a | n/a |
|
||||
| `workspace-repo-commit-protocol` | not observable | **FAIL** | pass |
|
||||
|
||||
### What the retrievals actually show
|
||||
|
||||
Attribution is the load-bearing detail, and it is only available because
|
||||
`attribute_sessions` maps a phase's tool calls back to the agent that made
|
||||
them:
|
||||
|
||||
Solveig (lead_researcher) -> skill:global/web-search-triage
|
||||
Olamide (report_writer) -> skill:global/scientific-writing-conventions
|
||||
|
||||
**Each agent reached for the skill bound to its own role, and neither reached
|
||||
for another role's.** That is a relevance judgement, not a sweep — an agent
|
||||
that fetched all four would have shown nothing except that it could.
|
||||
|
||||
Two skills went unread. `structured-paper-summary` scores `NotApplicable`: its
|
||||
holder passed over it and nothing in that phase could check whether it should
|
||||
have, so calling that a miss would punish correct triage.
|
||||
`workspace-repo-commit-protocol` scores **Fail**, and that verdict is the one
|
||||
to argue with: the agent never fetched the procedure, yet every write it made
|
||||
landed inside the checkout, so `boundary` passes. It behaved correctly without
|
||||
reading the rule. Under the rule as written — an applicable skill offered and
|
||||
not read is a Trigger failure — that is a fail, and the pass beside it is the
|
||||
honest counterweight rather than a contradiction.
|
||||
|
||||
### The regression the A/B existed to catch did not appear
|
||||
|
||||
Progressive disclosure can only cost Compliance: under `inline` the procedure
|
||||
sits in front of the model whether or not it noticed it applied. It did not
|
||||
cost anything measurable here. The `index` arm used **34% fewer tokens and 30
|
||||
fewer tool calls**, both arms passed the independent judge, and the deliverables
|
||||
came out slightly larger, not thinner:
|
||||
|
||||
research/speculative-decoding.md 4870 -> 6182
|
||||
research/evidence.md 10823 -> 11727
|
||||
research/evidence-check.md 3340 -> 5336
|
||||
research/questions.md 1894 -> 2402
|
||||
research/REPORT.md 9867 -> 7901
|
||||
|
||||
The `Edit` count is where the arms differ most (31 -> 5). That is a behavioural
|
||||
difference the measurement did not predict and cannot explain from one run
|
||||
each.
|
||||
|
||||
### Two readings this data does not support
|
||||
|
||||
**The prompt saving is small.** Skill bodies are a minority of a turn prompt,
|
||||
so withholding them cut 15-43% of the bytes, not the order of magnitude the
|
||||
framing suggests. Progressive disclosure is worth doing for Trigger, not for
|
||||
context economy.
|
||||
|
||||
**Every `tool.call` in a phase carries the same `created_at`** — the drain
|
||||
timestamp, not the call time. All 59 rows in run 10 read `12:48:12`. Ordering
|
||||
tool calls by that column produces a confident, entirely fabricated narrative;
|
||||
the first draft of this section said the report writer had fetched both skills,
|
||||
because both retrievals appeared inside its turn window. Attribution by
|
||||
`agent_id` is the real answer and it says something different.
|
||||
|
||||
**n = 1 per arm.** Two missions do not establish a rate. What they establish is
|
||||
that the axis now produces a signal at all, and that the control arm's
|
||||
structural blind spot is gone rather than papered over.
|
||||
|
||||
## Honest limits
|
||||
|
||||
- **Five runs, one tier, two workflows.** Nothing here generalises to the
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
-- Which arm of the skill-delivery A/B a mission ran.
|
||||
--
|
||||
-- 'inline' — every pinned skill's body is appended to the turn prompt.
|
||||
-- 'index' — the prompt carries name + when_to_use + uri; the agent fetches
|
||||
-- the bodies it judges relevant through the skills door.
|
||||
--
|
||||
-- Nullable, and NULL means 'inline'. Every mission that ran before this column
|
||||
-- existed ran inline, so a default would be a claim about history that a NULL
|
||||
-- correctly declines to make. `topology_exec::skill_delivery_mode` resolves
|
||||
-- NULL, an unrecognised value and a missing row identically, to the arm that
|
||||
-- requires nothing to be true.
|
||||
--
|
||||
-- Written once at launch by `mission_orchestrator`, which is the only place
|
||||
-- that knows whether the skills door installed — 'index' is refused without
|
||||
-- one, because an index the agent cannot fetch from is worse than no index.
|
||||
ALTER TABLE missions ADD COLUMN IF NOT EXISTS skill_delivery TEXT;
|
||||
@@ -21,6 +21,13 @@
|
||||
# REPO_ID repository to check out; required by the coding recipes, and
|
||||
# the only way the TDD and commit checks can ever fire — a
|
||||
# repo-less run writes markdown and commits nothing
|
||||
# DELIVERY skill delivery arm `inline` (default) or `index`
|
||||
# `index` sends name + when_to_use + a uri and makes the agent
|
||||
# fetch bodies through the skills door, which is the only arm
|
||||
# where Trigger is a question at all. Set per mission, so both
|
||||
# arms run against ONE server process — restarting between arms
|
||||
# would put a confound in the comparison that the numbers do
|
||||
# not show.
|
||||
# TIMEOUT seconds to wait (default 1800)
|
||||
# RETAIN_DAYS hold events this long so the run stays re-scorable (default 90)
|
||||
|
||||
@@ -30,6 +37,7 @@ API="${API:-http://127.0.0.1:8080}"
|
||||
PG="${PG:-clawmates-postgres-1}"
|
||||
OWNER="${OWNER:-om[email protected]}"
|
||||
TEMPLATE="${TEMPLATE:-research_only}"
|
||||
DELIVERY="${DELIVERY:-}"
|
||||
TIMEOUT="${TIMEOUT:-1800}"
|
||||
RETAIN_DAYS="${RETAIN_DAYS:-90}"
|
||||
|
||||
@@ -67,7 +75,7 @@ api() { # api <token> <METHOD> <path> [json]
|
||||
jqv() { python3 -c "import sys,json;d=json.load(sys.stdin);print(d$1)"; }
|
||||
|
||||
score() { # score <token> <mission-id>
|
||||
local token="$1" id="$2"
|
||||
local token="$1" id="$2" arm
|
||||
echo
|
||||
echo "── what the agents DID ─────────────────────────────────────"
|
||||
psql_ "select kind || ' ' || coalesce(target,'') ||
|
||||
@@ -77,6 +85,11 @@ score() { # score <token> <mission-id>
|
||||
order by id;"
|
||||
echo
|
||||
echo "── Skill-Use ───────────────────────────────────────────────"
|
||||
# Which arm ran, printed next to the scores it produced. A Trigger column of
|
||||
# "not observable" means something different under each arm, and a report
|
||||
# that does not say which one is not comparable to the next one.
|
||||
arm=$(psql_ "select coalesce(skill_delivery, 'inline (unrecorded)') from missions where id='$id';" | tr -d '[:space:]')
|
||||
echo "delivery arm: $arm"
|
||||
api "$token" GET "/api/missions/$id/skill-use" | python3 -m json.tool
|
||||
}
|
||||
|
||||
@@ -90,7 +103,7 @@ fi
|
||||
TITLE="${1:?title}"
|
||||
TASK="${2:?task description}"
|
||||
|
||||
body=$(python3 - "$TITLE" "$TASK" "$TEMPLATE" "${REPO_ID:-}" <<'PY'
|
||||
body=$(python3 - "$TITLE" "$TASK" "$TEMPLATE" "${REPO_ID:-}" "$DELIVERY" <<'PY'
|
||||
import json, sys
|
||||
req = {
|
||||
"title": sys.argv[1],
|
||||
@@ -99,6 +112,10 @@ req = {
|
||||
}
|
||||
if sys.argv[4]:
|
||||
req["repo_id"] = sys.argv[4]
|
||||
if sys.argv[5]:
|
||||
# The recipe merges `phase_teams` INTO this object rather than replacing
|
||||
# it, so an arm set here survives staffing.
|
||||
req["config"] = {"skill_delivery": sys.argv[5]}
|
||||
print(json.dumps(req))
|
||||
PY
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user