feat(skill-use): Trigger is observable — score it
The door made retrieval possible; this makes it *measured*. A skill that
arrives by retrieval leaves a recorded tool call, and until now the scorer
ignored it entirely — so the one axis the whole door was built for stayed
`NotObservable` even on a mission where three agents demonstrably reached
through it.
Taken from what run 8 actually recorded, not from the shape I imagined:
ReadMcpResourceTool {"uri":"skill:global/workspace-repo-commit-protocol",
"server":"clawmates_skills"}
`retrieved_skills` reads those URIs through `mcp_skills::parse_uri` — the
function that WROTE them — rather than a second matcher, because two
implementations of one format drift and the drift shows up as a skill
silently scoring nothing.
Trigger is now `Pass` for a skill the agent reached for, and
`NotObservable` for one that was inlined — with a reason that names the
fix rather than the transport: being handed a skill is not failing to
reach for one.
`score` also had to stop reading only the prompt. A skill retrieved and
never inlined is invisible to `skills_in_prompt`, and under progressive
disclosure that is EVERY skill — so the scorer would have reported zero
for the delivery model this axis exists to measure.
Listing the catalogue is browsing; reading a body is the reach. Only
`ReadMcpResourceTool` counts.
Co-Authored-By: Claude Opus 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_018i9Ten1LU4jUr5d7TAWda9
This commit is contained in:
co-authored by
Claude Opus 5
parent
4b160c5a1b
commit
b47ae7fa6b
@@ -144,23 +144,75 @@ impl<'a> Evidence<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Skills the agent RETRIEVED, in the order it reached for them.
|
||||
///
|
||||
/// This is Trigger, and it is observable the moment a skill arrives by
|
||||
/// retrieval instead of by inlining. Claude Code reads an MCP resource through
|
||||
/// `ReadMcpResourceTool`, and the recorded arguments carry the URI:
|
||||
///
|
||||
/// ```text
|
||||
/// ReadMcpResourceTool {"uri":"skill:global/workspace-repo-commit-protocol",
|
||||
/// "server":"clawmates_skills"}
|
||||
/// ```
|
||||
///
|
||||
/// The URI is parsed by `mcp_skills::parse_uri` — the same function that
|
||||
/// WROTE it — rather than by a second matcher here, for the reason every other
|
||||
/// reader in this module delegates: two implementations of one format drift,
|
||||
/// and the drift shows up as a skill silently scoring nothing.
|
||||
pub fn retrieved_skills(ev: &Evidence<'_>) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
for t in ev.tools {
|
||||
if t.tool != "ReadMcpResourceTool" {
|
||||
continue;
|
||||
}
|
||||
let Some(uri) = t.input.get("uri").and_then(|v| v.as_str()) else {
|
||||
continue;
|
||||
};
|
||||
if let Some((_, name)) = crate::mcp_skills::parse_uri(uri) {
|
||||
if !out.contains(&name) {
|
||||
out.push(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Score every skill a phase's prompt delivered, against what the agent produced.
|
||||
pub fn score(
|
||||
prompt: &str,
|
||||
evidence: &Evidence<'_>,
|
||||
source_kinds: &dyn Fn(&str) -> String,
|
||||
) -> Vec<SkillUse> {
|
||||
skills_in_prompt(prompt)
|
||||
let retrieved = retrieved_skills(evidence);
|
||||
// Delivered by either route. A skill that was retrieved and never inlined
|
||||
// is invisible to `skills_in_prompt`, and under progressive disclosure that
|
||||
// is EVERY skill — so scoring only the prompt would report zero for the
|
||||
// delivery model this axis exists to measure.
|
||||
let mut names = skills_in_prompt(prompt);
|
||||
for r in &retrieved {
|
||||
if !names.contains(r) {
|
||||
names.push(r.clone());
|
||||
}
|
||||
}
|
||||
names
|
||||
.into_iter()
|
||||
.map(|skill| {
|
||||
let (compliance, boundary) = check(&skill, evidence);
|
||||
SkillUse {
|
||||
source_kind: source_kinds(&skill),
|
||||
trigger: Verdict::NotObservable(
|
||||
"skills are inlined into the prompt, not retrieved — there is \
|
||||
no retrieval event to observe on the mission path"
|
||||
.into(),
|
||||
),
|
||||
trigger: if retrieved.contains(&skill) {
|
||||
// The agent reached for it. That is the paper's Trigger,
|
||||
// and it is now a recorded tool call like any other.
|
||||
Verdict::Pass
|
||||
} else {
|
||||
Verdict::NotObservable(
|
||||
"this skill was inlined into the prompt, not retrieved — \
|
||||
the agent was handed it, so there is no reaching-for to \
|
||||
observe. Serve it through the door instead and this \
|
||||
becomes a tool call"
|
||||
.into(),
|
||||
)
|
||||
},
|
||||
compliance,
|
||||
boundary,
|
||||
skill,
|
||||
@@ -892,6 +944,80 @@ mod tests {
|
||||
assert_eq!(fine[0].boundary, Verdict::Pass);
|
||||
}
|
||||
|
||||
/// Trigger, from the exact arguments a live mission recorded.
|
||||
///
|
||||
/// Run 8, 2026-08-21: three agents each reached through the door and read a
|
||||
/// skill body. Copied from `mission_events`, not invented — the shape of
|
||||
/// this payload is the whole basis of the axis.
|
||||
#[test]
|
||||
fn a_retrieved_skill_scores_trigger_as_a_pass() {
|
||||
let tools = acted(&[
|
||||
("ListMcpResourcesTool", None, json!({})),
|
||||
(
|
||||
"ReadMcpResourceTool",
|
||||
None,
|
||||
json!({"uri": "skill:global/workspace-repo-commit-protocol",
|
||||
"server": "clawmates_skills"}),
|
||||
),
|
||||
]);
|
||||
let ev = Evidence::new("", &tools);
|
||||
assert_eq!(
|
||||
retrieved_skills(&ev),
|
||||
vec!["workspace-repo-commit-protocol"]
|
||||
);
|
||||
|
||||
// Delivered by RETRIEVAL only — nothing in the prompt at all. Under
|
||||
// progressive disclosure this is every skill, so a scorer that reads
|
||||
// only the prompt would report nothing for the delivery model this
|
||||
// axis exists to measure.
|
||||
let scored = score("Task: do the thing", &ev, &builtin);
|
||||
assert_eq!(scored.len(), 1);
|
||||
assert_eq!(scored[0].skill, "workspace-repo-commit-protocol");
|
||||
assert_eq!(
|
||||
scored[0].trigger,
|
||||
Verdict::Pass,
|
||||
"the agent reached for it; that is the paper's Trigger"
|
||||
);
|
||||
}
|
||||
|
||||
/// An inlined skill is still `NotObservable`, and the reason now names the
|
||||
/// fix rather than the transport.
|
||||
#[test]
|
||||
fn an_inlined_skill_is_still_unobservable_for_trigger() {
|
||||
let prompt = rendered(&[("arxiv-daily", "x")]);
|
||||
let scored = score(&prompt, &narrative("did it"), &builtin);
|
||||
match &scored[0].trigger {
|
||||
Verdict::NotObservable(why) => assert!(
|
||||
why.contains("handed it"),
|
||||
"the reason must say WHY it cannot be seen — being handed a \
|
||||
skill is not failing to reach for one: {why}"
|
||||
),
|
||||
other => panic!("got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// A workspace-scoped URI resolves to the same name as a global one.
|
||||
#[test]
|
||||
fn both_uri_shapes_yield_the_skill_name() {
|
||||
let ws = uuid::Uuid::now_v7();
|
||||
let tools = acted(&[
|
||||
(
|
||||
"ReadMcpResourceTool",
|
||||
None,
|
||||
json!({"uri": format!("skill:workspace/{ws}/team-local-thing")}),
|
||||
),
|
||||
// Not a skill URI, and not a panic.
|
||||
("ReadMcpResourceTool", None, json!({"uri": "file:///etc/passwd"})),
|
||||
// A LIST call names no skill — only reads are retrievals.
|
||||
("ListMcpResourcesTool", None, json!({})),
|
||||
]);
|
||||
assert_eq!(
|
||||
retrieved_skills(&Evidence::new("", &tools)),
|
||||
vec!["team-local-thing"],
|
||||
"listing the catalogue is browsing; reading a body is the reach"
|
||||
);
|
||||
}
|
||||
|
||||
/// Agent-authored skills must stay visible as such in the report.
|
||||
#[test]
|
||||
fn writing_outside_the_mission_checkout_crosses_the_boundary() {
|
||||
|
||||
Reference in New Issue
Block a user