feat(skill-use): Trigger is observable — score it

The door made retrieval possible; this makes it *measured*. A skill that
arrives by retrieval leaves a recorded tool call, and until now the scorer
ignored it entirely — so the one axis the whole door was built for stayed
`NotObservable` even on a mission where three agents demonstrably reached
through it.

Taken from what run 8 actually recorded, not from the shape I imagined:

    ReadMcpResourceTool {"uri":"skill:global/workspace-repo-commit-protocol",
                         "server":"clawmates_skills"}

`retrieved_skills` reads those URIs through `mcp_skills::parse_uri` — the
function that WROTE them — rather than a second matcher, because two
implementations of one format drift and the drift shows up as a skill
silently scoring nothing.

Trigger is now `Pass` for a skill the agent reached for, and
`NotObservable` for one that was inlined — with a reason that names the
fix rather than the transport: being handed a skill is not failing to
reach for one.

`score` also had to stop reading only the prompt. A skill retrieved and
never inlined is invisible to `skills_in_prompt`, and under progressive
disclosure that is EVERY skill — so the scorer would have reported zero
for the delivery model this axis exists to measure.

Listing the catalogue is browsing; reading a body is the reach. Only
`ReadMcpResourceTool` counts.

Co-Authored-By: Claude Opus 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_018i9Ten1LU4jUr5d7TAWda9
This commit is contained in:
Omar Sobh
2026-08-21 12:37:49 -07:00
co-authored by Claude Opus 5
parent 4b160c5a1b
commit b47ae7fa6b
2 changed files with 133 additions and 7 deletions
+132 -6
View File
@@ -144,23 +144,75 @@ impl<'a> Evidence<'a> {
}
}
/// Skills the agent RETRIEVED, in the order it reached for them.
///
/// This is Trigger, and it is observable the moment a skill arrives by
/// retrieval instead of by inlining. Claude Code reads an MCP resource through
/// `ReadMcpResourceTool`, and the recorded arguments carry the URI:
///
/// ```text
/// ReadMcpResourceTool {"uri":"skill:global/workspace-repo-commit-protocol",
/// "server":"clawmates_skills"}
/// ```
///
/// The URI is parsed by `mcp_skills::parse_uri` — the same function that
/// WROTE it — rather than by a second matcher here, for the reason every other
/// reader in this module delegates: two implementations of one format drift,
/// and the drift shows up as a skill silently scoring nothing.
pub fn retrieved_skills(ev: &Evidence<'_>) -> Vec<String> {
let mut out = Vec::new();
for t in ev.tools {
if t.tool != "ReadMcpResourceTool" {
continue;
}
let Some(uri) = t.input.get("uri").and_then(|v| v.as_str()) else {
continue;
};
if let Some((_, name)) = crate::mcp_skills::parse_uri(uri) {
if !out.contains(&name) {
out.push(name);
}
}
}
out
}
/// Score every skill a phase's prompt delivered, against what the agent produced.
pub fn score(
prompt: &str,
evidence: &Evidence<'_>,
source_kinds: &dyn Fn(&str) -> String,
) -> Vec<SkillUse> {
skills_in_prompt(prompt)
let retrieved = retrieved_skills(evidence);
// Delivered by either route. A skill that was retrieved and never inlined
// is invisible to `skills_in_prompt`, and under progressive disclosure that
// is EVERY skill — so scoring only the prompt would report zero for the
// delivery model this axis exists to measure.
let mut names = skills_in_prompt(prompt);
for r in &retrieved {
if !names.contains(r) {
names.push(r.clone());
}
}
names
.into_iter()
.map(|skill| {
let (compliance, boundary) = check(&skill, evidence);
SkillUse {
source_kind: source_kinds(&skill),
trigger: Verdict::NotObservable(
"skills are inlined into the prompt, not retrieved — there is \
no retrieval event to observe on the mission path"
.into(),
),
trigger: if retrieved.contains(&skill) {
// The agent reached for it. That is the paper's Trigger,
// and it is now a recorded tool call like any other.
Verdict::Pass
} else {
Verdict::NotObservable(
"this skill was inlined into the prompt, not retrieved — \
the agent was handed it, so there is no reaching-for to \
observe. Serve it through the door instead and this \
becomes a tool call"
.into(),
)
},
compliance,
boundary,
skill,
@@ -892,6 +944,80 @@ mod tests {
assert_eq!(fine[0].boundary, Verdict::Pass);
}
/// Trigger, from the exact arguments a live mission recorded.
///
/// Run 8, 2026-08-21: three agents each reached through the door and read a
/// skill body. Copied from `mission_events`, not invented — the shape of
/// this payload is the whole basis of the axis.
#[test]
fn a_retrieved_skill_scores_trigger_as_a_pass() {
let tools = acted(&[
("ListMcpResourcesTool", None, json!({})),
(
"ReadMcpResourceTool",
None,
json!({"uri": "skill:global/workspace-repo-commit-protocol",
"server": "clawmates_skills"}),
),
]);
let ev = Evidence::new("", &tools);
assert_eq!(
retrieved_skills(&ev),
vec!["workspace-repo-commit-protocol"]
);
// Delivered by RETRIEVAL only — nothing in the prompt at all. Under
// progressive disclosure this is every skill, so a scorer that reads
// only the prompt would report nothing for the delivery model this
// axis exists to measure.
let scored = score("Task: do the thing", &ev, &builtin);
assert_eq!(scored.len(), 1);
assert_eq!(scored[0].skill, "workspace-repo-commit-protocol");
assert_eq!(
scored[0].trigger,
Verdict::Pass,
"the agent reached for it; that is the paper's Trigger"
);
}
/// An inlined skill is still `NotObservable`, and the reason now names the
/// fix rather than the transport.
#[test]
fn an_inlined_skill_is_still_unobservable_for_trigger() {
let prompt = rendered(&[("arxiv-daily", "x")]);
let scored = score(&prompt, &narrative("did it"), &builtin);
match &scored[0].trigger {
Verdict::NotObservable(why) => assert!(
why.contains("handed it"),
"the reason must say WHY it cannot be seen — being handed a \
skill is not failing to reach for one: {why}"
),
other => panic!("got {other:?}"),
}
}
/// A workspace-scoped URI resolves to the same name as a global one.
#[test]
fn both_uri_shapes_yield_the_skill_name() {
let ws = uuid::Uuid::now_v7();
let tools = acted(&[
(
"ReadMcpResourceTool",
None,
json!({"uri": format!("skill:workspace/{ws}/team-local-thing")}),
),
// Not a skill URI, and not a panic.
("ReadMcpResourceTool", None, json!({"uri": "file:///etc/passwd"})),
// A LIST call names no skill — only reads are retrievals.
("ListMcpResourcesTool", None, json!({})),
]);
assert_eq!(
retrieved_skills(&Evidence::new("", &tools)),
vec!["team-local-thing"],
"listing the catalogue is browsing; reading a body is the reach"
);
}
/// Agent-authored skills must stay visible as such in the report.
#[test]
fn writing_outside_the_mission_checkout_crosses_the_boundary() {