clawmates/crates/cm-decide/eval/skill-triage.json

{
  "_about": "Labelled skill-triage cases: for each mission task, which of the skills under skills/ an agent should read. Labelled by hand on 2026-09-21 against each skill's when_to_use line; role-pinned skills are marked positive when the task is the kind of work the pin is for (a Rust coding task pins the Rust and commit-protocol skills). The set is the bar every triage backend is measured against — disputed rows are the eval's business, not the backend's.",
  "cases": [
    {"id": "fc-latency-brief", "task": "Write a research brief on Firecracker microVM cold-start latency. Sweep the open web for measurements, prefer primary sources (papers, vendor docs, benchmark repos) over aggregators, and make every empirical claim traceable to a source. Deliver research/BRIEF.md.",
     "positives": ["web-search-triage", "scientific-writing-conventions", "structured-paper-summary"]},
    {"id": "continuous-research-digest", "task": "Process this week's harvest manifest for the Continuous Research mission: drop items we have covered before, rank the rest by signal, decide which matter to the platform and why, and write the digest entries into the valhalla vault under the usual conventions.",
     "positives": ["arxiv-daily", "duplicate-detection", "signal-to-noise-ranking", "paper-to-project-relevance", "executive-summary-writing", "structured-paper-summary", "obsidian-vault-conventions"]},
    {"id": "podcast-script", "task": "Script phase: turn research/analysis.md into script.md and episode.json for the Continuous Research episode — two hosts, plain speech, every claim from the analysis and nothing invented.",
     "positives": ["podcast-dialogue-writing"]},
    {"id": "axum-paginated-endpoint", "task": "Add GET /api/missions/{id}/evaluations to the Rust axum server, returning a paginated list of judge verdicts for the mission. Design the contract first, write the tests before the handler, keep commits small, and commit on the mission branch.",
     "positives": ["api-pagination-day-1", "openapi-contract-first", "tdd-red-green-refactor", "cargo-test-driven-development", "rust-error-handling", "rust-async-tokio-idioms", "write-rust-current-edition", "workspace-repo-commit-protocol", "small-focused-commits", "int-xx-marker-protocol"]},
    {"id": "slow-postgres-query", "task": "The query behind /api/world/live over mission_events has become slow. Find out why with EXPLAIN ANALYZE, decide whether an index fixes it, add it as a forward-only migration, and cover the query with an integration test against a real Postgres.",
     "positives": ["postgres-explain-analyze", "postgres-index-selection", "postgres-migrations-forward-only", "postgres-integration-testing", "workspace-repo-commit-protocol", "small-focused-commits"]},
    {"id": "criterion-baseline", "task": "Benchmark the add hot path with criterion and record the measured baseline (ns per iteration, harness, host) in BASELINE.md at the repository root. Commit the bench and the file.",
     "positives": ["criterion-benchmarking", "write-rust-current-edition", "workspace-repo-commit-protocol", "small-focused-commits"]},
    {"id": "security-scan", "task": "Security phase: run cargo audit against the advisory database and gitleaks over the full history, and write SECURITY.md naming each tool, its version, and the count of findings, with a line per finding.",
     "positives": ["cargo-audit-workflow", "secret-scanning-gitleaks", "workspace-repo-commit-protocol"]},
    {"id": "review-judge-pr", "task": "Review the diff that adds the commit-first round to the evaluator. Report defects, missing tests, and anything that weakens an existing guarantee; do not edit the code.",
     "positives": ["code-review-checklist"]},
    {"id": "map-unfamiliar-repo", "task": "Before changing anything, map the yc-software/qm repository: its crates, the entry points, how a request flows from the HTTP layer to the executor, and where a TurnExecutor seam would go. Write ARCHITECTURE.md.",
     "positives": ["ast-grep-repo-index", "request-lifecycle-tracing"]},
    {"id": "regression-forensics", "task": "Mission resume started dropping the runtime pairing code sometime in August. Find the commit that introduced the behaviour, explain why the code is the way it is, and propose the smallest fix.",
     "positives": ["git-log-forensics", "request-lifecycle-tracing"]},
    {"id": "react-metrics-panel", "task": "Build the per-agent metrics band for the agents page: a React 19 server component in the Next.js 15 app, styled with Tailwind v4, showing loading, empty, error and populated states, keyboard-navigable and screen-reader labelled. Commit on the mission branch.",
     "positives": ["react-19-server-components", "tailwind-v4-idioms", "component-4-state-model", "a11y-checklist", "workspace-repo-commit-protocol", "small-focused-commits"]},
    {"id": "playwright-login", "task": "Write Playwright end-to-end tests for the login flow: success, wrong password, and session expiry. Make them stable under CI's slower machines.",
     "positives": ["playwright-e2e-patterns", "workspace-repo-commit-protocol"]},
    {"id": "expo-missions-list", "task": "Start the mobile app: an Expo React Native project showing the missions list as a fast scrollable feed, with platform-appropriate navigation on iOS and Android.",
     "positives": ["expo-managed-vs-bare", "rn-flashlist-perf", "mobile-platform-conventions", "component-4-state-model", "workspace-repo-commit-protocol"]},
    {"id": "mobile-e2e-setup", "task": "Set up end-to-end runs for the mobile app on the iOS simulator and an Android emulator, and get the login test green on both.",
     "positives": ["mobile-e2e-and-simulators", "playwright-e2e-patterns"]},
    {"id": "cuda-reduction", "task": "Write a CUDA reduction kernel for the histogram step, profile it, reason about memory coalescing and occupancy, place it on the roofline, and show with a benchmark that it beats the current implementation.",
     "positives": ["gpu-kernel-authoring", "gpu-coalescing-and-occupancy", "gpu-profiling-workflow", "roofline-model", "criterion-benchmarking", "workspace-repo-commit-protocol"]},
    {"id": "threejs-world-scene", "task": "Plan the three.js scene graph for the Gource-style World view, write the particle shader in GLSL with a WGSL port, and fix the frame drops the replay scrubber causes.",
     "positives": ["scene-graph-planning", "shader-authoring-glsl-wgsl", "threejs-perf-and-teardown", "webgl-frame-profiling"]},
    {"id": "agent-level-up", "task": "Inspect the researcher agent's .brain, decide whether its definition matches what it actually does, propose a level-up changing its skills and model, and show with baseline metrics whether the last change helped.",
     "positives": ["brain-file-reading", "level-up-proposal-shape", "metrics-baseline-comparison"]},
    {"id": "planner-decompose", "task": "Planner: read research/IMPLEMENTATION_BRIEF.md and decompose it into INT-XX items with the marker protocol every coder role must follow; output the task list, do not implement anything.",
     "positives": ["decompose-int-items", "int-xx-marker-protocol"]},
    {"id": "paper-draft-novelty", "task": "Draft the evaluation section of the paper from our measured skill-retrieval numbers, and check whether the files-arm delivery idea is novel before we claim it.",
     "positives": ["scientific-writing-conventions", "prior-art-search"]},
    {"id": "migration-with-backfill", "task": "Add a nullable mission_id column to auth_sessions as a forward-only migration with a partial index, backfill nothing, and prove with an integration test against Postgres that inserts and the cascade delete behave.",
     "positives": ["postgres-migrations-forward-only", "postgres-integration-testing", "workspace-repo-commit-protocol", "small-focused-commits"]}
  ]
}