diff --git a/crates/cm-api/src/continuous_research.rs b/crates/cm-api/src/continuous_research.rs index ba1d144..feb35e9 100644 --- a/crates/cm-api/src/continuous_research.rs +++ b/crates/cm-api/src/continuous_research.rs @@ -142,25 +142,46 @@ pub fn write_manifest( /// What the decision model said about one harvested paper. `topic_tags` /// was written as `[]` on every manifest line from the day the manifest -/// existed — a slot the agents were told to read and nothing filled. This -/// fills it: which of the mission's topics the paper belongs to (a Choice, -/// every topic ≥ 0.3 kept, `none` when it fits none), and how much it -/// matters to an agent platform (a four-level Score), so the ranking phase -/// starts from a calibrated number rather than from the title alone. +/// existed — a slot the agents were told to read and nothing filled. +/// +/// **A score has to discriminate within the population it scores.** The +/// first version asked "how relevant is this to an agent platform" on a +/// four-level scale, and MEASURED on the ten papers of mission 01a0c940 it +/// answered 2.93–3.00 — a spread of 0.07, no ranking information at all. +/// Of course: the harvest runs the operator's own arXiv topic queries, so +/// every paper in it is about agents by construction. "How actionable is +/// it" saturated the same way (spread 0.20). What did discriminate on the +/// same ten abstracts was the strength of the evidence behind the claims +/// (1.36–3.00, spread 1.64) and what KIND of paper it is. So the manifest +/// carries those two and no relevance number: a digest phase ranking +/// already-relevant papers needs to know which ones measured something. #[derive(Debug, Clone, serde::Serialize, Default)] pub struct PaperTriage { pub topic_tags: Vec, - /// 0 = unrelated … 3 = directly about what the platform does. `None` - /// when no triage ran (no key, or the call failed). - pub relevance: Option, - pub relevance_confidence: Option, + /// `benchmark` | `method` | `measurement` | `survey` | `position`, and + /// how peaked that choice was — a paper the model cannot place is one + /// the reader should look at rather than trust the label for. + pub kind: Option, + pub kind_confidence: Option, + /// 0 = position piece, no experiments … 3 = measured on real systems + /// with ablations. `None` when no triage ran (no key, or a failed call). + pub evidence: Option, + pub evidence_confidence: Option, } -const RELEVANCE_LEVELS: [&str; 4] = [ - "Unrelated to LLM agents, agent platforms, or their evaluation", - "Adjacent: language models or systems work an agent platform might one day draw on", - "Relevant: about agents, tools, memory, skills, judging, sandboxing, or multi-agent orchestration", - "Directly applicable: a method, measurement, or failure mode a platform running autonomous coding and research agents should act on", +const EVIDENCE_LEVELS: [&str; 4] = [ + "Position, opinion, or framework description; no experiments.", + "Illustrative examples, a demo, or a single small case study.", + "Benchmarked with numbers, on a suite the authors assembled.", + "Measured on real systems or at scale, with ablations or failure analysis.", +]; + +const PAPER_KINDS: [(&str, &str); 5] = [ + ("benchmark", "Introduces a dataset or benchmark to measure something"), + ("method", "Proposes a technique, architecture, or algorithm"), + ("measurement", "Measures the behaviour of existing systems without proposing a new one"), + ("survey", "Reviews or categorises a body of existing work"), + ("position", "Argues a viewpoint or proposes an agenda"), ]; /// Triage every harvested paper in one call each. Best-effort: a missing key @@ -191,10 +212,17 @@ pub async fn triage_papers( }, ), ( - "relevance".to_string(), + "kind".to_string(), + Question::choice( + "What kind of paper is this?", + PAPER_KINDS.map(|(k, d)| (k, Some(d))), + ), + ), + ( + "evidence".to_string(), Question::score( - "How relevant is this paper to a platform that runs autonomous LLM coding and research agents?", - RELEVANCE_LEVELS, + "How strong is the evidence behind this paper's claims?", + EVIDENCE_LEVELS, ), ), ] @@ -221,9 +249,13 @@ pub async fn triage_papers( tags.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); t.topic_tags = tags.into_iter().map(|(k, _)| k).collect(); } - if let Some(Answer::Score { score, confidence, .. }) = d.answers.get("relevance") { - t.relevance = Some((*score * 100.0).round() / 100.0); - t.relevance_confidence = Some((*confidence * 100.0).round() / 100.0); + if let Some(Answer::Choice { choice, confidence, .. }) = d.answers.get("kind") { + t.kind = Some(choice.clone()); + t.kind_confidence = Some((*confidence * 100.0).round() / 100.0); + } + if let Some(Answer::Score { score, confidence, .. }) = d.answers.get("evidence") { + t.evidence = Some((*score * 100.0).round() / 100.0); + t.evidence_confidence = Some((*confidence * 100.0).round() / 100.0); } } Ok(Err(e)) => eprintln!("continuous_research: triage of {} failed: {e}", p.arxiv_id), @@ -232,12 +264,26 @@ pub async fn triage_papers( out.push(t); } let tagged = out.iter().filter(|t| !t.topic_tags.is_empty()).count(); + let scores: Vec = out.iter().filter_map(|t| t.evidence).collect(); eprintln!( - "continuous_research: triaged {} paper(s) with {}: {tagged} tagged, {} scored", + "continuous_research: triaged {} paper(s) with {}: {tagged} tagged, {} with evidence scored", papers.len(), jev.name(), - out.iter().filter(|t| t.relevance.is_some()).count() + scores.len() ); + // A score that came back the same for every paper ranked nothing. Said + // out loud because the first version of this question did exactly that + // and looked like a working feature — ten confident numbers, no + // information. See `PaperTriage`. + let spread = cm_decide::patterns::spread(&scores); + if scores.len() > 2 && spread < cm_decide::patterns::SATURATED_BELOW { + eprintln!( + "continuous_research: WARNING — evidence scores span only {spread:.2} across \ + {} papers. The question is not separating this harvest; the ranking phase \ + gets no signal from it.", + scores.len() + ); + } out } @@ -257,9 +303,9 @@ fn readable_topic(query: &str) -> String { /// The manifest lines for a set of freshly shelved papers. /// -/// Shape matches what `templates/teams/continuous_research.toml` documents: -/// `{ source, url, title, snippet, first_seen, topic_tags }`, plus -/// `relevance` since 2026-09-21 (see [`PaperTriage`]). `triage` is +/// Shape matches what `skills/research/arxiv-daily.md` documents: +/// `{ source, url, title, snippet, first_seen, topic_tags }`, plus `kind` +/// and `evidence` since 2026-09-22 (see [`PaperTriage`]). `triage` is /// positional with `papers`; shorter means the rest are untriaged. pub fn manifest_lines( papers: &[crate::papers::Paper], @@ -278,10 +324,14 @@ pub fn manifest_lines( "snippet": p.summary.chars().take(400).collect::(), "first_seen": first_seen, "topic_tags": t.topic_tags, - "relevance": t.relevance.map(|r| json!({ - "score": r, - "confidence": t.relevance_confidence, - "scale": "0 unrelated … 3 directly applicable", + "kind": t.kind.as_ref().map(|k| json!({ + "is": k, + "confidence": t.kind_confidence, + })), + "evidence": t.evidence.map(|e| json!({ + "score": e, + "confidence": t.evidence_confidence, + "scale": "0 position piece, no experiments … 3 measured on real systems with ablations", })), }) .to_string() @@ -361,22 +411,25 @@ mod tests { let out = manifest_lines(std::slice::from_ref(&p), "2026-08-17", &[]); assert_eq!(out.lines().count(), 1); let v: serde_json::Value = serde_json::from_str(&out).expect("each line is JSON"); - for key in ["source", "url", "title", "snippet", "first_seen", "topic_tags", "relevance"] { + for key in ["source", "url", "title", "snippet", "first_seen", "topic_tags", "kind", "evidence"] { assert!(v.get(key).is_some(), "missing {key} in {v}"); } - // Untriaged: the slot is there and empty, as it always was. + // Untriaged: the slots are there and empty, as they always were. assert_eq!(v["topic_tags"], serde_json::json!([])); - assert!(v["relevance"].is_null()); - // Triaged: the tags and the score land on the line. + assert!(v["kind"].is_null() && v["evidence"].is_null()); + // Triaged: the tags, the kind and the evidence score land on the line. let t = PaperTriage { topic_tags: vec!["agent memory long-term".into()], - relevance: Some(2.4), - relevance_confidence: Some(0.7), + kind: Some("benchmark".into()), + kind_confidence: Some(0.91), + evidence: Some(1.36), + evidence_confidence: Some(0.62), }; let out = manifest_lines(std::slice::from_ref(&p), "2026-08-17", std::slice::from_ref(&t)); let v: serde_json::Value = serde_json::from_str(&out).unwrap(); assert_eq!(v["topic_tags"][0], "agent memory long-term"); - assert_eq!(v["relevance"]["score"], 2.4); + assert_eq!(v["kind"]["is"], "benchmark"); + assert_eq!(v["evidence"]["score"], 1.36); assert_eq!(v["source"], "arxiv:2401.12345"); assert!( v["snippet"].as_str().unwrap().chars().count() <= 400, diff --git a/crates/cm-decide/src/patterns.rs b/crates/cm-decide/src/patterns.rs index b665079..1729aec 100644 --- a/crates/cm-decide/src/patterns.rs +++ b/crates/cm-decide/src/patterns.rs @@ -55,6 +55,28 @@ pub fn composite(parts: &[(f64, usize, f64)]) -> f64 { / total } +/// How much a Score actually separated a population: `max - min`, in +/// levels. A dimension that returns the same score for everything ranks +/// nothing, however confident each answer is — measured on a real harvest, +/// "how relevant is this paper" spanned 0.07 of a 3-level scale because the +/// corpus was selected to be relevant, while "how strong is the evidence" +/// spanned 1.64 on the same ten papers. Callers that rank should report +/// this and say so when it collapses; see `SATURATED_BELOW`. +pub fn spread(scores: &[f64]) -> f64 { + match ( + scores.iter().cloned().fold(f64::INFINITY, f64::min), + scores.iter().cloned().fold(f64::NEG_INFINITY, f64::max), + ) { + (lo, hi) if lo.is_finite() && hi.is_finite() => hi - lo, + _ => 0.0, + } +} + +/// A spread under this, on a population of more than a couple of items, is +/// a question that is not discriminating: act on it as a defect in the +/// question, not as a fact about the population. +pub const SATURATED_BELOW: f64 = 0.5; + /// Rerank: order candidates by a per-candidate Noul, highest first. pub fn rerank(mut items: Vec<(T, f64)>) -> Vec<(T, f64)> { items.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); @@ -80,6 +102,18 @@ mod tests { assert!((c - 0.85).abs() < 1e-9); } + #[test] + fn spread_reports_what_a_score_separated() { + // The measured numbers: relevance on ten harvested papers, then + // evidence on the same ten. + let relevance = [3.0, 2.99, 2.99, 2.98, 2.97, 2.97, 2.94, 3.0, 2.99, 3.0]; + let evidence = [3.0, 2.93, 2.86, 2.85, 2.82, 2.59, 2.88, 2.14, 2.16, 1.36]; + assert!(spread(&relevance) < SATURATED_BELOW, "{}", spread(&relevance)); + assert!(spread(&evidence) > SATURATED_BELOW, "{}", spread(&evidence)); + assert_eq!(spread(&[]), 0.0); + assert_eq!(spread(&[1.5]), 0.0); + } + #[test] fn rerank_is_descending() { let r = rerank(vec![("a", 0.2), ("b", 0.9), ("c", 0.5)]); diff --git a/skills/research/arxiv-daily.md b/skills/research/arxiv-daily.md index 3e42115..a60d114 100644 --- a/skills/research/arxiv-daily.md +++ b/skills/research/arxiv-daily.md @@ -16,9 +16,21 @@ Your input is the result: ``` ContinuousResearch//harvest.jsonl - { source, url, title, snippet, first_seen, topic_tags } + { source, url, title, snippet, first_seen, topic_tags, kind, evidence } ``` +`topic_tags`, `kind` and `evidence` are filled by a decision model before you +see the file. `kind.is` is one of benchmark / method / measurement / survey / +position, with the confidence of that call beside it — a low confidence means +the paper does not sit cleanly in one, so read before you trust the label. +`evidence.score` runs 0 (a position piece with no experiments) to 3 (measured +on real systems with ablations); it is the one dimension that separates papers +in a harvest, because every paper in the file already matched your topics. +There is deliberately no "relevance" number: measured on a real harvest it +came back 2.93–3.00 for every paper, which ranks nothing. All three keys are +absent or empty when the triage could not run; that is not a signal about the +paper. + One line per paper that is **new since the last run**. Papers already covered are not in it — by design, not omission.