diff --git a/crates/cm-api/src/continuous_research.rs b/crates/cm-api/src/continuous_research.rs index eafc588..ba1d144 100644 --- a/crates/cm-api/src/continuous_research.rs +++ b/crates/cm-api/src/continuous_research.rs @@ -128,32 +128,161 @@ pub fn write_manifest( checkout: &std::path::Path, papers: &[crate::papers::Paper], date: &str, + triage: &[PaperTriage], ) -> Result { let rel = manifest_path(date); let abs = checkout.join(&rel); if let Some(parent) = abs.parent() { std::fs::create_dir_all(parent).map_err(|e| format!("create {}: {e}", parent.display()))?; } - let body = manifest_lines(papers, date); + let body = manifest_lines(papers, date, triage); std::fs::write(&abs, format!("{body}\n")).map_err(|e| format!("write {}: {e}", abs.display()))?; Ok(abs) } +/// What the decision model said about one harvested paper. `topic_tags` +/// was written as `[]` on every manifest line from the day the manifest +/// existed — a slot the agents were told to read and nothing filled. This +/// fills it: which of the mission's topics the paper belongs to (a Choice, +/// every topic ≥ 0.3 kept, `none` when it fits none), and how much it +/// matters to an agent platform (a four-level Score), so the ranking phase +/// starts from a calibrated number rather than from the title alone. +#[derive(Debug, Clone, serde::Serialize, Default)] +pub struct PaperTriage { + pub topic_tags: Vec, + /// 0 = unrelated … 3 = directly about what the platform does. `None` + /// when no triage ran (no key, or the call failed). + pub relevance: Option, + pub relevance_confidence: Option, +} + +const RELEVANCE_LEVELS: [&str; 4] = [ + "Unrelated to LLM agents, agent platforms, or their evaluation", + "Adjacent: language models or systems work an agent platform might one day draw on", + "Relevant: about agents, tools, memory, skills, judging, sandboxing, or multi-agent orchestration", + "Directly applicable: a method, measurement, or failure mode a platform running autonomous coding and research agents should act on", +]; + +/// Triage every harvested paper in one call each. Best-effort: a missing key +/// or a failed call leaves that paper's tags empty and relevance `None`, the +/// state the manifest has always been in. +pub async fn triage_papers( + papers: &[crate::papers::Paper], + topics: &[String], +) -> Vec { + use cm_decide::{Answer, Decider as _, Question}; + let Some(jev) = cm_decide::jev::Jev::from_env() else { + return vec![PaperTriage::default(); papers.len()]; + }; + // Topics are arXiv query strings; the option NAME the model sees is the + // readable form (`all:"agent memory" AND all:"long-term"` → `agent memory + // long-term`), the value the query itself for precision. + let mut criteria: std::collections::BTreeMap> = topics + .iter() + .map(|t| (readable_topic(t), Some(t.clone()))) + .collect(); + criteria.insert("none".into(), Some("Fits none of the listed topics".into())); + let questions: std::collections::BTreeMap = [ + ( + "topic".to_string(), + Question::Choice { + instructions: "Which of these research topics is this paper about?".into(), + criteria, + }, + ), + ( + "relevance".to_string(), + Question::score( + "How relevant is this paper to a platform that runs autonomous LLM coding and research agents?", + RELEVANCE_LEVELS, + ), + ), + ] + .into_iter() + .collect(); + + let mut out = Vec::with_capacity(papers.len()); + for p in papers { + let state = format!("Title: {}\n\nAbstract: {}", p.title, p.summary); + let decided = tokio::time::timeout( + std::time::Duration::from_secs(10), + jev.decide(&state, &questions), + ) + .await; + let mut t = PaperTriage::default(); + match decided { + Ok(Ok(d)) => { + if let Some(Answer::Choice { probabilities, .. }) = d.answers.get("topic") { + let mut tags: Vec<(String, f64)> = probabilities + .iter() + .filter(|(k, p)| k.as_str() != "none" && **p >= 0.3) + .map(|(k, p)| (k.clone(), *p)) + .collect(); + tags.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + t.topic_tags = tags.into_iter().map(|(k, _)| k).collect(); + } + if let Some(Answer::Score { score, confidence, .. }) = d.answers.get("relevance") { + t.relevance = Some((*score * 100.0).round() / 100.0); + t.relevance_confidence = Some((*confidence * 100.0).round() / 100.0); + } + } + Ok(Err(e)) => eprintln!("continuous_research: triage of {} failed: {e}", p.arxiv_id), + Err(_) => eprintln!("continuous_research: triage of {} timed out", p.arxiv_id), + } + out.push(t); + } + let tagged = out.iter().filter(|t| !t.topic_tags.is_empty()).count(); + eprintln!( + "continuous_research: triaged {} paper(s) with {}: {tagged} tagged, {} scored", + papers.len(), + jev.name(), + out.iter().filter(|t| t.relevance.is_some()).count() + ); + out +} + +/// `all:"agent memory" AND all:"long-term"` → `agent memory long-term`. +fn readable_topic(query: &str) -> String { + let words: Vec<&str> = query + .split(|c: char| c == '"' || c.is_whitespace() || c == '(' || c == ')') + .filter(|w| !w.is_empty()) + .filter(|w| !matches!(*w, "AND" | "OR" | "NOT")) + .map(|w| w.strip_prefix("all:").unwrap_or(w)) + .map(|w| w.strip_prefix("ti:").unwrap_or(w)) + .map(|w| w.strip_prefix("abs:").unwrap_or(w)) + .filter(|w| !w.is_empty()) + .collect(); + words.join(" ") +} + /// The manifest lines for a set of freshly shelved papers. /// /// Shape matches what `templates/teams/continuous_research.toml` documents: -/// `{ source, url, title, snippet, first_seen, topic_tags }`. -pub fn manifest_lines(papers: &[crate::papers::Paper], first_seen: &str) -> String { +/// `{ source, url, title, snippet, first_seen, topic_tags }`, plus +/// `relevance` since 2026-09-21 (see [`PaperTriage`]). `triage` is +/// positional with `papers`; shorter means the rest are untriaged. +pub fn manifest_lines( + papers: &[crate::papers::Paper], + first_seen: &str, + triage: &[PaperTriage], +) -> String { papers .iter() - .map(|p| { + .enumerate() + .map(|(i, p)| { + let t = triage.get(i).cloned().unwrap_or_default(); json!({ "source": p.source_id(), "url": format!("https://arxiv.org/abs/{}", p.arxiv_id), "title": p.title, "snippet": p.summary.chars().take(400).collect::(), "first_seen": first_seen, - "topic_tags": [], + "topic_tags": t.topic_tags, + "relevance": t.relevance.map(|r| json!({ + "score": r, + "confidence": t.relevance_confidence, + "scale": "0 unrelated … 3 directly applicable", + })), }) .to_string() }) @@ -198,6 +327,18 @@ mod tests { } } + #[test] + fn arxiv_queries_become_readable_option_names() { + assert_eq!( + readable_topic(r#"all:"agent memory" AND all:"long-term""#), + "agent memory long-term" + ); + assert_eq!( + readable_topic(r#"all:"agentic topology" OR all:"multi-agent topology""#), + "agentic topology multi-agent topology" + ); + } + #[test] fn the_date_stamp_is_zero_padded() { let d = today(); @@ -217,12 +358,25 @@ mod tests { published: "2026-08-17".into(), pdf_url: "https://arxiv.org/pdf/2401.12345".into(), }; - let out = manifest_lines(std::slice::from_ref(&p), "2026-08-17"); + let out = manifest_lines(std::slice::from_ref(&p), "2026-08-17", &[]); assert_eq!(out.lines().count(), 1); let v: serde_json::Value = serde_json::from_str(&out).expect("each line is JSON"); - for key in ["source", "url", "title", "snippet", "first_seen", "topic_tags"] { + for key in ["source", "url", "title", "snippet", "first_seen", "topic_tags", "relevance"] { assert!(v.get(key).is_some(), "missing {key} in {v}"); } + // Untriaged: the slot is there and empty, as it always was. + assert_eq!(v["topic_tags"], serde_json::json!([])); + assert!(v["relevance"].is_null()); + // Triaged: the tags and the score land on the line. + let t = PaperTriage { + topic_tags: vec!["agent memory long-term".into()], + relevance: Some(2.4), + relevance_confidence: Some(0.7), + }; + let out = manifest_lines(std::slice::from_ref(&p), "2026-08-17", std::slice::from_ref(&t)); + let v: serde_json::Value = serde_json::from_str(&out).unwrap(); + assert_eq!(v["topic_tags"][0], "agent memory long-term"); + assert_eq!(v["relevance"]["score"], 2.4); assert_eq!(v["source"], "arxiv:2401.12345"); assert!( v["snippet"].as_str().unwrap().chars().count() <= 400, diff --git a/crates/cm-api/src/mission_memory.rs b/crates/cm-api/src/mission_memory.rs index 774e20d..af90235 100644 --- a/crates/cm-api/src/mission_memory.rs +++ b/crates/cm-api/src/mission_memory.rs @@ -27,6 +27,15 @@ use uuid::Uuid; /// tried" without becoming the prompt. pub const RECALL_K: usize = 3; +/// How many BM25 candidates the reranker sees. Wider than `RECALL_K` so a +/// relevant verdict that keyword overlap ranked fourth can still make the +/// brief; narrow enough that one call stays one call. +pub const CANDIDATES: usize = 8; + +/// Below this a candidate is dropped even if fewer than `RECALL_K` remain: +/// a brief that carries an irrelevant verdict is worse than a shorter one. +pub const RELEVANT_AT: f64 = 0.3; + /// The heading the recalled lines go under. Named here because the scorer and /// the prompt-order tests read it back. pub const SECTION_HEADING: &str = "# What past missions on this repository learned"; @@ -112,18 +121,88 @@ fn remember_in( /// The past verdicts most relevant to `query` (the phase's task text). /// Empty when the repo has no brain yet, which is every repo's first mission. -pub fn recall(repo_id: Uuid, query: &str) -> Vec { - recall_in(&cm_runtime::brain::brain_dir(), repo_id, query) +/// +/// Two stages when a decision model is configured: BM25 proposes +/// `CANDIDATES`, one Noul per candidate — "is this past verdict relevant +/// to the task?" — reorders them and drops the ones below `RELEVANT_AT`. +/// Keyword overlap is what BM25 measures, and a verdict about MICROVM.md +/// shares words with every task that mentions a file; the rerank is the +/// vendor's own pattern and costs one ~200 ms call. Without a key the +/// BM25 order stands, as before. +pub async fn recall(repo_id: Uuid, query: &str) -> Vec { + let candidates = recall_in(&cm_runtime::brain::brain_dir(), repo_id, query, CANDIDATES); + match cm_decide::jev::Jev::from_env() { + Some(jev) if candidates.len() > 1 => rerank(&jev, query, candidates).await, + _ => candidates.into_iter().take(RECALL_K).collect(), + } } -fn recall_in(dir: &Path, repo_id: Uuid, query: &str) -> Vec { +async fn rerank(jev: &cm_decide::jev::Jev, query: &str, candidates: Vec) -> Vec { + use cm_decide::{Answer, Decider as _, Question}; + let questions: std::collections::BTreeMap = candidates + .iter() + .enumerate() + .map(|(i, c)| { + ( + format!("c{i}"), + Question::noul(format!( + "This earlier judge verdict is relevant to the task and would help an \ + agent doing it: {c}" + )), + ) + }) + .collect(); + let decided = tokio::time::timeout( + std::time::Duration::from_secs(10), + jev.decide(query, &questions), + ) + .await; + let decision = match decided { + Ok(Ok(d)) => d, + Ok(Err(e)) => { + eprintln!("mission_memory: rerank failed ({e}); keeping the BM25 order"); + return candidates.into_iter().take(RECALL_K).collect(); + } + Err(_) => { + eprintln!("mission_memory: rerank timed out; keeping the BM25 order"); + return candidates.into_iter().take(RECALL_K).collect(); + } + }; + let scored: Vec<(String, f64)> = candidates + .into_iter() + .enumerate() + .map(|(i, c)| { + let p = match decision.answers.get(&format!("c{i}")) { + Some(Answer::Noul { noul }) => *noul, + _ => 0.0, + }; + (c, p) + }) + .collect(); + let kept: Vec = cm_decide::patterns::rerank(scored) + .into_iter() + .filter(|(_, p)| *p >= RELEVANT_AT) + .take(RECALL_K) + .map(|(c, _)| c) + .collect(); + eprintln!( + "mission_memory: reranked {} candidate(s) with {}, kept {} ({} ms)", + questions.len(), + decision.model, + kept.len(), + decision.latency.as_millis() + ); + kept +} + +fn recall_in(dir: &Path, repo_id: Uuid, query: &str, k: usize) -> Vec { let path = brain_path(dir, repo_id); if !path.exists() { return Vec::new(); } match ClawBrain::open_or_create(&path, &format!("repo_{repo_id}")) { Ok(brain) => brain - .recall(query, RECALL_K) + .recall(query, k) .into_iter() // `remember` stores "role: text"; the role is ours and not a lesson. .map(|m| m.strip_prefix("judge: ").map(str::to_string).unwrap_or(m)) @@ -227,13 +306,13 @@ mod tests { None, ); remember_in(&dir, repo, Uuid::now_v7(), "benchmark", "a baseline is recorded", &v); - let got = recall_in(&dir, repo, "record a performance baseline for the hot path"); + let got = recall_in(&dir, repo, "record a performance baseline for the hot path", RECALL_K); assert_eq!(got.len(), 1, "{got:?}"); assert!(got[0].starts_with("MET — benchmark phase"), "{}", got[0]); assert!(!got[0].starts_with("judge: ")); // A repo with no history recalls nothing and creates no file. let other = Uuid::now_v7(); - assert!(recall_in(&dir, other, "anything").is_empty()); + assert!(recall_in(&dir, other, "anything", RECALL_K).is_empty()); assert!(!brain_path(&dir, other).exists()); let _ = std::fs::remove_dir_all(&dir); } diff --git a/crates/cm-api/src/mission_orchestrator.rs b/crates/cm-api/src/mission_orchestrator.rs index 2ae248e..cbd431f 100644 --- a/crates/cm-api/src/mission_orchestrator.rs +++ b/crates/cm-api/src/mission_orchestrator.rs @@ -129,7 +129,12 @@ pub async fn on_launch( // rightly refuses the branch. See `write_manifest`. if mission.template_kind == crate::continuous_research::TEMPLATE_KIND { let date = crate::continuous_research::today(); - match crate::continuous_research::write_manifest(&path, &harvested, &date) { + // Tag and score each paper before the agents see the list; + // an untriaged manifest (no key) is the old, empty-tags one. + let topics = crate::continuous_research::topics_for(&mission.config); + let triage = + crate::continuous_research::triage_papers(&harvested, &topics).await; + match crate::continuous_research::write_manifest(&path, &harvested, &date, &triage) { Ok(at) => eprintln!( "mission_orchestrator: wrote {} paper(s) to {}", harvested.len(), diff --git a/crates/cm-api/src/phase_runner.rs b/crates/cm-api/src/phase_runner.rs index 5311c7a..7410e7d 100644 --- a/crates/cm-api/src/phase_runner.rs +++ b/crates/cm-api/src/phase_runner.rs @@ -1254,9 +1254,9 @@ async fn launch_phase( // Appended to the task rather than the identity so all three executors // carry it: the task text is the one thing they share. let task = match p.repo_id { - Some(repo) => match crate::mission_memory::section(&crate::mission_memory::recall( - repo, &task, - )) { + Some(repo) => match crate::mission_memory::section( + &crate::mission_memory::recall(repo, &task).await, + ) { Some(memory) => { eprintln!( "phase_runner: phase {phase_id} brief carries project memory for repo {repo}"