clawmates/crates/cm-api/src/papers.rs

//! Finding papers, shelving them, and cataloguing them.
//!
//! The library has three parts and it matters which is which:
//!
//! - **arXiv** is where papers are *found*.
//! - **The blob store** is the *shelf* — the PDF itself lives there.
//! - **The vault** is the *card catalogue* — a markdown note per paper, with
//!   the metadata and a pointer to the shelf.
//!
//! Plus [`crate::corpus`], which is the list of checkmarks: it is what stops
//! the same paper being fetched twice across weekly runs. That list is the
//! reason this can be a *continuous* job rather than one that redoes itself
//! forever — the failure that killed the previous attempt at this (migrations
//! 0030-0044, dropped in 0053).
//!
//! # The contract that ties it together
//!
//! Every note this module writes carries `source_id: arxiv:NNNN.NNNNN` in its
//! frontmatter. `corpus::parse_note` reads exactly that key, so re-indexing
//! the vault re-derives the checkmark list from the notes themselves. The
//! catalogue is authoritative; the index is rebuildable from it. If the
//! database were lost, a re-index of the vault would restore what we have.

use serde::{Deserialize, Serialize};

/// One paper as arXiv describes it.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Paper {
    /// Bare arXiv id, e.g. `2401.12345` — no version suffix.
    pub arxiv_id: String,
    pub title: String,
    pub authors: Vec<String>,
    pub summary: String,
    pub published: String,
    pub pdf_url: String,
}

impl Paper {
    /// The checkmark key. Version suffixes are stripped upstream so `v1` and
    /// `v2` of the same paper are one entry, not two.
    pub fn source_id(&self) -> String {
        format!("arxiv:{}", self.arxiv_id)
    }

    /// Where the PDF is shelved in the blob store.
    pub fn blob_key(&self) -> String {
        format!("papers/arxiv/{}.pdf", self.arxiv_id)
    }

    /// Where the catalogue note goes in the vault.
    ///
    /// Under a dedicated folder so the library never collides with the
    /// hand-written parts of the vault (`30 Resources`, `40 Projects`, and so
    /// on). A human should always be able to tell which notes a machine wrote.
    pub fn note_path(&self) -> String {
        format!("60 Papers/arxiv-{}.md", self.arxiv_id)
    }
}

/// Strip an arXiv version suffix: `2401.12345v3` -> `2401.12345`.
///
/// Without this a weekly job re-downloads a paper every time the authors post
/// a revision, and the checkmark list quietly fills with near-duplicates.
pub fn normalize_arxiv_id(raw: &str) -> String {
    let id = raw.rsplit('/').next().unwrap_or(raw);
    match id.find('v') {
        // Only a trailing `vN` counts; the `v` in a word must not truncate.
        Some(i) if id[i + 1..].chars().all(|c| c.is_ascii_digit()) && i + 1 < id.len() => {
            id[..i].to_string()
        }
        _ => id.to_string(),
    }
}

/// Parse arXiv's Atom feed.
///
/// Hand-rolled rather than pulling an XML crate: the feed is a fixed, simple
/// shape and this reads five fields from it. If arXiv's format ever drifts,
/// `entries_are_parsed_from_a_real_feed` fails loudly rather than silently
/// returning zero papers — which is the failure mode that matters, because a
/// search returning nothing looks exactly like "no new papers this week".
pub fn parse_atom(xml: &str) -> Vec<Paper> {
    let mut out = Vec::new();
    for chunk in xml.split("<entry>").skip(1) {
        let entry = chunk.split("</entry>").next().unwrap_or(chunk);
        let field = |tag: &str| -> Option<String> {
            let open = format!("<{tag}>");
            let close = format!("</{tag}>");
            let start = entry.find(&open)? + open.len();
            let end = entry[start..].find(&close)? + start;
            Some(unescape(entry[start..end].trim()))
        };

        let Some(raw_id) = field("id") else { continue };
        let arxiv_id = normalize_arxiv_id(&raw_id);
        if arxiv_id.is_empty() {
            continue;
        }
        let Some(title) = field("title") else { continue };

        let authors = entry
            .split("<author>")
            .skip(1)
            .filter_map(|a| {
                let start = a.find("<name>")? + 6;
                let end = a[start..].find("</name>")? + start;
                Some(unescape(a[start..end].trim()))
            })
            .collect();

        // The PDF link is an attribute, not an element.
        let pdf_url = entry
            .split("<link")
            .find(|l| l.contains("title=\"pdf\""))
            .and_then(|l| {
                let start = l.find("href=\"")? + 6;
                let end = l[start..].find('"')? + start;
                Some(l[start..end].to_string())
            })
            .unwrap_or_else(|| format!("https://arxiv.org/pdf/{arxiv_id}"));

        out.push(Paper {
            title: title.split_whitespace().collect::<Vec<_>>().join(" "),
            summary: field("summary")
                .unwrap_or_default()
                .split_whitespace()
                .collect::<Vec<_>>()
                .join(" "),
            published: field("published").unwrap_or_default(),
            authors,
            pdf_url,
            arxiv_id,
        });
    }
    out
}

fn unescape(s: &str) -> String {
    s.replace("&amp;", "&")
        .replace("&lt;", "<")
        .replace("&gt;", ">")
        .replace("&quot;", "\"")
        .replace("&#39;", "'")
}

/// Turn an operator topic into an arXiv `search_query`.
///
/// A bare topic is NOT a search. Passed through unfielded, arXiv matched
/// essentially nothing and `sortBy=submittedDate` then returned the newest
/// submissions across the whole archive — so a run for "speculative decoding"
/// shelved Galois extensions, a quantum black hole microstate, and blazar dark
/// matter in IceCube. Measured against the live API:
///
/// ```text
///   speculative decoding          -> pixel-space diffusion, simplicial actions
///   all:"speculative decoding"    -> S2-MoE self-speculative decoding, DARTree
/// ```
///
/// So the phrase is quoted into `all:` (title, abstract, authors, comments) and
/// constrained to `cat:cs.*` — this library exists to serve software projects,
/// and without the category bound the archive's physics and maths volume
/// dominates every recency-sorted result.
///
/// A topic that already looks fielded (`cat:`, `ti:`, `abs:`, `all:`) is passed
/// through untouched, so an operator who knows arXiv's syntax keeps full control.
pub fn arxiv_query(topic: &str) -> String {
    let t = topic.trim();
    const FIELDED: &[&str] = &["all:", "ti:", "abs:", "au:", "cat:", "co:", "jr:"];
    // Only a topic that STARTS with a field prefix is treated as hand-written
    // arXiv syntax. Also accepting anything containing " AND "/" OR " was the
    // first version, and a test caught it immediately: `agent" OR cat:hep-th`
    // passed straight through, so a topic string could escape the phrase and
    // rewrite the category bound. A natural-language topic may legitimately
    // contain the word "and" too.
    if FIELDED.iter().any(|p| t.starts_with(p)) {
        return t.to_string();
    }
    // Quotes make it a phrase; without them "vector index pruning" matches any
    // paper containing all three words anywhere, which is most of cs.
    let escaped = t.replace('"', "");
    format!("all:\"{escaped}\" AND cat:cs.*")
}

/// The looser form of a topic: every term required, but not adjacent.
///
/// A quoted phrase is precise and brittle. "hybrid retrieval BM25 dense" is a
/// perfectly good topic and appears verbatim in no paper on arXiv — measured, 0
/// hits — while requiring the same four terms anywhere returns exactly the
/// hybrid-retrieval evaluations the topic was asking for. Used only when the
/// phrase finds nothing, so an exact match still wins when one exists.
pub fn arxiv_query_broad(topic: &str) -> String {
    let terms: Vec<String> = topic
        .split_whitespace()
        .map(|w| w.trim_matches(|c: char| !c.is_alphanumeric() && c != '-'))
        .filter(|w| !w.is_empty())
        .map(|w| format!("all:{w}"))
        .collect();
    if terms.is_empty() {
        return arxiv_query(topic);
    }
    format!("{} AND cat:cs.*", terms.join(" AND "))
}

/// Search arXiv. `max_results` is capped to keep one run bounded.
pub async fn search(query: &str, max_results: usize) -> Result<Vec<Paper>, String> {
    let found = search_with(&arxiv_query(query), max_results).await?;
    if !found.is_empty() {
        return Ok(found);
    }
    // The phrase matched nothing. Before reporting a quiet day — which the whole
    // pipeline treats as a real and legitimate outcome — try the same terms
    // unquoted. A topic the operator writes as prose often is not a literal
    // phrase in any title, and silently harvesting zero because of punctuation
    // would be indistinguishable from a genuinely quiet field.
    let broad = arxiv_query_broad(query);
    if broad == arxiv_query(query) {
        return Ok(found);
    }
    eprintln!("papers: no exact phrase match for {query:?} — retrying as {broad}");
    search_with(&broad, max_results).await
}

async fn search_with(search_query: &str, max_results: usize) -> Result<Vec<Paper>, String> {
    let max = max_results.clamp(1, 50);
    let url = format!(
        "https://export.arxiv.org/api/query?search_query={}&start=0&max_results={max}\
         &sortBy=submittedDate&sortOrder=descending",
        urlencoding(search_query)
    );
    let body = reqwest::Client::new()
        .get(&url)
        .header("User-Agent", "clawmates-papers/0.1 (research library)")
        .timeout(std::time::Duration::from_secs(60))
        .send()
        .await
        .map_err(|e| format!("arxiv query: {e}"))?
        .text()
        .await
        .map_err(|e| format!("arxiv body: {e}"))?;
    Ok(parse_atom(&body))
}

/// Download the PDF. Returns the bytes; the caller decides where to shelve it.
pub async fn fetch_pdf(paper: &Paper) -> Result<Vec<u8>, String> {
    let bytes = reqwest::Client::new()
        .get(&paper.pdf_url)
        .header("User-Agent", "clawmates-papers/0.1 (research library)")
        .timeout(std::time::Duration::from_secs(180))
        .send()
        .await
        .map_err(|e| format!("fetch pdf {}: {e}", paper.arxiv_id))?
        .bytes()
        .await
        .map_err(|e| format!("read pdf {}: {e}", paper.arxiv_id))?;

    // A PDF starts with `%PDF`. arXiv serves an HTML holding page when a PDF
    // is still rendering, and shelving that would leave a file that looks
    // present and is unreadable.
    if !bytes.starts_with(b"%PDF") {
        return Err(format!(
            "{} did not return a PDF ({} bytes, starts {:?})",
            paper.pdf_url,
            bytes.len(),
            String::from_utf8_lossy(&bytes[..bytes.len().min(16)])
        ));
    }
    Ok(bytes.to_vec())
}

/// The catalogue note for a shelved paper.
///
/// `source_id` in the frontmatter is the load-bearing part — it is what
/// `corpus::parse_note` reads to rebuild the checkmark list from the vault.
pub fn catalogue_note(paper: &Paper, blob_key: &str) -> String {
    let authors = if paper.authors.is_empty() {
        "unknown".to_string()
    } else {
        paper.authors.join(", ")
    };
    format!(
        "---\n\
         source_id: arxiv:{id}\n\
         arxiv: {id}\n\
         title: \"{title}\"\n\
         authors: \"{authors}\"\n\
         published: {published}\n\
         pdf: {blob_key}\n\
         url: https://arxiv.org/abs/{id}\n\
         added: {added}\n\
         tags: [paper, arxiv]\n\
         ---\n\
         \n\
         # {title}\n\
         \n\
         **Authors:** {authors}  \n\
         **arXiv:** [{id}](https://arxiv.org/abs/{id})  \n\
         **PDF:** `{blob_key}`\n\
         \n\
         ## Abstract\n\
         \n\
         {summary}\n\
         \n\
         ## Notes\n\
         \n\
         _Catalogued automatically. Add your own notes below._\n",
        id = paper.arxiv_id,
        title = paper.title.replace('"', "'"),
        authors = authors,
        published = paper.published,
        blob_key = blob_key,
        added = paper.published,
        summary = paper.summary,
    )
}

fn urlencoding(s: &str) -> String {
    s.bytes()
        .map(|b| match b {
            b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
                (b as char).to_string()
            }
            b' ' => "+".to_string(),
            _ => format!("%{b:02X}"),
        })
        .collect()
}

#[cfg(test)]
mod tests {

    /// A bare topic must become a PHRASE search bound to cs — unfielded, arXiv
    /// matched nothing and recency-sort returned the whole archive, so a run
    /// for "speculative decoding" shelved blazar dark matter in IceCube.
    #[test]
    fn a_bare_topic_becomes_a_fielded_phrase_query() {
        let q = arxiv_query("speculative decoding");
        assert_eq!(q, "all:\"speculative decoding\" AND cat:cs.*");
        assert!(q.contains('"'), "unquoted, the words match separately");
        assert!(q.contains("cat:cs.*"), "without a category bound physics wins");
    }

    /// An operator who writes arXiv syntax keeps control — wrapping their query
    /// in another `all:"..."` would search for the literal text of their query.
    #[test]
    fn an_already_fielded_topic_is_left_alone() {
        for q in [
            "cat:cs.IR AND all:\"dense retrieval\"",
            "ti:\"world model\"",
            "abs:hnsw OR abs:\"vector index\"",
        ] {
            assert_eq!(arxiv_query(q), q, "{q} must pass through untouched");
        }
    }

    /// The broad form requires every term but not adjacency. Measured: the
    /// phrase "hybrid retrieval BM25 dense" has 0 hits on arXiv; the same four
    /// terms unquoted return the hybrid-retrieval evaluations that were asked
    /// for. Without the fallback that topic silently harvests nothing, which is
    /// indistinguishable from a genuinely quiet day.
    #[test]
    fn the_broad_form_requires_every_term_without_adjacency() {
        let q = arxiv_query_broad("hybrid retrieval BM25 dense");
        assert_eq!(
            q,
            "all:hybrid AND all:retrieval AND all:BM25 AND all:dense AND cat:cs.*"
        );
        assert!(!q.contains('"'), "the broad form must not be a phrase: {q}");
        assert!(q.contains("cat:cs.*"), "still category-bound: {q}");
    }

    /// Punctuation must not leak into a term and must not empty the query.
    #[test]
    fn the_broad_form_strips_punctuation_and_never_empties() {
        assert_eq!(
            arxiv_query_broad("retrieval-augmented, generation!"),
            "all:retrieval-augmented AND all:generation AND cat:cs.*",
            "hyphens are part of a term; trailing punctuation is not"
        );
        // Nothing usable left: fall back to the phrase form rather than
        // emitting a bare `cat:cs.*`, which would match all of computer science.
        let q = arxiv_query_broad("!!!");
        assert!(q.contains("all:"), "must never degrade to a bare category: {q}");
    }

    /// Quotes in a topic would terminate the phrase early and corrupt the query.
    #[test]
    fn quotes_in_a_topic_cannot_break_out_of_the_phrase() {
        let q = arxiv_query("agent\" OR cat:hep-th");
        assert_eq!(q.matches('"').count(), 2, "exactly one balanced phrase: {q}");
        assert!(q.ends_with("cat:cs.*"), "{q}");
    }

    use super::*;

    /// A revision must not read as a new paper.
    #[test]
    fn version_suffixes_are_stripped() {
        assert_eq!(normalize_arxiv_id("http://arxiv.org/abs/2401.12345v3"), "2401.12345");
        assert_eq!(normalize_arxiv_id("2401.12345v1"), "2401.12345");
        assert_eq!(normalize_arxiv_id("2401.12345"), "2401.12345");
        // Old-style ids contain letters and a slash.
        assert_eq!(normalize_arxiv_id("http://arxiv.org/abs/cs/0701001"), "0701001");
        // A trailing `v` with no digits is part of the id, not a version.
        assert_eq!(normalize_arxiv_id("2401.1234v"), "2401.1234v");
    }

    /// Parsed against the real shape of arXiv's Atom feed. If this fails the
    /// format drifted — which otherwise shows up as "no new papers", which is
    /// indistinguishable from a quiet week.
    #[test]
    fn entries_are_parsed_from_a_real_feed() {
        let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
  <entry>
    <id>http://arxiv.org/abs/2401.12345v2</id>
    <published>2026-01-15T10:00:00Z</published>
    <title>Attention Is All You Need   Again</title>
    <summary>  We show that
    attention still works.  </summary>
    <author><name>Ada Lovelace</name></author>
    <author><name>Alan Turing</name></author>
    <link href="http://arxiv.org/abs/2401.12345v2" rel="alternate" type="text/html"/>
    <link title="pdf" href="http://arxiv.org/pdf/2401.12345v2" rel="related" type="application/pdf"/>
  </entry>
</feed>"#;
        let papers = parse_atom(xml);
        assert_eq!(papers.len(), 1);
        let p = &papers[0];
        assert_eq!(p.arxiv_id, "2401.12345", "version stripped");
        assert_eq!(p.title, "Attention Is All You Need Again", "whitespace collapsed");
        assert_eq!(p.summary, "We show that attention still works.");
        assert_eq!(p.authors, vec!["Ada Lovelace", "Alan Turing"]);
        assert_eq!(p.pdf_url, "http://arxiv.org/pdf/2401.12345v2");
        assert_eq!(p.source_id(), "arxiv:2401.12345");
        assert_eq!(p.blob_key(), "papers/arxiv/2401.12345.pdf");
        assert_eq!(p.note_path(), "60 Papers/arxiv-2401.12345.md");
    }

    #[test]
    fn an_empty_feed_yields_no_papers_rather_than_panicking() {
        assert!(parse_atom("<feed></feed>").is_empty());
        assert!(parse_atom("").is_empty());
    }

    #[test]
    fn xml_entities_are_unescaped() {
        let xml = r#"<feed><entry><id>http://arxiv.org/abs/1v1</id>
        <title>Cats &amp; Dogs &lt;3</title><summary>a &quot;quote&quot;</summary>
        </entry></feed>"#;
        let p = &parse_atom(xml)[0];
        assert_eq!(p.title, "Cats & Dogs <3");
        assert_eq!(p.summary, "a \"quote\"");
    }

    /// The note must carry the identity `corpus::parse_note` reads, or the
    /// catalogue cannot rebuild the checkmark list and the library forgets
    /// itself the moment the database is lost.
    #[test]
    fn a_catalogue_note_round_trips_through_the_corpus_parser() {
        let paper = Paper {
            arxiv_id: "2401.12345".into(),
            title: "A \"Quoted\" Title".into(),
            authors: vec!["Ada Lovelace".into()],
            summary: "Summary text.".into(),
            published: "2026-01-15T10:00:00Z".into(),
            pdf_url: "http://arxiv.org/pdf/2401.12345".into(),
        };
        let note = catalogue_note(&paper, &paper.blob_key());

        let parsed = crate::corpus::parse_note(&paper.note_path(), &note);
        assert_eq!(
            parsed.declared_source_id.as_deref(),
            Some("arxiv:2401.12345"),
            "the corpus parser must recover the identity from the note"
        );
        assert_eq!(parsed.title.as_deref(), Some("A 'Quoted' Title"));
        assert!(note.contains("papers/arxiv/2401.12345.pdf"), "note points at the shelf");
    }

    #[test]
    fn queries_are_url_encoded() {
        assert_eq!(urlencoding("all:agent topologies"), "all%3Aagent+topologies");
    }
}