//! A second run must not re-download what the first run already shelved. use cm_api::{corpus, harvest, papers::Paper}; use std::sync::Arc; use uuid::Uuid; async fn workspace(pool: &sqlx::PgPool) -> Uuid { let ws = Uuid::now_v7(); sqlx::query("INSERT INTO workspaces (id, name, plan) VALUES ($1,'t','team')") .bind(ws) .execute(pool) .await .unwrap(); ws } fn paper(id: &str) -> Paper { Paper { arxiv_id: id.into(), title: format!("Paper {id}"), authors: vec!["Ada Lovelace".into()], summary: "A summary.".into(), published: "2026-01-15T10:00:00Z".into(), // Deliberately unreachable: if the skip works, this is never fetched. pdf_url: "http://127.0.0.1:1/never.pdf".into(), } } /// The load-bearing behaviour. Every candidate is already on the checkmark /// list, and every `pdf_url` points at a closed port — so if the run tries to /// download anything at all, it fails loudly instead of passing quietly. #[tokio::test] async fn papers_we_already_hold_are_never_downloaded_again() { let pool = cm_testkit::test_pool().await; let ws = workspace(&pool).await; let tmp = tempfile::tempdir().unwrap(); let blobs: Arc = Arc::new(cm_files::LocalBlobStore::new(tmp.path().join("blobs"))); let vault = tmp.path().join("vault"); let candidates = vec![paper("2401.11111"), paper("2401.22222")]; for p in &candidates { corpus::record( &pool, ws, "lib", "source", &p.source_id(), Some(&p.title), None, None, "h", None, ) .await .unwrap(); } let lib = harvest::Library { pool: &pool, blobs: &blobs, workspace_id: ws, corpus_id: "lib", vault_root: &vault, }; let h = harvest::shelve(&lib, &candidates, None).await.unwrap(); assert_eq!(h.candidates, 2); assert_eq!(h.already_had, 2, "both were already held"); assert!(h.shelved.is_empty()); assert!( h.failed.is_empty(), "nothing should have been fetched at all, but got: {:?}", h.failed ); assert!(h.healthy(), "a fully-known batch is a healthy quiet week"); assert!(!h.added_anything(), "and it added nothing"); assert!(!vault.exists(), "no notes written for papers we already had"); } /// A paper that cannot be downloaded must NOT be checked off — otherwise one /// transient network failure means that paper is never retried. #[tokio::test] async fn a_failed_download_leaves_the_paper_unseen_for_next_time() { let pool = cm_testkit::test_pool().await; let ws = workspace(&pool).await; let tmp = tempfile::tempdir().unwrap(); let blobs: Arc = Arc::new(cm_files::LocalBlobStore::new(tmp.path().join("blobs"))); let vault = tmp.path().join("vault"); let candidates = vec![paper("2401.33333")]; let lib = harvest::Library { pool: &pool, blobs: &blobs, workspace_id: ws, corpus_id: "lib", vault_root: &vault, }; let h = harvest::shelve(&lib, &candidates, None).await.unwrap(); assert_eq!(h.already_had, 0); assert!(h.shelved.is_empty()); assert_eq!(h.failed.len(), 1, "the unreachable fetch must be reported"); assert!(!h.healthy(), "a failed fetch is not a quiet week"); assert!( !corpus::seen(&pool, ws, "lib", "arxiv:2401.33333") .await .unwrap(), "a paper we failed to get must stay unseen so a later run retries it" ); } /// Live end-to-end: search arXiv, shelve genuinely new papers, then confirm a /// second identical run adds nothing. Ignored by default (network + Postgres): /// `cargo test -p cm-api --test harvest_run live_ -- --ignored --nocapture` #[tokio::test] #[ignore] async fn live_end_to_end_run_then_rerun() { let pool = cm_testkit::test_pool().await; let ws = workspace(&pool).await; let tmp = tempfile::tempdir().unwrap(); let blobs: Arc = Arc::new(cm_files::LocalBlobStore::new(tmp.path().join("blobs"))); let vault = tmp.path().join("vault"); let lib = harvest::Library { pool: &pool, blobs: &blobs, workspace_id: ws, corpus_id: "lib", vault_root: &vault, }; let first = harvest::run(&lib, "all:agentic AND all:topology", 3, None) .await .unwrap(); println!("RUN1 {}", first.summary()); for n in &first.notes_written { println!(" note: {n}"); } assert!(first.healthy(), "failures: {:?}", first.failed); assert!(first.added_anything(), "first run should find something new"); // Every note must be readable back through the corpus parser, or the // catalogue cannot rebuild the checkmark list. for rel in &first.notes_written { let text = std::fs::read_to_string(vault.join(rel)).unwrap(); let parsed = corpus::parse_note(rel, &text); assert!( parsed .declared_source_id .as_deref() .is_some_and(|s| s.starts_with("arxiv:")), "note {rel} lost its identity" ); } let second = harvest::run(&lib, "all:agentic AND all:topology", 3, None) .await .unwrap(); println!("RUN2 {}", second.summary()); assert!(second.healthy()); assert!( !second.added_anything(), "a rerun must add nothing — got {:?}", second.shelved ); assert_eq!(second.already_had, second.candidates); }