clawmates/crates/cm-api/tests/harvest_run.rs

//! A second run must not re-download what the first run already shelved.

use cm_api::{corpus, harvest, papers::Paper};
use std::sync::Arc;
use uuid::Uuid;

async fn workspace(pool: &sqlx::PgPool) -> Uuid {
    let ws = Uuid::now_v7();
    sqlx::query("INSERT INTO workspaces (id, name, plan) VALUES ($1,'t','team')")
        .bind(ws)
        .execute(pool)
        .await
        .unwrap();
    ws
}

fn paper(id: &str) -> Paper {
    Paper {
        arxiv_id: id.into(),
        title: format!("Paper {id}"),
        authors: vec!["Ada Lovelace".into()],
        summary: "A summary.".into(),
        published: "2026-01-15T10:00:00Z".into(),
        // Deliberately unreachable: if the skip works, this is never fetched.
        pdf_url: "http://127.0.0.1:1/never.pdf".into(),
    }
}

/// The load-bearing behaviour. Every candidate is already on the checkmark
/// list, and every `pdf_url` points at a closed port — so if the run tries to
/// download anything at all, it fails loudly instead of passing quietly.
#[tokio::test]
async fn papers_we_already_hold_are_never_downloaded_again() {
    let pool = cm_testkit::test_pool().await;
    let ws = workspace(&pool).await;
    let tmp = tempfile::tempdir().unwrap();
    let blobs: Arc<dyn cm_files::BlobStore> =
        Arc::new(cm_files::LocalBlobStore::new(tmp.path().join("blobs")));
    let vault = tmp.path().join("vault");

    let candidates = vec![paper("2401.11111"), paper("2401.22222")];
    for p in &candidates {
        corpus::record(
            &pool, ws, "lib", "source", &p.source_id(),
            Some(&p.title), None, None, "h", None,
        )
        .await
        .unwrap();
    }

    let lib = harvest::Library {
        pool: &pool, blobs: &blobs, workspace_id: ws,
        corpus_id: "lib", vault_root: &vault,
    };
    let h = harvest::shelve(&lib, &candidates, None).await.unwrap();

    assert_eq!(h.candidates, 2);
    assert_eq!(h.already_had, 2, "both were already held");
    assert!(h.shelved.is_empty());
    assert!(
        h.failed.is_empty(),
        "nothing should have been fetched at all, but got: {:?}",
        h.failed
    );
    assert!(h.healthy(), "a fully-known batch is a healthy quiet week");
    assert!(!h.added_anything(), "and it added nothing");
    assert!(!vault.exists(), "no notes written for papers we already had");
}

/// A paper that cannot be downloaded must NOT be checked off — otherwise one
/// transient network failure means that paper is never retried.
#[tokio::test]
async fn a_failed_download_leaves_the_paper_unseen_for_next_time() {
    let pool = cm_testkit::test_pool().await;
    let ws = workspace(&pool).await;
    let tmp = tempfile::tempdir().unwrap();
    let blobs: Arc<dyn cm_files::BlobStore> =
        Arc::new(cm_files::LocalBlobStore::new(tmp.path().join("blobs")));
    let vault = tmp.path().join("vault");

    let candidates = vec![paper("2401.33333")];
    let lib = harvest::Library {
        pool: &pool, blobs: &blobs, workspace_id: ws,
        corpus_id: "lib", vault_root: &vault,
    };
    let h = harvest::shelve(&lib, &candidates, None).await.unwrap();

    assert_eq!(h.already_had, 0);
    assert!(h.shelved.is_empty());
    assert_eq!(h.failed.len(), 1, "the unreachable fetch must be reported");
    assert!(!h.healthy(), "a failed fetch is not a quiet week");

    assert!(
        !corpus::seen(&pool, ws, "lib", "arxiv:2401.33333")
            .await
            .unwrap(),
        "a paper we failed to get must stay unseen so a later run retries it"
    );
}

/// Live end-to-end: search arXiv, shelve genuinely new papers, then confirm a
/// second identical run adds nothing. Ignored by default (network + Postgres):
/// `cargo test -p cm-api --test harvest_run live_ -- --ignored --nocapture`
#[tokio::test]
#[ignore]
async fn live_end_to_end_run_then_rerun() {
    let pool = cm_testkit::test_pool().await;
    let ws = workspace(&pool).await;
    let tmp = tempfile::tempdir().unwrap();
    let blobs: Arc<dyn cm_files::BlobStore> =
        Arc::new(cm_files::LocalBlobStore::new(tmp.path().join("blobs")));
    let vault = tmp.path().join("vault");

    let lib = harvest::Library {
        pool: &pool, blobs: &blobs, workspace_id: ws,
        corpus_id: "lib", vault_root: &vault,
    };
    let first = harvest::run(&lib, "all:agentic AND all:topology", 3, None)
        .await
        .unwrap();
    println!("RUN1 {}", first.summary());
    for n in &first.notes_written {
        println!("  note: {n}");
    }
    assert!(first.healthy(), "failures: {:?}", first.failed);
    assert!(first.added_anything(), "first run should find something new");

    // Every note must be readable back through the corpus parser, or the
    // catalogue cannot rebuild the checkmark list.
    for rel in &first.notes_written {
        let text = std::fs::read_to_string(vault.join(rel)).unwrap();
        let parsed = corpus::parse_note(rel, &text);
        assert!(
            parsed
                .declared_source_id
                .as_deref()
                .is_some_and(|s| s.starts_with("arxiv:")),
            "note {rel} lost its identity"
        );
    }

    let second = harvest::run(&lib, "all:agentic AND all:topology", 3, None)
        .await
        .unwrap();
    println!("RUN2 {}", second.summary());
    assert!(second.healthy());
    assert!(
        !second.added_anything(),
        "a rerun must add nothing — got {:?}",
        second.shelved
    );
    assert_eq!(second.already_had, second.candidates);
}

/// THE REAL RUN. Clones the live vault, harvests our current topics, pushes a
/// branch. Ignored by default — needs network, Postgres and GITEA_TOKEN:
/// `GITEA_TOKEN=… VAULT_URL=… cargo test -p cm-api --test harvest_run \
///  live_library_run -- --ignored --nocapture`
#[tokio::test]
#[ignore]
async fn live_library_run() {
    let pool = cm_testkit::test_pool().await;
    let ws = workspace(&pool).await;
    let tmp = tempfile::tempdir().unwrap();
    let blobs: Arc<dyn cm_files::BlobStore> =
        Arc::new(cm_files::LocalBlobStore::new(tmp.path().join("shelf")));
    let url = std::env::var("VAULT_URL").unwrap();

    let topics = cm_api::library::default_topics();
    for t in &topics {
        println!("topic: {t}");
    }

    let run = cm_api::library::run_to_vault(
        &pool, &blobs, ws, "valhalla-vault", &url,
        tmp.path(), &topics, 2, None,
    )
    .await
    .unwrap();

    println!("\nRESULT {}", run.harvest.summary());
    println!("branch: {}  pushed: {}", run.branch, run.pushed);
    if let Some(e) = &run.error {
        println!("error: {e}");
    }
    for n in &run.harvest.notes_written {
        println!("  note: {n}");
    }
    for (sid, why) in &run.harvest.failed {
        println!("  FAILED {sid}: {why}");
    }

    // Every shelved paper must have its PDF really on the shelf.
    for sid in &run.harvest.shelved {
        let id = sid.trim_start_matches("arxiv:");
        let key = format!("papers/arxiv/{id}.pdf");
        let bytes = blobs.get(&key).await.expect("pdf on the shelf");
        assert!(bytes.starts_with(b"%PDF"), "{key} is not a PDF");
        println!("  shelf: {key} ({} bytes)", bytes.len());
    }
    assert!(run.harvest.healthy(), "failures: {:?}", run.harvest.failed);
}