clawmates/crates/cm-api/tests/corpus_vault.rs

//! Indexing the vault must be idempotent, or a continuous mission cannot tell
//! new work from work it already did.
//!
//! These run against a real Postgres via cm-testkit. The vault fixture is
//! shaped from the actual `valhalla-vault`: 416 notes, only 145 with
//! frontmatter, none carrying arxiv/doi/url, plus repo-sync notes whose
//! frontmatter churns on every sync.

use cm_api::corpus;
use uuid::Uuid;

async fn workspace(pool: &sqlx::PgPool) -> Uuid {
    let ws = Uuid::now_v7();
    sqlx::query("INSERT INTO workspaces (id, name, plan) VALUES ($1,'t','team')")
        .bind(ws)
        .execute(pool)
        .await
        .unwrap();
    ws
}

/// A real mission row. `corpus_items.mission_id` has a foreign key, which is
/// deliberate: attribution to a mission that does not exist is not
/// attribution. The first version of the test below used a bare UUID and was
/// correctly rejected.
async fn mission(pool: &sqlx::PgPool, ws: Uuid) -> Uuid {
    let id = Uuid::now_v7();
    sqlx::query(
        "INSERT INTO missions (id, workspace_id, title, template_kind, schedule, status, config)
         VALUES ($1,$2,'library','research_only','{}'::jsonb,'running','{}'::jsonb)",
    )
    .bind(id)
    .bind(ws)
    .execute(pool)
    .await
    .unwrap();
    id
}

fn seed_vault(root: &std::path::Path) {
    std::fs::create_dir_all(root.join("50 APESS 2026/Lectures")).unwrap();
    std::fs::create_dir_all(root.join("Repos")).unwrap();
    std::fs::create_dir_all(root.join("Daily")).unwrap();
    // Course note: has frontmatter, but `source:` is a local path.
    std::fs::write(
        root.join("50 APESS 2026/Lectures/agentic.md"),
        "---\nsource: \"/Users/quantum/Downloads/Material/x.pdf\"\ntype: lecture\n---\n# Agentic Design\n\nbody\n",
    )
    .unwrap();
    // Repo-sync note: frontmatter churns, prose does not.
    std::fs::write(
        root.join("Repos/zeroclaw.md"),
        "---\nnode: tank\nupdated: 2026-08-01\nsize_kb: 12\n---\n# ZeroClaw\n\nmirror\n",
    )
    .unwrap();
    // Plain note: no frontmatter at all — the majority case.
    std::fs::write(root.join("Daily/2026-08-01.md"), "# Monday\n\nnotes\n").unwrap();
}

#[tokio::test]
async fn indexing_an_unchanged_vault_is_a_no_op() {
    let pool = cm_testkit::test_pool().await;
    let ws = workspace(&pool).await;
    let tmp = tempfile::tempdir().unwrap();
    seed_vault(tmp.path());

    let first = corpus::index_vault(&pool, ws, "vault", tmp.path())
        .await
        .unwrap();
    assert_eq!(first.scanned, 3);
    assert_eq!(first.inserted, 3);
    assert_eq!(first.unchanged, 0);

    // The decisive assertion: a second pass over an untouched vault must add
    // and change nothing. Without this, every run looks like new work.
    let second = corpus::index_vault(&pool, ws, "vault", tmp.path())
        .await
        .unwrap();
    assert_eq!(second.scanned, 3);
    assert_eq!(second.inserted, 0, "re-index must not insert");
    assert_eq!(second.updated, 0, "re-index must not update");
    assert_eq!(second.unchanged, 3);
}

#[tokio::test]
async fn a_repo_sync_touching_only_frontmatter_is_not_an_edit() {
    let pool = cm_testkit::test_pool().await;
    let ws = workspace(&pool).await;
    let tmp = tempfile::tempdir().unwrap();
    seed_vault(tmp.path());
    corpus::index_vault(&pool, ws, "vault", tmp.path())
        .await
        .unwrap();

    // Exactly what a repo sync does: bump `updated`/`size_kb`, prose untouched.
    std::fs::write(
        tmp.path().join("Repos/zeroclaw.md"),
        "---\nnode: tank\nupdated: 2026-08-03\nsize_kb: 14\n---\n# ZeroClaw\n\nmirror\n",
    )
    .unwrap();

    let stats = corpus::index_vault(&pool, ws, "vault", tmp.path())
        .await
        .unwrap();
    assert_eq!(stats.updated, 0, "frontmatter churn is not an edit");
    assert_eq!(stats.unchanged, 3);

    // A real prose edit must still be seen.
    std::fs::write(
        tmp.path().join("Repos/zeroclaw.md"),
        "---\nnode: tank\nupdated: 2026-08-03\n---\n# ZeroClaw\n\nREWRITTEN\n",
    )
    .unwrap();
    let stats = corpus::index_vault(&pool, ws, "vault", tmp.path())
        .await
        .unwrap();
    assert_eq!(stats.updated, 1, "a genuine edit must be visible");
}

#[tokio::test]
async fn a_hand_edited_note_survives_a_rebuild() {
    let pool = cm_testkit::test_pool().await;
    let ws = workspace(&pool).await;
    let tmp = tempfile::tempdir().unwrap();
    seed_vault(tmp.path());
    corpus::index_vault(&pool, ws, "vault", tmp.path())
        .await
        .unwrap();

    // The vault is authoritative: a human renames a note by hand.
    std::fs::remove_file(tmp.path().join("Daily/2026-08-01.md")).unwrap();
    std::fs::write(tmp.path().join("Daily/renamed.md"), "# Monday\n\nnotes\n").unwrap();

    let stats = corpus::index_vault(&pool, ws, "vault", tmp.path())
        .await
        .unwrap();
    assert_eq!(stats.scanned, 3);
    assert_eq!(stats.inserted, 1, "the renamed note is indexed under its new path");
    // The stale row is left alone rather than deleted — the index is derived
    // and rebuildable, and losing coverage history is worse than a stale row.
    assert!(corpus::seen(&pool, ws, "vault", "note:Daily/renamed.md")
        .await
        .unwrap());
}

#[tokio::test]
async fn unseen_filters_candidates_in_one_round_trip() {
    let pool = cm_testkit::test_pool().await;
    let ws = workspace(&pool).await;

    corpus::record(
        &pool, ws, "vault", "source", "arxiv:2401.11111",
        Some("Known"), None, None, "h", None,
    )
    .await
    .unwrap();

    let candidates = vec![
        "arxiv:2401.11111".to_string(), // already read
        "arxiv:2401.22222".to_string(),
        "doi:10.1000/new".to_string(),
    ];
    let fresh = corpus::unseen(&pool, ws, "vault", &candidates).await.unwrap();
    assert_eq!(fresh, vec!["arxiv:2401.22222", "doi:10.1000/new"]);

    assert!(corpus::seen(&pool, ws, "vault", "arxiv:2401.11111").await.unwrap());
    assert!(!corpus::seen(&pool, ws, "vault", "arxiv:2401.22222").await.unwrap());
    // A different corpus must not inherit another's seen-set.
    assert!(!corpus::seen(&pool, ws, "other", "arxiv:2401.11111").await.unwrap());
}

/// The first mission to find a source keeps the credit, so "did THIS run
/// contribute anything new" stays answerable across repeated runs.
#[tokio::test]
async fn re_recording_a_source_does_not_reassign_it() {
    let pool = cm_testkit::test_pool().await;
    let ws = workspace(&pool).await;

    let inserted = corpus::record(
        &pool, ws, "vault", "source", "arxiv:2401.33333",
        Some("Paper"), None, None, "h1", None,
    )
    .await
    .unwrap();
    assert!(inserted, "first sighting is an insert");

    let inserted_again = corpus::record(
        &pool, ws, "vault", "source", "arxiv:2401.33333",
        Some("Paper"), None, None, "h2", None,
    )
    .await
    .unwrap();
    assert!(!inserted_again, "a second sighting is not new work");
}
/// Idempotence against the real vault rather than a fixture.
///
/// Ignored by default because it needs a checkout: run with
/// `VAULT=/path/to/valhalla-vault cargo test -p cm-api --test corpus_vault \
///  index_the_real_vault -- --ignored --nocapture`.
///
/// Measured 2026-08-03 on the live vault:
///   PASS1 { scanned: 416, inserted: 416, updated: 0, unchanged: 0 }
///   PASS2 { scanned: 416, inserted: 0,   updated: 0, unchanged: 416 }
#[tokio::test]
#[ignore]
async fn index_the_real_vault() {
    let pool = cm_testkit::test_pool().await;
    let ws = uuid::Uuid::now_v7();
    sqlx::query("INSERT INTO workspaces (id, name, plan) VALUES ($1,'t','team')")
        .bind(ws).execute(&pool).await.unwrap();
    let root = std::path::Path::new(&std::env::var("VAULT").unwrap()).to_path_buf();
    let a = cm_api::corpus::index_vault(&pool, ws, "valhalla-vault", &root).await.unwrap();
    println!("PASS1 {a:?}");
    let b = cm_api::corpus::index_vault(&pool, ws, "valhalla-vault", &root).await.unwrap();
    println!("PASS2 {b:?}");
    assert_eq!(b.inserted, 0);
    assert_eq!(b.updated, 0);
    assert_eq!(b.unchanged, a.scanned);
}

/// Live arXiv check. Ignored by default (needs network); run with
/// `cargo test -p cm-api --test corpus_vault live_arxiv -- --ignored --nocapture`.
///
/// Guards the one failure that hides: if arXiv's feed format drifts, parsing
/// returns zero papers, which looks exactly like "no new papers this week".
#[tokio::test]
#[ignore]
async fn live_arxiv_search_and_fetch() {
    let papers = cm_api::papers::search("all:agentic topologies", 3)
        .await
        .expect("arxiv search");
    println!("found {} papers", papers.len());
    assert!(!papers.is_empty(), "arXiv returned nothing — format drift?");

    for p in &papers {
        println!("  {} | {}", p.source_id(), &p.title[..p.title.len().min(60)]);
        assert!(!p.arxiv_id.is_empty());
        assert!(!p.title.is_empty());
        assert!(!p.arxiv_id.contains('v'), "version must be stripped: {}", p.arxiv_id);
    }

    let pdf = cm_api::papers::fetch_pdf(&papers[0]).await.expect("fetch pdf");
    println!("pdf bytes: {}", pdf.len());
    assert!(pdf.starts_with(b"%PDF"));
    assert!(pdf.len() > 10_000, "suspiciously small pdf: {}", pdf.len());
}

/// A rerun must not be able to claim credit for work an earlier run did.
///
/// This is the verification predicate for a continuous mission: "did THIS run
/// contribute anything new". If a rerun could re-record an existing source
/// under its own mission id, every run would report success forever — the
/// failure that killed the 0030-0044 generation of this feature.
#[tokio::test]
async fn a_rerun_cannot_claim_an_earlier_missions_work() {
    let pool = cm_testkit::test_pool().await;
    let ws = workspace(&pool).await;
    let first_mission = mission(&pool, ws).await;
    let second_mission = mission(&pool, ws).await;

    corpus::record(
        &pool, ws, "lib", "source", "arxiv:2401.55555",
        Some("Paper"), None, None, "h1", Some(first_mission),
    )
    .await
    .unwrap();

    // The second mission sees the same paper and re-records it.
    corpus::record(
        &pool, ws, "lib", "source", "arxiv:2401.55555",
        Some("Paper"), None, None, "h2", Some(second_mission),
    )
    .await
    .unwrap();

    assert_eq!(
        corpus::contributed(&pool, ws, "lib", first_mission).await.unwrap(),
        1,
        "the finder keeps the credit"
    );
    assert_eq!(
        corpus::contributed(&pool, ws, "lib", second_mission).await.unwrap(),
        0,
        "a rerun that found nothing new must report zero, not one"
    );
}