//! A library run end to end: clone the vault, harvest, push the catalogue. //! //! [`harvest`](crate::harvest) writes catalogue notes into a directory. This //! puts that directory somewhere real: a checkout of the vault repo, with the //! new notes committed and pushed. //! //! # Never `main` //! //! The vault is a live Obsidian vault that a human edits and syncs. Pushing //! straight to `main` races that sync and can lose hand-written work. Every //! run lands on its own branch, exactly like the mission delivery path that //! was validated 20/20 earlier — a human merges when they have looked at it. //! //! # The PDFs do not go here //! //! Only notes are committed. PDFs are shelved in the blob store, because a //! few hundred papers is gigabytes and a vault that size is painful to clone //! and slow to open. The note carries the blob key, so the catalogue always //! knows where its shelf is. use std::path::{Path, PathBuf}; use std::sync::Arc; use uuid::Uuid; use crate::harvest::{self, Harvest, Library}; use crate::mission_workspace; /// What a full run produced, including whether it reached the forge. #[derive(Debug, Clone)] pub struct LibraryRun { pub harvest: Harvest, pub branch: String, /// `true` only when the push was observed to succeed. A run that shelved /// papers but could not push still has the PDFs and the checkmarks; the /// notes are simply not on the forge yet. pub pushed: bool, pub error: Option, } fn git_identity() -> [(&'static str, String); 4] { let (name, email) = crate::mission_delivery::commit_identity(); [ ("GIT_AUTHOR_NAME", name.clone()), ("GIT_AUTHOR_EMAIL", email.clone()), ("GIT_COMMITTER_NAME", name), ("GIT_COMMITTER_EMAIL", email), ] } async fn git(repo: &Path, args: &[&str]) -> Result { let mut cmd = tokio::process::Command::new("git"); cmd.arg("-C").arg(repo); cmd.args(["-c", &format!("safe.directory={}", repo.display())]); cmd.args(args); for (k, v) in git_identity() { cmd.env(k, v); } let out = cmd.output().await.map_err(|e| format!("spawn git: {e}"))?; if !out.status.success() { return Err(format!( "git {} → {}: {}", args.first().copied().unwrap_or("?"), out.status, mission_workspace::redact_token(&String::from_utf8_lossy(&out.stderr)) .chars() .take(300) .collect::() )); } Ok(String::from_utf8_lossy(&out.stdout).into_owned()) } /// Clone the vault fresh into `work_root`, returning the checkout path. /// /// Fresh each run rather than reused: a library run is short, the vault is /// small (measured 6.9 MB / 416 notes), and a stale checkout is how the /// mission path lost work three times this week. pub async fn clone_vault(clone_url: &str, work_root: &Path) -> Result { let path = work_root.join("vault"); if path.exists() { tokio::fs::remove_dir_all(&path) .await .map_err(|e| format!("clear {}: {e}", path.display()))?; } tokio::fs::create_dir_all(work_root) .await .map_err(|e| format!("mkdir {}: {e}", work_root.display()))?; let auth = mission_workspace::with_ambient_auth(clone_url); let out = tokio::process::Command::new("git") .args(["clone", "--quiet", "--depth", "1", &auth]) .arg(&path) .output() .await .map_err(|e| format!("spawn git clone: {e}"))?; if !out.status.success() { return Err(format!( "clone vault → {}: {}", out.status, mission_workspace::redact_token(&String::from_utf8_lossy(&out.stderr)) .chars() .take(300) .collect::() )); } // The token must not stay in .git/config: the checkout may be handed to a // container later, and a credential in a file an agent can read is a // credential an agent has. mission_workspace::scrub_remote_credentials(&path, &auth); Ok(path) } /// One complete library run. #[allow(clippy::too_many_arguments)] pub async fn run_to_vault( pool: &sqlx::PgPool, blobs: &Arc, workspace_id: Uuid, corpus_id: &str, clone_url: &str, work_root: &Path, queries: &[String], per_query: usize, mission_id: Option, ) -> Result { let vault = clone_vault(clone_url, work_root).await?; let lib = Library { pool, blobs, workspace_id, corpus_id, vault_root: &vault, }; // Accumulate across queries. Topics overlap — "agentic topology" and // "multi-agent orchestration" return some of the same papers — and the // checkmark list dedupes across them within a single run as well as // between runs, because each shelve records before the next query starts. let mut total = Harvest::default(); for q in queries { let h = harvest::run(&lib, q, per_query, mission_id).await?; total.candidates += h.candidates; total.already_had += h.already_had; total.shelved.extend(h.shelved); total.failed.extend(h.failed); total.notes_written.extend(h.notes_written); } // The TAIL of the uuid, not the head. UUIDv7 leads with a 48-bit // timestamp, so two ids minted in the same millisecond share their first // 12 hex characters exactly — the branch-name collision that hit mission // 019fc42b earlier. The tail is the random part. let branch = format!("clawmates/library-{}", branch_suffix(Uuid::now_v7())); if total.notes_written.is_empty() { // A quiet run is a success with nothing to push. Creating an empty // branch every week would be noise. return Ok(LibraryRun { harvest: total, branch, pushed: false, error: None, }); } git(&vault, &["checkout", "-B", &branch]).await?; git(&vault, &["add", "--", "60 Papers"]).await?; let message = format!( "library: {} new paper(s)\n\n{}\n\nShelved in the blob store; this commit is the catalogue.", total.shelved.len(), total .shelved .iter() .map(|s| format!("- {s}")) .collect::>() .join("\n") ); git(&vault, &["commit", "--no-verify", "-m", &message]).await?; let auth = mission_workspace::with_ambient_auth(clone_url); let refspec = format!("HEAD:refs/heads/{branch}"); match git(&vault, &["push", &auth, &refspec]).await { Ok(_) => Ok(LibraryRun { harvest: total, branch, pushed: true, error: None, }), Err(e) => Ok(LibraryRun { harvest: total, branch, pushed: false, error: Some(e), }), } } /// Distinct-per-run branch suffix. See the note at the call site: taking the /// head of a UUIDv7 yields the timestamp, which collides. fn branch_suffix(id: Uuid) -> String { let s = id.simple().to_string(); s[s.len() - 12..].to_string() } /// The topics this library currently tracks. /// /// Drawn from what the project is actually working on: `papers/dynamic- /// agentic-topologies.md` (topology search and evolution, citing ADAS, /// Darwin-Gödel and SwarmAgentic), plus the problems this week's work ran /// into — verifying what an agent actually did, and giving a long-running /// agent memory of what it has already covered. pub fn default_topics() -> Vec { [ "all:\"agentic topology\" OR all:\"multi-agent topology\"", "all:\"multi-agent orchestration\" AND all:LLM", "all:\"agent memory\" AND all:\"long-term\"", "all:\"LLM agent\" AND all:verification", "all:\"prompt injection\" AND all:agent", ] .iter() .map(|s| s.to_string()) .collect() } #[cfg(test)] mod tests { use super::*; #[test] fn topics_are_non_empty_and_arxiv_shaped() { let topics = default_topics(); assert!(topics.len() >= 3); for t in &topics { assert!(t.contains("all:"), "arXiv field prefix missing in {t:?}"); assert!(!t.trim().is_empty()); } } /// Two runs in the same millisecond must not collide. /// /// This caught a real repeat of the mission-path bug (019fc42b): UUIDv7 /// leads with a 48-bit timestamp, so the FIRST 12 hex characters of two /// ids minted together are identical. Taking the tail fixes it. Looping /// rather than sampling twice, because a one-shot check passes by luck /// whenever the millisecond happens to tick between the two calls. #[test] fn every_run_gets_a_distinct_branch() { let ids: Vec = (0..100).map(|_| branch_suffix(Uuid::now_v7())).collect(); let unique: std::collections::HashSet<&String> = ids.iter().collect(); assert_eq!(unique.len(), ids.len(), "branch suffixes collided: {ids:?}"); // And the head-based scheme really does collide, so this test has teeth. let heads: Vec = (0..100) .map(|_| Uuid::now_v7().simple().to_string()[..12].to_string()) .collect(); let head_unique: std::collections::HashSet<&String> = heads.iter().collect(); assert!( head_unique.len() < heads.len(), "the head of a UUIDv7 was expected to collide but did not" ); } }