feat(library): clone the vault, harvest our topics, push the catalogue
Completes the loop: the notes now land in the real vault. Topics come from what the project is actually working on — papers/dynamic-agentic- topologies.md (topology search, ADAS/Darwin-Godel/SwarmAgentic) plus the two problems this week ran into, verifying what an agent did and giving a long-running agent memory of what it covered. Never pushes to main. The vault is a live Obsidian vault a human edits and syncs; pushing to main races that sync and can lose hand-written work. Every run lands on its own branch for a human to merge, the same rule the mission delivery path was validated 20/20 under. PDFs are NOT committed. A few hundred papers is gigabytes and would make the vault painful to clone and slow to open, so they stay on the blob store shelf and the note carries the key. My own test caught me repeating this week's branch-collision bug: I named branches from the HEAD of a UUIDv7, which is a 48-bit timestamp, so two runs in the same millisecond produce the identical name — exactly what hit mission 019fc42b. Fixed by taking the tail. The test now loops 100 ids instead of sampling two (a one-shot check passes by luck whenever the millisecond ticks between calls) and additionally asserts the head-based scheme DOES collide, so it cannot rot into a no-op. Live against the real vault: 10 candidates, 1 already held, 9 shelved, 0 failed branch clawmates/library-019fc82292e8, pushed 9 notes verified on the forge, 9 PDFs verified %PDF on the shelf (the "1 already held" is cross-topic dedupe inside a single run) Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
Omar Sobh
co-authored by
Claude Opus 5
Claude Opus 5
parent
30eaa50c50
commit
09c6496725
@@ -0,0 +1,263 @@
|
||||
//! A library run end to end: clone the vault, harvest, push the catalogue.
|
||||
//!
|
||||
//! [`harvest`](crate::harvest) writes catalogue notes into a directory. This
|
||||
//! puts that directory somewhere real: a checkout of the vault repo, with the
|
||||
//! new notes committed and pushed.
|
||||
//!
|
||||
//! # Never `main`
|
||||
//!
|
||||
//! The vault is a live Obsidian vault that a human edits and syncs. Pushing
|
||||
//! straight to `main` races that sync and can lose hand-written work. Every
|
||||
//! run lands on its own branch, exactly like the mission delivery path that
|
||||
//! was validated 20/20 earlier — a human merges when they have looked at it.
|
||||
//!
|
||||
//! # The PDFs do not go here
|
||||
//!
|
||||
//! Only notes are committed. PDFs are shelved in the blob store, because a
|
||||
//! few hundred papers is gigabytes and a vault that size is painful to clone
|
||||
//! and slow to open. The note carries the blob key, so the catalogue always
|
||||
//! knows where its shelf is.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::harvest::{self, Harvest, Library};
|
||||
use crate::mission_workspace;
|
||||
|
||||
/// What a full run produced, including whether it reached the forge.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LibraryRun {
|
||||
pub harvest: Harvest,
|
||||
pub branch: String,
|
||||
/// `true` only when the push was observed to succeed. A run that shelved
|
||||
/// papers but could not push still has the PDFs and the checkmarks; the
|
||||
/// notes are simply not on the forge yet.
|
||||
pub pushed: bool,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
fn git_identity() -> [(&'static str, String); 4] {
|
||||
let (name, email) = crate::mission_delivery::commit_identity();
|
||||
[
|
||||
("GIT_AUTHOR_NAME", name.clone()),
|
||||
("GIT_AUTHOR_EMAIL", email.clone()),
|
||||
("GIT_COMMITTER_NAME", name),
|
||||
("GIT_COMMITTER_EMAIL", email),
|
||||
]
|
||||
}
|
||||
|
||||
async fn git(repo: &Path, args: &[&str]) -> Result<String, String> {
|
||||
let mut cmd = tokio::process::Command::new("git");
|
||||
cmd.arg("-C").arg(repo);
|
||||
cmd.args(["-c", &format!("safe.directory={}", repo.display())]);
|
||||
cmd.args(args);
|
||||
for (k, v) in git_identity() {
|
||||
cmd.env(k, v);
|
||||
}
|
||||
let out = cmd.output().await.map_err(|e| format!("spawn git: {e}"))?;
|
||||
if !out.status.success() {
|
||||
return Err(format!(
|
||||
"git {} → {}: {}",
|
||||
args.first().copied().unwrap_or("?"),
|
||||
out.status,
|
||||
mission_workspace::redact_token(&String::from_utf8_lossy(&out.stderr))
|
||||
.chars()
|
||||
.take(300)
|
||||
.collect::<String>()
|
||||
));
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
|
||||
}
|
||||
|
||||
/// Clone the vault fresh into `work_root`, returning the checkout path.
|
||||
///
|
||||
/// Fresh each run rather than reused: a library run is short, the vault is
|
||||
/// small (measured 6.9 MB / 416 notes), and a stale checkout is how the
|
||||
/// mission path lost work three times this week.
|
||||
pub async fn clone_vault(clone_url: &str, work_root: &Path) -> Result<PathBuf, String> {
|
||||
let path = work_root.join("vault");
|
||||
if path.exists() {
|
||||
tokio::fs::remove_dir_all(&path)
|
||||
.await
|
||||
.map_err(|e| format!("clear {}: {e}", path.display()))?;
|
||||
}
|
||||
tokio::fs::create_dir_all(work_root)
|
||||
.await
|
||||
.map_err(|e| format!("mkdir {}: {e}", work_root.display()))?;
|
||||
|
||||
let auth = mission_workspace::with_ambient_auth(clone_url);
|
||||
let out = tokio::process::Command::new("git")
|
||||
.args(["clone", "--quiet", "--depth", "1", &auth])
|
||||
.arg(&path)
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| format!("spawn git clone: {e}"))?;
|
||||
if !out.status.success() {
|
||||
return Err(format!(
|
||||
"clone vault → {}: {}",
|
||||
out.status,
|
||||
mission_workspace::redact_token(&String::from_utf8_lossy(&out.stderr))
|
||||
.chars()
|
||||
.take(300)
|
||||
.collect::<String>()
|
||||
));
|
||||
}
|
||||
// The token must not stay in .git/config: the checkout may be handed to a
|
||||
// container later, and a credential in a file an agent can read is a
|
||||
// credential an agent has.
|
||||
mission_workspace::scrub_remote_credentials(&path, &auth);
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
/// One complete library run.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn run_to_vault(
|
||||
pool: &sqlx::PgPool,
|
||||
blobs: &Arc<dyn cm_files::BlobStore>,
|
||||
workspace_id: Uuid,
|
||||
corpus_id: &str,
|
||||
clone_url: &str,
|
||||
work_root: &Path,
|
||||
queries: &[String],
|
||||
per_query: usize,
|
||||
mission_id: Option<Uuid>,
|
||||
) -> Result<LibraryRun, String> {
|
||||
let vault = clone_vault(clone_url, work_root).await?;
|
||||
let lib = Library {
|
||||
pool,
|
||||
blobs,
|
||||
workspace_id,
|
||||
corpus_id,
|
||||
vault_root: &vault,
|
||||
};
|
||||
|
||||
// Accumulate across queries. Topics overlap — "agentic topology" and
|
||||
// "multi-agent orchestration" return some of the same papers — and the
|
||||
// checkmark list dedupes across them within a single run as well as
|
||||
// between runs, because each shelve records before the next query starts.
|
||||
let mut total = Harvest::default();
|
||||
for q in queries {
|
||||
let h = harvest::run(&lib, q, per_query, mission_id).await?;
|
||||
total.candidates += h.candidates;
|
||||
total.already_had += h.already_had;
|
||||
total.shelved.extend(h.shelved);
|
||||
total.failed.extend(h.failed);
|
||||
total.notes_written.extend(h.notes_written);
|
||||
}
|
||||
|
||||
// The TAIL of the uuid, not the head. UUIDv7 leads with a 48-bit
|
||||
// timestamp, so two ids minted in the same millisecond share their first
|
||||
// 12 hex characters exactly — the branch-name collision that hit mission
|
||||
// 019fc42b earlier. The tail is the random part.
|
||||
let branch = format!("clawmates/library-{}", branch_suffix(Uuid::now_v7()));
|
||||
|
||||
if total.notes_written.is_empty() {
|
||||
// A quiet run is a success with nothing to push. Creating an empty
|
||||
// branch every week would be noise.
|
||||
return Ok(LibraryRun {
|
||||
harvest: total,
|
||||
branch,
|
||||
pushed: false,
|
||||
error: None,
|
||||
});
|
||||
}
|
||||
|
||||
git(&vault, &["checkout", "-B", &branch]).await?;
|
||||
git(&vault, &["add", "--", "60 Papers"]).await?;
|
||||
let message = format!(
|
||||
"library: {} new paper(s)\n\n{}\n\nShelved in the blob store; this commit is the catalogue.",
|
||||
total.shelved.len(),
|
||||
total
|
||||
.shelved
|
||||
.iter()
|
||||
.map(|s| format!("- {s}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
);
|
||||
git(&vault, &["commit", "--no-verify", "-m", &message]).await?;
|
||||
|
||||
let auth = mission_workspace::with_ambient_auth(clone_url);
|
||||
let refspec = format!("HEAD:refs/heads/{branch}");
|
||||
match git(&vault, &["push", &auth, &refspec]).await {
|
||||
Ok(_) => Ok(LibraryRun {
|
||||
harvest: total,
|
||||
branch,
|
||||
pushed: true,
|
||||
error: None,
|
||||
}),
|
||||
Err(e) => Ok(LibraryRun {
|
||||
harvest: total,
|
||||
branch,
|
||||
pushed: false,
|
||||
error: Some(e),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Distinct-per-run branch suffix. See the note at the call site: taking the
|
||||
/// head of a UUIDv7 yields the timestamp, which collides.
|
||||
fn branch_suffix(id: Uuid) -> String {
|
||||
let s = id.simple().to_string();
|
||||
s[s.len() - 12..].to_string()
|
||||
}
|
||||
|
||||
/// The topics this library currently tracks.
|
||||
///
|
||||
/// Drawn from what the project is actually working on: `papers/dynamic-
|
||||
/// agentic-topologies.md` (topology search and evolution, citing ADAS,
|
||||
/// Darwin-Gödel and SwarmAgentic), plus the problems this week's work ran
|
||||
/// into — verifying what an agent actually did, and giving a long-running
|
||||
/// agent memory of what it has already covered.
|
||||
pub fn default_topics() -> Vec<String> {
|
||||
[
|
||||
"all:\"agentic topology\" OR all:\"multi-agent topology\"",
|
||||
"all:\"multi-agent orchestration\" AND all:LLM",
|
||||
"all:\"agent memory\" AND all:\"long-term\"",
|
||||
"all:\"LLM agent\" AND all:verification",
|
||||
"all:\"prompt injection\" AND all:agent",
|
||||
]
|
||||
.iter()
|
||||
.map(|s| s.to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn topics_are_non_empty_and_arxiv_shaped() {
|
||||
let topics = default_topics();
|
||||
assert!(topics.len() >= 3);
|
||||
for t in &topics {
|
||||
assert!(t.contains("all:"), "arXiv field prefix missing in {t:?}");
|
||||
assert!(!t.trim().is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
/// Two runs in the same millisecond must not collide.
|
||||
///
|
||||
/// This caught a real repeat of the mission-path bug (019fc42b): UUIDv7
|
||||
/// leads with a 48-bit timestamp, so the FIRST 12 hex characters of two
|
||||
/// ids minted together are identical. Taking the tail fixes it. Looping
|
||||
/// rather than sampling twice, because a one-shot check passes by luck
|
||||
/// whenever the millisecond happens to tick between the two calls.
|
||||
#[test]
|
||||
fn every_run_gets_a_distinct_branch() {
|
||||
let ids: Vec<String> = (0..100).map(|_| branch_suffix(Uuid::now_v7())).collect();
|
||||
let unique: std::collections::HashSet<&String> = ids.iter().collect();
|
||||
assert_eq!(unique.len(), ids.len(), "branch suffixes collided: {ids:?}");
|
||||
|
||||
// And the head-based scheme really does collide, so this test has teeth.
|
||||
let heads: Vec<String> = (0..100)
|
||||
.map(|_| Uuid::now_v7().simple().to_string()[..12].to_string())
|
||||
.collect();
|
||||
let head_unique: std::collections::HashSet<&String> = heads.iter().collect();
|
||||
assert!(
|
||||
head_unique.len() < heads.len(),
|
||||
"the head of a UUIDv7 was expected to collide but did not"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user