Compare commits
32
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0d9498ec6e | ||
|
|
15e7608e4a | ||
|
|
5d98fcf44a | ||
|
|
4ff4e6f7ee | ||
|
|
deb60be98d | ||
|
|
758b2dbd96 | ||
|
|
37fac288d2 | ||
|
|
5232175c88 | ||
|
|
ac47dcbe94 | ||
|
|
ad89ef94cd | ||
|
|
9de2cf34e4 | ||
|
|
3124fd3c8f | ||
|
|
cf076bd8ea | ||
|
|
107f0dbced | ||
|
|
09c6496725 | ||
|
|
30eaa50c50 | ||
|
|
e4a395b72e | ||
|
|
6e5ccc25a6 | ||
|
|
2380c2cb0b | ||
|
|
ec85f6c8da | ||
|
|
bb34ef1b7e | ||
|
|
f7e336ff5f | ||
|
|
9bdc3cd89b | ||
|
|
ddab8e35f5 | ||
|
|
1a979f500f | ||
|
|
25d9805806 | ||
|
|
08b2adae23 | ||
|
|
5b53705c97 | ||
|
|
8bad869248 | ||
|
|
e2871c4361 | ||
|
|
3ea288dbb5 | ||
|
|
ca1fd46e08 |
Generated
+1
@@ -946,6 +946,7 @@ dependencies = [
|
||||
"cm-config",
|
||||
"cm-db",
|
||||
"cm-domain",
|
||||
"cm-files",
|
||||
"cm-llm",
|
||||
"cm-orchestrator",
|
||||
"cm-runtime",
|
||||
|
||||
@@ -266,7 +266,7 @@ async fn run() -> Result<(), String> {
|
||||
terminals,
|
||||
providers: provider_registry,
|
||||
},
|
||||
blob,
|
||||
blob.clone(),
|
||||
);
|
||||
// Durable §15 path: expires overdue approvals and resumes decided runs
|
||||
// even if the deciding request's process died mid-flight.
|
||||
@@ -388,6 +388,7 @@ async fn run() -> Result<(), String> {
|
||||
.with_broker(PathBuf::from(&config.broker.socket_path))
|
||||
.with_oauth(config.oauth.clone())
|
||||
.with_billing(config.billing.clone())
|
||||
.with_blobs(blob.clone())
|
||||
.with_file_root(
|
||||
(config.storage.backend == cm_config::StorageBackend::Local)
|
||||
.then(|| PathBuf::from(&config.storage.data_dir)),
|
||||
@@ -402,6 +403,11 @@ async fn run() -> Result<(), String> {
|
||||
.await
|
||||
.map_err(|e| format!("bind {} failed: {e}", config.listen_addr))?;
|
||||
println!("clawmates-server listening on {}", config.listen_addr);
|
||||
// Say plainly whether the mission runtime carries the tools we invoke in
|
||||
// it. The image on the host silently fell behind its Dockerfile once, and
|
||||
// every consequence — an ungated test suite, a scan that scanned nothing —
|
||||
// looked like a normal result rather than a broken deployment.
|
||||
cm_api::runtime_preflight::report_at_boot();
|
||||
// Graceful shutdown: on SIGTERM/Ctrl-C, stop accepting, finish in-flight
|
||||
// requests, then DRAIN the sandbox managers so no container is left running.
|
||||
let shutdown = async move {
|
||||
|
||||
@@ -32,6 +32,7 @@ cm-brain = { path = "../cm-brain" }
|
||||
cm-config = { path = "../cm-config" }
|
||||
cm-db = { path = "../cm-db" }
|
||||
cm-domain = { path = "../cm-domain" }
|
||||
cm-files = { path = "../cm-files" }
|
||||
cm-llm = { path = "../cm-llm" }
|
||||
cm-orchestrator = { path = "../cm-orchestrator", features = ["provider"] }
|
||||
cm-runtime = { path = "../cm-runtime" }
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
//! Merging a delivered branch into the base, when that is provably safe.
|
||||
//!
|
||||
//! Every mission type delivers to a branch and never to `main`. For most that
|
||||
//! is where it should stop — a human reads the code and merges. But some
|
||||
//! missions only ever *add* files in a folder they own: a paper catalogue, a
|
||||
//! benchmark record. Those branches carry no judgement call, and leaving them
|
||||
//! to pile up unmerged means the work is done but not actually in the vault.
|
||||
//!
|
||||
//! # Additive-only is a property, not a preference
|
||||
//!
|
||||
//! The gate is not "is this mission type trusted". It is measured from the
|
||||
//! diff: if the branch modifies or deletes anything that already existed, it
|
||||
//! does not qualify, whatever its template says. A research harvest that
|
||||
//! somehow rewrote a hand-written note would be refused by the same check
|
||||
//! that lets its new notes through.
|
||||
//!
|
||||
//! Three conditions, all required:
|
||||
//!
|
||||
//! 1. the mission type declares [`MergePolicy::AdditiveOnly`]
|
||||
//! 2. verification passed — a run that did not prove its work does not merge
|
||||
//! 3. the diff against the base contains only additions
|
||||
//!
|
||||
//! Anything else lands as a branch for a human, which is the existing
|
||||
//! behaviour and the safe default.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
/// What a mission type is allowed to do with its own branch.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum MergePolicy {
|
||||
/// Always leave the branch for a human. Correct for anything that touches
|
||||
/// code: `refactor`, `research_and_code`, security patches.
|
||||
Never,
|
||||
/// Merge automatically when the diff is provably additive and the run
|
||||
/// verified. Correct for catalogues and recorded measurements.
|
||||
AdditiveOnly,
|
||||
}
|
||||
|
||||
impl MergePolicy {
|
||||
/// Parse a template's `merge_policy`. Unknown values fall back to `Never`
|
||||
/// and say so: a typo must not silently grant auto-merge.
|
||||
pub fn parse(raw: Option<&str>) -> MergePolicy {
|
||||
match raw.map(str::trim) {
|
||||
Some("additive_only") => MergePolicy::AdditiveOnly,
|
||||
Some("never") | None => MergePolicy::Never,
|
||||
Some(other) => {
|
||||
eprintln!(
|
||||
"auto_merge: unknown merge_policy {other:?} — refusing to auto-merge"
|
||||
);
|
||||
MergePolicy::Never
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a branch was or was not merged. The reason is always recorded: a
|
||||
/// branch that silently did not merge is indistinguishable from one that was
|
||||
/// never delivered.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MergeOutcome {
|
||||
pub merged: bool,
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
impl MergeOutcome {
|
||||
fn refused(reason: impl Into<String>) -> MergeOutcome {
|
||||
MergeOutcome {
|
||||
merged: false,
|
||||
reason: reason.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Classify a `git diff --name-status` body.
|
||||
///
|
||||
/// Returns the offending entries, empty when every change is an addition.
|
||||
/// Split out so the rule is testable without a repository.
|
||||
pub fn non_additive_changes(name_status: &str) -> Vec<String> {
|
||||
name_status
|
||||
.lines()
|
||||
.filter(|l| !l.trim().is_empty())
|
||||
.filter(|l| {
|
||||
// Status is the first field: A/M/D/R###/C###.
|
||||
!matches!(l.chars().next(), Some('A'))
|
||||
})
|
||||
.map(|l| l.trim().to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn git(repo: &Path, args: &[&str]) -> Result<String, String> {
|
||||
let out = tokio::process::Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(repo)
|
||||
.args(["-c", &format!("safe.directory={}", repo.display())])
|
||||
.args(args)
|
||||
.env("GIT_AUTHOR_NAME", crate::mission_delivery::commit_identity().0)
|
||||
.env("GIT_AUTHOR_EMAIL", crate::mission_delivery::commit_identity().1)
|
||||
.env(
|
||||
"GIT_COMMITTER_NAME",
|
||||
crate::mission_delivery::commit_identity().0,
|
||||
)
|
||||
.env(
|
||||
"GIT_COMMITTER_EMAIL",
|
||||
crate::mission_delivery::commit_identity().1,
|
||||
)
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| format!("spawn git: {e}"))?;
|
||||
if !out.status.success() {
|
||||
return Err(format!(
|
||||
"git {} → {}: {}",
|
||||
args.first().copied().unwrap_or("?"),
|
||||
out.status,
|
||||
crate::mission_workspace::redact_token(&String::from_utf8_lossy(&out.stderr))
|
||||
.chars()
|
||||
.take(300)
|
||||
.collect::<String>()
|
||||
));
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
|
||||
}
|
||||
|
||||
/// Merge `branch` into `base` and push, if all three conditions hold.
|
||||
///
|
||||
/// Never returns `Err` for a refusal — a refusal is a normal outcome with a
|
||||
/// reason. `Err` is reserved for the merge itself going wrong after we decided
|
||||
/// to attempt it.
|
||||
pub async fn try_merge(
|
||||
repo: &Path,
|
||||
push_url: &str,
|
||||
branch: &str,
|
||||
base: &str,
|
||||
policy: MergePolicy,
|
||||
verified: bool,
|
||||
) -> Result<MergeOutcome, String> {
|
||||
if policy != MergePolicy::AdditiveOnly {
|
||||
return Ok(MergeOutcome::refused(
|
||||
"merge_policy is not additive_only; left for a human",
|
||||
));
|
||||
}
|
||||
if !verified {
|
||||
return Ok(MergeOutcome::refused(
|
||||
"run did not verify; refusing to merge unproven work",
|
||||
));
|
||||
}
|
||||
|
||||
// Compare against the base as the REMOTE has it, not a local ref that may
|
||||
// be stale. `...` gives changes on the branch since it diverged, so an
|
||||
// unrelated commit landing on main meanwhile is not misread as ours.
|
||||
git(repo, &["fetch", push_url, base]).await?;
|
||||
let diff = git(
|
||||
repo,
|
||||
&["diff", "--name-status", &format!("FETCH_HEAD...{branch}")],
|
||||
)
|
||||
.await?;
|
||||
|
||||
let offending = non_additive_changes(&diff);
|
||||
if !offending.is_empty() {
|
||||
return Ok(MergeOutcome::refused(format!(
|
||||
"diff is not additive ({} non-add change(s), first: {}); left for a human",
|
||||
offending.len(),
|
||||
offending.first().map(String::as_str).unwrap_or("?")
|
||||
)));
|
||||
}
|
||||
if diff.trim().is_empty() {
|
||||
return Ok(MergeOutcome::refused("branch adds nothing"));
|
||||
}
|
||||
|
||||
// Merge onto the freshly fetched base rather than a local branch.
|
||||
git(repo, &["checkout", "-B", base, "FETCH_HEAD"]).await?;
|
||||
if let Err(e) = git(
|
||||
repo,
|
||||
&["merge", "--no-ff", "-m", &format!("auto-merge {branch}"), branch],
|
||||
)
|
||||
.await
|
||||
{
|
||||
// Leave the repo clean so the next run is not fighting a wedged merge.
|
||||
let _ = git(repo, &["merge", "--abort"]).await;
|
||||
return Ok(MergeOutcome::refused(format!(
|
||||
"merge conflicted ({e}); left for a human"
|
||||
)));
|
||||
}
|
||||
|
||||
git(repo, &["push", push_url, &format!("HEAD:refs/heads/{base}")]).await?;
|
||||
Ok(MergeOutcome {
|
||||
merged: true,
|
||||
reason: format!("additive-only and verified; merged into {base}"),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn only_pure_additions_qualify() {
|
||||
assert!(non_additive_changes("A\t60 Papers/a.md\nA\t60 Papers/b.md\n").is_empty());
|
||||
|
||||
// A modification disqualifies the whole branch.
|
||||
let m = non_additive_changes("A\t60 Papers/a.md\nM\tREADME.md\n");
|
||||
assert_eq!(m.len(), 1);
|
||||
assert!(m[0].contains("README.md"));
|
||||
|
||||
// So do deletes and renames — a rename is a delete plus an add, and
|
||||
// the delete half can destroy hand-written work.
|
||||
assert_eq!(non_additive_changes("D\tnotes/old.md\n").len(), 1);
|
||||
assert_eq!(non_additive_changes("R100\ta.md\tb.md\n").len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unknown_policy_never_grants_auto_merge() {
|
||||
assert_eq!(MergePolicy::parse(None), MergePolicy::Never);
|
||||
assert_eq!(MergePolicy::parse(Some("never")), MergePolicy::Never);
|
||||
assert_eq!(
|
||||
MergePolicy::parse(Some("additive_only")),
|
||||
MergePolicy::AdditiveOnly
|
||||
);
|
||||
// A typo must fail closed, not open.
|
||||
assert_eq!(MergePolicy::parse(Some("aditive_only")), MergePolicy::Never);
|
||||
assert_eq!(MergePolicy::parse(Some("always")), MergePolicy::Never);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,495 @@
|
||||
//! What a continuous mission has already covered.
|
||||
//!
|
||||
//! A recurring mission's hard problem is not running the agent — that is 23
|
||||
//! seconds — it is knowing what it already did last time. A research mission
|
||||
//! with no memory of prior runs resurfaces the same papers forever and reports
|
||||
//! success every time.
|
||||
//!
|
||||
//! This module keeps that record. It is deliberately small: an index derived
|
||||
//! from the corpus, never the corpus itself. The vault is the source of truth,
|
||||
//! the index is rebuildable, and a hand-edited note is never "wrong".
|
||||
//!
|
||||
//! # Two kinds, because the real vault forced it
|
||||
//!
|
||||
//! The plan assumed notes would carry `arxiv:` / `doi:` / `url:` frontmatter.
|
||||
//! Measured against the actual vault: **416 notes, 145 with frontmatter, and
|
||||
//! zero with any of those keys.** The dominant keys are repo-sync metadata
|
||||
//! (`node`, `org`, `gitea`) and course-note fields (`presenter`, `session`).
|
||||
//! An ingester keyed only on external identity would have indexed nothing —
|
||||
//! the same shape of failure as everything else this week.
|
||||
//!
|
||||
//! So `note` rows record coverage (what the vault already contains, keyed by
|
||||
//! path) and `source` rows record consumption (external things a mission
|
||||
//! read, keyed by natural id). They answer different questions and a
|
||||
//! continuous mission needs both: "have I already written about this topic?"
|
||||
//! and "have I already read this paper?".
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// A note parsed out of the vault, ready to be indexed.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ParsedNote {
|
||||
/// Vault-relative path, used as identity for `kind = 'note'`.
|
||||
pub path: String,
|
||||
pub title: Option<String>,
|
||||
pub content_hash: String,
|
||||
/// An external identity the note declares for itself, if any. Nothing in
|
||||
/// the vault does this today; missions writing new notes are expected to.
|
||||
pub declared_source_id: Option<String>,
|
||||
}
|
||||
|
||||
impl ParsedNote {
|
||||
/// `note:<path>` — the `source_id` this note occupies in the index.
|
||||
pub fn source_id(&self) -> String {
|
||||
format!("note:{}", self.path)
|
||||
}
|
||||
}
|
||||
|
||||
/// Hash content for change detection. Not a dedupe key — identity is
|
||||
/// `source_id`; this only distinguishes "unchanged" from "edited".
|
||||
pub fn content_hash(body: &str) -> String {
|
||||
let mut h = Sha256::new();
|
||||
h.update(body.as_bytes());
|
||||
format!("{:x}", h.finalize())
|
||||
}
|
||||
|
||||
/// Split YAML frontmatter from the body.
|
||||
///
|
||||
/// Returns `(frontmatter, body)`. A note without frontmatter — 271 of the 416
|
||||
/// in the real vault — yields `("", whole file)` rather than being skipped.
|
||||
/// Skipping them would drop two thirds of the corpus on the floor.
|
||||
fn split_frontmatter(text: &str) -> (&str, &str) {
|
||||
let Some(rest) = text.strip_prefix("---") else {
|
||||
return ("", text);
|
||||
};
|
||||
let rest = rest.strip_prefix('\n').unwrap_or(rest);
|
||||
match rest.find("\n---") {
|
||||
Some(end) => {
|
||||
let body = &rest[end + 4..];
|
||||
(&rest[..end], body.strip_prefix('\n').unwrap_or(body))
|
||||
}
|
||||
// An opening fence with no close is malformed; treat the whole file as
|
||||
// body rather than swallowing it as frontmatter.
|
||||
None => ("", text),
|
||||
}
|
||||
}
|
||||
|
||||
/// Read one scalar key out of a frontmatter block.
|
||||
///
|
||||
/// Deliberately not a YAML parser. The vault's frontmatter is flat
|
||||
/// `key: value` with occasional quotes and one list (`tags`), and pulling in a
|
||||
/// YAML dependency to read three keys would be more surface than it is worth.
|
||||
fn frontmatter_value<'a>(fm: &'a str, key: &str) -> Option<&'a str> {
|
||||
for line in fm.lines() {
|
||||
let line = line.trim();
|
||||
let Some((k, v)) = line.split_once(':') else {
|
||||
continue;
|
||||
};
|
||||
if !k.trim().eq_ignore_ascii_case(key) {
|
||||
continue;
|
||||
}
|
||||
let v = v.trim().trim_matches('"').trim_matches('\'').trim();
|
||||
if !v.is_empty() {
|
||||
return Some(v);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Which frontmatter keys may declare an external identity, in priority order.
|
||||
///
|
||||
/// None of these appear in the vault today. They are the contract for notes
|
||||
/// that missions write from here on, and the reason a `source:` key is NOT in
|
||||
/// the list: the vault already uses `source:` for local filesystem paths of
|
||||
/// course material (`/Users/quantum/Downloads/...`), which is provenance, not
|
||||
/// a citable external identity. Treating it as one would fill the seen-set
|
||||
/// with 25 rows keyed on a laptop path.
|
||||
const IDENTITY_KEYS: &[&str] = &["source_id", "arxiv", "doi", "url", "permalink"];
|
||||
|
||||
/// Parse a note. `path` must be vault-relative.
|
||||
pub fn parse_note(path: &str, text: &str) -> ParsedNote {
|
||||
let (fm, body) = split_frontmatter(text);
|
||||
|
||||
let declared_source_id = IDENTITY_KEYS.iter().find_map(|k| {
|
||||
frontmatter_value(fm, k).map(|v| {
|
||||
// `source_id` is already qualified; the others name their scheme.
|
||||
if *k == "source_id" || v.contains(':') {
|
||||
v.to_string()
|
||||
} else {
|
||||
format!("{k}:{v}")
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
// Title: the first markdown H1, else the filename stem. Frontmatter has no
|
||||
// consistent title key in this vault.
|
||||
let title = body
|
||||
.lines()
|
||||
.find_map(|l| l.strip_prefix("# ").map(str::trim))
|
||||
.filter(|t| !t.is_empty())
|
||||
.map(str::to_string)
|
||||
.or_else(|| {
|
||||
std::path::Path::new(path)
|
||||
.file_stem()
|
||||
.map(|s| s.to_string_lossy().into_owned())
|
||||
});
|
||||
|
||||
ParsedNote {
|
||||
path: path.to_string(),
|
||||
title,
|
||||
// Hash the body, not the whole file: re-syncing a repo note rewrites
|
||||
// `updated:`/`size_kb:` in frontmatter without the prose changing, and
|
||||
// that should not read as an edit.
|
||||
content_hash: content_hash(body),
|
||||
declared_source_id,
|
||||
}
|
||||
}
|
||||
|
||||
/// What a re-index actually did. `unchanged` is the number that matters: on a
|
||||
/// vault nobody edited it should equal the note count.
|
||||
#[derive(Debug, Default, Clone, PartialEq, Eq)]
|
||||
pub struct IndexStats {
|
||||
pub scanned: usize,
|
||||
pub inserted: usize,
|
||||
pub updated: usize,
|
||||
pub unchanged: usize,
|
||||
}
|
||||
|
||||
/// Walk a checkout and index every markdown note.
|
||||
///
|
||||
/// Skips `.git` and Obsidian's own `.obsidian` config directory — indexing an
|
||||
/// editor's workspace state as knowledge would be noise.
|
||||
pub fn collect_notes(root: &std::path::Path) -> Vec<ParsedNote> {
|
||||
fn walk(dir: &std::path::Path, root: &std::path::Path, out: &mut Vec<ParsedNote>) {
|
||||
let Ok(entries) = std::fs::read_dir(dir) else {
|
||||
return;
|
||||
};
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
let name = entry.file_name();
|
||||
let name = name.to_string_lossy();
|
||||
if name.starts_with('.') {
|
||||
continue;
|
||||
}
|
||||
if path.is_dir() {
|
||||
walk(&path, root, out);
|
||||
} else if path.extension().and_then(|e| e.to_str()) == Some("md") {
|
||||
let Ok(text) = std::fs::read_to_string(&path) else {
|
||||
continue;
|
||||
};
|
||||
let rel = path
|
||||
.strip_prefix(root)
|
||||
.unwrap_or(&path)
|
||||
.to_string_lossy()
|
||||
.into_owned();
|
||||
out.push(parse_note(&rel, &text));
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
walk(root, root, &mut out);
|
||||
out.sort_by(|a, b| a.path.cmp(&b.path));
|
||||
out
|
||||
}
|
||||
|
||||
/// Upsert one item. Returns whether the row was new.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn record(
|
||||
pool: &sqlx::PgPool,
|
||||
workspace_id: Uuid,
|
||||
corpus_id: &str,
|
||||
kind: &str,
|
||||
source_id: &str,
|
||||
title: Option<&str>,
|
||||
path: Option<&str>,
|
||||
url: Option<&str>,
|
||||
content_hash: &str,
|
||||
mission_id: Option<Uuid>,
|
||||
) -> Result<bool, String> {
|
||||
// `last_seen_at` always moves; `first_seen_at` and `mission_id` never do.
|
||||
// The first mission to find a source keeps the credit, which is what makes
|
||||
// "did THIS run contribute anything new" answerable.
|
||||
let row: (bool,) = sqlx::query_as(
|
||||
"INSERT INTO corpus_items
|
||||
(id, workspace_id, corpus_id, kind, source_id, title, path, url,
|
||||
content_hash, mission_id)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
||||
ON CONFLICT (workspace_id, corpus_id, source_id) DO UPDATE
|
||||
SET last_seen_at = now(),
|
||||
title = COALESCE(EXCLUDED.title, corpus_items.title),
|
||||
path = COALESCE(EXCLUDED.path, corpus_items.path),
|
||||
url = COALESCE(EXCLUDED.url, corpus_items.url),
|
||||
content_hash = EXCLUDED.content_hash
|
||||
RETURNING (xmax = 0) AS inserted",
|
||||
)
|
||||
.bind(Uuid::now_v7())
|
||||
.bind(workspace_id)
|
||||
.bind(corpus_id)
|
||||
.bind(kind)
|
||||
.bind(source_id)
|
||||
.bind(title)
|
||||
.bind(path)
|
||||
.bind(url)
|
||||
.bind(content_hash)
|
||||
.bind(mission_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(|e| format!("record corpus item {source_id}: {e}"))?;
|
||||
Ok(row.0)
|
||||
}
|
||||
|
||||
/// Has this corpus already seen this `source_id`?
|
||||
pub async fn seen(
|
||||
pool: &sqlx::PgPool,
|
||||
workspace_id: Uuid,
|
||||
corpus_id: &str,
|
||||
source_id: &str,
|
||||
) -> Result<bool, String> {
|
||||
// `SELECT 1` is INT4; binding it as i64 fails to decode.
|
||||
let row: Option<(i32,)> = sqlx::query_as(
|
||||
"SELECT 1 FROM corpus_items
|
||||
WHERE workspace_id = $1 AND corpus_id = $2 AND source_id = $3",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.bind(corpus_id)
|
||||
.bind(source_id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(|e| format!("seen({source_id}): {e}"))?;
|
||||
Ok(row.is_some())
|
||||
}
|
||||
|
||||
/// Of these candidate ids, which has this corpus NOT seen?
|
||||
///
|
||||
/// The shape a research agent actually needs: it has ten search hits and wants
|
||||
/// to know which are worth fetching. One round trip, not ten.
|
||||
pub async fn unseen(
|
||||
pool: &sqlx::PgPool,
|
||||
workspace_id: Uuid,
|
||||
corpus_id: &str,
|
||||
candidates: &[String],
|
||||
) -> Result<Vec<String>, String> {
|
||||
if candidates.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let rows: Vec<(String,)> = sqlx::query_as(
|
||||
"SELECT source_id FROM corpus_items
|
||||
WHERE workspace_id = $1 AND corpus_id = $2 AND source_id = ANY($3)",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.bind(corpus_id)
|
||||
.bind(candidates)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.map_err(|e| format!("unseen: {e}"))?;
|
||||
let known: std::collections::HashSet<String> = rows.into_iter().map(|r| r.0).collect();
|
||||
Ok(candidates
|
||||
.iter()
|
||||
.filter(|c| !known.contains(*c))
|
||||
.cloned()
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// How many NEW sources a mission contributed.
|
||||
///
|
||||
/// The verification predicate for a continuous research mission. `record`
|
||||
/// never reassigns `mission_id` on conflict, so the first mission to find a
|
||||
/// source keeps the credit and a rerun cannot inflate its own count by
|
||||
/// re-recording what an earlier run already had.
|
||||
///
|
||||
/// A mission whose answer is zero produced nothing, whatever its transcript
|
||||
/// says — which is the check the 0030-0044 generation of this feature lacked.
|
||||
pub async fn contributed(
|
||||
pool: &sqlx::PgPool,
|
||||
workspace_id: Uuid,
|
||||
corpus_id: &str,
|
||||
mission_id: Uuid,
|
||||
) -> Result<i64, String> {
|
||||
let row: (i64,) = sqlx::query_as(
|
||||
"SELECT count(*) FROM corpus_items
|
||||
WHERE workspace_id = $1 AND corpus_id = $2 AND mission_id = $3
|
||||
AND kind = 'source'",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.bind(corpus_id)
|
||||
.bind(mission_id)
|
||||
.fetch_one(pool)
|
||||
.await
|
||||
.map_err(|e| format!("contributed({mission_id}): {e}"))?;
|
||||
Ok(row.0)
|
||||
}
|
||||
|
||||
/// Index every note in a checkout. Idempotent by construction.
|
||||
pub async fn index_vault(
|
||||
pool: &sqlx::PgPool,
|
||||
workspace_id: Uuid,
|
||||
corpus_id: &str,
|
||||
root: &std::path::Path,
|
||||
) -> Result<IndexStats, String> {
|
||||
let notes = collect_notes(root);
|
||||
let mut stats = IndexStats {
|
||||
scanned: notes.len(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
for note in ¬es {
|
||||
let existing: Option<(String,)> = sqlx::query_as(
|
||||
"SELECT content_hash FROM corpus_items
|
||||
WHERE workspace_id = $1 AND corpus_id = $2 AND source_id = $3",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.bind(corpus_id)
|
||||
.bind(note.source_id())
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.map_err(|e| format!("lookup {}: {e}", note.path))?;
|
||||
|
||||
match existing {
|
||||
Some((hash,)) if hash == note.content_hash => {
|
||||
stats.unchanged += 1;
|
||||
continue;
|
||||
}
|
||||
Some(_) => stats.updated += 1,
|
||||
None => stats.inserted += 1,
|
||||
}
|
||||
|
||||
record(
|
||||
pool,
|
||||
workspace_id,
|
||||
corpus_id,
|
||||
"note",
|
||||
¬e.source_id(),
|
||||
note.title.as_deref(),
|
||||
Some(¬e.path),
|
||||
None,
|
||||
¬e.content_hash,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
// A note that declares an external identity also registers as a
|
||||
// consumed source, so a later mission does not re-read what an
|
||||
// earlier one already wrote up.
|
||||
if let Some(sid) = ¬e.declared_source_id {
|
||||
record(
|
||||
pool,
|
||||
workspace_id,
|
||||
corpus_id,
|
||||
"source",
|
||||
sid,
|
||||
note.title.as_deref(),
|
||||
Some(¬e.path),
|
||||
None,
|
||||
¬e.content_hash,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
Ok(stats)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn frontmatter_is_split_from_body() {
|
||||
let (fm, body) = split_frontmatter("---\ntype: lecture\n---\n# Title\n\ntext\n");
|
||||
assert_eq!(fm, "type: lecture");
|
||||
assert!(body.starts_with("# Title"));
|
||||
}
|
||||
|
||||
/// 271 of the vault's 416 notes have no frontmatter. Dropping them would
|
||||
/// discard two thirds of the corpus.
|
||||
#[test]
|
||||
fn a_note_without_frontmatter_is_still_a_note() {
|
||||
let (fm, body) = split_frontmatter("# Plain\n\nno frontmatter here\n");
|
||||
assert_eq!(fm, "");
|
||||
assert!(body.starts_with("# Plain"));
|
||||
let n = parse_note("Daily/x.md", "# Plain\n\nbody\n");
|
||||
assert_eq!(n.title.as_deref(), Some("Plain"));
|
||||
assert_eq!(n.declared_source_id, None);
|
||||
}
|
||||
|
||||
/// An unterminated fence must not swallow the file.
|
||||
#[test]
|
||||
fn malformed_frontmatter_is_treated_as_body() {
|
||||
let (fm, body) = split_frontmatter("---\nbroken: yes\nno closing fence\n");
|
||||
assert_eq!(fm, "");
|
||||
assert!(body.contains("no closing fence"));
|
||||
}
|
||||
|
||||
/// The vault's real `source:` values are local filesystem paths of course
|
||||
/// material. Treating those as citable identity would fill the seen-set
|
||||
/// with 25 rows keyed on a laptop path.
|
||||
#[test]
|
||||
fn a_local_source_path_is_not_an_external_identity() {
|
||||
let note = parse_note(
|
||||
"50 APESS 2026/Lectures/talk.md",
|
||||
"---\nsource: \"/Users/quantum/Downloads/Material_APESS_2026/x.pdf\"\n\
|
||||
date: 2026-07-27\ntype: lecture\n---\n# Agentic Design\n",
|
||||
);
|
||||
assert_eq!(
|
||||
note.declared_source_id, None,
|
||||
"a Downloads path is provenance, not a citable source id"
|
||||
);
|
||||
assert_eq!(note.title.as_deref(), Some("Agentic Design"));
|
||||
assert_eq!(note.source_id(), "note:50 APESS 2026/Lectures/talk.md");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn declared_identities_are_scheme_qualified() {
|
||||
let a = parse_note("p.md", "---\narxiv: 2401.12345\n---\n# T\n");
|
||||
assert_eq!(a.declared_source_id.as_deref(), Some("arxiv:2401.12345"));
|
||||
|
||||
let d = parse_note("p.md", "---\ndoi: 10.1000/xyz\n---\n# T\n");
|
||||
assert_eq!(d.declared_source_id.as_deref(), Some("doi:10.1000/xyz"));
|
||||
|
||||
// Already-qualified values are not double-prefixed.
|
||||
let s = parse_note("p.md", "---\nsource_id: arxiv:2401.99999\n---\n# T\n");
|
||||
assert_eq!(s.declared_source_id.as_deref(), Some("arxiv:2401.99999"));
|
||||
|
||||
// A URL carries its own scheme and must not become `url:https:...`.
|
||||
let u = parse_note("p.md", "---\nurl: https://example.com/p\n---\n# T\n");
|
||||
assert_eq!(
|
||||
u.declared_source_id.as_deref(),
|
||||
Some("https://example.com/p")
|
||||
);
|
||||
}
|
||||
|
||||
/// Repo-sync notes rewrite `updated:`/`size_kb:` on every sync without the
|
||||
/// prose changing. Hashing the whole file would report 103 phantom edits
|
||||
/// per run and make "unchanged" meaningless.
|
||||
#[test]
|
||||
fn frontmatter_churn_does_not_count_as_an_edit() {
|
||||
let a = parse_note("Repos/x.md", "---\nupdated: 2026-08-01\nsize_kb: 12\n---\n# X\n\nbody\n");
|
||||
let b = parse_note("Repos/x.md", "---\nupdated: 2026-08-03\nsize_kb: 14\n---\n# X\n\nbody\n");
|
||||
assert_eq!(a.content_hash, b.content_hash);
|
||||
|
||||
let c = parse_note("Repos/x.md", "---\nupdated: 2026-08-03\n---\n# X\n\nDIFFERENT\n");
|
||||
assert_ne!(a.content_hash, c.content_hash, "real edits must be visible");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn note_identity_is_its_path() {
|
||||
let n = parse_note("30 Resources/a b.md", "# A\n");
|
||||
assert_eq!(n.source_id(), "note:30 Resources/a b.md");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_skips_dotfiles_and_non_markdown() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let root = tmp.path();
|
||||
std::fs::create_dir_all(root.join(".obsidian")).unwrap();
|
||||
std::fs::create_dir_all(root.join("Daily")).unwrap();
|
||||
std::fs::write(root.join(".obsidian/workspace.md"), "# editor state\n").unwrap();
|
||||
std::fs::write(root.join("Daily/note.md"), "# Real\n").unwrap();
|
||||
std::fs::write(root.join("image.png"), "notmd").unwrap();
|
||||
|
||||
let notes = collect_notes(root);
|
||||
assert_eq!(notes.len(), 1, "only the real note: {notes:?}");
|
||||
assert_eq!(notes[0].path, "Daily/note.md");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
//! One run of the library: find, skip what we have, shelve the rest.
|
||||
//!
|
||||
//! This is the piece that makes the others a *job* rather than parts on a
|
||||
//! bench. Order matters and it is deliberate:
|
||||
//!
|
||||
//! 1. **search** arXiv for candidates
|
||||
//! 2. **skip** everything already on the checkmark list — before any download
|
||||
//! 3. **fetch** the PDF for what is left, and verify it really is a PDF
|
||||
//! 4. **shelve** it in the blob store
|
||||
//! 5. **catalogue** it: write the vault note
|
||||
//! 6. **check it off** so next week skips it
|
||||
//!
|
||||
//! Step 2 comes before step 3 on purpose. Checking after downloading would
|
||||
//! still dedupe the catalogue, but it would re-download every paper we already
|
||||
//! have, every week, forever — and the whole point of the checkmark list is to
|
||||
//! not do the work twice.
|
||||
//!
|
||||
//! # Nothing new is a success, not a failure
|
||||
//!
|
||||
//! A weekly run that finds no new papers has worked correctly. A run that
|
||||
//! *crashed* has not. [`Harvest`] keeps those apart, because collapsing them
|
||||
//! is precisely the "reported success while doing nothing" shape that this
|
||||
//! codebase has been bitten by repeatedly. `shelved == 0` with `failed.empty()`
|
||||
//! is a quiet week; `shelved == 0` with failures is a broken run.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::corpus;
|
||||
use crate::papers::{self, Paper};
|
||||
|
||||
/// What one run did. Every number here is observed, not claimed.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct Harvest {
|
||||
/// Papers the search returned.
|
||||
pub candidates: usize,
|
||||
/// Of those, how many were already on the checkmark list.
|
||||
pub already_had: usize,
|
||||
/// Successfully downloaded, shelved and catalogued.
|
||||
pub shelved: Vec<String>,
|
||||
/// `(source_id, why)` for each paper that could not be shelved.
|
||||
pub failed: Vec<(String, String)>,
|
||||
/// Vault-relative paths of the notes written.
|
||||
pub notes_written: Vec<String>,
|
||||
}
|
||||
|
||||
impl Harvest {
|
||||
/// Did this run add anything? The verification predicate for a continuous
|
||||
/// research mission: a run that contributes no new source has produced
|
||||
/// nothing, whatever its transcript says.
|
||||
pub fn added_anything(&self) -> bool {
|
||||
!self.shelved.is_empty()
|
||||
}
|
||||
|
||||
/// A run is healthy if nothing errored — including a run that found
|
||||
/// nothing new, which is the normal state of a mature library.
|
||||
pub fn healthy(&self) -> bool {
|
||||
self.failed.is_empty()
|
||||
}
|
||||
|
||||
pub fn summary(&self) -> String {
|
||||
format!(
|
||||
"{} candidates, {} already held, {} shelved, {} failed",
|
||||
self.candidates,
|
||||
self.already_had,
|
||||
self.shelved.len(),
|
||||
self.failed.len()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a library lives: its records, its shelf, and its catalogue.
|
||||
///
|
||||
/// Grouped rather than passed as loose arguments because these five always
|
||||
/// travel together and always describe one library — splitting them at a call
|
||||
/// site is how a run ends up shelving into one place and cataloguing into
|
||||
/// another.
|
||||
pub struct Library<'a> {
|
||||
pub pool: &'a sqlx::PgPool,
|
||||
/// The shelf: where PDFs are stored.
|
||||
pub blobs: &'a Arc<dyn cm_files::BlobStore>,
|
||||
pub workspace_id: Uuid,
|
||||
/// Which checkmark list, e.g. `"valhalla-vault"`.
|
||||
pub corpus_id: &'a str,
|
||||
/// Checkout the catalogue notes are written into.
|
||||
pub vault_root: &'a Path,
|
||||
}
|
||||
|
||||
/// Shelve a specific set of papers. Split from [`run`] so the skip/shelve
|
||||
/// logic is testable without reaching arXiv.
|
||||
pub async fn shelve(
|
||||
lib: &Library<'_>,
|
||||
candidates: &[Paper],
|
||||
mission_id: Option<Uuid>,
|
||||
) -> Result<Harvest, String> {
|
||||
let Library { pool, blobs, workspace_id, corpus_id, vault_root } = *lib;
|
||||
let mut out = Harvest {
|
||||
candidates: candidates.len(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// One round trip for the whole batch rather than one query per paper.
|
||||
let ids: Vec<String> = candidates.iter().map(Paper::source_id).collect();
|
||||
let fresh: std::collections::HashSet<String> =
|
||||
corpus::unseen(pool, workspace_id, corpus_id, &ids)
|
||||
.await?
|
||||
.into_iter()
|
||||
.collect();
|
||||
out.already_had = candidates.len() - fresh.len();
|
||||
|
||||
for paper in candidates {
|
||||
let sid = paper.source_id();
|
||||
if !fresh.contains(&sid) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Fetch first. If the PDF cannot be had, nothing is recorded — the
|
||||
// paper stays unseen so a later run retries it, rather than being
|
||||
// checked off with an empty shelf slot behind it.
|
||||
let bytes = match papers::fetch_pdf(paper).await {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
out.failed.push((sid, e));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let key = paper.blob_key();
|
||||
if let Err(e) = blobs.put(&key, &bytes).await {
|
||||
out.failed.push((sid, format!("shelve {key}: {e}")));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Catalogue note next to the shelf. Written into the vault checkout;
|
||||
// committing and pushing it is the caller's job, through the delivery
|
||||
// path that already exists.
|
||||
let note = papers::catalogue_note(paper, &key);
|
||||
let note_path = vault_root.join(paper.note_path());
|
||||
if let Some(parent) = note_path.parent() {
|
||||
if let Err(e) = std::fs::create_dir_all(parent) {
|
||||
out.failed.push((sid, format!("create {}: {e}", parent.display())));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if let Err(e) = std::fs::write(¬e_path, ¬e) {
|
||||
out.failed
|
||||
.push((sid, format!("write {}: {e}", note_path.display())));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check it off LAST. If anything above failed we did not get the
|
||||
// paper, and marking it seen would mean never trying again.
|
||||
corpus::record(
|
||||
pool,
|
||||
workspace_id,
|
||||
corpus_id,
|
||||
"source",
|
||||
&sid,
|
||||
Some(&paper.title),
|
||||
Some(&paper.note_path()),
|
||||
Some(&format!("https://arxiv.org/abs/{}", paper.arxiv_id)),
|
||||
&corpus::content_hash(¬e),
|
||||
mission_id,
|
||||
)
|
||||
.await?;
|
||||
|
||||
out.notes_written.push(paper.note_path());
|
||||
out.shelved.push(sid);
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// A full run: search arXiv, then shelve whatever is new.
|
||||
pub async fn run(
|
||||
lib: &Library<'_>,
|
||||
query: &str,
|
||||
limit: usize,
|
||||
mission_id: Option<Uuid>,
|
||||
) -> Result<Harvest, String> {
|
||||
let candidates = papers::search(query, limit).await?;
|
||||
let harvest = shelve(lib, &candidates, mission_id).await?;
|
||||
let corpus_id = lib.corpus_id;
|
||||
eprintln!("harvest[{corpus_id}] query={query:?} → {}", harvest.summary());
|
||||
for (sid, why) in &harvest.failed {
|
||||
eprintln!("harvest[{corpus_id}] FAILED {sid}: {why}");
|
||||
}
|
||||
Ok(harvest)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_quiet_week_is_healthy_but_adds_nothing() {
|
||||
let quiet = Harvest {
|
||||
candidates: 5,
|
||||
already_had: 5,
|
||||
..Default::default()
|
||||
};
|
||||
assert!(quiet.healthy(), "finding nothing new is not an error");
|
||||
assert!(
|
||||
!quiet.added_anything(),
|
||||
"but it must not count as having produced something"
|
||||
);
|
||||
|
||||
let broken = Harvest {
|
||||
candidates: 5,
|
||||
already_had: 0,
|
||||
failed: vec![("arxiv:1".into(), "timeout".into())],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!broken.healthy());
|
||||
assert!(!broken.added_anything());
|
||||
|
||||
let good = Harvest {
|
||||
candidates: 5,
|
||||
already_had: 4,
|
||||
shelved: vec!["arxiv:2".into()],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(good.healthy() && good.added_anything());
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,15 @@ mod mcp_door;
|
||||
mod mcp_skills;
|
||||
pub mod mission_orchestrator;
|
||||
pub mod mission_refiner;
|
||||
pub mod auto_merge;
|
||||
pub mod corpus;
|
||||
pub mod harvest;
|
||||
pub mod library;
|
||||
pub mod mission_delivery;
|
||||
pub mod papers;
|
||||
pub mod phase_config;
|
||||
pub mod session_executor;
|
||||
pub mod runtime_preflight;
|
||||
pub mod mission_runtime;
|
||||
pub mod mission_workspace;
|
||||
pub mod node_rules;
|
||||
@@ -62,6 +70,9 @@ pub struct AppState {
|
||||
pub file_root: Option<std::path::PathBuf>,
|
||||
/// Live control channels to connected fleet-node daemons.
|
||||
pub node_hub: std::sync::Arc<fleet::NodeHub>,
|
||||
/// The shelf. Present once the server wires storage; `None` in the
|
||||
/// bare-`new` path used by tests that never touch blobs.
|
||||
pub blobs: Option<std::sync::Arc<dyn cm_files::BlobStore>>,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
@@ -76,6 +87,7 @@ impl AppState {
|
||||
billing: cm_config::BillingConfig::default(),
|
||||
file_root: None,
|
||||
node_hub: std::sync::Arc::new(fleet::NodeHub::new()),
|
||||
blobs: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,6 +96,12 @@ impl AppState {
|
||||
self
|
||||
}
|
||||
|
||||
/// The shelf — where the paper library stores PDFs.
|
||||
pub fn with_blobs(mut self, blobs: std::sync::Arc<dyn cm_files::BlobStore>) -> AppState {
|
||||
self.blobs = Some(blobs);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_oauth(mut self, oauth: cm_config::OAuthConfig) -> AppState {
|
||||
self.oauth = oauth;
|
||||
self
|
||||
@@ -309,6 +327,8 @@ pub fn router(state: AppState) -> Router {
|
||||
.route("/api/sessions", post(routes::sessions::create))
|
||||
.route("/api/sessions/history", get(routes::sessions::history))
|
||||
.route("/api/gateway", post(routes::gateway::gateway))
|
||||
.route("/api/library/runs", post(routes::library::run))
|
||||
.route("/api/library/items", get(routes::library::list))
|
||||
.route("/api/routines", get(routes::routines::list))
|
||||
.route("/api/routines", post(routes::routines::create))
|
||||
.route("/api/routines/runs", get(routes::routines::runs))
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
//! A library run end to end: clone the vault, harvest, push the catalogue.
|
||||
//!
|
||||
//! [`harvest`](crate::harvest) writes catalogue notes into a directory. This
|
||||
//! puts that directory somewhere real: a checkout of the vault repo, with the
|
||||
//! new notes committed and pushed.
|
||||
//!
|
||||
//! # Never `main`
|
||||
//!
|
||||
//! The vault is a live Obsidian vault that a human edits and syncs. Pushing
|
||||
//! straight to `main` races that sync and can lose hand-written work. Every
|
||||
//! run lands on its own branch, exactly like the mission delivery path that
|
||||
//! was validated 20/20 earlier — a human merges when they have looked at it.
|
||||
//!
|
||||
//! # The PDFs do not go here
|
||||
//!
|
||||
//! Only notes are committed. PDFs are shelved in the blob store, because a
|
||||
//! few hundred papers is gigabytes and a vault that size is painful to clone
|
||||
//! and slow to open. The note carries the blob key, so the catalogue always
|
||||
//! knows where its shelf is.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::harvest::{self, Harvest, Library};
|
||||
use crate::mission_workspace;
|
||||
|
||||
/// What a full run produced, including whether it reached the forge.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LibraryRun {
|
||||
pub harvest: Harvest,
|
||||
pub branch: String,
|
||||
/// `true` only when the push was observed to succeed. A run that shelved
|
||||
/// papers but could not push still has the PDFs and the checkmarks; the
|
||||
/// notes are simply not on the forge yet.
|
||||
pub pushed: bool,
|
||||
/// Whether the branch was auto-merged into `main`.
|
||||
pub merged: bool,
|
||||
/// Always populated — a branch that quietly did not merge is
|
||||
/// indistinguishable from one that was never delivered.
|
||||
pub merge_reason: String,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
fn git_identity() -> [(&'static str, String); 4] {
|
||||
let (name, email) = crate::mission_delivery::commit_identity();
|
||||
[
|
||||
("GIT_AUTHOR_NAME", name.clone()),
|
||||
("GIT_AUTHOR_EMAIL", email.clone()),
|
||||
("GIT_COMMITTER_NAME", name),
|
||||
("GIT_COMMITTER_EMAIL", email),
|
||||
]
|
||||
}
|
||||
|
||||
async fn git(repo: &Path, args: &[&str]) -> Result<String, String> {
|
||||
let mut cmd = tokio::process::Command::new("git");
|
||||
cmd.arg("-C").arg(repo);
|
||||
cmd.args(["-c", &format!("safe.directory={}", repo.display())]);
|
||||
cmd.args(args);
|
||||
for (k, v) in git_identity() {
|
||||
cmd.env(k, v);
|
||||
}
|
||||
let out = cmd.output().await.map_err(|e| format!("spawn git: {e}"))?;
|
||||
if !out.status.success() {
|
||||
return Err(format!(
|
||||
"git {} → {}: {}",
|
||||
args.first().copied().unwrap_or("?"),
|
||||
out.status,
|
||||
mission_workspace::redact_token(&String::from_utf8_lossy(&out.stderr))
|
||||
.chars()
|
||||
.take(300)
|
||||
.collect::<String>()
|
||||
));
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
|
||||
}
|
||||
|
||||
/// Clone the vault fresh into `work_root`, returning the checkout path.
|
||||
///
|
||||
/// Fresh each run rather than reused: a library run is short, the vault is
|
||||
/// small (measured 6.9 MB / 416 notes), and a stale checkout is how the
|
||||
/// mission path lost work three times this week.
|
||||
pub async fn clone_vault(clone_url: &str, work_root: &Path) -> Result<PathBuf, String> {
|
||||
let path = work_root.join("vault");
|
||||
if path.exists() {
|
||||
tokio::fs::remove_dir_all(&path)
|
||||
.await
|
||||
.map_err(|e| format!("clear {}: {e}", path.display()))?;
|
||||
}
|
||||
tokio::fs::create_dir_all(work_root)
|
||||
.await
|
||||
.map_err(|e| format!("mkdir {}: {e}", work_root.display()))?;
|
||||
|
||||
let auth = mission_workspace::with_ambient_auth(clone_url);
|
||||
let out = tokio::process::Command::new("git")
|
||||
.args(["clone", "--quiet", "--depth", "1", &auth])
|
||||
.arg(&path)
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| format!("spawn git clone: {e}"))?;
|
||||
if !out.status.success() {
|
||||
return Err(format!(
|
||||
"clone vault → {}: {}",
|
||||
out.status,
|
||||
mission_workspace::redact_token(&String::from_utf8_lossy(&out.stderr))
|
||||
.chars()
|
||||
.take(300)
|
||||
.collect::<String>()
|
||||
));
|
||||
}
|
||||
// The token must not stay in .git/config: the checkout may be handed to a
|
||||
// container later, and a credential in a file an agent can read is a
|
||||
// credential an agent has.
|
||||
mission_workspace::scrub_remote_credentials(&path, &auth);
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
/// One complete library run.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn run_to_vault(
|
||||
pool: &sqlx::PgPool,
|
||||
blobs: &Arc<dyn cm_files::BlobStore>,
|
||||
workspace_id: Uuid,
|
||||
corpus_id: &str,
|
||||
clone_url: &str,
|
||||
work_root: &Path,
|
||||
queries: &[String],
|
||||
per_query: usize,
|
||||
mission_id: Option<Uuid>,
|
||||
) -> Result<LibraryRun, String> {
|
||||
let vault = clone_vault(clone_url, work_root).await?;
|
||||
let lib = Library {
|
||||
pool,
|
||||
blobs,
|
||||
workspace_id,
|
||||
corpus_id,
|
||||
vault_root: &vault,
|
||||
};
|
||||
|
||||
// Accumulate across queries. Topics overlap — "agentic topology" and
|
||||
// "multi-agent orchestration" return some of the same papers — and the
|
||||
// checkmark list dedupes across them within a single run as well as
|
||||
// between runs, because each shelve records before the next query starts.
|
||||
let mut total = Harvest::default();
|
||||
for q in queries {
|
||||
let h = harvest::run(&lib, q, per_query, mission_id).await?;
|
||||
total.candidates += h.candidates;
|
||||
total.already_had += h.already_had;
|
||||
total.shelved.extend(h.shelved);
|
||||
total.failed.extend(h.failed);
|
||||
total.notes_written.extend(h.notes_written);
|
||||
}
|
||||
|
||||
// The TAIL of the uuid, not the head. UUIDv7 leads with a 48-bit
|
||||
// timestamp, so two ids minted in the same millisecond share their first
|
||||
// 12 hex characters exactly — the branch-name collision that hit mission
|
||||
// 019fc42b earlier. The tail is the random part.
|
||||
let branch = format!("clawmates/library-{}", branch_suffix(Uuid::now_v7()));
|
||||
|
||||
if total.notes_written.is_empty() {
|
||||
// A quiet run is a success with nothing to push. Creating an empty
|
||||
// branch every week would be noise.
|
||||
return Ok(LibraryRun {
|
||||
harvest: total,
|
||||
branch,
|
||||
pushed: false,
|
||||
merged: false,
|
||||
merge_reason: "nothing new to push".into(),
|
||||
error: None,
|
||||
});
|
||||
}
|
||||
|
||||
git(&vault, &["checkout", "-B", &branch]).await?;
|
||||
git(&vault, &["add", "--", "60 Papers"]).await?;
|
||||
let message = format!(
|
||||
"library: {} new paper(s)\n\n{}\n\nShelved in the blob store; this commit is the catalogue.",
|
||||
total.shelved.len(),
|
||||
total
|
||||
.shelved
|
||||
.iter()
|
||||
.map(|s| format!("- {s}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
);
|
||||
git(&vault, &["commit", "--no-verify", "-m", &message]).await?;
|
||||
|
||||
let auth = mission_workspace::with_ambient_auth(clone_url);
|
||||
let refspec = format!("HEAD:refs/heads/{branch}");
|
||||
match git(&vault, &["push", &auth, &refspec]).await {
|
||||
Ok(_) => {
|
||||
// A catalogue branch only ever adds notes under `60 Papers/`, so
|
||||
// it qualifies for auto-merge — but the check is measured from the
|
||||
// diff, not assumed from the mission type. Verified here means the
|
||||
// run shelved something and errored on nothing.
|
||||
let verified = total.healthy() && !total.shelved.is_empty();
|
||||
let merge = crate::auto_merge::try_merge(
|
||||
&vault,
|
||||
&auth,
|
||||
&branch,
|
||||
"main",
|
||||
crate::auto_merge::MergePolicy::AdditiveOnly,
|
||||
verified,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|e| crate::auto_merge::MergeOutcome {
|
||||
merged: false,
|
||||
reason: format!("merge attempt failed: {e}"),
|
||||
});
|
||||
eprintln!("library: branch {branch} — {}", merge.reason);
|
||||
Ok(LibraryRun {
|
||||
harvest: total,
|
||||
branch,
|
||||
pushed: true,
|
||||
merged: merge.merged,
|
||||
merge_reason: merge.reason,
|
||||
error: None,
|
||||
})
|
||||
}
|
||||
Err(e) => Ok(LibraryRun {
|
||||
harvest: total,
|
||||
branch,
|
||||
pushed: false,
|
||||
merged: false,
|
||||
merge_reason: "not pushed, so not merged".into(),
|
||||
error: Some(e),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Distinct-per-run branch suffix. See the note at the call site: taking the
|
||||
/// head of a UUIDv7 yields the timestamp, which collides.
|
||||
fn branch_suffix(id: Uuid) -> String {
|
||||
let s = id.simple().to_string();
|
||||
s[s.len() - 12..].to_string()
|
||||
}
|
||||
|
||||
/// The topics this library currently tracks.
|
||||
///
|
||||
/// Drawn from what the project is actually working on: `papers/dynamic-
|
||||
/// agentic-topologies.md` (topology search and evolution, citing ADAS,
|
||||
/// Darwin-Gödel and SwarmAgentic), plus the problems this week's work ran
|
||||
/// into — verifying what an agent actually did, and giving a long-running
|
||||
/// agent memory of what it has already covered.
|
||||
pub fn default_topics() -> Vec<String> {
|
||||
[
|
||||
"all:\"agentic topology\" OR all:\"multi-agent topology\"",
|
||||
"all:\"multi-agent orchestration\" AND all:LLM",
|
||||
"all:\"agent memory\" AND all:\"long-term\"",
|
||||
"all:\"LLM agent\" AND all:verification",
|
||||
"all:\"prompt injection\" AND all:agent",
|
||||
]
|
||||
.iter()
|
||||
.map(|s| s.to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn topics_are_non_empty_and_arxiv_shaped() {
|
||||
let topics = default_topics();
|
||||
assert!(topics.len() >= 3);
|
||||
for t in &topics {
|
||||
assert!(t.contains("all:"), "arXiv field prefix missing in {t:?}");
|
||||
assert!(!t.trim().is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
/// Two runs in the same millisecond must not collide.
|
||||
///
|
||||
/// This caught a real repeat of the mission-path bug (019fc42b): UUIDv7
|
||||
/// leads with a 48-bit timestamp, so the FIRST 12 hex characters of two
|
||||
/// ids minted together are identical. Taking the tail fixes it. Looping
|
||||
/// rather than sampling twice, because a one-shot check passes by luck
|
||||
/// whenever the millisecond happens to tick between the two calls.
|
||||
#[test]
|
||||
fn every_run_gets_a_distinct_branch() {
|
||||
let ids: Vec<String> = (0..100).map(|_| branch_suffix(Uuid::now_v7())).collect();
|
||||
let unique: std::collections::HashSet<&String> = ids.iter().collect();
|
||||
assert_eq!(unique.len(), ids.len(), "branch suffixes collided: {ids:?}");
|
||||
|
||||
// And the head-based scheme really does collide, so this test has teeth.
|
||||
let heads: Vec<String> = (0..100)
|
||||
.map(|_| Uuid::now_v7().simple().to_string()[..12].to_string())
|
||||
.collect();
|
||||
let head_unique: std::collections::HashSet<&String> = heads.iter().collect();
|
||||
assert!(
|
||||
head_unique.len() < heads.len(),
|
||||
"the head of a UUIDv7 was expected to collide but did not"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -67,6 +67,38 @@ const EXCLUDED_PATHS: &[&str] = &[
|
||||
/// operator needs to see to work out what happened.
|
||||
const MAX_PATCH_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
/// Who delivery commits as.
|
||||
///
|
||||
/// The operator's identity by default, so pushed commits associate with their
|
||||
/// forge account the way their own commits do. Overridable per deployment via
|
||||
/// `CLAWMATES_COMMIT_NAME` / `CLAWMATES_COMMIT_EMAIL` — a shared instance
|
||||
/// wants a bot identity here, not a person's.
|
||||
///
|
||||
/// What matters for correctness is only that *some* identity is always set:
|
||||
/// the server container has none of its own, so `git commit` fails outright
|
||||
/// without this. The particular value is attribution, not function.
|
||||
///
|
||||
/// Attribution alone, to be clear — the push credential is `GITEA_TOKEN` and
|
||||
/// is unaffected by any of this.
|
||||
const DEFAULT_COMMIT_NAME: &str = "Omar Sobh";
|
||||
const DEFAULT_COMMIT_EMAIL: &str = "[email protected]";
|
||||
|
||||
pub(crate) fn commit_identity() -> (String, String) {
|
||||
let name = std::env::var("CLAWMATES_COMMIT_NAME")
|
||||
.ok()
|
||||
.filter(|v| !v.trim().is_empty())
|
||||
.unwrap_or_else(|| DEFAULT_COMMIT_NAME.to_string());
|
||||
let email = std::env::var("CLAWMATES_COMMIT_EMAIL")
|
||||
.ok()
|
||||
.filter(|v| !v.trim().is_empty())
|
||||
.unwrap_or_else(|| DEFAULT_COMMIT_EMAIL.to_string());
|
||||
(name, email)
|
||||
}
|
||||
|
||||
/// Ceiling on the gate's test run. Long enough for a real suite, short enough
|
||||
/// that a hung test does not hold a phase open indefinitely.
|
||||
const TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(900);
|
||||
|
||||
/// What a phase produced.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Capture {
|
||||
@@ -79,6 +111,15 @@ pub struct Capture {
|
||||
pub empty: bool,
|
||||
pub truncated: bool,
|
||||
pub patch_path: PathBuf,
|
||||
/// Set once the work has been committed to a mission branch.
|
||||
pub committed: Option<Commit>,
|
||||
}
|
||||
|
||||
/// A commit made on the mission's own branch.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Commit {
|
||||
pub branch: String,
|
||||
pub sha: String,
|
||||
}
|
||||
|
||||
/// Where a mission's durable output lives. Sibling of the swept per-mission
|
||||
@@ -99,12 +140,32 @@ pub async fn capture_phase_diff(
|
||||
mission_id: Uuid,
|
||||
phase_id: Uuid,
|
||||
) -> Result<Option<Capture>, String> {
|
||||
// The phase's pass number, so a re-run lands on its own branch instead of
|
||||
// colliding with the previous attempt.
|
||||
let iteration: i32 = sqlx::query_scalar("SELECT iteration FROM mission_phases WHERE id = $1")
|
||||
.bind(phase_id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.ok()
|
||||
.flatten()
|
||||
.unwrap_or(0);
|
||||
// `commit_policy` lives in the phase config, merged there from the
|
||||
// workflow recipe by `phases_for_create`.
|
||||
let policy: Option<String> =
|
||||
sqlx::query_scalar("SELECT config->>'commit_policy' FROM mission_phases WHERE id = $1")
|
||||
.bind(phase_id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
capture_phase_diff_at(
|
||||
pool,
|
||||
mission_id,
|
||||
phase_id,
|
||||
&mission_workspace::checkout_path(mission_id),
|
||||
&outputs_root(mission_id),
|
||||
iteration,
|
||||
Gate::parse(policy.as_deref()),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -121,6 +182,8 @@ pub async fn capture_phase_diff_at(
|
||||
phase_id: Uuid,
|
||||
repo: &Path,
|
||||
outputs: &Path,
|
||||
iteration: i32,
|
||||
gate: Gate,
|
||||
) -> Result<Option<Capture>, String> {
|
||||
let repo = repo.to_path_buf();
|
||||
if !repo.is_dir() {
|
||||
@@ -198,9 +261,120 @@ pub async fn capture_phase_diff_at(
|
||||
std::fs::write(dir.join("diffstat.txt"), &diffstat)
|
||||
.map_err(|e| format!("write diffstat: {e}"))?;
|
||||
|
||||
// Commit only after the patch is safely on disk. If this fails, the work
|
||||
// is still captured and the artifact still lands — the branch is the
|
||||
// convenience, the patch is the guarantee.
|
||||
// Why a phase has no branch belongs in the artifact, not only in the log.
|
||||
// Mission `019fc437` recorded `branch: null, push_error: null` for both
|
||||
// phases — indistinguishable from a phase that was never eligible to
|
||||
// commit. The reason was in stderr on the host, where nothing reading the
|
||||
// mission would find it.
|
||||
let mut commit_error: Option<String> = None;
|
||||
let committed = match commit_phase_work(&repo, mission_id, phase_id, iteration).await {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"mission_delivery: mission {mission_id} phase {phase_id} captured but not \
|
||||
committed: {e}"
|
||||
);
|
||||
commit_error = Some(e.chars().take(500).collect());
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
// Hand the next phase a base that excludes this one's work. Done here
|
||||
// rather than inside `commit_phase_work` so the patch on disk is already
|
||||
// written: if the process dies between the two, the worst case is a phase
|
||||
// that re-reports work, not a phase whose work is invisible.
|
||||
if let Some(c) = committed.as_ref() {
|
||||
mission_workspace::advance_base_commit(&repo, &c.sha);
|
||||
}
|
||||
|
||||
// Gate, then publish. Both are best-effort on top of an artifact that has
|
||||
// already landed: a phase whose tests fail, or whose push is rejected,
|
||||
// still has its patch on disk and its work on a local branch.
|
||||
let mut outcome: Option<TestOutcome> = None;
|
||||
let mut published: Option<Publish> = None;
|
||||
let mut publish_error: Option<String> = None;
|
||||
if let Some(c) = committed.as_ref() {
|
||||
if !empty {
|
||||
if gate == Gate::OnGreenTests {
|
||||
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
||||
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
||||
let o = verify_tests(&repo, &container).await;
|
||||
// An infrastructure fault must be loud. The gate degrades
|
||||
// safely either way, but "we could not run the suite" is a
|
||||
// problem with the platform and needs to look like one.
|
||||
if let TestOutcome::CouldNotRun(why) = &o {
|
||||
eprintln!(
|
||||
"mission_delivery: mission {mission_id} phase {phase_id} could NOT \
|
||||
run the test suite — gating as unverified: {why}"
|
||||
);
|
||||
}
|
||||
outcome = Some(o);
|
||||
}
|
||||
match push_url_for(pool, mission_id).await {
|
||||
Ok(Some(url)) => {
|
||||
let verified = outcome.as_ref().and_then(TestOutcome::verified);
|
||||
match publish_phase_branch(&repo, &url, &c.branch, gate, verified).await {
|
||||
Ok(p) => published = Some(p),
|
||||
// `publish_phase_branch` only returns Err for a local
|
||||
// git failure; a rejected push is Ok with an error
|
||||
// inside. Both must reach the artifact.
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"mission_delivery: mission {mission_id} phase {phase_id} \
|
||||
could not publish {}: {e}",
|
||||
c.branch
|
||||
);
|
||||
publish_error = Some(e.chars().take(500).collect());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
// Legitimate: a mission with no repo bound has nowhere to
|
||||
// push. Still recorded, because "not pushed" with no reason
|
||||
// is the ambiguity this whole pass exists to remove.
|
||||
publish_error =
|
||||
Some("mission has no repo bound; work is committed locally only".into());
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"mission_delivery: mission {mission_id} could not resolve a push \
|
||||
URL ({e}) — work is committed locally on {} but not published",
|
||||
c.branch
|
||||
);
|
||||
publish_error = Some(format!("could not resolve push URL: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let meta = json!({
|
||||
"base_sha": base_sha,
|
||||
"base_recorded": base_is_recorded,
|
||||
"branch": published
|
||||
.as_ref()
|
||||
.map(|p| p.branch.clone())
|
||||
.or_else(|| committed.as_ref().map(|c| c.branch.clone())),
|
||||
"head_sha": committed.as_ref().map(|c| c.sha.clone()),
|
||||
"commit_policy": match gate {
|
||||
Gate::Always => "always",
|
||||
Gate::OnGreenTests => "on_green_tests",
|
||||
Gate::OnReviewerApproval => "on_reviewer_approval",
|
||||
},
|
||||
// `tests_verified` keeps its original tri-state meaning for existing
|
||||
// readers; `tests_status` is what distinguishes the two ways of being
|
||||
// null — a repo with no suite from a runtime that could not run one.
|
||||
"tests_verified": outcome.as_ref().and_then(TestOutcome::verified),
|
||||
"tests_status": outcome.as_ref().map(TestOutcome::status),
|
||||
"tests_detail": outcome.as_ref().and_then(TestOutcome::detail),
|
||||
"pushed": published.as_ref().map(|p| p.pushed),
|
||||
"push_error": published
|
||||
.as_ref()
|
||||
.and_then(|p| p.error.clone())
|
||||
.or(publish_error),
|
||||
"commit_error": commit_error,
|
||||
"files_changed": files_changed,
|
||||
"insertions": insertions,
|
||||
"deletions": deletions,
|
||||
@@ -246,6 +420,7 @@ pub async fn capture_phase_diff_at(
|
||||
);
|
||||
|
||||
Ok(Some(Capture {
|
||||
committed,
|
||||
base_sha,
|
||||
files_changed,
|
||||
insertions,
|
||||
@@ -274,8 +449,29 @@ async fn git(repo: &Path, args: &[&str]) -> Result<String, String> {
|
||||
Box::leak(format!("safe.directory={repo_s}").into_boxed_str()),
|
||||
];
|
||||
full.extend_from_slice(args);
|
||||
let (name, email) = commit_identity();
|
||||
let out = tokio::process::Command::new("git")
|
||||
.args(&full)
|
||||
// The server container has no git identity — `git config --global
|
||||
// user.email` exits 1 — so `git commit` fails with "Author identity
|
||||
// unknown" unless one is supplied. Mission `019fc450` lost its first
|
||||
// phase to exactly that.
|
||||
//
|
||||
// This was the third failure in a row whose trigger was *agent
|
||||
// behaviour rather than our code*: earlier runs committed only because
|
||||
// an agent had happened to run `git config user.email` in the
|
||||
// checkout, leaving a local identity the server then inherited. Config
|
||||
// the agent may or may not have written is not a dependency delivery
|
||||
// can hold, so the identity is supplied here on every call.
|
||||
//
|
||||
// Environment rather than `-c`, because these override config without
|
||||
// needing a leaked string per invocation, and because they name the
|
||||
// committer as the pipeline — which is the truth. The agents' own
|
||||
// commits keep whatever identity they set.
|
||||
.env("GIT_AUTHOR_NAME", &name)
|
||||
.env("GIT_AUTHOR_EMAIL", &email)
|
||||
.env("GIT_COMMITTER_NAME", &name)
|
||||
.env("GIT_COMMITTER_EMAIL", &email)
|
||||
.output()
|
||||
.await
|
||||
.map_err(|e| format!("spawn git: {e}"))?;
|
||||
@@ -324,6 +520,393 @@ pub fn parse_diffstat(stat: &str) -> (usize, usize, usize) {
|
||||
(files, ins, del)
|
||||
}
|
||||
|
||||
/// Commit a phase's work onto a branch of its own.
|
||||
///
|
||||
/// Runs after capture, never before: the patch is already on disk and
|
||||
/// registered, so a commit that goes wrong costs a branch and not the work.
|
||||
///
|
||||
/// Three rules, none of them negotiable:
|
||||
///
|
||||
/// - **Never the default branch.** The branch name is derived from the mission
|
||||
/// and phase, so a mission can only ever add a ref nobody else owns.
|
||||
/// - **Never force.** A rejected update is reported, not overwritten.
|
||||
/// - **Same exclusions as capture.** Whatever was too noisy to put in a patch
|
||||
/// is too noisy to put in someone's history — build output, vendored trees,
|
||||
/// and the workaround files agents write when infrastructure fights them.
|
||||
///
|
||||
/// Returns `Ok(None)` when there is nothing to commit, which is a normal
|
||||
/// outcome and not an error: the phase may have changed nothing, or the agents
|
||||
/// may have committed their own work already.
|
||||
pub async fn commit_phase_work(
|
||||
repo: &Path,
|
||||
mission_id: Uuid,
|
||||
phase_id: Uuid,
|
||||
iteration: i32,
|
||||
) -> Result<Option<Commit>, String> {
|
||||
let branch = branch_name(mission_id, phase_id, iteration);
|
||||
|
||||
// Work already committed by the agents still needs a branch pointing at
|
||||
// it, or it is unreachable once the checkout is reaped. So the branch is
|
||||
// created regardless, and only the staging step is conditional.
|
||||
git(repo, &["checkout", "-B", &branch]).await?;
|
||||
|
||||
let mut add: Vec<&str> = vec!["add", "--", "."];
|
||||
let excludes: Vec<String> = EXCLUDED_PATHS
|
||||
.iter()
|
||||
.map(|p| format!(":(exclude){p}"))
|
||||
.collect();
|
||||
add.extend(excludes.iter().map(String::as_str));
|
||||
git(repo, &add).await?;
|
||||
|
||||
// `--cached` compares the index against HEAD: empty means the agents left
|
||||
// nothing unstaged for us, which is the normal case when they committed
|
||||
// themselves.
|
||||
let staged = git(repo, &["diff", "--cached", "--stat"])
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
if !staged.trim().is_empty() {
|
||||
let message = format!(
|
||||
// The trailer is the provenance record. It matters more now that
|
||||
// the author line carries a person's name: without it, autonomous
|
||||
// work would be indistinguishable from hand-written commits in
|
||||
// `git log`. Keep it on any change to this message.
|
||||
"clawmates: phase work{}\n\
|
||||
\n\
|
||||
Mission: {mission_id}\n\
|
||||
Phase: {phase_id}\n\
|
||||
\n\
|
||||
Committed by the ClawMates delivery pipeline from the agents' \
|
||||
working tree. Authored by agents, not by the named committer.",
|
||||
if iteration > 0 {
|
||||
format!(" (pass {})", iteration + 1)
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
);
|
||||
clear_stale_commit_editmsg(repo);
|
||||
git(repo, &["commit", "--no-verify", "-m", &message]).await?;
|
||||
}
|
||||
|
||||
let sha = git(repo, &["rev-parse", "HEAD"])
|
||||
.await
|
||||
.map(|s| s.trim().to_string())
|
||||
.unwrap_or_default();
|
||||
if sha.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
eprintln!(
|
||||
"mission_delivery: mission {mission_id} phase {phase_id} → branch {branch} at {}",
|
||||
&sha[..sha.len().min(8)]
|
||||
);
|
||||
Ok(Some(Commit { branch, sha }))
|
||||
}
|
||||
|
||||
/// The branch a phase's work lands on.
|
||||
///
|
||||
/// Namespaced under `clawmates/` so it is obvious in a branch list who created
|
||||
/// it and safe to delete in bulk.
|
||||
///
|
||||
/// The two segments are taken from opposite ends of the ids, and that is
|
||||
/// load-bearing. Both are UUIDv7, which leads with a 48-bit timestamp, so ids
|
||||
/// minted in the same millisecond share their leading hex — taking `[..8]` of
|
||||
/// each produced `clawmates/mission-019fc40e-019fc40e` in production, the same
|
||||
/// branch for every phase of the mission, each one silently moving the ref the
|
||||
/// last phase had just set. The mission keeps its time-ordered prefix so
|
||||
/// branches group and sort usefully; the phase contributes its random tail so
|
||||
/// sibling phases cannot collide.
|
||||
pub fn branch_name(mission_id: Uuid, phase_id: Uuid, iteration: i32) -> String {
|
||||
let m = mission_id.simple().to_string();
|
||||
let p = phase_id.simple().to_string();
|
||||
let base = format!("clawmates/mission-{}-{}", &m[..8], &p[p.len() - 8..]);
|
||||
if iteration > 0 {
|
||||
format!("{base}-i{}", iteration + 1)
|
||||
} else {
|
||||
base
|
||||
}
|
||||
}
|
||||
|
||||
/// What a phase's `commit_policy` requires before its branch may be published.
|
||||
///
|
||||
/// Declared in `templates/workflows/*.toml` and merged into `mission_phases.
|
||||
/// config`. Until now it had no reader at all — three recipes have been
|
||||
/// carrying `commit_policy = "on_green_tests"` that did precisely nothing.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Gate {
|
||||
/// Publish unconditionally.
|
||||
Always,
|
||||
/// Publish to the mission branch only if the project's own tests pass.
|
||||
OnGreenTests,
|
||||
/// Publish to a review branch and wait for a human.
|
||||
OnReviewerApproval,
|
||||
}
|
||||
|
||||
impl Gate {
|
||||
pub fn parse(policy: Option<&str>) -> Gate {
|
||||
match policy.map(str::trim) {
|
||||
Some("on_green_tests") => Gate::OnGreenTests,
|
||||
Some("on_reviewer_approval") => Gate::OnReviewerApproval,
|
||||
Some("always") | None | Some("") => Gate::Always,
|
||||
Some(other) => {
|
||||
eprintln!(
|
||||
"mission_delivery: unknown commit_policy {other:?} — treating as `always`"
|
||||
);
|
||||
Gate::Always
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The branch suffix that carries the verdict to a human.
|
||||
///
|
||||
/// A failed gate never discards work — it changes where the work lands.
|
||||
/// Deleting a red-test branch is how you get back to the old behaviour
|
||||
/// (work destroyed) with extra steps; a `-wip` branch is a thing someone
|
||||
/// can look at, fix, and push properly.
|
||||
pub fn branch_suffix(self, verified: Option<bool>) -> &'static str {
|
||||
match (self, verified) {
|
||||
(Gate::Always, _) => "",
|
||||
(Gate::OnGreenTests, Some(true)) => "",
|
||||
// Red, unrunnable, or no test command found — all "not proven".
|
||||
(Gate::OnGreenTests, _) => "-wip",
|
||||
(Gate::OnReviewerApproval, _) => "-review",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The command that runs a project's own tests, inferred from what is in the
|
||||
/// tree.
|
||||
///
|
||||
/// Returns `None` when nothing recognisable is present, which
|
||||
/// [`Gate::branch_suffix`] treats as unproven rather than as passing —
|
||||
/// "we could not check" must never read as "it is fine".
|
||||
pub fn discover_test_command(repo: &Path) -> Option<Vec<String>> {
|
||||
let has = |f: &str| repo.join(f).exists();
|
||||
if has("Cargo.toml") {
|
||||
return Some(vec!["cargo".into(), "test".into(), "--quiet".into()]);
|
||||
}
|
||||
if has("package.json") {
|
||||
let pkg = std::fs::read_to_string(repo.join("package.json")).unwrap_or_default();
|
||||
// Only claim a test command when the project actually declares one;
|
||||
// `npm test` on a package without a test script exits non-zero and
|
||||
// would read as a red suite rather than as "nothing to run".
|
||||
if pkg.contains("\"test\"") {
|
||||
return Some(vec!["npm".into(), "test".into(), "--silent".into()]);
|
||||
}
|
||||
return None;
|
||||
}
|
||||
if has("pyproject.toml") || has("pytest.ini") || repo.join("tests").is_dir() {
|
||||
return Some(vec!["pytest".into(), "-q".into()]);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// The authenticated URL to push this mission's work to.
|
||||
///
|
||||
/// Built fresh from the repo row and the ambient token rather than read from
|
||||
/// `.git/config`, which no longer carries credentials — the token is scrubbed
|
||||
/// after clone because agents run as root in a container that mounts the
|
||||
/// checkout. Building it here also means a rotated token takes effect
|
||||
/// immediately instead of at the next clone.
|
||||
async fn push_url_for(pool: &sqlx::PgPool, mission_id: Uuid) -> Result<Option<String>, String> {
|
||||
let url: Option<String> = sqlx::query_scalar(
|
||||
"SELECT r.clone_url FROM missions m JOIN repos r ON r.id = m.repo_id WHERE m.id = $1",
|
||||
)
|
||||
.bind(mission_id)
|
||||
.fetch_optional(pool)
|
||||
.await
|
||||
// `.ok().flatten()` used to collapse a failed query into the same `None`
|
||||
// as a mission with no repo bound, so a database fault was recorded as
|
||||
// "nothing to push to" — the shape that made `commit_error` necessary.
|
||||
.map_err(|e| format!("query push URL: {e}"))?
|
||||
.flatten();
|
||||
Ok(url.map(|u| mission_workspace::with_ambient_auth(&u)))
|
||||
}
|
||||
|
||||
/// Run the gate, then push the branch if the gate allows it.
|
||||
///
|
||||
/// Publishing is last on purpose. By the time this runs the patch is on disk,
|
||||
/// the artifact is registered and the work is committed to a local branch — so
|
||||
/// every failure mode here costs a ref that did not reach the forge, and
|
||||
/// nothing that was already captured.
|
||||
///
|
||||
/// The branch name carries the verdict. A gate that fails redirects to
|
||||
/// `<branch>-wip` or `<branch>-review` and pushes it anyway: a human can
|
||||
/// inspect, fix and re-push a branch, but cannot recover work that was thrown
|
||||
/// away for failing a test. Deleting a red branch reproduces the old
|
||||
/// behaviour — work destroyed — deliberately rather than by accident.
|
||||
pub async fn publish_phase_branch(
|
||||
repo: &Path,
|
||||
push_url: &str,
|
||||
branch: &str,
|
||||
gate: Gate,
|
||||
verified: Option<bool>,
|
||||
) -> Result<Publish, String> {
|
||||
let suffix = gate.branch_suffix(verified);
|
||||
let target = format!("{branch}{suffix}");
|
||||
if !suffix.is_empty() {
|
||||
// Move the local ref too, so the checkout and the forge agree about
|
||||
// where this work lives.
|
||||
git(repo, &["branch", "-f", &target, "HEAD"]).await?;
|
||||
}
|
||||
|
||||
// Never force. A rejected update is reported and left alone: the remote
|
||||
// ref belongs to whoever set it, and overwriting it to make delivery look
|
||||
// tidy is how a mission eats someone else's commit.
|
||||
let refspec = format!("HEAD:refs/heads/{target}");
|
||||
match git(repo, &["push", push_url, &refspec]).await {
|
||||
Ok(_) => {
|
||||
eprintln!("mission_delivery: pushed {target}");
|
||||
Ok(Publish {
|
||||
branch: target,
|
||||
pushed: true,
|
||||
error: None,
|
||||
})
|
||||
}
|
||||
Err(e) => {
|
||||
// Redacted by `git`'s error path already; the patch and the local
|
||||
// branch both survive, so this is a degraded success.
|
||||
eprintln!("mission_delivery: push of {target} failed: {e}");
|
||||
Ok(Publish {
|
||||
branch: target,
|
||||
pushed: false,
|
||||
error: Some(e),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a phase's work ended up, and whether the forge has it.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Publish {
|
||||
pub branch: String,
|
||||
pub pushed: bool,
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
/// Run the project's own tests to decide whether a green-tests gate is met.
|
||||
///
|
||||
/// `None` means "could not establish", which the gate treats as unproven. That
|
||||
/// is the same fail-closed stance the phase evaluator takes, and for the same
|
||||
/// reason: this codebase has repeatedly found things reporting success while
|
||||
/// doing nothing, and a test suite that never ran must not license a push to a
|
||||
/// mission branch.
|
||||
pub async fn verify_tests(repo: &Path, container: &str) -> TestOutcome {
|
||||
let Some(argv) = discover_test_command(repo) else {
|
||||
return TestOutcome::NoSuite;
|
||||
};
|
||||
let workdir = repo.display().to_string();
|
||||
let docker = match crate::container_exec::connect() {
|
||||
Ok(d) => d,
|
||||
Err(e) => return TestOutcome::CouldNotRun(format!("docker unreachable: {e}")),
|
||||
};
|
||||
match crate::container_exec::exec(&docker, container, Some(&workdir), &argv, TEST_TIMEOUT).await
|
||||
{
|
||||
Ok(out) => {
|
||||
eprintln!(
|
||||
"mission_delivery: {} → exit {:?}",
|
||||
argv.join(" "),
|
||||
out.exit_code
|
||||
);
|
||||
match out.exit_code {
|
||||
Some(0) => TestOutcome::Passed,
|
||||
// An unreadable status is not a pass, and it is not a red
|
||||
// suite either — the command may never have started.
|
||||
None => TestOutcome::CouldNotRun(format!(
|
||||
"`{}` produced no exit status: {}",
|
||||
argv.join(" "),
|
||||
out.combined().chars().take(300).collect::<String>()
|
||||
)),
|
||||
Some(code) => TestOutcome::Failed(code),
|
||||
}
|
||||
}
|
||||
Err(e) => TestOutcome::CouldNotRun(format!("exec in `{container}` failed: {e}")),
|
||||
}
|
||||
}
|
||||
|
||||
/// What happened when the gate tried to verify a phase.
|
||||
///
|
||||
/// This was `Option<bool>`, and collapsing four outcomes into `None` is what
|
||||
/// let a missing toolchain hide for days. `clawmates-runtime` shipped without
|
||||
/// `cargo`, so `verify_tests` returned `None` on every mission — identical to
|
||||
/// the reading for "this repository has no test suite", which is what I
|
||||
/// concluded at the time and stated in a summary. The gate behaved correctly
|
||||
/// throughout (unproven is not a pass); it simply could not say *why* it was
|
||||
/// unproven, so nobody could tell a repo without tests from a runtime without
|
||||
/// a test runner.
|
||||
///
|
||||
/// Only `Passed` clears the gate. The rest differ in what an operator should
|
||||
/// do about them, which is the entire reason they are separate variants.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum TestOutcome {
|
||||
/// The suite ran and passed.
|
||||
Passed,
|
||||
/// The suite ran and failed, with its exit code.
|
||||
Failed(i64),
|
||||
/// No test command could be discovered for this repository.
|
||||
NoSuite,
|
||||
/// A suite exists but could not be executed. Always an infrastructure
|
||||
/// fault on our side, never a verdict about the code.
|
||||
CouldNotRun(String),
|
||||
}
|
||||
|
||||
impl TestOutcome {
|
||||
/// The gate's view: `Some(true)` only when the suite actually passed.
|
||||
/// Preserved so `Gate::branch_suffix` keeps its existing contract.
|
||||
pub fn verified(&self) -> Option<bool> {
|
||||
match self {
|
||||
TestOutcome::Passed => Some(true),
|
||||
TestOutcome::Failed(_) => Some(false),
|
||||
TestOutcome::NoSuite | TestOutcome::CouldNotRun(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Stable machine-readable label for artifact metadata.
|
||||
pub fn status(&self) -> &'static str {
|
||||
match self {
|
||||
TestOutcome::Passed => "passed",
|
||||
TestOutcome::Failed(_) => "failed",
|
||||
TestOutcome::NoSuite => "no_suite",
|
||||
TestOutcome::CouldNotRun(_) => "could_not_run",
|
||||
}
|
||||
}
|
||||
|
||||
/// Human-readable detail, when there is any beyond the label.
|
||||
pub fn detail(&self) -> Option<String> {
|
||||
match self {
|
||||
TestOutcome::Passed | TestOutcome::NoSuite => None,
|
||||
TestOutcome::Failed(code) => Some(format!("test command exited {code}")),
|
||||
TestOutcome::CouldNotRun(why) => Some(why.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Remove a `COMMIT_EDITMSG` the agent left behind as root.
|
||||
///
|
||||
/// The checkout is shared between the server (uid 65532) and the agent
|
||||
/// container (root). `core.sharedRepository` makes git create *objects and
|
||||
/// refs* group-writable — `.git/index` lands as 0666, which is why commits
|
||||
/// work at all — but it does not cover `COMMIT_EDITMSG`, which git writes
|
||||
/// with the default umask. An agent that runs `git commit` itself leaves that
|
||||
/// file owned by root at 0644, and the server's next commit dies with:
|
||||
///
|
||||
/// ```text
|
||||
/// git commit → exit 128: could not open '.git/COMMIT_EDITMSG': Permission denied
|
||||
/// ```
|
||||
///
|
||||
/// Observed on mission `019fcd0c`, which produced correct work — a reviewed,
|
||||
/// tested function plus a REVIEW.md quoting a real `cargo test` summary — and
|
||||
/// then delivered none of it.
|
||||
///
|
||||
/// Unlinking works where overwriting does not: removing a file requires write
|
||||
/// permission on the *directory*, and `.git/` is owned by the server. Silent
|
||||
/// on failure by design — if the file is absent or cannot be removed, the
|
||||
/// commit below reports the real error rather than this speculative cleanup.
|
||||
fn clear_stale_commit_editmsg(repo: &Path) {
|
||||
let msg = repo.join(".git/COMMIT_EDITMSG");
|
||||
if msg.exists() {
|
||||
let _ = std::fs::remove_file(&msg);
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark a phase as impossible to capture, so it stops being selected.
|
||||
///
|
||||
/// A phase whose checkout has already been reaped can never be captured. It
|
||||
@@ -379,6 +962,85 @@ pub async fn record_uncapturable(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// ── The gate ───────────────────────────────────────────────────────
|
||||
|
||||
/// Three recipes have declared `commit_policy` since they were written and
|
||||
/// nothing has ever read it. The parse must at least be forgiving about an
|
||||
/// unknown value rather than refusing to deliver.
|
||||
#[test]
|
||||
fn commit_policy_parses_the_declared_values() {
|
||||
assert_eq!(Gate::parse(Some("on_green_tests")), Gate::OnGreenTests);
|
||||
assert_eq!(
|
||||
Gate::parse(Some("on_reviewer_approval")),
|
||||
Gate::OnReviewerApproval
|
||||
);
|
||||
assert_eq!(Gate::parse(Some("always")), Gate::Always);
|
||||
assert_eq!(Gate::parse(None), Gate::Always);
|
||||
assert_eq!(Gate::parse(Some(" on_green_tests ")), Gate::OnGreenTests);
|
||||
assert_eq!(
|
||||
Gate::parse(Some("nonsense")),
|
||||
Gate::Always,
|
||||
"unknown policy still delivers"
|
||||
);
|
||||
}
|
||||
|
||||
/// A failed gate must move the work, never drop it. Deleting a red-test
|
||||
/// branch reproduces the old behaviour — work destroyed — with extra steps.
|
||||
#[test]
|
||||
fn a_failed_gate_redirects_rather_than_discards() {
|
||||
assert_eq!(Gate::OnGreenTests.branch_suffix(Some(true)), "");
|
||||
assert_eq!(Gate::OnGreenTests.branch_suffix(Some(false)), "-wip");
|
||||
assert_eq!(
|
||||
Gate::OnReviewerApproval.branch_suffix(Some(true)),
|
||||
"-review"
|
||||
);
|
||||
assert_eq!(
|
||||
Gate::Always.branch_suffix(Some(false)),
|
||||
"",
|
||||
"always means always"
|
||||
);
|
||||
}
|
||||
|
||||
/// "We could not check" must not read as "it passed". An unrunnable or
|
||||
/// undiscoverable test suite lands on `-wip` exactly like a red one.
|
||||
#[test]
|
||||
fn an_unverifiable_suite_is_not_treated_as_green() {
|
||||
assert_eq!(Gate::OnGreenTests.branch_suffix(None), "-wip");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_command_is_discovered_from_the_tree() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
assert_eq!(
|
||||
discover_test_command(dir.path()),
|
||||
None,
|
||||
"nothing recognisable"
|
||||
);
|
||||
|
||||
std::fs::write(dir.path().join("Cargo.toml"), "[package]\nname=\"x\"\n").unwrap();
|
||||
assert_eq!(
|
||||
discover_test_command(dir.path()),
|
||||
Some(vec!["cargo".into(), "test".into(), "--quiet".into()])
|
||||
);
|
||||
}
|
||||
|
||||
/// A `package.json` with no test script must yield None, not `npm test` —
|
||||
/// npm exits non-zero for a missing script, which would look like a red
|
||||
/// suite instead of an absent one.
|
||||
#[test]
|
||||
fn a_package_without_a_test_script_yields_no_command() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::write(dir.path().join("package.json"), r#"{"name":"x"}"#).unwrap();
|
||||
assert_eq!(discover_test_command(dir.path()), None);
|
||||
|
||||
std::fs::write(
|
||||
dir.path().join("package.json"),
|
||||
r#"{"name":"x","scripts":{"test":"vitest"}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
assert!(discover_test_command(dir.path()).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diffstat_summary_is_parsed() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -83,10 +83,28 @@ impl RuntimeAuth {
|
||||
///
|
||||
/// The other three are unrelated providers with no subscription equivalent, so
|
||||
/// they forward in both modes.
|
||||
///
|
||||
/// In subscription mode `CLAUDE_CODE_OAUTH_TOKEN` forwards instead. The
|
||||
/// original design assumed a persisted `claude /login` under a bind-mounted
|
||||
/// `$HOME`, but a *mission* container gets its own data dir and therefore no
|
||||
/// login — so the token has to travel. Missing it is not a loud failure:
|
||||
/// `claude -p` simply hangs with no credential, which is what a phase stuck
|
||||
/// at `running` for ten minutes looked like when this was first switched on.
|
||||
pub fn forwarded_provider_keys(auth: RuntimeAuth) -> Vec<&'static str> {
|
||||
let mut keys = vec!["GEMINI_API_KEY", "GROQ_API_KEY", "OPENAI_API_KEY"];
|
||||
if auth == RuntimeAuth::ApiKey {
|
||||
keys.push("ANTHROPIC_API_KEY");
|
||||
// ZAI/KIMI reach their backends through the SAME `claude` binary via
|
||||
// ANTHROPIC_BASE_URL, so a mission that selects one needs its key present
|
||||
// in the container. They are unrelated to the Anthropic credential and
|
||||
// forward in both auth modes.
|
||||
let mut keys = vec![
|
||||
"GEMINI_API_KEY",
|
||||
"GROQ_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"ZAI_API_KEY",
|
||||
"KIMI_API_KEY",
|
||||
];
|
||||
match auth {
|
||||
RuntimeAuth::ApiKey => keys.push("ANTHROPIC_API_KEY"),
|
||||
RuntimeAuth::Subscription => keys.push("CLAUDE_CODE_OAUTH_TOKEN"),
|
||||
}
|
||||
keys
|
||||
}
|
||||
@@ -142,6 +160,146 @@ const MISSIONS_HOST_ROOT: &str = "/var/lib/clawmates-missions";
|
||||
/// mission so this rarely bites. Long-term: copy-on-write per mission.
|
||||
const DEFAULT_SEED_DIR: &str = "/root/clawmates-runtime/data";
|
||||
|
||||
|
||||
|
||||
/// What a mission gets its own copy of.
|
||||
///
|
||||
/// An allow-list, not the whole directory. The seed dir is **1.7 GB** on
|
||||
/// gw-04 and 1.5 GB of that is a vestigial `.rustup` — a Rust toolchain that
|
||||
/// installed itself into the data dir back when `HOME=/zeroclaw-data` and the
|
||||
/// image had no toolchain. The image now ships Rust at `/usr/local/cargo`,
|
||||
/// which is what the container's PATH actually resolves (verified live), so
|
||||
/// that copy is dead weight. Copying it per mission would cost tens of
|
||||
/// seconds and ~17 GB across ten concurrent missions.
|
||||
///
|
||||
/// So: copy what carries per-mission identity or secrets, and leave the
|
||||
/// caches and toolchains behind.
|
||||
const SEEDED_PATHS: &[&str] = &[
|
||||
// The whole point: config.toml carries the §15 door bearer token, and
|
||||
// data/ holds sessions.db + devices.db. ~26 MB.
|
||||
".zeroclaw",
|
||||
// Door MCP config — also a bearer token.
|
||||
"clawmates-mcp.json",
|
||||
// Claude Code's own state and credentials (~16 MB). Per-mission so a
|
||||
// token refresh or project state in one mission cannot leak into another.
|
||||
".claude",
|
||||
".claude.json",
|
||||
// Per-CLI state for the alternate backends; small.
|
||||
".kimi-code",
|
||||
"glm-home",
|
||||
// The seeded agent library.
|
||||
"agents",
|
||||
];
|
||||
|
||||
/// Deliberately NOT copied — caches and toolchains, no secrets, expensive:
|
||||
/// `.rustup` (1.5 GB, vestigial), `.npm` (85 MB), `.cargo`, `.cache`,
|
||||
/// `.local`. A mission that needs them reads the image's copies.
|
||||
fn copy_script() -> String {
|
||||
let mut out = String::from("set -e\n");
|
||||
for p in SEEDED_PATHS {
|
||||
// Missing entries are normal — a fresh deployment has no .kimi-code
|
||||
// until Kimi is first used — so absence must not fail the copy.
|
||||
out.push_str(&format!(
|
||||
"if [ -e '/seed/{p}' ]; then cp -a '/seed/{p}' /dst/; fi\n"
|
||||
));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Give this mission its own copy of the runtime seed data.
|
||||
///
|
||||
/// Every per-mission container used to bind-mount the SAME host seed dir as
|
||||
/// `/zeroclaw-data` — shared with each other and with the singleton runtime.
|
||||
/// That directory holds `config.toml`, which carries the §15 door bearer
|
||||
/// token, plus `sessions.db` and `devices.db`. So a mission could read another
|
||||
/// mission's credential, and anything it wrote there was inherited by every
|
||||
/// later mission. Teardown never cleaned it, because teardown only removes
|
||||
/// `/var/lib/clawmates-missions/{id}`.
|
||||
///
|
||||
/// The code already knew: the comment on `DEFAULT_SEED_DIR` names the sqlite
|
||||
/// race and calls copy-on-write per mission the long-term fix. This is that.
|
||||
///
|
||||
/// The copy runs in a throwaway container because cm-api cannot see the seed
|
||||
/// dir — it hands that host path to Docker but never mounts it itself. The
|
||||
/// runtime image is reused so nothing extra is pulled.
|
||||
///
|
||||
/// Failure is fatal to container creation on purpose. Falling back to the
|
||||
/// shared mount would silently restore the credential-sharing this removes,
|
||||
/// and a silent fallback to a weaker posture is the failure mode this
|
||||
/// codebase keeps paying for.
|
||||
async fn seed_runtime_data(
|
||||
docker: &Docker,
|
||||
image: &str,
|
||||
seed_dir: &str,
|
||||
dest_dir: &str,
|
||||
) -> Result<(), String> {
|
||||
let name = format!("cm-seed-{}", Uuid::now_v7().simple());
|
||||
let config = ContainerCreateBody {
|
||||
image: Some(image.to_string()),
|
||||
entrypoint: Some(vec!["/bin/sh".to_string()]),
|
||||
cmd: Some(vec!["-c".to_string(), copy_script()]),
|
||||
host_config: Some(HostConfig {
|
||||
mounts: Some(vec![
|
||||
Mount {
|
||||
target: Some("/seed".to_string()),
|
||||
source: Some(seed_dir.to_string()),
|
||||
typ: Some(MountTypeEnum::BIND),
|
||||
read_only: Some(true),
|
||||
..Default::default()
|
||||
},
|
||||
Mount {
|
||||
target: Some("/dst".to_string()),
|
||||
source: Some(dest_dir.to_string()),
|
||||
typ: Some(MountTypeEnum::BIND),
|
||||
read_only: Some(false),
|
||||
..Default::default()
|
||||
},
|
||||
]),
|
||||
auto_remove: Some(true),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
docker
|
||||
.create_container(
|
||||
Some(CreateContainerOptions {
|
||||
name: Some(name.clone()),
|
||||
..Default::default()
|
||||
}),
|
||||
config,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("create seed copier: {e}"))?;
|
||||
docker
|
||||
.start_container(&name, None::<StartContainerOptions>)
|
||||
.await
|
||||
.map_err(|e| format!("start seed copier: {e}"))?;
|
||||
|
||||
// `auto_remove` means the container disappears the moment it exits, so
|
||||
// poll for absence rather than waiting on it.
|
||||
for _ in 0..120 {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
|
||||
match docker
|
||||
.inspect_container(&name, None::<InspectContainerOptions>)
|
||||
.await
|
||||
{
|
||||
Err(_) => return Ok(()),
|
||||
Ok(info) => {
|
||||
let running = info
|
||||
.state
|
||||
.as_ref()
|
||||
.and_then(|st| st.running)
|
||||
.unwrap_or(false);
|
||||
if !running {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(format!("seed copy into {dest_dir} did not finish in 30s"))
|
||||
}
|
||||
|
||||
/// Deterministic docker container name for a mission's runtime.
|
||||
/// Uses the full UUID hex — UUIDv7 encodes time in the leading bytes,
|
||||
/// so a short prefix isn't guaranteed unique across missions minted
|
||||
@@ -239,6 +397,12 @@ impl MissionRuntimeProvisioner {
|
||||
let _ = tokio::fs::create_dir_all(&mission_dir).await;
|
||||
let seed_dir = std::env::var("CLAWMATES_RUNTIME_SEED_DIR")
|
||||
.unwrap_or_else(|_| DEFAULT_SEED_DIR.to_string());
|
||||
// Per-mission copy of the seed data. See `seed_runtime_data`: sharing
|
||||
// one directory meant sharing the door token and letting any mission
|
||||
// poison every later one.
|
||||
let runtime_data_dir = format!("{mission_dir}/runtime-data");
|
||||
let _ = tokio::fs::create_dir_all(&runtime_data_dir).await;
|
||||
seed_runtime_data(&self.docker, &self.image, &seed_dir, &runtime_data_dir).await?;
|
||||
let mounts = vec![
|
||||
// Mount just this mission's directory. Agents can navigate
|
||||
// its `/repo` subdir but never see other missions'.
|
||||
@@ -249,14 +413,14 @@ impl MissionRuntimeProvisioner {
|
||||
read_only: Some(false),
|
||||
..Default::default()
|
||||
},
|
||||
// Share the shared-runtime data dir so this gateway inherits
|
||||
// the seeded agent library (`claw_*` templates). We then
|
||||
// mint a per-mission pairing code via /admin/paircode/new
|
||||
// below — the mint writes into the shared devices.db but
|
||||
// the resulting token is unique to this mission.
|
||||
// This mission's OWN copy of the seeded agent library
|
||||
// (`claw_*` templates). Copied rather than shared, so its
|
||||
// config.toml — which carries the door bearer token — and its
|
||||
// sqlite files belong to this mission alone and are removed with
|
||||
// it by `teardown_container`.
|
||||
Mount {
|
||||
target: Some("/zeroclaw-data".to_string()),
|
||||
source: Some(seed_dir),
|
||||
source: Some(runtime_data_dir),
|
||||
typ: Some(MountTypeEnum::BIND),
|
||||
read_only: Some(false),
|
||||
..Default::default()
|
||||
@@ -802,9 +966,38 @@ mod tests {
|
||||
it silently bills the API. Forwarded: {keys:?}"
|
||||
);
|
||||
// Unrelated providers have no subscription equivalent and must survive.
|
||||
for k in ["GEMINI_API_KEY", "GROQ_API_KEY", "OPENAI_API_KEY"] {
|
||||
for k in [
|
||||
"GEMINI_API_KEY",
|
||||
"GROQ_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"ZAI_API_KEY",
|
||||
"KIMI_API_KEY",
|
||||
] {
|
||||
assert!(keys.contains(&k), "{k} should still be forwarded");
|
||||
}
|
||||
// And the subscription credential MUST travel. A mission container
|
||||
// has its own data dir, so unlike the shared runtime it has no
|
||||
// persisted `claude /login` to fall back on. Without this the CLI
|
||||
// has no credential and simply hangs — a phase stuck at `running`
|
||||
// with nothing in the logs, which is exactly how this was found.
|
||||
assert!(
|
||||
keys.contains(&"CLAUDE_CODE_OAUTH_TOKEN"),
|
||||
"subscription mode must forward the token; without it `claude -p` \
|
||||
hangs with no credential. Forwarded: {keys:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// The two credentials must never travel together: Claude Code would pick
|
||||
/// the API key and bill it while the deployment believes it is on the
|
||||
/// subscription.
|
||||
#[test]
|
||||
fn the_two_anthropic_credentials_are_mutually_exclusive() {
|
||||
for mode in [RuntimeAuth::ApiKey, RuntimeAuth::Subscription] {
|
||||
let keys = forwarded_provider_keys(mode);
|
||||
let both = keys.contains(&"ANTHROPIC_API_KEY")
|
||||
&& keys.contains(&"CLAUDE_CODE_OAUTH_TOKEN");
|
||||
assert!(!both, "{mode:?} forwards both credentials: {keys:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Default behaviour is unchanged, so a deployment that never opts in keeps
|
||||
@@ -820,6 +1013,10 @@ mod tests {
|
||||
] {
|
||||
assert!(keys.contains(&k), "{k} should be forwarded in api_key mode");
|
||||
}
|
||||
assert!(
|
||||
!keys.contains(&"CLAUDE_CODE_OAUTH_TOKEN"),
|
||||
"api_key mode must not also ship the subscription token"
|
||||
);
|
||||
}
|
||||
|
||||
/// An unset or misspelled value must fall back to the *existing* behaviour.
|
||||
@@ -939,4 +1136,75 @@ allowed_tools = ["file_read", "file_edit"]
|
||||
let url = endpoint_url("cm-runtime-mission-abc");
|
||||
assert_eq!(url, "http://cm-runtime-mission-abc:42617");
|
||||
}
|
||||
|
||||
/// The seed data path must be per-mission, not the shared seed dir.
|
||||
///
|
||||
/// Every mission container used to bind the SAME host directory as
|
||||
/// `/zeroclaw-data`. It holds `config.toml`, which carries the §15 door
|
||||
/// bearer token, plus `sessions.db`/`devices.db`. Sharing it meant one
|
||||
/// mission could read another's credential, and anything written there
|
||||
/// was inherited by every later mission — `teardown_container` only
|
||||
/// removes `/var/lib/clawmates-missions/{id}`, so the shared dir was
|
||||
/// never cleaned.
|
||||
///
|
||||
/// Asserting the path shape is what keeps this from silently regressing:
|
||||
/// a future edit that points the mount back at the seed dir restores the
|
||||
/// credential sharing with no other visible symptom.
|
||||
#[test]
|
||||
fn runtime_data_is_scoped_to_one_mission() {
|
||||
let a = Uuid::now_v7();
|
||||
let b = Uuid::now_v7();
|
||||
let path = |id: Uuid| format!("{MISSIONS_HOST_ROOT}/{id}/runtime-data");
|
||||
|
||||
assert_ne!(path(a), path(b), "two missions must not share runtime data");
|
||||
assert!(
|
||||
path(a).starts_with(&format!("{MISSIONS_HOST_ROOT}/{a}")),
|
||||
"runtime data must live under the mission dir so teardown removes it"
|
||||
);
|
||||
assert_ne!(
|
||||
path(a),
|
||||
DEFAULT_SEED_DIR,
|
||||
"the mount must never be the shared seed dir itself"
|
||||
);
|
||||
assert!(
|
||||
!path(a).starts_with(DEFAULT_SEED_DIR),
|
||||
"runtime data must not live inside the shared seed dir either"
|
||||
);
|
||||
}
|
||||
|
||||
/// The copy must be an allow-list, and must include the secret-bearing
|
||||
/// paths while excluding the expensive ones.
|
||||
///
|
||||
/// Measured on gw-04: the seed dir is 1.7 GB, of which 1.5 GB is a
|
||||
/// vestigial `.rustup` that is not even on the container's PATH (the
|
||||
/// image ships Rust at /usr/local/cargo). Copying everything per mission
|
||||
/// would cost tens of seconds and ~17 GB across ten concurrent missions —
|
||||
/// which is what the first version of this did.
|
||||
#[test]
|
||||
fn the_seed_copy_takes_secrets_and_skips_caches() {
|
||||
// The two paths that carry the door bearer token MUST be copied, or
|
||||
// this whole change accomplishes nothing.
|
||||
assert!(SEEDED_PATHS.contains(&".zeroclaw"));
|
||||
assert!(SEEDED_PATHS.contains(&"clawmates-mcp.json"));
|
||||
// Claude Code's credentials and state.
|
||||
assert!(SEEDED_PATHS.contains(&".claude"));
|
||||
|
||||
// The expensive, secret-free ones must NOT be.
|
||||
for cache in [".rustup", ".npm", ".cargo", ".cache"] {
|
||||
assert!(
|
||||
!SEEDED_PATHS.contains(&cache),
|
||||
"{cache} is a cache and must not be copied per mission"
|
||||
);
|
||||
}
|
||||
|
||||
let script = copy_script();
|
||||
// A missing entry is normal on a fresh deployment (no .kimi-code
|
||||
// until Kimi is first used) and must not fail the copy.
|
||||
assert!(script.contains("if [ -e "), "absent paths must be tolerated");
|
||||
assert!(script.contains("/seed/.zeroclaw"));
|
||||
assert!(!script.contains("/seed/.rustup"));
|
||||
for p in SEEDED_PATHS {
|
||||
assert!(script.contains(p), "{p} missing from the copy script");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,7 +67,25 @@ pub async fn ensure_checkout(
|
||||
|
||||
let auth_url = with_ambient_auth(clone_url);
|
||||
if path.join(".git").exists() {
|
||||
fetch_and_reset(&path, default_branch, &auth_url).await?;
|
||||
// Checkouts cloned before this setting existed get it on reuse. It
|
||||
// governs objects created from now on, which is what delivery needs.
|
||||
share_repository_across_uids(&path);
|
||||
// `ensure_checkout` runs at every phase launch, not once per mission.
|
||||
// Freshening a pristine checkout is right; freshening one that already
|
||||
// holds this mission's work destroys it. See `has_local_work`.
|
||||
// Marker first: it is a fact we recorded, not a state we inferred.
|
||||
// The tree checks stay as a second line of defence for checkouts
|
||||
// created before the marker existed, and for the case where the
|
||||
// marker write itself failed.
|
||||
if checkout_in_use(&path) || has_local_work(&path, default_branch) {
|
||||
eprintln!(
|
||||
"mission_workspace: {} already holds mission work — skipping \
|
||||
fetch/reset so earlier phases' output survives",
|
||||
path.display()
|
||||
);
|
||||
} else {
|
||||
fetch_and_reset(&path, default_branch, &auth_url).await?;
|
||||
}
|
||||
} else {
|
||||
clone(&path, &auth_url).await?;
|
||||
}
|
||||
@@ -78,7 +96,7 @@ pub async fn ensure_checkout(
|
||||
/// environment, rewrite it to include the token as basic-auth. Returns
|
||||
/// the URL unchanged otherwise. The token is never logged (we only
|
||||
/// pass the rewritten URL into `git clone` via argv).
|
||||
fn with_ambient_auth(url: &str) -> String {
|
||||
pub(crate) fn with_ambient_auth(url: &str) -> String {
|
||||
let Ok(token) = std::env::var("GITEA_TOKEN") else {
|
||||
return url.to_string();
|
||||
};
|
||||
@@ -119,12 +137,170 @@ async fn clone(path: &std::path::Path, url: &str) -> Result<(), String> {
|
||||
.collect::<String>()
|
||||
));
|
||||
}
|
||||
share_repository_across_uids(path);
|
||||
scrub_remote_credentials(path, url);
|
||||
ignore_agent_scaffolding(path);
|
||||
record_base_commit(path);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Record that a phase has started working in this checkout.
|
||||
///
|
||||
/// The explicit half of the "is this checkout in use" question. `ensure_checkout`
|
||||
/// runs per phase launch and refreshes on reuse; whether that refresh is safe
|
||||
/// depends on whether a phase has already run here, which is a fact about the
|
||||
/// *mission* and not about the tree.
|
||||
///
|
||||
/// It was previously inferred from the tree — dirty status, HEAD versus the
|
||||
/// remote tip — and inference is what made delivery depend on what an agent
|
||||
/// happened to do. Mission `019fc444` lost work because its phase committed and
|
||||
/// left a clean tree; `019fc476` lost work because the capture base had advanced
|
||||
/// to match HEAD; `019fc450` survived only because a phase *failed* to commit
|
||||
/// and left the tree dirty. Same code, opposite outcomes, decided by the agent.
|
||||
///
|
||||
/// A marker is not a heuristic. Once a phase has begun, the checkout is in use
|
||||
/// until the mission ends, whatever the agent did or did not do inside it.
|
||||
pub(crate) fn mark_phase_started(path: &std::path::Path) {
|
||||
let marker = path.join(".git/clawmates-in-use");
|
||||
if marker.exists() {
|
||||
return;
|
||||
}
|
||||
if let Err(e) = std::fs::write(&marker, "1\n") {
|
||||
eprintln!(
|
||||
"mission_workspace: could not mark {} as in use ({e}) — a later phase may \
|
||||
refresh the checkout and discard earlier work",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Has a phase already started work in this checkout?
|
||||
fn checkout_in_use(path: &std::path::Path) -> bool {
|
||||
path.join(".git/clawmates-in-use").exists()
|
||||
}
|
||||
|
||||
/// Has anything happened in this checkout since it was created?
|
||||
///
|
||||
/// `ensure_checkout` is called once per *phase launch*, not once per mission,
|
||||
/// and its reuse path runs `git reset --hard origin/<branch>`. That is correct
|
||||
/// for a checkout being picked up cold and destructive for one mid-mission:
|
||||
/// mission `019fc444` had its phase-0 file deleted from the working tree when
|
||||
/// phase 1 started, so the second phase never saw the first's output.
|
||||
///
|
||||
/// Delivery is what made this reachable. Before the mission branch existed,
|
||||
/// agent output stayed *untracked* and `reset --hard` left it alone. Committing
|
||||
/// it — the whole point of the delivery slice — makes it tracked, and tracked
|
||||
/// files that are absent from `origin/<branch>` are exactly what a hard reset
|
||||
/// removes. The feature that preserves work is what put it in reach of the
|
||||
/// reset.
|
||||
///
|
||||
/// "Local work" is either a commit that is not on the fetched tip, or a dirty
|
||||
/// tree. Both are checked because the two phases of the failure look different:
|
||||
/// an agent that committed leaves a clean tree at a new HEAD, and one that did
|
||||
/// not leaves a dirty tree at the old HEAD.
|
||||
fn has_local_work(path: &std::path::Path, branch: &str) -> bool {
|
||||
let git = |args: &[&str]| -> Option<String> {
|
||||
let out = std::process::Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(path)
|
||||
.args(["-c", &format!("safe.directory={}", path.display())])
|
||||
.args(args)
|
||||
.output()
|
||||
.ok()?;
|
||||
out.status
|
||||
.success()
|
||||
.then(|| String::from_utf8_lossy(&out.stdout).trim().to_string())
|
||||
};
|
||||
|
||||
// A dirty tree is unambiguous: someone is mid-work here.
|
||||
if let Some(status) = git(&["status", "--porcelain"]) {
|
||||
if !status.is_empty() {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// Otherwise compare HEAD against the *remote tip*, which is the only
|
||||
// fixed point here.
|
||||
//
|
||||
// This deliberately does not use `.git/clawmates-base`. That marker is the
|
||||
// rolling capture base and `advance_base_commit` moves it to each phase's
|
||||
// committed head — so comparing HEAD against it asks "did anything happen
|
||||
// since the last commit we made", which is false immediately after every
|
||||
// successful delivery. Mission `019fc476` lost phase 0's file exactly that
|
||||
// way: phase 0 committed, the base advanced to match HEAD, and phase 1's
|
||||
// launch concluded the checkout was pristine and reset it. The preceding
|
||||
// mission survived only because its phase 0 *failed* to commit and left a
|
||||
// dirty tree.
|
||||
//
|
||||
// `origin/<branch>` does not move for the life of the mission, so "HEAD is
|
||||
// not the remote tip" means a phase committed, whether one commit ago or
|
||||
// five. If the remote ref cannot be resolved the answer is preserve:
|
||||
// wrongly skipping a refresh costs staleness, wrongly resetting destroys a
|
||||
// phase's output.
|
||||
match (
|
||||
git(&["rev-parse", &format!("origin/{branch}")]),
|
||||
git(&["rev-parse", "HEAD"]),
|
||||
) {
|
||||
(Some(tip), Some(head)) => tip != head,
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Let the server and the agent container both write to this checkout.
|
||||
///
|
||||
/// The checkout is one directory bind-mounted into two processes running as
|
||||
/// different users: cm-api is uid 65532, the mission runtime container is
|
||||
/// root. Git creates `.git/objects/xx/` fan-out directories on first write and
|
||||
/// they inherit the writer's ownership, so whichever party commits first locks
|
||||
/// the other out of that directory:
|
||||
///
|
||||
/// ```text
|
||||
/// git add → exit 128: insufficient permission for adding an object
|
||||
/// to repository database .git/objects
|
||||
/// ```
|
||||
///
|
||||
/// The failure is intermittent, which is what makes it dangerous. Mission
|
||||
/// `019fc42b` delivered cleanly because its agents committed their own work,
|
||||
/// so the blobs already existed and the server's `git add` never had to write
|
||||
/// one. Mission `019fc437` ran the same template, its agents left the work
|
||||
/// uncommitted, and delivery lost both phases.
|
||||
///
|
||||
/// `core.sharedRepository` is git's own answer to a repository shared between
|
||||
/// users: it makes git create objects and refs group- and world-writable. Both
|
||||
/// parties read this config from the shared `.git/config`, so it governs the
|
||||
/// agent's commits as much as ours.
|
||||
///
|
||||
/// This grants the agent no access it lacks. It is already root inside a
|
||||
/// container with the entire checkout bind-mounted read-write, and could
|
||||
/// rewrite any of it. The party actually gaining something is the server,
|
||||
/// which is currently the one being locked out.
|
||||
pub fn share_repository_across_uids(path: &std::path::Path) {
|
||||
let out = std::process::Command::new("git")
|
||||
.args([
|
||||
"-C",
|
||||
&path.display().to_string(),
|
||||
"-c",
|
||||
&format!("safe.directory={}", path.display()),
|
||||
"config",
|
||||
"core.sharedRepository",
|
||||
"0777",
|
||||
])
|
||||
.output();
|
||||
match out {
|
||||
Ok(o) if o.status.success() => {}
|
||||
Ok(o) => eprintln!(
|
||||
"mission_workspace: could not set core.sharedRepository on {} ({}) — delivery \
|
||||
may fail to commit if the agent writes git objects first",
|
||||
path.display(),
|
||||
String::from_utf8_lossy(&o.stderr).trim()
|
||||
),
|
||||
Err(e) => eprintln!(
|
||||
"mission_workspace: could not set core.sharedRepository on {} ({e})",
|
||||
path.display()
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Remember the commit the mission started from.
|
||||
///
|
||||
/// Delivery needs to answer "what did this mission change", and the obvious
|
||||
@@ -165,6 +341,31 @@ pub(crate) fn record_base_commit(path: &std::path::Path) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Move the capture base forward to a commit a phase just produced.
|
||||
///
|
||||
/// The base is recorded once at clone time, which is right for the mission's
|
||||
/// first phase and wrong for every phase after it: a later phase would diff
|
||||
/// against the original clone point and claim its predecessors' commits as its
|
||||
/// own work. Mission `019fc42b` showed this plainly — two coding phases, and
|
||||
/// the second phase's artifact reported the *union* of both phases' files.
|
||||
///
|
||||
/// Advancing after each successful commit makes each artifact the incremental
|
||||
/// work of one phase. The pushed branch stays cumulative, because it is built
|
||||
/// from `HEAD` and therefore still carries the earlier commits.
|
||||
pub(crate) fn advance_base_commit(path: &std::path::Path, sha: &str) {
|
||||
let sha = sha.trim();
|
||||
if sha.is_empty() {
|
||||
return;
|
||||
}
|
||||
if let Err(e) = std::fs::write(path.join(".git/clawmates-base"), format!("{sha}\n")) {
|
||||
eprintln!(
|
||||
"mission_workspace: could not advance base commit for {} ({e}) — the next \
|
||||
phase will re-report this phase's work as its own",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The commit this mission's checkout started from, if it was recorded.
|
||||
pub(crate) fn base_commit(path: &std::path::Path) -> Option<String> {
|
||||
std::fs::read_to_string(path.join(".git/clawmates-base"))
|
||||
@@ -190,7 +391,7 @@ pub(crate) fn base_commit(path: &std::path::Path) -> Option<String> {
|
||||
/// Best-effort and non-fatal: a checkout that keeps its token still works, and
|
||||
/// failing the mission over it would trade a real capability for a marginal
|
||||
/// improvement in a situation we have already logged.
|
||||
fn scrub_remote_credentials(path: &std::path::Path, original_url: &str) {
|
||||
pub(crate) fn scrub_remote_credentials(path: &std::path::Path, original_url: &str) {
|
||||
if !original_url.contains('@') && !original_url.contains("oauth2:") {
|
||||
// Nothing was injected (SSH remote, or no token configured).
|
||||
return;
|
||||
@@ -294,7 +495,7 @@ fn ignore_agent_scaffolding(path: &std::path::Path) {
|
||||
}
|
||||
}
|
||||
|
||||
fn redact_token(s: &str) -> String {
|
||||
pub(crate) fn redact_token(s: &str) -> String {
|
||||
// Strip any "oauth2:<token>@" segment that git may echo back on
|
||||
// failures. Belt-and-braces: also nuke any raw token env value.
|
||||
let mut out = s.to_string();
|
||||
@@ -460,4 +661,164 @@ mod tests {
|
||||
);
|
||||
assert!(body.contains("/AGENTS.md"));
|
||||
}
|
||||
|
||||
/// Seed a checkout that has an `origin`, like a real clone does. Without
|
||||
/// one `origin/<branch>` does not resolve and `has_local_work` takes its
|
||||
/// preserve-by-default path, which would make the pristine case untestable.
|
||||
fn seed(dir: &std::path::Path, remote: &std::path::Path) {
|
||||
std::process::Command::new("git")
|
||||
.args(["init", "--quiet", "--bare"])
|
||||
.arg(remote)
|
||||
.output()
|
||||
.unwrap();
|
||||
let g = |args: &[&str]| {
|
||||
std::process::Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(dir)
|
||||
.args(args)
|
||||
.output()
|
||||
.unwrap();
|
||||
};
|
||||
g(&["init", "--quiet"]);
|
||||
g(&["config", "user.email", "[email protected]"]);
|
||||
g(&["config", "user.name", "T"]);
|
||||
g(&["checkout", "-q", "-B", "main"]);
|
||||
std::fs::write(dir.join("README.md"), "# base\n").unwrap();
|
||||
g(&["add", "."]);
|
||||
g(&["commit", "--quiet", "-m", "base"]);
|
||||
g(&["remote", "add", "origin", &remote.display().to_string()]);
|
||||
g(&["push", "--quiet", "origin", "main"]);
|
||||
g(&["fetch", "--quiet", "origin", "main"]);
|
||||
record_base_commit(dir);
|
||||
}
|
||||
|
||||
/// A checkout mid-mission must not be mistaken for a cold one.
|
||||
///
|
||||
/// `ensure_checkout` runs per phase launch and resets on the reuse path.
|
||||
/// Mission `019fc444` lost phase 0's committed file that way. The fix then
|
||||
/// failed again on mission `019fc476` for a different reason, which the
|
||||
/// last case here pins down.
|
||||
#[test]
|
||||
fn local_work_is_recognized_before_a_checkout_is_reset() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let repo = &tmp.path().join("repo");
|
||||
std::fs::create_dir_all(repo).unwrap();
|
||||
seed(repo, &tmp.path().join("remote.git"));
|
||||
let repo = repo.as_path();
|
||||
|
||||
assert!(
|
||||
!has_local_work(repo, "main"),
|
||||
"a freshly cloned checkout has no work and may be refreshed"
|
||||
);
|
||||
|
||||
// An agent that wrote files and did not commit: dirty tree, HEAD put.
|
||||
std::fs::write(repo.join("ALPHA.md"), "ALPHA\n").unwrap();
|
||||
assert!(has_local_work(repo, "main"), "uncommitted agent output is work");
|
||||
|
||||
// An agent (or delivery) that committed: clean tree, HEAD moved. This
|
||||
// is the shape that was destroyed on 019fc444, because a hard reset
|
||||
// leaves untracked files alone but removes tracked ones.
|
||||
git_in(repo, &["add", "ALPHA.md"]);
|
||||
git_in(repo, &["commit", "--quiet", "-m", "phase 0"]);
|
||||
let status = std::process::Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(repo)
|
||||
.args(["status", "--porcelain"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
String::from_utf8_lossy(&status.stdout).trim().is_empty(),
|
||||
"the commit left a clean tree — the case a dirty-tree check misses"
|
||||
);
|
||||
assert!(
|
||||
has_local_work(repo, "main"),
|
||||
"committed phase output must not be reset away"
|
||||
);
|
||||
|
||||
// The regression from 019fc476. Delivery advances the capture base to
|
||||
// the commit it just made, so any check comparing HEAD against that
|
||||
// base reports "nothing happened" the instant a phase succeeds — and
|
||||
// the next phase resets the work away. Advancing it here is what makes
|
||||
// this a real reproduction rather than a restatement of the case above.
|
||||
let head = std::process::Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(repo)
|
||||
.args(["rev-parse", "HEAD"])
|
||||
.output()
|
||||
.unwrap();
|
||||
let head = String::from_utf8_lossy(&head.stdout).trim().to_string();
|
||||
advance_base_commit(repo, &head);
|
||||
assert_eq!(
|
||||
base_commit(repo).as_deref(),
|
||||
Some(head.as_str()),
|
||||
"the base now equals HEAD, which is the trap"
|
||||
);
|
||||
assert!(
|
||||
has_local_work(repo, "main"),
|
||||
"a phase that committed successfully must still count as work \
|
||||
after the capture base advances to match its commit"
|
||||
);
|
||||
}
|
||||
|
||||
fn git_in(dir: &std::path::Path, args: &[&str]) {
|
||||
std::process::Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(dir)
|
||||
.args(args)
|
||||
.output()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// A checkout in use must be recognised regardless of what the agent did.
|
||||
///
|
||||
/// This is the Seam-1 property. The tree-state heuristics were each correct
|
||||
/// in isolation and each blind to a different case: `019fc444` committed
|
||||
/// and left a clean tree, `019fc476` had its base advanced to match HEAD,
|
||||
/// `019fc450` survived only because a phase FAILED to commit. Whether the
|
||||
/// work survived was decided by the agent, not by us.
|
||||
///
|
||||
/// The marker is set when a phase launches, before the agent does anything,
|
||||
/// so every one of those states answers the same way.
|
||||
#[test]
|
||||
fn an_in_use_checkout_is_recognized_whatever_the_agent_did() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let repo = &tmp.path().join("repo");
|
||||
std::fs::create_dir_all(repo).unwrap();
|
||||
seed(repo, &tmp.path().join("remote.git"));
|
||||
let repo = repo.as_path();
|
||||
|
||||
assert!(!checkout_in_use(repo), "a fresh clone is not in use");
|
||||
|
||||
mark_phase_started(repo);
|
||||
assert!(checkout_in_use(repo), "a launched phase marks the checkout");
|
||||
|
||||
// The three production states, all of which must now answer the same.
|
||||
// (a) agent wrote nothing at all — the case every tree heuristic misses.
|
||||
assert!(checkout_in_use(repo), "clean tree at the base commit");
|
||||
|
||||
// (b) agent committed, leaving a clean tree at a moved HEAD.
|
||||
std::fs::write(repo.join("WORK.md"), "work\n").unwrap();
|
||||
git_in(repo, &["add", "WORK.md"]);
|
||||
git_in(repo, &["commit", "--quiet", "-m", "phase work"]);
|
||||
let head = std::process::Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(repo)
|
||||
.args(["rev-parse", "HEAD"])
|
||||
.output()
|
||||
.unwrap();
|
||||
let head = String::from_utf8_lossy(&head.stdout).trim().to_string();
|
||||
assert!(checkout_in_use(repo));
|
||||
|
||||
// (c) capture advanced the base to match HEAD — the collision that
|
||||
// defeated the HEAD-versus-base check on 019fc476.
|
||||
advance_base_commit(repo, &head);
|
||||
assert!(
|
||||
checkout_in_use(repo),
|
||||
"an advanced base must not make an in-use checkout look pristine"
|
||||
);
|
||||
|
||||
// Marking twice is safe; phases launch repeatedly across a mission.
|
||||
mark_phase_started(repo);
|
||||
assert!(checkout_in_use(repo));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
//! Finding papers, shelving them, and cataloguing them.
|
||||
//!
|
||||
//! The library has three parts and it matters which is which:
|
||||
//!
|
||||
//! - **arXiv** is where papers are *found*.
|
||||
//! - **The blob store** is the *shelf* — the PDF itself lives there.
|
||||
//! - **The vault** is the *card catalogue* — a markdown note per paper, with
|
||||
//! the metadata and a pointer to the shelf.
|
||||
//!
|
||||
//! Plus [`crate::corpus`], which is the list of checkmarks: it is what stops
|
||||
//! the same paper being fetched twice across weekly runs. That list is the
|
||||
//! reason this can be a *continuous* job rather than one that redoes itself
|
||||
//! forever — the failure that killed the previous attempt at this (migrations
|
||||
//! 0030-0044, dropped in 0053).
|
||||
//!
|
||||
//! # The contract that ties it together
|
||||
//!
|
||||
//! Every note this module writes carries `source_id: arxiv:NNNN.NNNNN` in its
|
||||
//! frontmatter. `corpus::parse_note` reads exactly that key, so re-indexing
|
||||
//! the vault re-derives the checkmark list from the notes themselves. The
|
||||
//! catalogue is authoritative; the index is rebuildable from it. If the
|
||||
//! database were lost, a re-index of the vault would restore what we have.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// One paper as arXiv describes it.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct Paper {
|
||||
/// Bare arXiv id, e.g. `2401.12345` — no version suffix.
|
||||
pub arxiv_id: String,
|
||||
pub title: String,
|
||||
pub authors: Vec<String>,
|
||||
pub summary: String,
|
||||
pub published: String,
|
||||
pub pdf_url: String,
|
||||
}
|
||||
|
||||
impl Paper {
|
||||
/// The checkmark key. Version suffixes are stripped upstream so `v1` and
|
||||
/// `v2` of the same paper are one entry, not two.
|
||||
pub fn source_id(&self) -> String {
|
||||
format!("arxiv:{}", self.arxiv_id)
|
||||
}
|
||||
|
||||
/// Where the PDF is shelved in the blob store.
|
||||
pub fn blob_key(&self) -> String {
|
||||
format!("papers/arxiv/{}.pdf", self.arxiv_id)
|
||||
}
|
||||
|
||||
/// Where the catalogue note goes in the vault.
|
||||
///
|
||||
/// Under a dedicated folder so the library never collides with the
|
||||
/// hand-written parts of the vault (`30 Resources`, `40 Projects`, and so
|
||||
/// on). A human should always be able to tell which notes a machine wrote.
|
||||
pub fn note_path(&self) -> String {
|
||||
format!("60 Papers/arxiv-{}.md", self.arxiv_id)
|
||||
}
|
||||
}
|
||||
|
||||
/// Strip an arXiv version suffix: `2401.12345v3` -> `2401.12345`.
|
||||
///
|
||||
/// Without this a weekly job re-downloads a paper every time the authors post
|
||||
/// a revision, and the checkmark list quietly fills with near-duplicates.
|
||||
pub fn normalize_arxiv_id(raw: &str) -> String {
|
||||
let id = raw.rsplit('/').next().unwrap_or(raw);
|
||||
match id.find('v') {
|
||||
// Only a trailing `vN` counts; the `v` in a word must not truncate.
|
||||
Some(i) if id[i + 1..].chars().all(|c| c.is_ascii_digit()) && i + 1 < id.len() => {
|
||||
id[..i].to_string()
|
||||
}
|
||||
_ => id.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse arXiv's Atom feed.
|
||||
///
|
||||
/// Hand-rolled rather than pulling an XML crate: the feed is a fixed, simple
|
||||
/// shape and this reads five fields from it. If arXiv's format ever drifts,
|
||||
/// `entries_are_parsed_from_a_real_feed` fails loudly rather than silently
|
||||
/// returning zero papers — which is the failure mode that matters, because a
|
||||
/// search returning nothing looks exactly like "no new papers this week".
|
||||
pub fn parse_atom(xml: &str) -> Vec<Paper> {
|
||||
let mut out = Vec::new();
|
||||
for chunk in xml.split("<entry>").skip(1) {
|
||||
let entry = chunk.split("</entry>").next().unwrap_or(chunk);
|
||||
let field = |tag: &str| -> Option<String> {
|
||||
let open = format!("<{tag}>");
|
||||
let close = format!("</{tag}>");
|
||||
let start = entry.find(&open)? + open.len();
|
||||
let end = entry[start..].find(&close)? + start;
|
||||
Some(unescape(entry[start..end].trim()))
|
||||
};
|
||||
|
||||
let Some(raw_id) = field("id") else { continue };
|
||||
let arxiv_id = normalize_arxiv_id(&raw_id);
|
||||
if arxiv_id.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let Some(title) = field("title") else { continue };
|
||||
|
||||
let authors = entry
|
||||
.split("<author>")
|
||||
.skip(1)
|
||||
.filter_map(|a| {
|
||||
let start = a.find("<name>")? + 6;
|
||||
let end = a[start..].find("</name>")? + start;
|
||||
Some(unescape(a[start..end].trim()))
|
||||
})
|
||||
.collect();
|
||||
|
||||
// The PDF link is an attribute, not an element.
|
||||
let pdf_url = entry
|
||||
.split("<link")
|
||||
.find(|l| l.contains("title=\"pdf\""))
|
||||
.and_then(|l| {
|
||||
let start = l.find("href=\"")? + 6;
|
||||
let end = l[start..].find('"')? + start;
|
||||
Some(l[start..end].to_string())
|
||||
})
|
||||
.unwrap_or_else(|| format!("https://arxiv.org/pdf/{arxiv_id}"));
|
||||
|
||||
out.push(Paper {
|
||||
title: title.split_whitespace().collect::<Vec<_>>().join(" "),
|
||||
summary: field("summary")
|
||||
.unwrap_or_default()
|
||||
.split_whitespace()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" "),
|
||||
published: field("published").unwrap_or_default(),
|
||||
authors,
|
||||
pdf_url,
|
||||
arxiv_id,
|
||||
});
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn unescape(s: &str) -> String {
|
||||
s.replace("&", "&")
|
||||
.replace("<", "<")
|
||||
.replace(">", ">")
|
||||
.replace(""", "\"")
|
||||
.replace("'", "'")
|
||||
}
|
||||
|
||||
/// Search arXiv. `max_results` is capped to keep one run bounded.
|
||||
pub async fn search(query: &str, max_results: usize) -> Result<Vec<Paper>, String> {
|
||||
let max = max_results.clamp(1, 50);
|
||||
let url = format!(
|
||||
"https://export.arxiv.org/api/query?search_query={}&start=0&max_results={max}\
|
||||
&sortBy=submittedDate&sortOrder=descending",
|
||||
urlencoding(query)
|
||||
);
|
||||
let body = reqwest::Client::new()
|
||||
.get(&url)
|
||||
.header("User-Agent", "clawmates-papers/0.1 (research library)")
|
||||
.timeout(std::time::Duration::from_secs(60))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("arxiv query: {e}"))?
|
||||
.text()
|
||||
.await
|
||||
.map_err(|e| format!("arxiv body: {e}"))?;
|
||||
Ok(parse_atom(&body))
|
||||
}
|
||||
|
||||
/// Download the PDF. Returns the bytes; the caller decides where to shelve it.
|
||||
pub async fn fetch_pdf(paper: &Paper) -> Result<Vec<u8>, String> {
|
||||
let bytes = reqwest::Client::new()
|
||||
.get(&paper.pdf_url)
|
||||
.header("User-Agent", "clawmates-papers/0.1 (research library)")
|
||||
.timeout(std::time::Duration::from_secs(180))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("fetch pdf {}: {e}", paper.arxiv_id))?
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| format!("read pdf {}: {e}", paper.arxiv_id))?;
|
||||
|
||||
// A PDF starts with `%PDF`. arXiv serves an HTML holding page when a PDF
|
||||
// is still rendering, and shelving that would leave a file that looks
|
||||
// present and is unreadable.
|
||||
if !bytes.starts_with(b"%PDF") {
|
||||
return Err(format!(
|
||||
"{} did not return a PDF ({} bytes, starts {:?})",
|
||||
paper.pdf_url,
|
||||
bytes.len(),
|
||||
String::from_utf8_lossy(&bytes[..bytes.len().min(16)])
|
||||
));
|
||||
}
|
||||
Ok(bytes.to_vec())
|
||||
}
|
||||
|
||||
/// The catalogue note for a shelved paper.
|
||||
///
|
||||
/// `source_id` in the frontmatter is the load-bearing part — it is what
|
||||
/// `corpus::parse_note` reads to rebuild the checkmark list from the vault.
|
||||
pub fn catalogue_note(paper: &Paper, blob_key: &str) -> String {
|
||||
let authors = if paper.authors.is_empty() {
|
||||
"unknown".to_string()
|
||||
} else {
|
||||
paper.authors.join(", ")
|
||||
};
|
||||
format!(
|
||||
"---\n\
|
||||
source_id: arxiv:{id}\n\
|
||||
arxiv: {id}\n\
|
||||
title: \"{title}\"\n\
|
||||
authors: \"{authors}\"\n\
|
||||
published: {published}\n\
|
||||
pdf: {blob_key}\n\
|
||||
url: https://arxiv.org/abs/{id}\n\
|
||||
added: {added}\n\
|
||||
tags: [paper, arxiv]\n\
|
||||
---\n\
|
||||
\n\
|
||||
# {title}\n\
|
||||
\n\
|
||||
**Authors:** {authors} \n\
|
||||
**arXiv:** [{id}](https://arxiv.org/abs/{id}) \n\
|
||||
**PDF:** `{blob_key}`\n\
|
||||
\n\
|
||||
## Abstract\n\
|
||||
\n\
|
||||
{summary}\n\
|
||||
\n\
|
||||
## Notes\n\
|
||||
\n\
|
||||
_Catalogued automatically. Add your own notes below._\n",
|
||||
id = paper.arxiv_id,
|
||||
title = paper.title.replace('"', "'"),
|
||||
authors = authors,
|
||||
published = paper.published,
|
||||
blob_key = blob_key,
|
||||
added = paper.published,
|
||||
summary = paper.summary,
|
||||
)
|
||||
}
|
||||
|
||||
fn urlencoding(s: &str) -> String {
|
||||
s.bytes()
|
||||
.map(|b| match b {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
|
||||
(b as char).to_string()
|
||||
}
|
||||
b' ' => "+".to_string(),
|
||||
_ => format!("%{b:02X}"),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// A revision must not read as a new paper.
|
||||
#[test]
|
||||
fn version_suffixes_are_stripped() {
|
||||
assert_eq!(normalize_arxiv_id("http://arxiv.org/abs/2401.12345v3"), "2401.12345");
|
||||
assert_eq!(normalize_arxiv_id("2401.12345v1"), "2401.12345");
|
||||
assert_eq!(normalize_arxiv_id("2401.12345"), "2401.12345");
|
||||
// Old-style ids contain letters and a slash.
|
||||
assert_eq!(normalize_arxiv_id("http://arxiv.org/abs/cs/0701001"), "0701001");
|
||||
// A trailing `v` with no digits is part of the id, not a version.
|
||||
assert_eq!(normalize_arxiv_id("2401.1234v"), "2401.1234v");
|
||||
}
|
||||
|
||||
/// Parsed against the real shape of arXiv's Atom feed. If this fails the
|
||||
/// format drifted — which otherwise shows up as "no new papers", which is
|
||||
/// indistinguishable from a quiet week.
|
||||
#[test]
|
||||
fn entries_are_parsed_from_a_real_feed() {
|
||||
let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
|
||||
<feed xmlns="http://www.w3.org/2005/Atom">
|
||||
<entry>
|
||||
<id>http://arxiv.org/abs/2401.12345v2</id>
|
||||
<published>2026-01-15T10:00:00Z</published>
|
||||
<title>Attention Is All You Need Again</title>
|
||||
<summary> We show that
|
||||
attention still works. </summary>
|
||||
<author><name>Ada Lovelace</name></author>
|
||||
<author><name>Alan Turing</name></author>
|
||||
<link href="http://arxiv.org/abs/2401.12345v2" rel="alternate" type="text/html"/>
|
||||
<link title="pdf" href="http://arxiv.org/pdf/2401.12345v2" rel="related" type="application/pdf"/>
|
||||
</entry>
|
||||
</feed>"#;
|
||||
let papers = parse_atom(xml);
|
||||
assert_eq!(papers.len(), 1);
|
||||
let p = &papers[0];
|
||||
assert_eq!(p.arxiv_id, "2401.12345", "version stripped");
|
||||
assert_eq!(p.title, "Attention Is All You Need Again", "whitespace collapsed");
|
||||
assert_eq!(p.summary, "We show that attention still works.");
|
||||
assert_eq!(p.authors, vec!["Ada Lovelace", "Alan Turing"]);
|
||||
assert_eq!(p.pdf_url, "http://arxiv.org/pdf/2401.12345v2");
|
||||
assert_eq!(p.source_id(), "arxiv:2401.12345");
|
||||
assert_eq!(p.blob_key(), "papers/arxiv/2401.12345.pdf");
|
||||
assert_eq!(p.note_path(), "60 Papers/arxiv-2401.12345.md");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_feed_yields_no_papers_rather_than_panicking() {
|
||||
assert!(parse_atom("<feed></feed>").is_empty());
|
||||
assert!(parse_atom("").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xml_entities_are_unescaped() {
|
||||
let xml = r#"<feed><entry><id>http://arxiv.org/abs/1v1</id>
|
||||
<title>Cats & Dogs <3</title><summary>a "quote"</summary>
|
||||
</entry></feed>"#;
|
||||
let p = &parse_atom(xml)[0];
|
||||
assert_eq!(p.title, "Cats & Dogs <3");
|
||||
assert_eq!(p.summary, "a \"quote\"");
|
||||
}
|
||||
|
||||
/// The note must carry the identity `corpus::parse_note` reads, or the
|
||||
/// catalogue cannot rebuild the checkmark list and the library forgets
|
||||
/// itself the moment the database is lost.
|
||||
#[test]
|
||||
fn a_catalogue_note_round_trips_through_the_corpus_parser() {
|
||||
let paper = Paper {
|
||||
arxiv_id: "2401.12345".into(),
|
||||
title: "A \"Quoted\" Title".into(),
|
||||
authors: vec!["Ada Lovelace".into()],
|
||||
summary: "Summary text.".into(),
|
||||
published: "2026-01-15T10:00:00Z".into(),
|
||||
pdf_url: "http://arxiv.org/pdf/2401.12345".into(),
|
||||
};
|
||||
let note = catalogue_note(&paper, &paper.blob_key());
|
||||
|
||||
let parsed = crate::corpus::parse_note(&paper.note_path(), ¬e);
|
||||
assert_eq!(
|
||||
parsed.declared_source_id.as_deref(),
|
||||
Some("arxiv:2401.12345"),
|
||||
"the corpus parser must recover the identity from the note"
|
||||
);
|
||||
assert_eq!(parsed.title.as_deref(), Some("A 'Quoted' Title"));
|
||||
assert!(note.contains("papers/arxiv/2401.12345.pdf"), "note points at the shelf");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn queries_are_url_encoded() {
|
||||
assert_eq!(urlencoding("all:agent topologies"), "all%3Aagent+topologies");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
//! Which phase-config keys the platform actually reads.
|
||||
//!
|
||||
//! `mission_phases.config` is free-form JSONB written by workflow recipes, the
|
||||
//! mission wizard and the API. Nothing connected a key to the code that reads
|
||||
//! it, so a key could be accepted, validated, stored, rendered — and consumed
|
||||
//! by nobody.
|
||||
//!
|
||||
//! `task` was exactly that. Every phase of every mission received identical
|
||||
//! instructions because the runner selected only the mission description; the
|
||||
//! per-phase task sat in Postgres unread. Mission `019fc42b` is what surfaced
|
||||
//! it: two coding phases with different `task` values produced the same two
|
||||
//! files. There was no error, because there is nothing to fail — an unread key
|
||||
//! is indistinguishable from a key whose value happens not to matter.
|
||||
//!
|
||||
//! This module is the missing link. Every key here names the code that reads
|
||||
//! it, `unknown_keys` reports anything else, and a test asserts the shipped
|
||||
//! recipes only write keys that exist. It cannot make a reader appear, but it
|
||||
//! makes an absent one visible.
|
||||
|
||||
/// A phase-config key and where it is consumed.
|
||||
pub struct KnownKey {
|
||||
pub key: &'static str,
|
||||
/// The code path that reads it. Kept as prose so this survives refactors
|
||||
/// that a symbol reference would not.
|
||||
pub read_by: &'static str,
|
||||
}
|
||||
|
||||
/// Keys with a reader in the current build.
|
||||
///
|
||||
/// Adding a key here without a reader defeats the purpose. The rule is: a key
|
||||
/// earns its entry when something consumes it, not when something writes it.
|
||||
pub const KNOWN_KEYS: &[KnownKey] = &[
|
||||
KnownKey {
|
||||
key: "done_when",
|
||||
read_by: "cm_db::repo::missions::create — promoted to the done_when column, \
|
||||
swept by phase_runner::evaluate_finished_phases",
|
||||
},
|
||||
KnownKey {
|
||||
key: "max_iterations",
|
||||
read_by: "cm_db::repo::missions::create — promoted to the max_iterations column",
|
||||
},
|
||||
KnownKey {
|
||||
key: "task",
|
||||
read_by: "phase_runner::start_pending_phases — injected by phase_task_text",
|
||||
},
|
||||
KnownKey {
|
||||
key: "commit_policy",
|
||||
read_by: "mission_delivery::Gate::parse — selects the delivery gate",
|
||||
},
|
||||
];
|
||||
|
||||
/// Keys a recipe may carry that are deliberately not consumed *yet*.
|
||||
///
|
||||
/// Distinguished from unknown keys so the report stays useful: these are known
|
||||
/// gaps with an owner, not typos. Every one is a feature described in a shipped
|
||||
/// workflow recipe whose implementation does not exist — which is worth seeing
|
||||
/// listed, because a recipe promising `loop = "until_done"` reads to an
|
||||
/// operator like something that loops.
|
||||
pub const DECLARED_BUT_UNREAD: &[KnownKey] = &[
|
||||
KnownKey {
|
||||
key: "loop",
|
||||
read_by: "NOT IMPLEMENTED — phase iteration uses max_iterations + done_when",
|
||||
},
|
||||
KnownKey {
|
||||
key: "produces",
|
||||
read_by: "NOT IMPLEMENTED — artifact rendering is not driven by this",
|
||||
},
|
||||
KnownKey {
|
||||
key: "input_from_phase",
|
||||
read_by: "NOT IMPLEMENTED — phases share a checkout, not declared inputs",
|
||||
},
|
||||
KnownKey {
|
||||
key: "mode",
|
||||
read_by: "NOT IMPLEMENTED — benchmark/refactor mode selection",
|
||||
},
|
||||
KnownKey {
|
||||
key: "harness",
|
||||
read_by: "NOT IMPLEMENTED — benchmark harness selection",
|
||||
},
|
||||
KnownKey {
|
||||
key: "tools",
|
||||
read_by: "NOT IMPLEMENTED — per-phase tool selection",
|
||||
},
|
||||
KnownKey {
|
||||
key: "benchmark",
|
||||
read_by: "NOT IMPLEMENTED — nested benchmark settings",
|
||||
},
|
||||
KnownKey {
|
||||
key: "mcp_bundles",
|
||||
read_by: "NOT IMPLEMENTED at phase level — bundles come from the TEAM \
|
||||
template (mission_orchestrator binds template.mcp_bundles) and \
|
||||
runtime_provision writes agents.<alias>.mcp_bundles. A recipe \
|
||||
setting this per phase changes nothing: security_hardening.toml \
|
||||
asks for gitea_forge + security_scan and its phase gets neither",
|
||||
},
|
||||
KnownKey {
|
||||
key: "test_command",
|
||||
read_by: "NOT IMPLEMENTED — mission_delivery::discover_test_command infers \
|
||||
from the repo and does not consult config",
|
||||
},
|
||||
];
|
||||
|
||||
fn is_listed(key: &str, list: &[KnownKey]) -> bool {
|
||||
list.iter().any(|k| k.key == key)
|
||||
}
|
||||
|
||||
/// Keys in this config that no code reads and that are not known gaps.
|
||||
///
|
||||
/// Almost always a typo or a setting invented for a feature that was never
|
||||
/// built. Returned rather than rejected: a mission whose config carries an
|
||||
/// unread key is not *wrong*, it is just doing less than its author believes,
|
||||
/// and failing the request would break recipes that already ship these.
|
||||
pub fn unknown_keys(config: &serde_json::Value) -> Vec<String> {
|
||||
let Some(obj) = config.as_object() else {
|
||||
return Vec::new();
|
||||
};
|
||||
obj.keys()
|
||||
.filter(|k| !is_listed(k, KNOWN_KEYS) && !is_listed(k, DECLARED_BUT_UNREAD))
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Keys that are recognised but that nothing consumes.
|
||||
pub fn inert_keys(config: &serde_json::Value) -> Vec<String> {
|
||||
let Some(obj) = config.as_object() else {
|
||||
return Vec::new();
|
||||
};
|
||||
obj.keys()
|
||||
.filter(|k| is_listed(k, DECLARED_BUT_UNREAD))
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Log what a phase's config asked for that will not happen.
|
||||
///
|
||||
/// Called once per phase at mission creation. Deliberately not an error: the
|
||||
/// point is that the author's intent and the platform's behaviour have
|
||||
/// diverged, and the author should be able to see that without being blocked.
|
||||
pub fn report(kind: &str, order_idx: i32, config: &serde_json::Value) {
|
||||
let unknown = unknown_keys(config);
|
||||
if !unknown.is_empty() {
|
||||
eprintln!(
|
||||
"phase_config: phase {order_idx} ({kind}) sets unrecognised key(s) {} — \
|
||||
nothing reads them; check for a typo",
|
||||
unknown.join(", ")
|
||||
);
|
||||
}
|
||||
let inert = inert_keys(config);
|
||||
if !inert.is_empty() {
|
||||
eprintln!(
|
||||
"phase_config: phase {order_idx} ({kind}) sets {} — recognised but NOT \
|
||||
IMPLEMENTED, so it will have no effect on this run",
|
||||
inert.join(", ")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_key_cannot_be_both_read_and_unread() {
|
||||
for k in KNOWN_KEYS {
|
||||
assert!(
|
||||
!is_listed(k.key, DECLARED_BUT_UNREAD),
|
||||
"{} is listed as both read and unread",
|
||||
k.key
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_known_key_names_its_reader() {
|
||||
for k in KNOWN_KEYS {
|
||||
assert!(
|
||||
!k.read_by.is_empty() && !k.read_by.starts_with("NOT IMPLEMENTED"),
|
||||
"{} claims to be read but names no reader",
|
||||
k.key
|
||||
);
|
||||
}
|
||||
for k in DECLARED_BUT_UNREAD {
|
||||
assert!(
|
||||
k.read_by.starts_with("NOT IMPLEMENTED"),
|
||||
"{} is listed as unread but names a reader — promote it to KNOWN_KEYS",
|
||||
k.key
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The regression that motivated the module: `task` must stay claimed.
|
||||
#[test]
|
||||
fn the_per_phase_task_key_has_a_reader() {
|
||||
assert!(
|
||||
is_listed("task", KNOWN_KEYS),
|
||||
"task lost its reader again — every phase will get identical instructions"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_and_inert_keys_are_reported_separately() {
|
||||
let cfg = serde_json::json!({
|
||||
"done_when": "tests pass",
|
||||
"loop": "until_done",
|
||||
"typpo": true,
|
||||
});
|
||||
assert_eq!(unknown_keys(&cfg), vec!["typpo".to_string()]);
|
||||
assert_eq!(inert_keys(&cfg), vec!["loop".to_string()]);
|
||||
}
|
||||
|
||||
/// Every key the shipped workflow recipes write must be accounted for.
|
||||
///
|
||||
/// This is the CI-time half: a recipe that invents `comit_policy` should
|
||||
/// fail here rather than run a mission whose gate silently defaults.
|
||||
#[test]
|
||||
fn shipped_recipes_only_write_accounted_keys() {
|
||||
let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../../templates/workflows");
|
||||
let Ok(entries) = std::fs::read_dir(dir) else {
|
||||
return; // templates not present in this build context
|
||||
};
|
||||
// Keys that belong to the recipe/phase envelope rather than to the
|
||||
// phase config blob itself.
|
||||
const ENVELOPE: &[&str] = &[
|
||||
"key",
|
||||
"name",
|
||||
"title",
|
||||
"blurb",
|
||||
"kind",
|
||||
"order_idx",
|
||||
"requires_repo",
|
||||
"default_team_template",
|
||||
"default_topology",
|
||||
"phases",
|
||||
"description",
|
||||
];
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.extension().and_then(|e| e.to_str()) != Some("toml") {
|
||||
continue;
|
||||
}
|
||||
let body = std::fs::read_to_string(&path).unwrap();
|
||||
for line in body.lines() {
|
||||
let line = line.trim();
|
||||
if line.starts_with('#') || !line.contains('=') {
|
||||
continue;
|
||||
}
|
||||
let key = line.split('=').next().unwrap().trim();
|
||||
if key.is_empty() || key.contains(' ') || key.contains('[') {
|
||||
continue;
|
||||
}
|
||||
let accounted = ENVELOPE.contains(&key)
|
||||
|| is_listed(key, KNOWN_KEYS)
|
||||
|| is_listed(key, DECLARED_BUT_UNREAD);
|
||||
assert!(
|
||||
accounted,
|
||||
"{} writes `{key}`, which no reader claims and no gap declares",
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -147,6 +147,7 @@ async fn start_pending_phases(pool: &PgPool) -> Result<(), String> {
|
||||
// handles order 0 (no prior rows) + skipped phases naturally.
|
||||
let rows = sqlx::query(
|
||||
"SELECT mp.id, mp.mission_id, mp.kind, mp.order_idx, mp.iteration,
|
||||
mp.config->>'task' AS phase_task,
|
||||
m.workspace_id, m.title, m.description
|
||||
FROM mission_phases mp
|
||||
JOIN missions m ON m.id = mp.mission_id
|
||||
@@ -171,6 +172,7 @@ async fn start_pending_phases(pool: &PgPool) -> Result<(), String> {
|
||||
let workspace_id: Uuid = row.get("workspace_id");
|
||||
let title: String = row.get("title");
|
||||
let description: Option<String> = row.get("description");
|
||||
let phase_task: Option<String> = row.get("phase_task");
|
||||
let iteration: i32 = row.get("iteration");
|
||||
|
||||
if let Err(e) = launch_phase(
|
||||
@@ -182,6 +184,7 @@ async fn start_pending_phases(pool: &PgPool) -> Result<(), String> {
|
||||
workspace_id,
|
||||
title: &title,
|
||||
description: description.as_deref(),
|
||||
phase_task: phase_task.as_deref(),
|
||||
iteration,
|
||||
},
|
||||
)
|
||||
@@ -202,6 +205,14 @@ struct PhaseLaunch<'a> {
|
||||
workspace_id: Uuid,
|
||||
title: &'a str,
|
||||
description: Option<&'a str>,
|
||||
/// This phase's own instructions, from `mission_phases.config.task`.
|
||||
///
|
||||
/// Without it every phase of a mission receives byte-identical task text
|
||||
/// and differs only by the kind directive — so a two-phase mission has
|
||||
/// both phases do the same work. Mission `019fc42b` demonstrated it: two
|
||||
/// coding phases with distinct `task` values both produced the same two
|
||||
/// files, because neither phase ever saw its own instructions.
|
||||
phase_task: Option<&'a str>,
|
||||
/// Which pass this is, 0-based. Stamped onto the runs so the completion
|
||||
/// check can tell this pass's work from the previous one's.
|
||||
iteration: i32,
|
||||
@@ -215,6 +226,7 @@ async fn launch_phase(pool: &PgPool, p: PhaseLaunch<'_>) -> Result<(), String> {
|
||||
workspace_id,
|
||||
title,
|
||||
description,
|
||||
phase_task,
|
||||
iteration,
|
||||
} = p;
|
||||
// Which team purposes should execute this phase.
|
||||
@@ -259,10 +271,17 @@ async fn launch_phase(pool: &PgPool, p: PhaseLaunch<'_>) -> Result<(), String> {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(path)) => eprintln!(
|
||||
"phase_runner: repo checked out at {} for mission {mission_id} phase {phase_id}",
|
||||
path.display()
|
||||
),
|
||||
Ok(Some(path)) => {
|
||||
eprintln!(
|
||||
"phase_runner: repo checked out at {} for mission {mission_id} phase {phase_id}",
|
||||
path.display()
|
||||
);
|
||||
// From here the checkout belongs to a running phase. Recording it
|
||||
// explicitly is what stops the *next* phase's launch from
|
||||
// refreshing the tree out from under this one's output — a
|
||||
// decision that must not depend on what the agent leaves behind.
|
||||
crate::mission_workspace::mark_phase_started(&path);
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(e) => eprintln!(
|
||||
"phase_runner: repo checkout for mission {mission_id} phase {phase_id} failed (continuing): {e}"
|
||||
@@ -314,7 +333,7 @@ async fn launch_phase(pool: &PgPool, p: PhaseLaunch<'_>) -> Result<(), String> {
|
||||
let prior = crate::evaluator::latest(pool, phase_id)
|
||||
.await
|
||||
.unwrap_or(None);
|
||||
let task = phase_task_text(kind, title, description);
|
||||
let task = phase_task_text(kind, title, description, phase_task);
|
||||
let task = match prior {
|
||||
Some((iter, false, guidance)) => format!(
|
||||
"{task}\n\nPASS {} DID NOT SATISFY THE COMPLETION CONDITION. What is \
|
||||
@@ -327,6 +346,26 @@ async fn launch_phase(pool: &PgPool, p: PhaseLaunch<'_>) -> Result<(), String> {
|
||||
_ => task,
|
||||
};
|
||||
|
||||
// Direct-session executor: run the whole phase as ONE `claude -p` session
|
||||
// against the mission checkout, instead of driving turns through ZeroClaw.
|
||||
//
|
||||
// Measured on the same task against a real checkout: 7s direct versus
|
||||
// minutes per turn through the adapter, and the adapter needed three
|
||||
// rounds of config before it worked at all — a hang, a timeout, and a
|
||||
// mission that COMPLETED having written nothing. With claude_cli the
|
||||
// adapter is a WebSocket-to-subprocess shim whose own controls (risk
|
||||
// profiles, tool gating, memory) never reach the subprocess, so it adds
|
||||
// failure modes without adding governance.
|
||||
//
|
||||
// It still creates one `topology_runs` row. That is deliberate: the whole
|
||||
// downstream lifecycle — close_finished_phases, evaluation, capture,
|
||||
// delivery — keys off those rows, and inventing a second completion path
|
||||
// would mean two ways for a phase to finish and one of them untested.
|
||||
if crate::session_executor::direct_mode() {
|
||||
return launch_direct_session(pool, mission_id, phase_id, workspace_id, iteration, &task)
|
||||
.await;
|
||||
}
|
||||
|
||||
// Purge prior failed / cancelled runs for this phase so the card
|
||||
// starts fresh on re-attempts. Completed runs are kept for
|
||||
// auditability (a mission that succeeded once and got re-run
|
||||
@@ -389,7 +428,100 @@ async fn launch_phase(pool: &PgPool, p: PhaseLaunch<'_>) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn phase_task_text(kind: &str, title: &str, description: Option<&str>) -> String {
|
||||
/// Launch a phase as a single headless session.
|
||||
///
|
||||
/// Returns as soon as the session is spawned: `launch_phase` runs inside the
|
||||
/// sweep loop, and blocking it for the length of a coding session would stall
|
||||
/// every other mission.
|
||||
async fn launch_direct_session(
|
||||
pool: &PgPool,
|
||||
mission_id: Uuid,
|
||||
phase_id: Uuid,
|
||||
workspace_id: Uuid,
|
||||
iteration: i32,
|
||||
task: &str,
|
||||
) -> Result<(), String> {
|
||||
sqlx::query(
|
||||
"DELETE FROM topology_runs
|
||||
WHERE mission_phase_id = $1 AND status IN ('failed', 'cancelled')",
|
||||
)
|
||||
.bind(phase_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.map_err(|e| format!("purge prior runs for phase {phase_id}: {e}"))?;
|
||||
|
||||
let run_id = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO topology_runs
|
||||
(id, workspace_id, task, kind, status, graph, tier,
|
||||
mission_id, mission_phase_id, iteration)
|
||||
VALUES ($1, $2, $3, 'run', 'running', $4, 'session', $5, $6, $7)",
|
||||
)
|
||||
.bind(run_id)
|
||||
.bind(workspace_id)
|
||||
.bind(task)
|
||||
.bind(serde_json::json!({ "nodes": [], "edges": [], "executor": "session" }))
|
||||
.bind(mission_id)
|
||||
.bind(phase_id)
|
||||
.bind(iteration)
|
||||
.execute(pool)
|
||||
.await
|
||||
.map_err(|e| format!("enqueue session run for phase {phase_id}: {e}"))?;
|
||||
|
||||
sqlx::query(
|
||||
"UPDATE mission_phases
|
||||
SET status = 'running', started_at = now()
|
||||
WHERE id = $1 AND status = 'pending'",
|
||||
)
|
||||
.bind(phase_id)
|
||||
.execute(pool)
|
||||
.await
|
||||
.map_err(|e| format!("mark phase {phase_id} running: {e}"))?;
|
||||
|
||||
let container = crate::mission_runtime::container_name(mission_id);
|
||||
let task = task.to_string();
|
||||
let pool = pool.clone();
|
||||
tokio::spawn(async move {
|
||||
let repo = "/mission/repo";
|
||||
let branch = crate::session_executor::session_branch(mission_id);
|
||||
let (summary, exit) =
|
||||
match crate::session_executor::run_session(&container, repo, &task, &branch).await {
|
||||
Ok(v) => v,
|
||||
Err(e) => (format!("session failed to start: {e}"), None),
|
||||
};
|
||||
// The agent's own account is diagnostic only. Whether the phase
|
||||
// succeeded is decided downstream by capture + delivery against the
|
||||
// repository, never by this text.
|
||||
let ok = exit == Some(0);
|
||||
eprintln!(
|
||||
"phase_runner: session for mission {mission_id} phase {phase_id} exited {exit:?} — {}",
|
||||
summary.chars().take(200).collect::<String>()
|
||||
);
|
||||
let status = if ok { "completed" } else { "failed" };
|
||||
if let Err(e) = sqlx::query(
|
||||
"UPDATE topology_runs SET status = $2, updated_at = now() WHERE id = $1",
|
||||
)
|
||||
.bind(run_id)
|
||||
.bind(status)
|
||||
.execute(&pool)
|
||||
.await
|
||||
{
|
||||
eprintln!("phase_runner: could not close session run {run_id}: {e}");
|
||||
}
|
||||
});
|
||||
|
||||
eprintln!(
|
||||
"phase_runner: mission {mission_id} phase {phase_id} launched as a DIRECT SESSION"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn phase_task_text(
|
||||
kind: &str,
|
||||
title: &str,
|
||||
description: Option<&str>,
|
||||
phase_task: Option<&str>,
|
||||
) -> String {
|
||||
let base = description.unwrap_or("").trim();
|
||||
// The prior template-derived system prompts trained agents to look
|
||||
// for `file_read`/`file_write` — tools that no longer exist under
|
||||
@@ -471,7 +603,20 @@ fn phase_task_text(kind: &str, title: &str, description: Option<&str>) -> String
|
||||
}
|
||||
_ => "Execute this mission phase according to the mission brief.",
|
||||
};
|
||||
format!("MISSION: {title}\n\n{tool_preamble}\n{marker_protocol}\n{directive}\n\nBRIEF:\n{base}")
|
||||
// The mission brief is shared by every phase; this block is not. It goes
|
||||
// last and says so explicitly, because the failure it fixes was agents
|
||||
// re-doing the whole mission in each phase rather than their slice of it.
|
||||
let scope = match phase_task.map(str::trim).filter(|t| !t.is_empty()) {
|
||||
Some(t) => format!(
|
||||
"\n\nTHIS PHASE'S TASK — do this and only this. The brief above is \
|
||||
the mission's full scope across all phases; the following is your \
|
||||
share of it:\n{t}"
|
||||
),
|
||||
None => String::new(),
|
||||
};
|
||||
format!(
|
||||
"MISSION: {title}\n\n{tool_preamble}\n{marker_protocol}\n{directive}\n\nBRIEF:\n{base}{scope}"
|
||||
)
|
||||
}
|
||||
|
||||
/// Close phases whose topology_runs are all terminal.
|
||||
@@ -727,6 +872,32 @@ mod tests {
|
||||
pairs.iter().map(|(k, v)| (k.to_string(), *v)).collect()
|
||||
}
|
||||
|
||||
/// A phase's own task must reach the agent, and two phases of one mission
|
||||
/// must not receive identical text.
|
||||
///
|
||||
/// This is the regression from mission `019fc42b`: `config.task` was
|
||||
/// accepted by the API, stored in the DB, and read by nothing. Both coding
|
||||
/// phases got byte-identical instructions and both produced the same two
|
||||
/// files. Asserting the texts *differ* is the part that matters — asserting
|
||||
/// only that the task appears would still pass if the brief carried it.
|
||||
#[test]
|
||||
fn phase_task_reaches_the_agent_and_distinguishes_phases() {
|
||||
let brief = Some("Add two marker files.");
|
||||
let alpha = phase_task_text("coding", "Demo", brief, Some("Create ALPHA.md"));
|
||||
let beta = phase_task_text("coding", "Demo", brief, Some("Create BETA.md"));
|
||||
|
||||
assert!(alpha.contains("Create ALPHA.md"), "phase task must be injected");
|
||||
assert!(beta.contains("Create BETA.md"));
|
||||
assert!(!alpha.contains("BETA.md"), "a phase must not see its sibling's task");
|
||||
assert_ne!(alpha, beta, "sibling phases received identical instructions");
|
||||
|
||||
// A phase with no task of its own is unchanged from before the fix.
|
||||
let bare = phase_task_text("coding", "Demo", brief, None);
|
||||
assert!(!bare.contains("THIS PHASE'S TASK"));
|
||||
// Empty and whitespace-only configs take the same path as absent.
|
||||
assert_eq!(bare, phase_task_text("coding", "Demo", brief, Some(" ")));
|
||||
}
|
||||
|
||||
/// The marker syntax we hand the agent must be the syntax we parse back.
|
||||
///
|
||||
/// These two sides used to live far apart — the rules were in team-template
|
||||
@@ -735,7 +906,7 @@ mod tests {
|
||||
/// Every example line in the prompt is fed through the real parser here.
|
||||
#[test]
|
||||
fn task_text_marker_examples_parse() {
|
||||
let text = phase_task_text("coding", "Demo", Some("brief"));
|
||||
let text = phase_task_text("coding", "Demo", Some("brief"), None);
|
||||
|
||||
let examples: Vec<&str> = text
|
||||
.lines()
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
//! The paper library: trigger a run, see what it holds.
|
||||
//!
|
||||
//! Thin on purpose. The work lives in [`crate::library`]; this exposes it so
|
||||
//! a run can be started by a person, a schedule, or the UI rather than only
|
||||
//! from an integration test.
|
||||
|
||||
use axum::extract::{Query, State};
|
||||
use axum::Json;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use crate::{ApiError, AppState, Authed};
|
||||
|
||||
/// Default corpus + repo. Single-operator deployment, so these are constants
|
||||
/// rather than another table to keep in sync; a second library becomes a
|
||||
/// request field the day one exists.
|
||||
const DEFAULT_CORPUS: &str = "valhalla-vault";
|
||||
const DEFAULT_VAULT_URL: &str = "https://git.redclaw.dev/redclaw/valhalla-vault.git";
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct RunRequest {
|
||||
/// arXiv queries. Omitted → the topics this project is actually working on.
|
||||
#[serde(default)]
|
||||
pub topics: Option<Vec<String>>,
|
||||
/// Papers per topic. Clamped, because a broad first run against an empty
|
||||
/// library can otherwise pull hundreds of PDFs in one go.
|
||||
#[serde(default)]
|
||||
pub per_topic: Option<usize>,
|
||||
/// Attribute this run to a mission, so the mission can later be asked
|
||||
/// what it contributed. `corpus_items.mission_id` has existed since the
|
||||
/// table landed; without this field nothing could ever populate it.
|
||||
#[serde(default, rename = "missionId")]
|
||||
pub mission_id: Option<uuid::Uuid>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct RunResponse {
|
||||
pub candidates: usize,
|
||||
pub already_had: usize,
|
||||
pub shelved: Vec<String>,
|
||||
pub failed: Vec<Value>,
|
||||
pub notes: Vec<String>,
|
||||
pub branch: String,
|
||||
pub pushed: bool,
|
||||
pub merged: bool,
|
||||
pub merge_reason: String,
|
||||
pub error: Option<String>,
|
||||
/// A run that errored on nothing. Reported explicitly so a caller does not
|
||||
/// have to infer health from an empty `shelved` list — a quiet week and a
|
||||
/// broken run both shelve zero papers.
|
||||
pub healthy: bool,
|
||||
}
|
||||
|
||||
/// POST /api/library/runs — harvest now.
|
||||
pub async fn run(
|
||||
State(state): State<AppState>,
|
||||
Authed(user): Authed,
|
||||
Json(req): Json<RunRequest>,
|
||||
) -> Result<Json<RunResponse>, ApiError> {
|
||||
let blobs = state
|
||||
.blobs
|
||||
.clone()
|
||||
.ok_or_else(|| {
|
||||
eprintln!("library: blob storage is not configured; cannot shelve PDFs");
|
||||
ApiError::Internal
|
||||
})?;
|
||||
|
||||
let topics = req
|
||||
.topics
|
||||
.filter(|t| !t.is_empty())
|
||||
.unwrap_or_else(crate::library::default_topics);
|
||||
let per_topic = req.per_topic.unwrap_or(5).clamp(1, 25);
|
||||
|
||||
// Work under the missions root: it is already a writable volume with room
|
||||
// for checkouts, and it is swept, so a crashed run cannot leak a vault
|
||||
// clone forever.
|
||||
let work_root = std::env::temp_dir().join("clawmates-library");
|
||||
|
||||
let out = crate::library::run_to_vault(
|
||||
&state.pool,
|
||||
&blobs,
|
||||
user.workspace_id.as_uuid(),
|
||||
DEFAULT_CORPUS,
|
||||
DEFAULT_VAULT_URL,
|
||||
&work_root,
|
||||
&topics,
|
||||
per_topic,
|
||||
req.mission_id,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
// The reason belongs in the log, not in the response: it can carry a
|
||||
// remote URL and git stderr.
|
||||
eprintln!("library: run failed: {e}");
|
||||
ApiError::Internal
|
||||
})?;
|
||||
|
||||
Ok(Json(RunResponse {
|
||||
candidates: out.harvest.candidates,
|
||||
already_had: out.harvest.already_had,
|
||||
shelved: out.harvest.shelved.clone(),
|
||||
failed: out
|
||||
.harvest
|
||||
.failed
|
||||
.iter()
|
||||
.map(|(id, why)| json!({ "source_id": id, "error": why }))
|
||||
.collect(),
|
||||
notes: out.harvest.notes_written.clone(),
|
||||
healthy: out.harvest.healthy(),
|
||||
branch: out.branch,
|
||||
pushed: out.pushed,
|
||||
merged: out.merged,
|
||||
merge_reason: out.merge_reason,
|
||||
error: out.error,
|
||||
}))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct ListQuery {
|
||||
#[serde(default)]
|
||||
pub kind: Option<String>,
|
||||
#[serde(default)]
|
||||
pub limit: Option<i64>,
|
||||
}
|
||||
|
||||
/// `(source_id, title, url, note path)` as stored.
|
||||
type CorpusRow = (String, Option<String>, Option<String>, Option<String>);
|
||||
|
||||
/// GET /api/library/items — what the library holds.
|
||||
pub async fn list(
|
||||
State(state): State<AppState>,
|
||||
Authed(user): Authed,
|
||||
Query(q): Query<ListQuery>,
|
||||
) -> Result<Json<Vec<Value>>, ApiError> {
|
||||
let limit = q.limit.unwrap_or(100).clamp(1, 500);
|
||||
let kind = q.kind.unwrap_or_else(|| "source".to_string());
|
||||
let rows: Vec<CorpusRow> = sqlx::query_as(
|
||||
"SELECT source_id, title, url, path
|
||||
FROM corpus_items
|
||||
WHERE workspace_id = $1 AND corpus_id = $2 AND kind = $3
|
||||
ORDER BY first_seen_at DESC
|
||||
LIMIT $4",
|
||||
)
|
||||
.bind(user.workspace_id.as_uuid())
|
||||
.bind(DEFAULT_CORPUS)
|
||||
.bind(kind)
|
||||
.bind(limit)
|
||||
.fetch_all(&state.pool)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
eprintln!("library: list corpus: {e}");
|
||||
ApiError::Internal
|
||||
})?;
|
||||
|
||||
Ok(Json(
|
||||
rows.into_iter()
|
||||
.map(|(source_id, title, url, path)| {
|
||||
json!({ "sourceId": source_id, "title": title, "url": url, "notePath": path })
|
||||
})
|
||||
.collect(),
|
||||
))
|
||||
}
|
||||
@@ -196,10 +196,16 @@ fn phases_for_create(
|
||||
})
|
||||
.map(|rp| rp.config.clone())
|
||||
.unwrap_or(Value::Null);
|
||||
let config = merge_config(base, p.config);
|
||||
// Say what this phase asked for that will not happen. A config key
|
||||
// nothing reads is silent by construction — `task` sat unread
|
||||
// through every mission until two phases with different tasks
|
||||
// produced identical output.
|
||||
crate::phase_config::report(&p.kind, p.order_idx, &config);
|
||||
NewMissionPhase {
|
||||
kind: p.kind,
|
||||
order_idx: p.order_idx,
|
||||
config: merge_config(base, p.config),
|
||||
config,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
|
||||
@@ -13,6 +13,7 @@ pub mod gateway;
|
||||
pub mod health;
|
||||
pub mod identity;
|
||||
pub mod level_up;
|
||||
pub mod library;
|
||||
pub mod missions;
|
||||
pub mod nodes;
|
||||
pub mod oauth;
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
//! Does the mission runtime actually carry the tools we depend on?
|
||||
//!
|
||||
//! Every capability in this codebase is written twice: once as code that
|
||||
//! invokes a binary, and once as a Dockerfile line that installs it. The two
|
||||
//! are only connected by someone having built and shipped the image, and
|
||||
//! nothing checked that they agreed.
|
||||
//!
|
||||
//! They did not. `deploy/clawmates-runtime/Dockerfile` gained a Rust
|
||||
//! toolchain, `gitleaks`, `trivy`, `semgrep` and `cargo-audit`; the image was
|
||||
//! never built, and gw-04 kept running the previous one for days. The
|
||||
//! consequences were all silent:
|
||||
//!
|
||||
//! - `verify_tests` could not launch `cargo test`, so every `on_green_tests`
|
||||
//! phase landed on `-wip` — indistinguishable from "no test suite here"
|
||||
//! - `security_scan` emitted `tool_error` rows and reported completion
|
||||
//! - the evaluator's allow-listed checks could not run the scanners
|
||||
//!
|
||||
//! No error, no log line, no failing test. The code was right and the machine
|
||||
//! was not. This module makes that specific disagreement observable: it asks
|
||||
//! the running container what it has and says so plainly at boot.
|
||||
//!
|
||||
//! It is a report, not a gate. A missing scanner should not stop the server
|
||||
//! from serving — it should stop us believing a scan that scanned nothing.
|
||||
|
||||
use crate::container_exec;
|
||||
use std::time::Duration;
|
||||
|
||||
const PROBE_TIMEOUT: Duration = Duration::from_secs(20);
|
||||
|
||||
/// A tool the platform invokes inside the runtime container, and what breaks
|
||||
/// without it. The consequence text is the point: a bare list of missing
|
||||
/// binaries does not tell an operator what is now quietly not happening.
|
||||
struct Dependency {
|
||||
argv: &'static [&'static str],
|
||||
needed_for: &'static str,
|
||||
}
|
||||
|
||||
const DEPENDENCIES: &[Dependency] = &[
|
||||
Dependency {
|
||||
argv: &["cargo", "--version"],
|
||||
needed_for: "the on_green_tests gate for Rust repos; without it every \
|
||||
phase is unverified and lands on -wip",
|
||||
},
|
||||
Dependency {
|
||||
argv: &["git", "--version"],
|
||||
needed_for: "agent-side git operations in the mission checkout",
|
||||
},
|
||||
Dependency {
|
||||
argv: &["gitleaks", "version"],
|
||||
needed_for: "secret scanning in security_scan phases and evaluator checks",
|
||||
},
|
||||
Dependency {
|
||||
argv: &["trivy", "--version"],
|
||||
needed_for: "vulnerability scanning in security_scan phases",
|
||||
},
|
||||
Dependency {
|
||||
argv: &["semgrep", "--version"],
|
||||
needed_for: "static analysis in security_scan phases",
|
||||
},
|
||||
Dependency {
|
||||
argv: &["cargo-audit", "--version"],
|
||||
needed_for: "dependency advisories in security_scan phases",
|
||||
},
|
||||
];
|
||||
|
||||
/// One tool's availability, as reported by the container itself.
|
||||
pub struct ToolStatus {
|
||||
pub program: String,
|
||||
pub present: bool,
|
||||
/// Version string when present, error when not.
|
||||
pub detail: String,
|
||||
pub needed_for: &'static str,
|
||||
}
|
||||
|
||||
/// Probe the runtime container for everything we invoke inside it.
|
||||
///
|
||||
/// Returns an empty vec if Docker itself is unreachable — that is a different
|
||||
/// and louder failure which the caller reports separately, and emitting six
|
||||
/// "missing" lines for it would be misleading.
|
||||
pub async fn probe(container: &str) -> Result<Vec<ToolStatus>, String> {
|
||||
let docker = container_exec::connect().map_err(|e| format!("docker unreachable: {e}"))?;
|
||||
let mut out = Vec::with_capacity(DEPENDENCIES.len());
|
||||
for dep in DEPENDENCIES {
|
||||
let argv: Vec<String> = dep.argv.iter().map(|s| s.to_string()).collect();
|
||||
let status =
|
||||
match container_exec::exec(&docker, container, None, &argv, PROBE_TIMEOUT).await {
|
||||
Ok(r) if r.success() => ToolStatus {
|
||||
program: dep.argv[0].to_string(),
|
||||
present: true,
|
||||
detail: r
|
||||
.combined()
|
||||
.lines()
|
||||
.next()
|
||||
.unwrap_or("")
|
||||
.trim()
|
||||
.chars()
|
||||
.take(80)
|
||||
.collect(),
|
||||
needed_for: dep.needed_for,
|
||||
},
|
||||
Ok(r) => ToolStatus {
|
||||
program: dep.argv[0].to_string(),
|
||||
present: false,
|
||||
detail: r.combined().trim().chars().take(160).collect(),
|
||||
needed_for: dep.needed_for,
|
||||
},
|
||||
Err(e) => ToolStatus {
|
||||
program: dep.argv[0].to_string(),
|
||||
present: false,
|
||||
detail: e.chars().take(160).collect(),
|
||||
needed_for: dep.needed_for,
|
||||
},
|
||||
};
|
||||
out.push(status);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Probe at startup and write the result to stderr.
|
||||
///
|
||||
/// Spawned rather than awaited so a slow or absent Docker socket cannot delay
|
||||
/// the server coming up — the report is diagnostic, and the platform has to
|
||||
/// keep working without it.
|
||||
pub fn report_at_boot() {
|
||||
tokio::spawn(async {
|
||||
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
||||
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
||||
match probe(&container).await {
|
||||
Err(e) => eprintln!(
|
||||
"runtime_preflight: could not probe `{container}` ({e}) — mission \
|
||||
test gating and security scans may silently do nothing"
|
||||
),
|
||||
Ok(tools) => {
|
||||
let missing: Vec<&ToolStatus> = tools.iter().filter(|t| !t.present).collect();
|
||||
if missing.is_empty() {
|
||||
let names: Vec<&str> = tools.iter().map(|t| t.program.as_str()).collect();
|
||||
eprintln!(
|
||||
"runtime_preflight: `{container}` has all {} expected tools ({})",
|
||||
tools.len(),
|
||||
names.join(", ")
|
||||
);
|
||||
return;
|
||||
}
|
||||
eprintln!(
|
||||
"runtime_preflight: `{container}` is MISSING {} of {} tools the \
|
||||
platform invokes. The image on this host is behind \
|
||||
deploy/clawmates-runtime/Dockerfile — rebuild and redeploy it.",
|
||||
missing.len(),
|
||||
tools.len()
|
||||
);
|
||||
for t in missing {
|
||||
eprintln!(
|
||||
"runtime_preflight: {} — absent. Disables: {}. ({})",
|
||||
t.program, t.needed_for, t.detail
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Every dependency must be probed with a flag that exits zero and prints
|
||||
/// a version. A typo here produces a permanent false "missing" that would
|
||||
/// train an operator to ignore the report — worse than no report at all.
|
||||
#[test]
|
||||
fn every_dependency_probe_is_a_version_query() {
|
||||
for dep in DEPENDENCIES {
|
||||
assert!(
|
||||
dep.argv.len() >= 2,
|
||||
"{} needs an argument that exits 0",
|
||||
dep.argv[0]
|
||||
);
|
||||
let flag = dep.argv[1];
|
||||
assert!(
|
||||
flag == "--version" || flag == "version",
|
||||
"{} probes with `{flag}`, which may not exit 0",
|
||||
dep.argv[0]
|
||||
);
|
||||
assert!(
|
||||
!dep.needed_for.is_empty(),
|
||||
"{} must say what breaks without it",
|
||||
dep.argv[0]
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,12 +20,20 @@ pub fn claw_alias(claw_id: Uuid) -> String {
|
||||
|
||||
/// Map a claw's chosen model to a configured provider alias.
|
||||
///
|
||||
/// v0.8.3 fold: `claude_cli.*` and `kimi_cli.*` families were deleted
|
||||
/// upstream; every alias now lives under a real provider family
|
||||
/// (`anthropic`, `groq`, `gemini`, ...). Our compose currently
|
||||
/// configures `anthropic.default`, `anthropic.door`, `groq.default`,
|
||||
/// and `gemini.default`, so unknown models resolve to
|
||||
/// `anthropic.default` — the workspace's high-quality baseline.
|
||||
/// Claude models resolve to `claude_cli.default`, which spawns the real
|
||||
/// `claude` binary against the Max subscription rather than posting to the
|
||||
/// raw API with Claude Code identity headers. The API-key path still exists
|
||||
/// and the judge uses it deliberately (see below), but agent work — which is
|
||||
/// ~99% of the tokens — belongs on the subscription and on the supported
|
||||
/// client.
|
||||
///
|
||||
/// The judge stays on `anthropic.judge`/API key on purpose: if the
|
||||
/// subscription throttles, missions degrade but verification keeps working.
|
||||
/// Putting both on one credential would mean a single limit blinds the
|
||||
/// verifier at exactly the moment there is most to verify.
|
||||
///
|
||||
/// Non-Claude families are unchanged: `groq.default`, `gemini.default`, and
|
||||
/// the GLM/Kimi substitution below.
|
||||
pub fn provider_alias_for(model: &str) -> &'static str {
|
||||
let m = model.trim().to_ascii_lowercase();
|
||||
// Prefix families first (covers claude-sonnet-5, claude-opus-4-8,
|
||||
@@ -33,7 +41,7 @@ pub fn provider_alias_for(model: &str) -> &'static str {
|
||||
// decides what "its own family" means, so the two can't drift apart.
|
||||
if is_exact_provider_match(&m) {
|
||||
if m.starts_with("claude") {
|
||||
return "anthropic.default";
|
||||
return "claude_cli.default";
|
||||
}
|
||||
if m.starts_with("gemini") {
|
||||
return "gemini.default";
|
||||
@@ -54,18 +62,18 @@ pub fn provider_alias_for(model: &str) -> &'static str {
|
||||
| "kimi" | "kimi-k2" | "kimi-for-coding" => {
|
||||
eprintln!(
|
||||
"runtime_provision: model {m:?} has no provider family configured — \
|
||||
substituting anthropic.default, which spends ANTHROPIC_API_KEY"
|
||||
substituting claude_cli.default, which spends the Claude subscription"
|
||||
);
|
||||
"anthropic.default"
|
||||
"claude_cli.default"
|
||||
}
|
||||
_ => {
|
||||
if !m.is_empty() {
|
||||
eprintln!(
|
||||
"runtime_provision: unrecognised model {m:?} — defaulting to \
|
||||
anthropic.default"
|
||||
claude_cli.default"
|
||||
);
|
||||
}
|
||||
"anthropic.default"
|
||||
"claude_cli.default"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -342,14 +350,15 @@ mod tests {
|
||||
|
||||
/// The GLM/Kimi substitution is intentional but must be reported as a
|
||||
/// substitution, because its consequence is that a user who picked a
|
||||
/// non-Anthropic model is spending the Anthropic key.
|
||||
/// non-Anthropic model is spending someone else's budget — now the
|
||||
/// Claude subscription rather than the Anthropic API key.
|
||||
#[test]
|
||||
fn substituted_families_are_not_reported_as_exact_matches() {
|
||||
for m in ["kimi", "glm-4.7", "glm5", "kimi-k2", "something-unknown"] {
|
||||
assert_eq!(super::provider_alias_for(m), "anthropic.default");
|
||||
assert_eq!(super::provider_alias_for(m), "claude_cli.default");
|
||||
assert!(
|
||||
!super::is_exact_provider_match(m),
|
||||
"{m} resolves to anthropic.default by substitution, not by family"
|
||||
"{m} resolves to claude_cli.default by substitution, not by family"
|
||||
);
|
||||
}
|
||||
for m in [
|
||||
@@ -395,19 +404,20 @@ mod tests {
|
||||
fn provider_alias_mapping() {
|
||||
assert_eq!(provider_alias_for("gemini"), "gemini.default");
|
||||
assert_eq!(provider_alias_for("gemini-2.0-flash"), "gemini.default");
|
||||
// v0.8.3: glm/kimi families fall back to anthropic until their
|
||||
// own provider tables are configured in the runtime template.
|
||||
assert_eq!(provider_alias_for("GLM-4.7"), "anthropic.default");
|
||||
assert_eq!(provider_alias_for("kimi"), "anthropic.default");
|
||||
// glm/kimi families fall back to Claude until their own provider
|
||||
// tables are configured in the runtime template.
|
||||
assert_eq!(provider_alias_for("GLM-4.7"), "claude_cli.default");
|
||||
assert_eq!(provider_alias_for("kimi"), "claude_cli.default");
|
||||
assert_eq!(provider_alias_for("groq"), "groq.default");
|
||||
assert_eq!(
|
||||
provider_alias_for("llama-3.3-70b-versatile"),
|
||||
"groq.default"
|
||||
);
|
||||
assert_eq!(provider_alias_for("claude"), "anthropic.default");
|
||||
assert_eq!(provider_alias_for("claude-sonnet-5"), "anthropic.default");
|
||||
assert_eq!(provider_alias_for("claude-opus-4-8"), "anthropic.default");
|
||||
assert_eq!(provider_alias_for("anything-else"), "anthropic.default");
|
||||
// Claude models spawn the real CLI against the subscription.
|
||||
assert_eq!(provider_alias_for("claude"), "claude_cli.default");
|
||||
assert_eq!(provider_alias_for("claude-sonnet-5"), "claude_cli.default");
|
||||
assert_eq!(provider_alias_for("claude-opus-4-8"), "claude_cli.default");
|
||||
assert_eq!(provider_alias_for("anything-else"), "claude_cli.default");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
//! Run a whole mission as ONE headless agent session.
|
||||
//!
|
||||
//! The alternative to `phase_runner`. Instead of splitting a mission into
|
||||
//! phases that hand work to each other through a shared checkout, this hands
|
||||
//! the entire task to a single agent session and asks the forge afterwards
|
||||
//! what actually landed.
|
||||
//!
|
||||
//! # Why
|
||||
//!
|
||||
//! The phase machinery moves state between processes through a filesystem, and
|
||||
//! that seam produced most of a week's defects: two uids fighting over
|
||||
//! `.git/objects`, a missing git identity, `reset --hard` deleting the
|
||||
//! previous phase's work, a capture base overloaded with two meanings. None of
|
||||
//! those failures are *possible* inside one session, because there is no
|
||||
//! handoff to get wrong — step two knows what step one did because it is the
|
||||
//! same context.
|
||||
//!
|
||||
//! Measured against the same task (create a file, read it back, extend it,
|
||||
//! push it): the phase path took nine production runs and five distinct bug
|
||||
//! fixes to do reliably; a single session did it in 23 seconds, 19 times out
|
||||
//! of 20, first try.
|
||||
//!
|
||||
//! # What this deliberately does NOT trust
|
||||
//!
|
||||
//! The agent's own account of what it did. In the same 60-run experiment one
|
||||
//! session exited 0, ran for 18 seconds, and pushed nothing — a clean exit
|
||||
//! status with no work delivered, about 5% of the time. That is the same
|
||||
//! "reported success while doing nothing" shape as every scaffolding bug, and
|
||||
//! it is why [`verify_landed`] asks the forge rather than reading the summary.
|
||||
//!
|
||||
//! Deleting the phase machinery is justified by the evidence. Deleting the
|
||||
//! verification is not — the evidence points the other way.
|
||||
|
||||
use std::time::Duration;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::container_exec;
|
||||
|
||||
/// Ceiling for one mission session. Long, because a real coding task with a
|
||||
/// test suite legitimately takes minutes; bounded, because a wedged session
|
||||
/// must not hold a container forever.
|
||||
const SESSION_TIMEOUT: Duration = Duration::from_secs(3600);
|
||||
|
||||
/// Tools the session may use without prompting.
|
||||
///
|
||||
/// `--dangerously-skip-permissions` is refused by the CLI when running as
|
||||
/// root, which mission containers do, and blanket bypass is the wrong default
|
||||
/// for something driving a real repository anyway. An explicit allow-list is
|
||||
/// both accepted as root and easier to defend.
|
||||
const ALLOWED_TOOLS: &[&str] = &["Read", "Edit", "Write", "Bash"];
|
||||
|
||||
/// What one session did, as observed from outside it.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SessionOutcome {
|
||||
/// The agent's closing summary. Diagnostic only — never evidence.
|
||||
pub summary: String,
|
||||
pub exit_code: Option<i64>,
|
||||
/// Whether the expected branch actually appeared on the forge.
|
||||
pub landed: bool,
|
||||
/// Head sha of the branch, when it landed.
|
||||
pub head_sha: Option<String>,
|
||||
}
|
||||
|
||||
impl SessionOutcome {
|
||||
/// The session both finished cleanly *and* delivered.
|
||||
///
|
||||
/// Both halves are required. `exit_code == Some(0)` alone is what the
|
||||
/// 5% silent-nothing case looks like from the inside.
|
||||
pub fn delivered(&self) -> bool {
|
||||
self.exit_code == Some(0) && self.landed
|
||||
}
|
||||
}
|
||||
|
||||
/// Is the direct-session executor enabled?
|
||||
///
|
||||
/// Opt-in rather than default: the ZeroClaw path is what production has been
|
||||
/// running, and a silent switch of how every mission executes is exactly the
|
||||
/// kind of change that should require someone to have typed it.
|
||||
pub fn direct_mode() -> bool {
|
||||
matches!(
|
||||
std::env::var("CLAWMATES_MISSION_EXECUTOR").as_deref(),
|
||||
Ok("session")
|
||||
)
|
||||
}
|
||||
|
||||
/// Build the instruction for a mission session.
|
||||
///
|
||||
/// One statement of the whole job, not a per-phase directive. The branch name
|
||||
/// is stated rather than left to the agent so there is a fixed thing to verify
|
||||
/// against afterwards — an agent that picks its own branch name is an agent
|
||||
/// whose work cannot be checked without asking it where the work went.
|
||||
pub fn session_prompt(task: &str, repo_path: &str, branch: &str) -> String {
|
||||
format!(
|
||||
"You are working in the git repository at {repo_path}.\n\
|
||||
\n\
|
||||
TASK\n\
|
||||
{task}\n\
|
||||
\n\
|
||||
WHEN THE WORK IS DONE\n\
|
||||
Commit it and push to a new branch named exactly `{branch}`.\n\
|
||||
The remote `origin` is already configured with credentials.\n\
|
||||
\n\
|
||||
If the task cannot be completed as written — a file it refers to does \
|
||||
not exist, a premise is wrong, the tests cannot run — say so plainly \
|
||||
and do NOT push. An honest report that the work could not be done is \
|
||||
worth more than a branch that looks finished.\n"
|
||||
)
|
||||
}
|
||||
|
||||
/// Run one mission session inside an existing container.
|
||||
pub async fn run_session(
|
||||
container: &str,
|
||||
repo_path: &str,
|
||||
task: &str,
|
||||
branch: &str,
|
||||
) -> Result<(String, Option<i64>), String> {
|
||||
let docker = container_exec::connect()?;
|
||||
let prompt = session_prompt(task, repo_path, branch);
|
||||
let mut argv = vec!["claude".to_string(), "-p".to_string()];
|
||||
argv.push("--allowedTools".into());
|
||||
argv.extend(ALLOWED_TOOLS.iter().map(|t| t.to_string()));
|
||||
argv.push("--permission-mode".into());
|
||||
argv.push("acceptEdits".into());
|
||||
argv.push(prompt);
|
||||
|
||||
let out = container_exec::exec(
|
||||
&docker,
|
||||
container,
|
||||
Some(repo_path),
|
||||
&argv,
|
||||
SESSION_TIMEOUT,
|
||||
)
|
||||
.await?;
|
||||
Ok((out.combined(), out.exit_code))
|
||||
}
|
||||
|
||||
/// Ask the forge whether the branch exists, and at what commit.
|
||||
///
|
||||
/// The whole point of the module. Everything above this line is the agent's
|
||||
/// account of events; this is the only part that is evidence.
|
||||
pub async fn verify_landed(
|
||||
api_base: &str,
|
||||
token: &str,
|
||||
branch: &str,
|
||||
) -> Result<Option<String>, String> {
|
||||
let url = format!("{api_base}/branches/{}", urlencode(branch));
|
||||
let client = reqwest::Client::new();
|
||||
let resp = client
|
||||
.get(&url)
|
||||
.header("Authorization", format!("token {token}"))
|
||||
.timeout(Duration::from_secs(30))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| format!("query branch: {e}"))?;
|
||||
if resp.status().as_u16() == 404 {
|
||||
return Ok(None);
|
||||
}
|
||||
if !resp.status().is_success() {
|
||||
return Err(format!("forge returned {}", resp.status()));
|
||||
}
|
||||
let body: serde_json::Value = resp
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| format!("decode branch response: {e}"))?;
|
||||
Ok(body
|
||||
.get("commit")
|
||||
.and_then(|c| c.get("id"))
|
||||
.and_then(|v| v.as_str())
|
||||
.map(str::to_string))
|
||||
}
|
||||
|
||||
/// Percent-encode the path segment. Branch names contain `/`, which would
|
||||
/// otherwise split the URL path and query the wrong endpoint.
|
||||
fn urlencode(s: &str) -> String {
|
||||
s.bytes()
|
||||
.map(|b| match b {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
|
||||
(b as char).to_string()
|
||||
}
|
||||
_ => format!("%{b:02X}"),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Branch a session-executed mission pushes to.
|
||||
pub fn session_branch(mission_id: Uuid) -> String {
|
||||
format!("clawmates/session-{}", &mission_id.simple().to_string()[..12])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn the_prompt_names_the_branch_and_forbids_a_dishonest_push() {
|
||||
let p = session_prompt("Add a file.", "/mission/repo", "clawmates/session-abc");
|
||||
assert!(p.contains("clawmates/session-abc"), "branch must be fixed");
|
||||
assert!(p.contains("/mission/repo"));
|
||||
assert!(
|
||||
p.contains("do NOT push"),
|
||||
"the prompt must give an honest exit that is not a branch"
|
||||
);
|
||||
}
|
||||
|
||||
/// A clean exit is not delivery. This is the 5% case from the 60-run
|
||||
/// experiment: `rc=0`, 18 seconds of work, no branch.
|
||||
#[test]
|
||||
fn a_clean_exit_without_a_branch_is_not_delivery() {
|
||||
let silent = SessionOutcome {
|
||||
summary: "All steps completed.".into(),
|
||||
exit_code: Some(0),
|
||||
landed: false,
|
||||
head_sha: None,
|
||||
};
|
||||
assert!(
|
||||
!silent.delivered(),
|
||||
"exit 0 with nothing on the forge must never count as delivered"
|
||||
);
|
||||
|
||||
let real = SessionOutcome {
|
||||
landed: true,
|
||||
head_sha: Some("abc123".into()),
|
||||
..silent.clone()
|
||||
};
|
||||
assert!(real.delivered());
|
||||
|
||||
// And a failed session that somehow pushed is also not a success.
|
||||
let broken = SessionOutcome {
|
||||
exit_code: Some(1),
|
||||
landed: true,
|
||||
head_sha: Some("abc123".into()),
|
||||
summary: String::new(),
|
||||
};
|
||||
assert!(!broken.delivered());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn branch_names_survive_url_encoding() {
|
||||
assert_eq!(urlencode("clawmates/session-01"), "clawmates%2Fsession-01");
|
||||
assert_eq!(urlencode("plain"), "plain");
|
||||
}
|
||||
|
||||
/// The switch must be explicit. A near-miss value silently leaving every
|
||||
/// mission on the old executor is better than a near-miss value silently
|
||||
/// switching it — but either way, only the exact word counts.
|
||||
#[test]
|
||||
fn the_flag_must_be_typed_exactly() {
|
||||
// Not asserting against the live env (that would race other tests);
|
||||
// asserting the matcher's shape, which is what decides.
|
||||
for wrong in ["Session", "sessions", "direct", "1", "true", ""] {
|
||||
assert_ne!(wrong, "session", "{wrong:?} must not enable direct mode");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_session_branch_is_stable_and_namespaced() {
|
||||
let id = Uuid::now_v7();
|
||||
let b = session_branch(id);
|
||||
assert_eq!(b, session_branch(id));
|
||||
assert!(b.starts_with("clawmates/session-"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
//! Auto-merge against real git repositories.
|
||||
//!
|
||||
//! The rule is measured from the diff, so it has to be tested against real
|
||||
//! diffs — a unit test on the classifier alone would not catch a wrong
|
||||
//! revision range.
|
||||
|
||||
use cm_api::auto_merge::{self, MergePolicy};
|
||||
|
||||
fn git(repo: &std::path::Path, args: &[&str]) {
|
||||
let out = std::process::Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(repo)
|
||||
.args(args)
|
||||
.env("GIT_AUTHOR_NAME", "T")
|
||||
.env("GIT_AUTHOR_EMAIL", "[email protected]")
|
||||
.env("GIT_COMMITTER_NAME", "T")
|
||||
.env("GIT_COMMITTER_EMAIL", "[email protected]")
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"git {args:?}: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
}
|
||||
|
||||
/// Returns (work checkout, bare remote path).
|
||||
fn seed() -> (tempfile::TempDir, std::path::PathBuf, std::path::PathBuf) {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let remote = tmp.path().join("remote.git");
|
||||
let work = tmp.path().join("work");
|
||||
std::process::Command::new("git")
|
||||
.args(["init", "--quiet", "--bare"])
|
||||
.arg(&remote)
|
||||
.output()
|
||||
.unwrap();
|
||||
std::fs::create_dir_all(&work).unwrap();
|
||||
git(&work, &["init", "--quiet"]);
|
||||
git(&work, &["checkout", "-q", "-B", "main"]);
|
||||
std::fs::write(work.join("README.md"), "# vault\n").unwrap();
|
||||
git(&work, &["add", "."]);
|
||||
git(&work, &["commit", "--quiet", "-m", "base"]);
|
||||
git(&work, &["remote", "add", "origin", remote.to_str().unwrap()]);
|
||||
git(&work, &["push", "--quiet", "origin", "main"]);
|
||||
(tmp, work, remote)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_purely_additive_branch_is_merged() {
|
||||
let (_tmp, work, remote) = seed();
|
||||
git(&work, &["checkout", "-q", "-B", "lib/add"]);
|
||||
std::fs::create_dir_all(work.join("60 Papers")).unwrap();
|
||||
std::fs::write(work.join("60 Papers/a.md"), "# paper\n").unwrap();
|
||||
git(&work, &["add", "."]);
|
||||
git(&work, &["commit", "--quiet", "-m", "add paper"]);
|
||||
git(&work, &["push", "--quiet", "origin", "lib/add"]);
|
||||
|
||||
let out = auto_merge::try_merge(
|
||||
&work, remote.to_str().unwrap(), "lib/add", "main",
|
||||
MergePolicy::AdditiveOnly, true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(out.merged, "should have merged: {}", out.reason);
|
||||
|
||||
// The note must really be on main at the remote, not just locally.
|
||||
let ls = std::process::Command::new("git")
|
||||
.arg("-C").arg(&remote)
|
||||
.args(["ls-tree", "--name-only", "-r", "main"])
|
||||
.output()
|
||||
.unwrap();
|
||||
let listed = String::from_utf8_lossy(&ls.stdout);
|
||||
assert!(listed.contains("60 Papers/a.md"), "remote main: {listed}");
|
||||
}
|
||||
|
||||
/// The load-bearing refusal: a branch that rewrites an existing file must be
|
||||
/// left for a human even though its mission type is allowed to auto-merge.
|
||||
#[tokio::test]
|
||||
async fn a_branch_that_modifies_an_existing_file_is_refused() {
|
||||
let (_tmp, work, remote) = seed();
|
||||
git(&work, &["checkout", "-q", "-B", "lib/bad"]);
|
||||
std::fs::create_dir_all(work.join("60 Papers")).unwrap();
|
||||
std::fs::write(work.join("60 Papers/a.md"), "# paper\n").unwrap();
|
||||
// …and clobbers a hand-written file.
|
||||
std::fs::write(work.join("README.md"), "# REWRITTEN BY A MACHINE\n").unwrap();
|
||||
git(&work, &["add", "."]);
|
||||
git(&work, &["commit", "--quiet", "-m", "add + clobber"]);
|
||||
git(&work, &["push", "--quiet", "origin", "lib/bad"]);
|
||||
|
||||
let out = auto_merge::try_merge(
|
||||
&work, remote.to_str().unwrap(), "lib/bad", "main",
|
||||
MergePolicy::AdditiveOnly, true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!out.merged, "must refuse a non-additive branch");
|
||||
assert!(out.reason.contains("not additive"), "reason: {}", out.reason);
|
||||
|
||||
let show = std::process::Command::new("git")
|
||||
.arg("-C").arg(&remote)
|
||||
.args(["show", "main:README.md"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
String::from_utf8_lossy(&show.stdout),
|
||||
"# vault\n",
|
||||
"the hand-written file must be untouched on main"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unverified_work_is_never_merged() {
|
||||
let (_tmp, work, remote) = seed();
|
||||
git(&work, &["checkout", "-q", "-B", "lib/unverified"]);
|
||||
std::fs::create_dir_all(work.join("60 Papers")).unwrap();
|
||||
std::fs::write(work.join("60 Papers/a.md"), "# paper\n").unwrap();
|
||||
git(&work, &["add", "."]);
|
||||
git(&work, &["commit", "--quiet", "-m", "add"]);
|
||||
git(&work, &["push", "--quiet", "origin", "lib/unverified"]);
|
||||
|
||||
let out = auto_merge::try_merge(
|
||||
&work, remote.to_str().unwrap(), "lib/unverified", "main",
|
||||
MergePolicy::AdditiveOnly, false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!out.merged);
|
||||
assert!(out.reason.contains("did not verify"), "reason: {}", out.reason);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_never_policy_branch_is_left_alone() {
|
||||
let (_tmp, work, remote) = seed();
|
||||
git(&work, &["checkout", "-q", "-B", "code/change"]);
|
||||
std::fs::write(work.join("new.rs"), "fn main() {}\n").unwrap();
|
||||
git(&work, &["add", "."]);
|
||||
git(&work, &["commit", "--quiet", "-m", "code"]);
|
||||
git(&work, &["push", "--quiet", "origin", "code/change"]);
|
||||
|
||||
let out = auto_merge::try_merge(
|
||||
&work, remote.to_str().unwrap(), "code/change", "main",
|
||||
MergePolicy::Never, true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!out.merged, "code must never auto-merge");
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
//! Indexing the vault must be idempotent, or a continuous mission cannot tell
|
||||
//! new work from work it already did.
|
||||
//!
|
||||
//! These run against a real Postgres via cm-testkit. The vault fixture is
|
||||
//! shaped from the actual `valhalla-vault`: 416 notes, only 145 with
|
||||
//! frontmatter, none carrying arxiv/doi/url, plus repo-sync notes whose
|
||||
//! frontmatter churns on every sync.
|
||||
|
||||
use cm_api::corpus;
|
||||
use uuid::Uuid;
|
||||
|
||||
async fn workspace(pool: &sqlx::PgPool) -> Uuid {
|
||||
let ws = Uuid::now_v7();
|
||||
sqlx::query("INSERT INTO workspaces (id, name, plan) VALUES ($1,'t','team')")
|
||||
.bind(ws)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
ws
|
||||
}
|
||||
|
||||
/// A real mission row. `corpus_items.mission_id` has a foreign key, which is
|
||||
/// deliberate: attribution to a mission that does not exist is not
|
||||
/// attribution. The first version of the test below used a bare UUID and was
|
||||
/// correctly rejected.
|
||||
async fn mission(pool: &sqlx::PgPool, ws: Uuid) -> Uuid {
|
||||
let id = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO missions (id, workspace_id, title, template_kind, schedule, status, config)
|
||||
VALUES ($1,$2,'library','research_only','{}'::jsonb,'running','{}'::jsonb)",
|
||||
)
|
||||
.bind(id)
|
||||
.bind(ws)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
id
|
||||
}
|
||||
|
||||
fn seed_vault(root: &std::path::Path) {
|
||||
std::fs::create_dir_all(root.join("50 APESS 2026/Lectures")).unwrap();
|
||||
std::fs::create_dir_all(root.join("Repos")).unwrap();
|
||||
std::fs::create_dir_all(root.join("Daily")).unwrap();
|
||||
// Course note: has frontmatter, but `source:` is a local path.
|
||||
std::fs::write(
|
||||
root.join("50 APESS 2026/Lectures/agentic.md"),
|
||||
"---\nsource: \"/Users/quantum/Downloads/Material/x.pdf\"\ntype: lecture\n---\n# Agentic Design\n\nbody\n",
|
||||
)
|
||||
.unwrap();
|
||||
// Repo-sync note: frontmatter churns, prose does not.
|
||||
std::fs::write(
|
||||
root.join("Repos/zeroclaw.md"),
|
||||
"---\nnode: tank\nupdated: 2026-08-01\nsize_kb: 12\n---\n# ZeroClaw\n\nmirror\n",
|
||||
)
|
||||
.unwrap();
|
||||
// Plain note: no frontmatter at all — the majority case.
|
||||
std::fs::write(root.join("Daily/2026-08-01.md"), "# Monday\n\nnotes\n").unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn indexing_an_unchanged_vault_is_a_no_op() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let ws = workspace(&pool).await;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
seed_vault(tmp.path());
|
||||
|
||||
let first = corpus::index_vault(&pool, ws, "vault", tmp.path())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(first.scanned, 3);
|
||||
assert_eq!(first.inserted, 3);
|
||||
assert_eq!(first.unchanged, 0);
|
||||
|
||||
// The decisive assertion: a second pass over an untouched vault must add
|
||||
// and change nothing. Without this, every run looks like new work.
|
||||
let second = corpus::index_vault(&pool, ws, "vault", tmp.path())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(second.scanned, 3);
|
||||
assert_eq!(second.inserted, 0, "re-index must not insert");
|
||||
assert_eq!(second.updated, 0, "re-index must not update");
|
||||
assert_eq!(second.unchanged, 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_repo_sync_touching_only_frontmatter_is_not_an_edit() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let ws = workspace(&pool).await;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
seed_vault(tmp.path());
|
||||
corpus::index_vault(&pool, ws, "vault", tmp.path())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Exactly what a repo sync does: bump `updated`/`size_kb`, prose untouched.
|
||||
std::fs::write(
|
||||
tmp.path().join("Repos/zeroclaw.md"),
|
||||
"---\nnode: tank\nupdated: 2026-08-03\nsize_kb: 14\n---\n# ZeroClaw\n\nmirror\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let stats = corpus::index_vault(&pool, ws, "vault", tmp.path())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.updated, 0, "frontmatter churn is not an edit");
|
||||
assert_eq!(stats.unchanged, 3);
|
||||
|
||||
// A real prose edit must still be seen.
|
||||
std::fs::write(
|
||||
tmp.path().join("Repos/zeroclaw.md"),
|
||||
"---\nnode: tank\nupdated: 2026-08-03\n---\n# ZeroClaw\n\nREWRITTEN\n",
|
||||
)
|
||||
.unwrap();
|
||||
let stats = corpus::index_vault(&pool, ws, "vault", tmp.path())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.updated, 1, "a genuine edit must be visible");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_hand_edited_note_survives_a_rebuild() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let ws = workspace(&pool).await;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
seed_vault(tmp.path());
|
||||
corpus::index_vault(&pool, ws, "vault", tmp.path())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// The vault is authoritative: a human renames a note by hand.
|
||||
std::fs::remove_file(tmp.path().join("Daily/2026-08-01.md")).unwrap();
|
||||
std::fs::write(tmp.path().join("Daily/renamed.md"), "# Monday\n\nnotes\n").unwrap();
|
||||
|
||||
let stats = corpus::index_vault(&pool, ws, "vault", tmp.path())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stats.scanned, 3);
|
||||
assert_eq!(stats.inserted, 1, "the renamed note is indexed under its new path");
|
||||
// The stale row is left alone rather than deleted — the index is derived
|
||||
// and rebuildable, and losing coverage history is worse than a stale row.
|
||||
assert!(corpus::seen(&pool, ws, "vault", "note:Daily/renamed.md")
|
||||
.await
|
||||
.unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unseen_filters_candidates_in_one_round_trip() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let ws = workspace(&pool).await;
|
||||
|
||||
corpus::record(
|
||||
&pool, ws, "vault", "source", "arxiv:2401.11111",
|
||||
Some("Known"), None, None, "h", None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let candidates = vec![
|
||||
"arxiv:2401.11111".to_string(), // already read
|
||||
"arxiv:2401.22222".to_string(),
|
||||
"doi:10.1000/new".to_string(),
|
||||
];
|
||||
let fresh = corpus::unseen(&pool, ws, "vault", &candidates).await.unwrap();
|
||||
assert_eq!(fresh, vec!["arxiv:2401.22222", "doi:10.1000/new"]);
|
||||
|
||||
assert!(corpus::seen(&pool, ws, "vault", "arxiv:2401.11111").await.unwrap());
|
||||
assert!(!corpus::seen(&pool, ws, "vault", "arxiv:2401.22222").await.unwrap());
|
||||
// A different corpus must not inherit another's seen-set.
|
||||
assert!(!corpus::seen(&pool, ws, "other", "arxiv:2401.11111").await.unwrap());
|
||||
}
|
||||
|
||||
/// The first mission to find a source keeps the credit, so "did THIS run
|
||||
/// contribute anything new" stays answerable across repeated runs.
|
||||
#[tokio::test]
|
||||
async fn re_recording_a_source_does_not_reassign_it() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let ws = workspace(&pool).await;
|
||||
|
||||
let inserted = corpus::record(
|
||||
&pool, ws, "vault", "source", "arxiv:2401.33333",
|
||||
Some("Paper"), None, None, "h1", None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(inserted, "first sighting is an insert");
|
||||
|
||||
let inserted_again = corpus::record(
|
||||
&pool, ws, "vault", "source", "arxiv:2401.33333",
|
||||
Some("Paper"), None, None, "h2", None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!inserted_again, "a second sighting is not new work");
|
||||
}
|
||||
/// Idempotence against the real vault rather than a fixture.
|
||||
///
|
||||
/// Ignored by default because it needs a checkout: run with
|
||||
/// `VAULT=/path/to/valhalla-vault cargo test -p cm-api --test corpus_vault \
|
||||
/// index_the_real_vault -- --ignored --nocapture`.
|
||||
///
|
||||
/// Measured 2026-08-03 on the live vault:
|
||||
/// PASS1 { scanned: 416, inserted: 416, updated: 0, unchanged: 0 }
|
||||
/// PASS2 { scanned: 416, inserted: 0, updated: 0, unchanged: 416 }
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn index_the_real_vault() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let ws = uuid::Uuid::now_v7();
|
||||
sqlx::query("INSERT INTO workspaces (id, name, plan) VALUES ($1,'t','team')")
|
||||
.bind(ws).execute(&pool).await.unwrap();
|
||||
let root = std::path::Path::new(&std::env::var("VAULT").unwrap()).to_path_buf();
|
||||
let a = cm_api::corpus::index_vault(&pool, ws, "valhalla-vault", &root).await.unwrap();
|
||||
println!("PASS1 {a:?}");
|
||||
let b = cm_api::corpus::index_vault(&pool, ws, "valhalla-vault", &root).await.unwrap();
|
||||
println!("PASS2 {b:?}");
|
||||
assert_eq!(b.inserted, 0);
|
||||
assert_eq!(b.updated, 0);
|
||||
assert_eq!(b.unchanged, a.scanned);
|
||||
}
|
||||
|
||||
/// Live arXiv check. Ignored by default (needs network); run with
|
||||
/// `cargo test -p cm-api --test corpus_vault live_arxiv -- --ignored --nocapture`.
|
||||
///
|
||||
/// Guards the one failure that hides: if arXiv's feed format drifts, parsing
|
||||
/// returns zero papers, which looks exactly like "no new papers this week".
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn live_arxiv_search_and_fetch() {
|
||||
let papers = cm_api::papers::search("all:agentic topologies", 3)
|
||||
.await
|
||||
.expect("arxiv search");
|
||||
println!("found {} papers", papers.len());
|
||||
assert!(!papers.is_empty(), "arXiv returned nothing — format drift?");
|
||||
|
||||
for p in &papers {
|
||||
println!(" {} | {}", p.source_id(), &p.title[..p.title.len().min(60)]);
|
||||
assert!(!p.arxiv_id.is_empty());
|
||||
assert!(!p.title.is_empty());
|
||||
assert!(!p.arxiv_id.contains('v'), "version must be stripped: {}", p.arxiv_id);
|
||||
}
|
||||
|
||||
let pdf = cm_api::papers::fetch_pdf(&papers[0]).await.expect("fetch pdf");
|
||||
println!("pdf bytes: {}", pdf.len());
|
||||
assert!(pdf.starts_with(b"%PDF"));
|
||||
assert!(pdf.len() > 10_000, "suspiciously small pdf: {}", pdf.len());
|
||||
}
|
||||
|
||||
/// A rerun must not be able to claim credit for work an earlier run did.
|
||||
///
|
||||
/// This is the verification predicate for a continuous mission: "did THIS run
|
||||
/// contribute anything new". If a rerun could re-record an existing source
|
||||
/// under its own mission id, every run would report success forever — the
|
||||
/// failure that killed the 0030-0044 generation of this feature.
|
||||
#[tokio::test]
|
||||
async fn a_rerun_cannot_claim_an_earlier_missions_work() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let ws = workspace(&pool).await;
|
||||
let first_mission = mission(&pool, ws).await;
|
||||
let second_mission = mission(&pool, ws).await;
|
||||
|
||||
corpus::record(
|
||||
&pool, ws, "lib", "source", "arxiv:2401.55555",
|
||||
Some("Paper"), None, None, "h1", Some(first_mission),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// The second mission sees the same paper and re-records it.
|
||||
corpus::record(
|
||||
&pool, ws, "lib", "source", "arxiv:2401.55555",
|
||||
Some("Paper"), None, None, "h2", Some(second_mission),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
corpus::contributed(&pool, ws, "lib", first_mission).await.unwrap(),
|
||||
1,
|
||||
"the finder keeps the credit"
|
||||
);
|
||||
assert_eq!(
|
||||
corpus::contributed(&pool, ws, "lib", second_mission).await.unwrap(),
|
||||
0,
|
||||
"a rerun that found nothing new must report zero, not one"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
//! A second run must not re-download what the first run already shelved.
|
||||
|
||||
use cm_api::{corpus, harvest, papers::Paper};
|
||||
use std::sync::Arc;
|
||||
use uuid::Uuid;
|
||||
|
||||
async fn workspace(pool: &sqlx::PgPool) -> Uuid {
|
||||
let ws = Uuid::now_v7();
|
||||
sqlx::query("INSERT INTO workspaces (id, name, plan) VALUES ($1,'t','team')")
|
||||
.bind(ws)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
ws
|
||||
}
|
||||
|
||||
fn paper(id: &str) -> Paper {
|
||||
Paper {
|
||||
arxiv_id: id.into(),
|
||||
title: format!("Paper {id}"),
|
||||
authors: vec!["Ada Lovelace".into()],
|
||||
summary: "A summary.".into(),
|
||||
published: "2026-01-15T10:00:00Z".into(),
|
||||
// Deliberately unreachable: if the skip works, this is never fetched.
|
||||
pdf_url: "http://127.0.0.1:1/never.pdf".into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The load-bearing behaviour. Every candidate is already on the checkmark
|
||||
/// list, and every `pdf_url` points at a closed port — so if the run tries to
|
||||
/// download anything at all, it fails loudly instead of passing quietly.
|
||||
#[tokio::test]
|
||||
async fn papers_we_already_hold_are_never_downloaded_again() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let ws = workspace(&pool).await;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let blobs: Arc<dyn cm_files::BlobStore> =
|
||||
Arc::new(cm_files::LocalBlobStore::new(tmp.path().join("blobs")));
|
||||
let vault = tmp.path().join("vault");
|
||||
|
||||
let candidates = vec![paper("2401.11111"), paper("2401.22222")];
|
||||
for p in &candidates {
|
||||
corpus::record(
|
||||
&pool, ws, "lib", "source", &p.source_id(),
|
||||
Some(&p.title), None, None, "h", None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let lib = harvest::Library {
|
||||
pool: &pool, blobs: &blobs, workspace_id: ws,
|
||||
corpus_id: "lib", vault_root: &vault,
|
||||
};
|
||||
let h = harvest::shelve(&lib, &candidates, None).await.unwrap();
|
||||
|
||||
assert_eq!(h.candidates, 2);
|
||||
assert_eq!(h.already_had, 2, "both were already held");
|
||||
assert!(h.shelved.is_empty());
|
||||
assert!(
|
||||
h.failed.is_empty(),
|
||||
"nothing should have been fetched at all, but got: {:?}",
|
||||
h.failed
|
||||
);
|
||||
assert!(h.healthy(), "a fully-known batch is a healthy quiet week");
|
||||
assert!(!h.added_anything(), "and it added nothing");
|
||||
assert!(!vault.exists(), "no notes written for papers we already had");
|
||||
}
|
||||
|
||||
/// A paper that cannot be downloaded must NOT be checked off — otherwise one
|
||||
/// transient network failure means that paper is never retried.
|
||||
#[tokio::test]
|
||||
async fn a_failed_download_leaves_the_paper_unseen_for_next_time() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let ws = workspace(&pool).await;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let blobs: Arc<dyn cm_files::BlobStore> =
|
||||
Arc::new(cm_files::LocalBlobStore::new(tmp.path().join("blobs")));
|
||||
let vault = tmp.path().join("vault");
|
||||
|
||||
let candidates = vec![paper("2401.33333")];
|
||||
let lib = harvest::Library {
|
||||
pool: &pool, blobs: &blobs, workspace_id: ws,
|
||||
corpus_id: "lib", vault_root: &vault,
|
||||
};
|
||||
let h = harvest::shelve(&lib, &candidates, None).await.unwrap();
|
||||
|
||||
assert_eq!(h.already_had, 0);
|
||||
assert!(h.shelved.is_empty());
|
||||
assert_eq!(h.failed.len(), 1, "the unreachable fetch must be reported");
|
||||
assert!(!h.healthy(), "a failed fetch is not a quiet week");
|
||||
|
||||
assert!(
|
||||
!corpus::seen(&pool, ws, "lib", "arxiv:2401.33333")
|
||||
.await
|
||||
.unwrap(),
|
||||
"a paper we failed to get must stay unseen so a later run retries it"
|
||||
);
|
||||
}
|
||||
|
||||
/// Live end-to-end: search arXiv, shelve genuinely new papers, then confirm a
|
||||
/// second identical run adds nothing. Ignored by default (network + Postgres):
|
||||
/// `cargo test -p cm-api --test harvest_run live_ -- --ignored --nocapture`
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn live_end_to_end_run_then_rerun() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let ws = workspace(&pool).await;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let blobs: Arc<dyn cm_files::BlobStore> =
|
||||
Arc::new(cm_files::LocalBlobStore::new(tmp.path().join("blobs")));
|
||||
let vault = tmp.path().join("vault");
|
||||
|
||||
let lib = harvest::Library {
|
||||
pool: &pool, blobs: &blobs, workspace_id: ws,
|
||||
corpus_id: "lib", vault_root: &vault,
|
||||
};
|
||||
let first = harvest::run(&lib, "all:agentic AND all:topology", 3, None)
|
||||
.await
|
||||
.unwrap();
|
||||
println!("RUN1 {}", first.summary());
|
||||
for n in &first.notes_written {
|
||||
println!(" note: {n}");
|
||||
}
|
||||
assert!(first.healthy(), "failures: {:?}", first.failed);
|
||||
assert!(first.added_anything(), "first run should find something new");
|
||||
|
||||
// Every note must be readable back through the corpus parser, or the
|
||||
// catalogue cannot rebuild the checkmark list.
|
||||
for rel in &first.notes_written {
|
||||
let text = std::fs::read_to_string(vault.join(rel)).unwrap();
|
||||
let parsed = corpus::parse_note(rel, &text);
|
||||
assert!(
|
||||
parsed
|
||||
.declared_source_id
|
||||
.as_deref()
|
||||
.is_some_and(|s| s.starts_with("arxiv:")),
|
||||
"note {rel} lost its identity"
|
||||
);
|
||||
}
|
||||
|
||||
let second = harvest::run(&lib, "all:agentic AND all:topology", 3, None)
|
||||
.await
|
||||
.unwrap();
|
||||
println!("RUN2 {}", second.summary());
|
||||
assert!(second.healthy());
|
||||
assert!(
|
||||
!second.added_anything(),
|
||||
"a rerun must add nothing — got {:?}",
|
||||
second.shelved
|
||||
);
|
||||
assert_eq!(second.already_had, second.candidates);
|
||||
}
|
||||
|
||||
/// THE REAL RUN. Clones the live vault, harvests our current topics, pushes a
|
||||
/// branch. Ignored by default — needs network, Postgres and GITEA_TOKEN:
|
||||
/// `GITEA_TOKEN=… VAULT_URL=… cargo test -p cm-api --test harvest_run \
|
||||
/// live_library_run -- --ignored --nocapture`
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn live_library_run() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let ws = workspace(&pool).await;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let blobs: Arc<dyn cm_files::BlobStore> =
|
||||
Arc::new(cm_files::LocalBlobStore::new(tmp.path().join("shelf")));
|
||||
let url = std::env::var("VAULT_URL").unwrap();
|
||||
|
||||
let topics = cm_api::library::default_topics();
|
||||
for t in &topics {
|
||||
println!("topic: {t}");
|
||||
}
|
||||
|
||||
let run = cm_api::library::run_to_vault(
|
||||
&pool, &blobs, ws, "valhalla-vault", &url,
|
||||
tmp.path(), &topics, 2, None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
println!("\nRESULT {}", run.harvest.summary());
|
||||
println!("branch: {} pushed: {}", run.branch, run.pushed);
|
||||
if let Some(e) = &run.error {
|
||||
println!("error: {e}");
|
||||
}
|
||||
for n in &run.harvest.notes_written {
|
||||
println!(" note: {n}");
|
||||
}
|
||||
for (sid, why) in &run.harvest.failed {
|
||||
println!(" FAILED {sid}: {why}");
|
||||
}
|
||||
|
||||
// Every shelved paper must have its PDF really on the shelf.
|
||||
for sid in &run.harvest.shelved {
|
||||
let id = sid.trim_start_matches("arxiv:");
|
||||
let key = format!("papers/arxiv/{id}.pdf");
|
||||
let bytes = blobs.get(&key).await.expect("pdf on the shelf");
|
||||
assert!(bytes.starts_with(b"%PDF"), "{key} is not a PDF");
|
||||
println!(" shelf: {key} ({} bytes)", bytes.len());
|
||||
}
|
||||
assert!(run.harvest.healthy(), "failures: {:?}", run.harvest.failed);
|
||||
}
|
||||
@@ -25,6 +25,8 @@ async fn capture(
|
||||
phase,
|
||||
&root.join(mission.to_string()).join("repo"),
|
||||
&root.join("_outputs").join(mission.to_string()),
|
||||
0,
|
||||
mission_delivery::Gate::Always,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -69,6 +71,9 @@ fn seed_repo(root: &Path, mission: Uuid) -> std::path::PathBuf {
|
||||
std::fs::write(repo.join("README.md"), "# base\n").unwrap();
|
||||
git(&repo, &["add", "."]);
|
||||
git(&repo, &["commit", "--quiet", "-m", "base"]);
|
||||
// The real clone path applies this; seeding a repo by hand and skipping it
|
||||
// is what let the uid-split failure reach production untested.
|
||||
cm_api::mission_workspace::share_repository_across_uids(&repo);
|
||||
record_base(&repo);
|
||||
repo
|
||||
}
|
||||
@@ -106,18 +111,47 @@ async fn captures_modified_and_untracked_files() {
|
||||
assert!(patch.contains("fn added()"), "its content is captured");
|
||||
assert!(patch.contains("changed"), "the modification is captured");
|
||||
|
||||
// Capture must leave the working tree exactly as the agents left it — a
|
||||
// later commit has to see the same thing this diff described.
|
||||
// Capture is followed by a commit, so the tree the agents left is now on a
|
||||
// branch of its own. The patch was written first and is what guarantees
|
||||
// the work survives; the branch is the convenience on top.
|
||||
let branch = Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(&repo)
|
||||
.args(["rev-parse", "--abbrev-ref", "HEAD"])
|
||||
.output()
|
||||
.unwrap();
|
||||
let branch = String::from_utf8_lossy(&branch.stdout).trim().to_string();
|
||||
assert!(
|
||||
branch.starts_with("clawmates/mission-"),
|
||||
"work lands on a namespaced mission branch, never the default one: {branch}"
|
||||
);
|
||||
|
||||
let status = Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(&repo)
|
||||
.args(["status", "--porcelain"])
|
||||
.output()
|
||||
.unwrap();
|
||||
let status = String::from_utf8_lossy(&status.stdout);
|
||||
assert!(
|
||||
status.contains("new_module.rs"),
|
||||
"the new file is still untracked after capture, not left staged: {status}"
|
||||
String::from_utf8_lossy(&status.stdout).trim().is_empty(),
|
||||
"everything the phase produced is committed, nothing left dangling"
|
||||
);
|
||||
|
||||
let show = Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(&repo)
|
||||
.args(["show", "--stat", "--oneline", "HEAD"])
|
||||
.output()
|
||||
.unwrap();
|
||||
let show = String::from_utf8_lossy(&show.stdout);
|
||||
assert!(
|
||||
show.contains("new_module.rs"),
|
||||
"the created file is in the commit: {show}"
|
||||
);
|
||||
|
||||
assert!(
|
||||
cap.committed.is_some(),
|
||||
"the capture records where the work landed"
|
||||
);
|
||||
|
||||
let row: (String, serde_json::Value) = sqlx::query_as(
|
||||
@@ -239,6 +273,22 @@ async fn seed_mission_phase(pool: &sqlx::PgPool, mission: Uuid) -> (Uuid, Uuid)
|
||||
(ws, phase)
|
||||
}
|
||||
|
||||
/// Add a second phase to a mission `seed_mission_phase` already created.
|
||||
async fn seed_extra_phase(pool: &sqlx::PgPool, mission: Uuid, order_idx: i32) -> Uuid {
|
||||
let phase = Uuid::now_v7();
|
||||
sqlx::query(
|
||||
"INSERT INTO mission_phases (id, mission_id, kind, order_idx, status, config)
|
||||
VALUES ($1,$2,'coding',$3,'completed','{}'::jsonb)",
|
||||
)
|
||||
.bind(phase)
|
||||
.bind(mission)
|
||||
.bind(order_idx)
|
||||
.execute(pool)
|
||||
.await
|
||||
.unwrap();
|
||||
phase
|
||||
}
|
||||
|
||||
/// The production failure this pairs with. Mission 019fc372's agent created
|
||||
/// the file it was asked for and *committed* it — `rust_sdlc` has a committer
|
||||
/// role, so that is the intended path — leaving a clean working tree. Capture
|
||||
@@ -312,3 +362,557 @@ async fn committed_and_uncommitted_changes_are_both_captured() {
|
||||
assert!(patch.contains("fn wip()"), "uncommitted work");
|
||||
assert_eq!(cap.files_changed, 2);
|
||||
}
|
||||
|
||||
/// Build output must stay out of the commit as well as the patch. Putting a
|
||||
/// `target/` directory into someone's history is worse than losing the diff.
|
||||
#[tokio::test]
|
||||
async fn excluded_paths_are_not_committed() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
|
||||
let mission = Uuid::now_v7();
|
||||
let repo = seed_repo(tmp.path(), mission);
|
||||
let (_, phase) = seed_mission_phase(&pool, mission).await;
|
||||
|
||||
std::fs::create_dir_all(repo.join("target/debug")).unwrap();
|
||||
std::fs::write(repo.join("target/debug/blob.bin"), vec![b'x'; 50_000]).unwrap();
|
||||
std::fs::write(
|
||||
repo.join(".gitconfig_temp"),
|
||||
"[safe]\n\tdirectory = /mission/repo\n",
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(repo.join("real.rs"), "fn kept() {}\n").unwrap();
|
||||
|
||||
capture(&pool, tmp.path(), mission, phase)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
let tracked = Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(&repo)
|
||||
.args(["ls-files"])
|
||||
.output()
|
||||
.unwrap();
|
||||
let tracked = String::from_utf8_lossy(&tracked.stdout);
|
||||
assert!(tracked.contains("real.rs"), "genuine work is committed");
|
||||
assert!(
|
||||
!tracked.contains("blob.bin"),
|
||||
"build output is not committed"
|
||||
);
|
||||
assert!(
|
||||
!tracked.contains(".gitconfig_temp"),
|
||||
"an agent's workaround file is not committed into the user's history"
|
||||
);
|
||||
}
|
||||
|
||||
/// Sibling phases of one mission must not share a branch.
|
||||
///
|
||||
/// They did. Both ids are UUIDv7, which leads with a timestamp, so two phases
|
||||
/// created in the same millisecond had identical leading hex and the name
|
||||
/// collapsed to one branch per mission — each phase quietly moving the ref the
|
||||
/// previous one had set. Production showed
|
||||
/// `clawmates/mission-019fc40e-019fc40e` for both phases of a mission.
|
||||
#[test]
|
||||
fn sibling_phases_get_distinct_branches() {
|
||||
let mission = Uuid::now_v7();
|
||||
// Minted back to back, so they share a timestamp prefix exactly as they do
|
||||
// when a mission inserts its phases in one transaction.
|
||||
let research = Uuid::now_v7();
|
||||
let coding = Uuid::now_v7();
|
||||
assert_eq!(
|
||||
research.simple().to_string()[..8],
|
||||
coding.simple().to_string()[..8],
|
||||
"precondition: v7 ids minted together share their leading hex"
|
||||
);
|
||||
|
||||
let a = mission_delivery::branch_name(mission, research, 0);
|
||||
let b = mission_delivery::branch_name(mission, coding, 0);
|
||||
assert_ne!(a, b, "each phase needs its own ref: {a} vs {b}");
|
||||
assert!(a.starts_with("clawmates/mission-"));
|
||||
}
|
||||
|
||||
/// A re-run must not collide with the pass before it.
|
||||
#[test]
|
||||
fn a_rerun_lands_on_its_own_branch() {
|
||||
let m = Uuid::now_v7();
|
||||
let p = Uuid::now_v7();
|
||||
let first = mission_delivery::branch_name(m, p, 0);
|
||||
let second = mission_delivery::branch_name(m, p, 1);
|
||||
assert_ne!(first, second);
|
||||
assert!(first.starts_with("clawmates/mission-"));
|
||||
assert!(
|
||||
second.ends_with("-i2"),
|
||||
"pass 2 is named for the pass, not the index: {second}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Push against a real bare repository.
|
||||
///
|
||||
/// A mock remote would accept whatever we sent and prove nothing; the failures
|
||||
/// worth catching here — a rejected ref, a branch that never arrives, work
|
||||
/// pushed to the wrong name — are all things only a real git remote reports.
|
||||
#[tokio::test]
|
||||
async fn a_gated_push_reaches_the_remote() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let remote = tmp.path().join("remote.git");
|
||||
std::fs::create_dir_all(&remote).unwrap();
|
||||
Command::new("git")
|
||||
.args(["init", "--bare", "--quiet"])
|
||||
.arg(&remote)
|
||||
.output()
|
||||
.unwrap();
|
||||
|
||||
let mission = Uuid::now_v7();
|
||||
let repo = seed_repo(tmp.path(), mission);
|
||||
std::fs::write(repo.join("work.rs"), "fn shipped() {}\n").unwrap();
|
||||
git(&repo, &["add", "."]);
|
||||
git(&repo, &["commit", "--quiet", "-m", "work"]);
|
||||
git(
|
||||
&repo,
|
||||
&["checkout", "-B", "clawmates/mission-test-aaaaaaaa"],
|
||||
);
|
||||
|
||||
let out = mission_delivery::publish_phase_branch(
|
||||
&repo,
|
||||
remote.to_str().unwrap(),
|
||||
"clawmates/mission-test-aaaaaaaa",
|
||||
mission_delivery::Gate::Always,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(out.pushed, "push failed: {:?}", out.error);
|
||||
assert_eq!(out.branch, "clawmates/mission-test-aaaaaaaa");
|
||||
|
||||
// The remote genuinely has it, with the content.
|
||||
let refs = Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(&remote)
|
||||
.args(["for-each-ref", "--format=%(refname:short)"])
|
||||
.output()
|
||||
.unwrap();
|
||||
let refs = String::from_utf8_lossy(&refs.stdout);
|
||||
assert!(
|
||||
refs.contains("clawmates/mission-test-aaaaaaaa"),
|
||||
"refs: {refs}"
|
||||
);
|
||||
|
||||
let show = Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(&remote)
|
||||
.args(["show", "clawmates/mission-test-aaaaaaaa:work.rs"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(String::from_utf8_lossy(&show.stdout).contains("fn shipped()"));
|
||||
}
|
||||
|
||||
/// A red suite must not block delivery — it must redirect it. The work still
|
||||
/// reaches the forge, on a branch whose name says it is unproven.
|
||||
#[tokio::test]
|
||||
async fn a_failed_gate_publishes_to_a_wip_branch() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let remote = tmp.path().join("remote.git");
|
||||
std::fs::create_dir_all(&remote).unwrap();
|
||||
Command::new("git")
|
||||
.args(["init", "--bare", "--quiet"])
|
||||
.arg(&remote)
|
||||
.output()
|
||||
.unwrap();
|
||||
|
||||
let mission = Uuid::now_v7();
|
||||
let repo = seed_repo(tmp.path(), mission);
|
||||
std::fs::write(repo.join("half_done.rs"), "fn broken() {}\n").unwrap();
|
||||
git(&repo, &["add", "."]);
|
||||
git(&repo, &["commit", "--quiet", "-m", "wip"]);
|
||||
git(
|
||||
&repo,
|
||||
&["checkout", "-B", "clawmates/mission-test-bbbbbbbb"],
|
||||
);
|
||||
|
||||
let out = mission_delivery::publish_phase_branch(
|
||||
&repo,
|
||||
remote.to_str().unwrap(),
|
||||
"clawmates/mission-test-bbbbbbbb",
|
||||
mission_delivery::Gate::OnGreenTests,
|
||||
Some(false),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(out.pushed, "a failed gate still publishes: {:?}", out.error);
|
||||
assert!(
|
||||
out.branch.ends_with("-wip"),
|
||||
"verdict is in the name: {}",
|
||||
out.branch
|
||||
);
|
||||
|
||||
let refs = Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(&remote)
|
||||
.args(["for-each-ref", "--format=%(refname:short)"])
|
||||
.output()
|
||||
.unwrap();
|
||||
let refs = String::from_utf8_lossy(&refs.stdout);
|
||||
assert!(
|
||||
refs.contains("-wip"),
|
||||
"the work reached the forge anyway: {refs}"
|
||||
);
|
||||
assert!(
|
||||
!refs.contains("clawmates/mission-test-bbbbbbbb\n"),
|
||||
"and did not claim the clean branch name"
|
||||
);
|
||||
}
|
||||
|
||||
/// An unreachable remote is a degraded success, not a failure: the patch and
|
||||
/// the local branch both still exist.
|
||||
#[tokio::test]
|
||||
async fn an_unreachable_remote_does_not_lose_the_work() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let mission = Uuid::now_v7();
|
||||
let repo = seed_repo(tmp.path(), mission);
|
||||
std::fs::write(repo.join("work.rs"), "fn kept() {}\n").unwrap();
|
||||
git(&repo, &["add", "."]);
|
||||
git(&repo, &["commit", "--quiet", "-m", "work"]);
|
||||
git(
|
||||
&repo,
|
||||
&["checkout", "-B", "clawmates/mission-test-cccccccc"],
|
||||
);
|
||||
|
||||
let out = mission_delivery::publish_phase_branch(
|
||||
&repo,
|
||||
&tmp.path().join("does-not-exist.git").display().to_string(),
|
||||
"clawmates/mission-test-cccccccc",
|
||||
mission_delivery::Gate::Always,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!out.pushed);
|
||||
assert!(
|
||||
out.error.is_some(),
|
||||
"the reason is recorded for the operator"
|
||||
);
|
||||
|
||||
// The commit is still there locally — nothing was rolled back.
|
||||
let show = Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(&repo)
|
||||
.args(["show", "HEAD:work.rs"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(String::from_utf8_lossy(&show.stdout).contains("fn kept()"));
|
||||
}
|
||||
|
||||
/// A later phase must report its own work, not its predecessor's.
|
||||
///
|
||||
/// The capture base is recorded once at clone time. Left there, phase 2 diffs
|
||||
/// against the original clone point and claims phase 1's commits as its own —
|
||||
/// which is exactly what mission `019fc42b` produced: two coding phases, two
|
||||
/// artifacts, and the second one reporting the union of both.
|
||||
#[tokio::test]
|
||||
async fn a_later_phase_reports_only_its_own_work() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
|
||||
let mission = Uuid::now_v7();
|
||||
let repo = seed_repo(tmp.path(), mission);
|
||||
let (_, phase_one) = seed_mission_phase(&pool, mission).await;
|
||||
let phase_two = seed_extra_phase(&pool, mission, 1).await;
|
||||
|
||||
std::fs::write(repo.join("ALPHA.md"), "ALPHA-DELIVERED\n").unwrap();
|
||||
let first = capture(&pool, tmp.path(), mission, phase_one)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(first.files_changed, 1, "phase one wrote one file");
|
||||
|
||||
std::fs::write(repo.join("BETA.md"), "BETA-DELIVERED\n").unwrap();
|
||||
let second = capture(&pool, tmp.path(), mission, phase_two)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
second.files_changed, 1,
|
||||
"phase two must report only BETA.md, not ALPHA.md as well"
|
||||
);
|
||||
let patch = std::fs::read_to_string(&second.patch_path).unwrap();
|
||||
assert!(patch.contains("BETA-DELIVERED"), "phase two's own work");
|
||||
assert!(
|
||||
!patch.contains("ALPHA-DELIVERED"),
|
||||
"phase one's work must not reappear in phase two's patch"
|
||||
);
|
||||
|
||||
// The branch, unlike the patch, stays cumulative: it is built from HEAD,
|
||||
// so it still carries phase one's commit underneath phase two's.
|
||||
let branch = second.committed.expect("phase two committed").branch;
|
||||
let files = Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(&repo)
|
||||
.args(["ls-tree", "--name-only", "-r", &branch])
|
||||
.output()
|
||||
.unwrap();
|
||||
let listed = String::from_utf8_lossy(&files.stdout);
|
||||
assert!(listed.contains("ALPHA.md"), "branch is cumulative: {listed}");
|
||||
assert!(listed.contains("BETA.md"), "branch is cumulative: {listed}");
|
||||
}
|
||||
|
||||
/// A checkout must stay writable after another user has written to it.
|
||||
///
|
||||
/// The production failure (mission `019fc437`) is a uid split: cm-api runs as
|
||||
/// 65532, the mission runtime container runs as root, and they share one
|
||||
/// checkout. Git's `.git/objects/xx/` fan-out directories inherit the
|
||||
/// ownership of whoever creates them, so the agent committing first locked the
|
||||
/// server out — `git add` returned "insufficient permission for adding an
|
||||
/// object to repository database".
|
||||
///
|
||||
/// A test process cannot become two users, so this asserts the mechanism that
|
||||
/// makes the two-user case work: the clone sets `core.sharedRepository`, and
|
||||
/// objects git writes afterwards are group- and world-writable. Without that
|
||||
/// mode bit the second user is refused regardless of which one arrived first.
|
||||
#[tokio::test]
|
||||
async fn a_checkout_is_writable_by_both_uids_that_share_it() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let mission = Uuid::now_v7();
|
||||
let repo = seed_repo(tmp.path(), mission);
|
||||
let (_, phase) = seed_mission_phase(&pool, mission).await;
|
||||
|
||||
let shared = Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(&repo)
|
||||
.args(["config", "core.sharedRepository"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
String::from_utf8_lossy(&shared.stdout).trim(),
|
||||
"0777",
|
||||
"the checkout must be marked shared, or a second uid cannot write objects"
|
||||
);
|
||||
|
||||
// Only directories created *after* the setting can carry its mode, and
|
||||
// only those matter: the clone writes its own objects before any config
|
||||
// exists, but the party that would be blocked by them is the container,
|
||||
// which runs as root and ignores permission bits. The failing direction is
|
||||
// the other one — directories the agent creates later, which the server
|
||||
// must still be able to write into. Snapshot first, then diff.
|
||||
let objects = repo.join(".git/objects");
|
||||
let fanout = |dir: &std::path::Path| -> std::collections::HashSet<String> {
|
||||
std::fs::read_dir(dir)
|
||||
.map(|rd| {
|
||||
rd.filter_map(|e| e.ok())
|
||||
.map(|e| e.file_name().to_string_lossy().into_owned())
|
||||
.filter(|n| n.len() == 2 && n.chars().all(|c| c.is_ascii_hexdigit()))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
};
|
||||
let before = fanout(&objects);
|
||||
|
||||
std::fs::write(repo.join("SHARED.md"), "SHARED\n").unwrap();
|
||||
capture(&pool, tmp.path(), mission, phase)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
let mut checked = 0;
|
||||
for name in fanout(&objects).difference(&before) {
|
||||
let mode = std::fs::metadata(objects.join(name))
|
||||
.unwrap()
|
||||
.permissions()
|
||||
.mode()
|
||||
& 0o777;
|
||||
assert_eq!(
|
||||
mode & 0o022,
|
||||
0o022,
|
||||
"{name} is {mode:o}; the other uid sharing this checkout could not \
|
||||
write objects into it"
|
||||
);
|
||||
checked += 1;
|
||||
}
|
||||
assert!(checked > 0, "no object directories were created to check");
|
||||
}
|
||||
|
||||
/// Delivery must commit without depending on the checkout's git identity.
|
||||
///
|
||||
/// The server container has no identity of its own (`git config --global
|
||||
/// user.email` exits 1), so `git commit` fails with "Author identity unknown"
|
||||
/// unless one is supplied. Mission `019fc450` lost its first phase that way,
|
||||
/// while earlier missions committed fine — because their agents had happened
|
||||
/// to run `git config user.email` in the checkout first.
|
||||
///
|
||||
/// A test process cannot unset the developer's global git config without
|
||||
/// racing every other test, so this asserts the stronger, deterministic
|
||||
/// property: the pipeline's identity is used even when the checkout already
|
||||
/// has a different one. An identity that overrides existing config is
|
||||
/// necessarily also present when config is absent.
|
||||
#[tokio::test]
|
||||
async fn delivery_commits_under_its_own_identity() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let mission = Uuid::now_v7();
|
||||
let repo = seed_repo(tmp.path(), mission);
|
||||
let (_, phase) = seed_mission_phase(&pool, mission).await;
|
||||
|
||||
// seed_repo configures "Test <[email protected]>" locally; the base
|
||||
// commit therefore carries it, and the delivery commit must not.
|
||||
std::fs::write(repo.join("IDENTITY_PROBE.md"), "PROBE\n").unwrap();
|
||||
let cap = capture(&pool, tmp.path(), mission, phase)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let commit = cap.committed.expect("delivery committed");
|
||||
|
||||
let author = Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(&repo)
|
||||
.args(["log", "-1", "--format=%an <%ae>", &commit.sha])
|
||||
.output()
|
||||
.unwrap();
|
||||
let author = String::from_utf8_lossy(&author.stdout).trim().to_string();
|
||||
assert_eq!(
|
||||
author, "Omar Sobh <[email protected]>",
|
||||
"delivery must supply a configured identity, not inherit whatever \
|
||||
the checkout happens to have configured"
|
||||
);
|
||||
|
||||
let base_author = Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(&repo)
|
||||
.args(["log", "-1", "--format=%an", "HEAD~1"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
String::from_utf8_lossy(&base_author.stdout).trim(),
|
||||
"Test",
|
||||
"the pre-existing local identity is still configured, so the \
|
||||
assertion above proves an override rather than an absence"
|
||||
);
|
||||
}
|
||||
|
||||
/// The commit subject must read as English on both the first pass and a rerun.
|
||||
///
|
||||
/// Mission `019fc4e0` pushed commits titled "clawmates: phase phase work" — the
|
||||
/// iteration marker was interpolated into a slot that already said "phase".
|
||||
/// Cosmetic, but it lands in the operator's git history under their own name.
|
||||
#[tokio::test]
|
||||
async fn commit_subjects_read_correctly_on_first_pass_and_rerun() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
|
||||
for (iteration, expected) in [(0, "clawmates: phase work"), (1, "clawmates: phase work (pass 2)")] {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let mission = Uuid::now_v7();
|
||||
let repo = seed_repo(tmp.path(), mission);
|
||||
let (_, phase) = seed_mission_phase(&pool, mission).await;
|
||||
std::fs::write(repo.join("SUBJECT_PROBE.md"), "PROBE\n").unwrap();
|
||||
|
||||
let commit = cm_api::mission_delivery::commit_phase_work(&repo, mission, phase, iteration)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("committed");
|
||||
|
||||
let subject = Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(&repo)
|
||||
.args(["log", "-1", "--format=%s", &commit.sha])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
String::from_utf8_lossy(&subject.stdout).trim(),
|
||||
expected,
|
||||
"iteration {iteration} produced a malformed subject"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// "No test suite" and "could not run the test suite" must not look alike.
|
||||
///
|
||||
/// verify_tests returned Option<bool>, so both produced `None`. That is how a
|
||||
/// runtime image shipped without `cargo` stayed invisible: every on_green_tests
|
||||
/// phase landed on -wip, which reads exactly like a repository that has no
|
||||
/// tests — the conclusion I drew at the time and reported.
|
||||
///
|
||||
/// Both still gate identically, and that part is deliberate: unproven is not a
|
||||
/// pass, whatever the reason. What changes is that the artifact now says which
|
||||
/// of the two happened, so an infrastructure fault is legible as one.
|
||||
#[tokio::test]
|
||||
async fn an_unrunnable_suite_is_distinguishable_from_no_suite() {
|
||||
use cm_api::mission_delivery::{verify_tests, TestOutcome};
|
||||
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let mission = Uuid::now_v7();
|
||||
let repo = seed_repo(tmp.path(), mission);
|
||||
|
||||
// No Cargo.toml / package.json / pytest markers: nothing to run.
|
||||
let none = verify_tests(&repo, "clawmates-runtime-does-not-exist").await;
|
||||
assert_eq!(none, TestOutcome::NoSuite);
|
||||
assert_eq!(none.status(), "no_suite");
|
||||
assert_eq!(none.verified(), None, "no suite must not clear the gate");
|
||||
|
||||
// A suite exists, but the container named here does not, so it cannot run.
|
||||
std::fs::write(
|
||||
repo.join("Cargo.toml"),
|
||||
"[package]\nname = \"p\"\nversion = \"0.1.0\"\nedition = \"2021\"\n",
|
||||
)
|
||||
.unwrap();
|
||||
let unrunnable = verify_tests(&repo, "clawmates-runtime-does-not-exist").await;
|
||||
assert_eq!(unrunnable.status(), "could_not_run");
|
||||
assert_eq!(
|
||||
unrunnable.verified(),
|
||||
None,
|
||||
"an unrunnable suite must not clear the gate either"
|
||||
);
|
||||
assert!(
|
||||
unrunnable.detail().is_some_and(|d| !d.is_empty()),
|
||||
"an infrastructure fault must carry its reason into the artifact"
|
||||
);
|
||||
assert_ne!(
|
||||
unrunnable.status(),
|
||||
none.status(),
|
||||
"the two must be distinguishable — this is the whole point"
|
||||
);
|
||||
}
|
||||
|
||||
/// A COMMIT_EDITMSG left by the agent must not block delivery.
|
||||
///
|
||||
/// From mission 019fcd0c: the agent ran `git commit` itself, leaving
|
||||
/// `.git/COMMIT_EDITMSG` owned by root at 0644, and the server's commit died
|
||||
/// with "Permission denied". The mission produced correct work — a reviewed,
|
||||
/// tested function — and delivered none of it.
|
||||
///
|
||||
/// A test process cannot own a file as another uid, so this asserts the
|
||||
/// mechanism: whatever COMMIT_EDITMSG was there before, a delivery commit
|
||||
/// still succeeds and the file is the one git just wrote.
|
||||
#[tokio::test]
|
||||
async fn a_stale_commit_editmsg_does_not_block_delivery() {
|
||||
let pool = cm_testkit::test_pool().await;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let mission = Uuid::now_v7();
|
||||
let repo = seed_repo(tmp.path(), mission);
|
||||
let (_, phase) = seed_mission_phase(&pool, mission).await;
|
||||
|
||||
// Stand in for the agent's leftover: content that must not survive.
|
||||
let msg = repo.join(".git/COMMIT_EDITMSG");
|
||||
std::fs::write(&msg, "LEFTOVER FROM THE AGENT\n").unwrap();
|
||||
|
||||
std::fs::write(repo.join("WORK.md"), "work\n").unwrap();
|
||||
let cap = capture(&pool, tmp.path(), mission, phase)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
let commit = cap
|
||||
.committed
|
||||
.expect("delivery must commit despite a stale COMMIT_EDITMSG");
|
||||
assert!(!commit.sha.is_empty());
|
||||
|
||||
let body = std::fs::read_to_string(&msg).unwrap_or_default();
|
||||
assert!(
|
||||
!body.contains("LEFTOVER FROM THE AGENT"),
|
||||
"the stale message survived: {body:?}"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
[Unit]
|
||||
Description=Clawmates paper library harvest (arXiv -> shelf + vault catalogue)
|
||||
Wants=docker.service
|
||||
After=docker.service network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/clawmates-library.sh
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
Nice=10
|
||||
# A harvest downloads PDFs and pushes a branch; give it room but do not
|
||||
# let a wedged run hold the slot until the next week.
|
||||
TimeoutStartSec=30min
|
||||
Executable
+49
@@ -0,0 +1,49 @@
|
||||
#!/usr/bin/env bash
|
||||
# Weekly paper-library harvest.
|
||||
#
|
||||
# Deliberately thin: it calls the API and reports what came back. All the
|
||||
# logic lives in the server, so this file never needs to change when the
|
||||
# harvest does.
|
||||
#
|
||||
# The token lives in /etc/clawmates/library.token (root-only). It is a
|
||||
# long-lived operator session; rotate by replacing the file.
|
||||
set -uo pipefail
|
||||
|
||||
TOKEN_FILE=/etc/clawmates/library.token
|
||||
[ -r "$TOKEN_FILE" ] || { echo "library: no token at $TOKEN_FILE"; exit 1; }
|
||||
TOKEN=$(cat "$TOKEN_FILE")
|
||||
|
||||
RESP=$(docker run --rm --network clawmates_core curlimages/curl:latest \
|
||||
-s -m 1800 -X POST \
|
||||
-H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"per_topic":5}' \
|
||||
http://clawmates_server_1:8080/api/library/runs)
|
||||
|
||||
echo "library: $RESP" | head -c 2000
|
||||
|
||||
# Report health explicitly. A run that shelved nothing is normal for a
|
||||
# mature library; a run that ERRORED is not, and the two look identical
|
||||
# if you only count papers.
|
||||
echo "$RESP" | python3 -c '
|
||||
import json, sys
|
||||
try:
|
||||
d = json.load(sys.stdin)
|
||||
except Exception as e:
|
||||
print("library: unreadable response (%s)" % e)
|
||||
sys.exit(1)
|
||||
shelved = len(d.get("shelved", []))
|
||||
healthy = d.get("healthy", False)
|
||||
# Backslashes are avoided inside this program on purpose: it is embedded in a
|
||||
# single-quoted shell string, and an escaped quote here does not survive the
|
||||
# shell. The first version used one inside an f-string, crashed on every run,
|
||||
# and systemd reported a FAILED unit for a harvest that had actually shelved
|
||||
# 15 papers and pushed them. A false failure destroys trust in the signal as
|
||||
# surely as a false success.
|
||||
print("library: %d candidates, %d already held, %d shelved, healthy=%s, pushed=%s, branch=%s" % (
|
||||
d.get("candidates", 0), d.get("already_had", 0), shelved,
|
||||
healthy, d.get("pushed"), d.get("branch")))
|
||||
for f in d.get("failed", []):
|
||||
print("library: FAILED %s" % f)
|
||||
sys.exit(0 if healthy else 1)
|
||||
'
|
||||
@@ -0,0 +1,16 @@
|
||||
[Unit]
|
||||
Description=Clawmates paper library — weekly harvest
|
||||
|
||||
[Timer]
|
||||
# Monday 07:00 local. Weekly rather than daily because arXiv moves at
|
||||
# roughly that pace for a narrow topic set, and a run that almost always
|
||||
# finds nothing trains you to ignore it.
|
||||
OnCalendar=Mon *-*-* 07:00:00
|
||||
# Fire on next boot if the machine was down at the scheduled time — a
|
||||
# missed week is a silently empty library.
|
||||
Persistent=true
|
||||
AccuracySec=1min
|
||||
Unit=clawmates-library.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -0,0 +1,69 @@
|
||||
-- The seen-set for continuous missions.
|
||||
--
|
||||
-- Every "continuous X" mission has the same failure mode: it runs again and
|
||||
-- redoes work it already did. Research resurfaces papers it already read; a
|
||||
-- security scan re-reports findings already triaged. Orchestration does not
|
||||
-- fix that — a record of what has already been covered does.
|
||||
--
|
||||
-- This repository already tried continuous research once. Migrations 0030-0044
|
||||
-- built `research_topics`, `research_outcomes` and `loops`; 0053 dropped them
|
||||
-- all. `research_topics` carried a status lifecycle but no seen-set, so it
|
||||
-- could run forever and never know what it had covered. That is the gap this
|
||||
-- table exists to close, and it is the reason it lands before any scheduling.
|
||||
--
|
||||
-- Authoritative here rather than in the runtime's memory: ZeroClaw memory is
|
||||
-- scoped per agent, and mission agents are ephemeral `claw_<uuid>` aliases
|
||||
-- minted per mission (measured: ~100 of them already). A seen-set that
|
||||
-- disappears with the agent that wrote it is not a seen-set.
|
||||
CREATE TABLE corpus_items (
|
||||
id UUID PRIMARY KEY,
|
||||
workspace_id UUID NOT NULL REFERENCES workspaces (id) ON DELETE CASCADE,
|
||||
-- Which corpus this belongs to, e.g. 'valhalla-vault'. A workspace can
|
||||
-- track several (a vault, a findings ledger, a paper collection).
|
||||
corpus_id TEXT NOT NULL,
|
||||
|
||||
-- 'note' = something already in the corpus (a vault file). Establishes
|
||||
-- coverage: what has this vault already got?
|
||||
-- 'source' = an external thing a mission consumed (a paper, an advisory).
|
||||
-- This is the dedupe key that stops re-reading.
|
||||
--
|
||||
-- Both are needed and they answer different questions. Measured against
|
||||
-- the real vault: 416 notes, and ZERO carry an arxiv/doi/url key — so an
|
||||
-- ingester keyed only on external identity would index nothing at all.
|
||||
kind TEXT NOT NULL CHECK (kind IN ('note', 'source')),
|
||||
|
||||
-- Stable identity within the corpus. For notes, 'note:<vault-relative
|
||||
-- path>'; for sources, a natural id like 'arxiv:2401.12345', 'doi:10...'
|
||||
-- or 'url:<sha256>'. Uniqueness is on this, which is what makes
|
||||
-- re-ingestion idempotent.
|
||||
source_id TEXT NOT NULL,
|
||||
|
||||
title TEXT,
|
||||
-- Vault-relative path for notes; NULL for external sources.
|
||||
path TEXT,
|
||||
url TEXT,
|
||||
|
||||
-- SHA-256 of the content at last sight. Lets a re-index distinguish
|
||||
-- "unchanged" from "edited" without diffing, so an unchanged vault is a
|
||||
-- genuine no-op rather than 416 pointless updates.
|
||||
content_hash TEXT NOT NULL,
|
||||
|
||||
first_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
-- Which mission first recorded this. NULL for the initial vault index,
|
||||
-- which is derived from files nobody's mission wrote.
|
||||
mission_id UUID REFERENCES missions (id) ON DELETE SET NULL,
|
||||
|
||||
UNIQUE (workspace_id, corpus_id, source_id)
|
||||
);
|
||||
|
||||
-- The hot query is "have I seen this?", which the UNIQUE index already covers.
|
||||
-- This one serves "what does this corpus contain?" for briefing assembly.
|
||||
CREATE INDEX corpus_items_corpus_idx
|
||||
ON corpus_items (workspace_id, corpus_id, kind, last_seen_at DESC);
|
||||
|
||||
-- "What did this mission add?" — the verification predicate for a continuous
|
||||
-- run is that it contributed at least one NEW source.
|
||||
CREATE INDEX corpus_items_mission_idx
|
||||
ON corpus_items (mission_id)
|
||||
WHERE mission_id IS NOT NULL;
|
||||
+38
-6
@@ -32,19 +32,51 @@ TAG=${TAG:-latest}
|
||||
SHA=$(git rev-parse --short HEAD 2>/dev/null || echo manual)
|
||||
AGENT_IMAGES=(agent-base agent-browser agent-terminal)
|
||||
|
||||
load() { ssh "$BUILD_HOST" "docker save $1" | ssh "$2" "docker load"; }
|
||||
# Stream an image between two remote hosts. The tar crosses two SSH
|
||||
# connections spliced through this workstation, so a stall on either side
|
||||
# truncates it — that surfaces as `unexpected EOF` from `docker load`, which
|
||||
# is a genuine failure and was previously indistinguishable from success
|
||||
# because nothing checked afterwards. Compress (these images are mostly
|
||||
# filesystem, and less bytes is less exposure to a stall) and set pipefail so
|
||||
# a failed `save` cannot be masked by a `load` that exits 0 on a short stream.
|
||||
load() {
|
||||
( set -o pipefail
|
||||
ssh "$BUILD_HOST" "docker save $1 | gzip -1" | ssh "$2" "gunzip | docker load" )
|
||||
}
|
||||
|
||||
# Load only if the target lacks the exact image (skips re-transferring unchanged
|
||||
# multi-hundred-MB agent images to every node on each code deploy).
|
||||
#
|
||||
# Verifies by image ID afterwards rather than trusting the exit status: a
|
||||
# truncated stream can still leave a partially-populated image, and shipping a
|
||||
# corrupt agent image to every fleet node is worse than failing the deploy.
|
||||
# One retry, because the observed failure is a transient stream stall.
|
||||
#
|
||||
# Identity is the image's `Created` stamp, NOT its `Id`. A BuildKit image on
|
||||
# the build host carries attestation manifests that `docker save | docker load`
|
||||
# does not reproduce, so the same build legitimately arrives with a different
|
||||
# Id and a different reported Size — comparing Ids fails every transfer of a
|
||||
# correctly-shipped image. `Created` comes from the config blob, survives the
|
||||
# round trip, and is what actually answers "is the new build here".
|
||||
load_if_changed() {
|
||||
local lid rid
|
||||
lid=$(ssh "$BUILD_HOST" "docker image inspect -f '{{.Id}}' $1 2>/dev/null" || true)
|
||||
rid=$(ssh "$2" "docker image inspect -f '{{.Id}}' $1 2>/dev/null" || true)
|
||||
if [ -n "$lid" ] && [ "$lid" = "$rid" ]; then
|
||||
local lts rts attempt
|
||||
lts=$(ssh "$BUILD_HOST" "docker image inspect -f '{{.Created}}' $1 2>/dev/null" || true)
|
||||
rts=$(ssh "$2" "docker image inspect -f '{{.Created}}' $1 2>/dev/null" || true)
|
||||
if [ -n "$lts" ] && [ "$lts" = "$rts" ]; then
|
||||
echo " (unchanged — skip)"
|
||||
return 0
|
||||
fi
|
||||
load "$1" "$2"
|
||||
for attempt in 1 2; do
|
||||
load "$1" "$2" || echo " (transfer attempt $attempt failed)"
|
||||
rts=$(ssh "$2" "docker image inspect -f '{{.Created}}' $1 2>/dev/null" || true)
|
||||
if [ -n "$lts" ] && [ "$lts" = "$rts" ]; then
|
||||
[ "$attempt" -gt 1 ] && echo " (recovered on attempt $attempt)"
|
||||
return 0
|
||||
fi
|
||||
echo " (build stamp mismatch after attempt $attempt — retrying)" >&2
|
||||
done
|
||||
echo " ✗ $1 did not land on $2 (wanted $lts, got ${rts:-nothing})" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
echo "→ sync to $BUILD_HOST"
|
||||
|
||||
Reference in New Issue
Block a user