Files
clawmates/crates/cm-api/src/mission_outputs.rs
T
Omar SobhandClaude Opus 5 548f977212 style: rustfmt the four files the repo-less mission fix touched
Found while removing .github/workflows/ci.yml: three of the four files in that
change were unformatted, and four of the diffs were newly introduced (the new
prompt tests and the tool_preamble format! call). Formatting only the files that
change already touched — a repo-wide `cargo fmt` would be 63 files of unrelated
churn and belongs in its own commit.

Mechanical; `cargo test -p cm-api --lib` stays at 322 passed.

Co-Authored-By: Claude Opus 5 <[email protected]>
2026-08-13 12:19:43 -07:00

534 lines
22 KiB
Rust

//! Capture for missions that have no repository.
//!
//! `mission_delivery` captures a phase's work by diffing a git checkout. A
//! mission with `repo_id IS NULL` — every `research_only` mission, because that
//! recipe sets `requires_repo = false` — has no checkout, so
//! `capture_finished_coding_phases` filters it out at the SQL level
//! (`AND m.repo_id IS NOT NULL`) and never reads the container at all.
//!
//! The agents still write files. The research directive tells them to save
//! findings under `/mission/repo/research/`, and it says so whether or not a
//! repo exists. So the work lands in the container's own filesystem, is never
//! collected, and is destroyed when the sweeper reaps the container.
//!
//! # What this cost, measured
//!
//! Mission `019fdc35` ("ClawHDF5 Research"): four agents, 9.5 minutes, **eight
//! research documents** — an HDF5 parser design, a Rust ecosystem survey, a
//! seven-crate dependency map, tracing and fuzzing strategy. `mission_artifacts`
//! held zero rows and the mission reported `completed`. One agent's own summary
//! recorded the situation exactly: *"No git repo — file is written."* It noticed,
//! wrote anyway, and the platform threw the result away without a word.
//!
//! Nothing survived but the summarizer's account of it — which is the agents'
//! description of the work, not the work.
//!
//! # Why a separate path rather than widening the diff capture
//!
//! There is no base commit to diff against and no branch to push, so every
//! concept `capture_phase_diff` is built on is absent. What a repo-less mission
//! produces is simply *files*, and the honest capture is to copy them out and
//! register each as an artifact. `_outputs/` is deliberately a SIBLING of the
//! mission directory and survives `teardown_container`, so artifacts registered
//! here outlive the reap that destroyed the originals.
use std::path::{Path, PathBuf};
use sqlx::{PgPool, Row};
use time::{Duration, OffsetDateTime};
use uuid::Uuid;
/// Directories never worth capturing, whatever an agent leaves behind.
///
/// Same intent as `mission_fs`'s exclusion list: a captured `.git` or
/// `node_modules` is noise that would bury the four documents that matter.
const SKIP_DIRS: &[&str] = &[
".git",
"node_modules",
"target",
".venv",
"venv",
"__pycache__",
".cache",
"dist",
"build",
];
/// How many phases to capture per tick, matching `CAPTURE_BATCH`.
const BATCH: i64 = 5;
/// How long a phase's outputs may stay uncollectable before the sweep stops
/// retrying and calls it empty.
///
/// Generous on purpose: the container is torn down asynchronously after a
/// phase, so an early tick can legitimately fail. What must NOT happen is
/// retrying forever — that is the state this constant exists to end.
const COLLECT_GRACE: Duration = Duration::minutes(10);
/// The artifact kind this path registers. Also the idempotency key: a phase with
/// one of these has already been captured.
pub const OUTPUT_KIND: &str = "document";
/// Filename of the marker written when a phase produced nothing.
const EMPTY_MARKER: &str = "NO-OUTPUT.md";
/// Capture the outputs of finished phases on missions that have no repo.
pub async fn capture_repo_less_phases(pool: &PgPool) -> Result<(), String> {
let rows = sqlx::query(
"SELECT mp.id, mp.mission_id, mp.kind, mp.config, mp.completed_at, m.runtime_kind
FROM mission_phases mp
JOIN missions m ON m.id = mp.mission_id
WHERE mp.status IN ('completed', 'failed')
AND m.repo_id IS NULL
-- microVM used to be excluded here because `run_phase_in_vm`
-- refused to boot without a checkout. It no longer does: a
-- repo-less mission gets an empty workspace at the same guest path,
-- and the collect unpacks it back onto the host — so those files are
-- already on disk and `collect_into` reads them instead of asking a
-- container that never existed.
AND NOT EXISTS (
SELECT 1 FROM mission_artifacts a
WHERE a.mission_id = mp.mission_id
AND a.phase_id = mp.id
AND a.kind = $2
)
ORDER BY mp.completed_at DESC NULLS LAST
LIMIT $1",
)
.bind(BATCH)
.bind(OUTPUT_KIND)
.fetch_all(pool)
.await
.map_err(|e| format!("select repo-less phases to capture: {e}"))?;
for row in rows {
let phase_id: Uuid = row.get("id");
let mission_id: Uuid = row.get("mission_id");
let kind: String = row.get("kind");
let config: serde_json::Value = row.get("config");
let completed_at: Option<OffsetDateTime> = row.get("completed_at");
let runtime_kind: String = row.get("runtime_kind");
let dest = outputs_dir(mission_id, phase_id);
let captured = match collect_into(mission_id, &dest, &runtime_kind).await {
Ok(files) => files,
Err(e) => {
// Retryable, but BOUNDED. A bare `continue` here is how a phase
// whose collect can never succeed stayed `completed` with zero
// artifacts forever: the fail-empty rule and the NO-OUTPUT
// marker both live below this point, so neither was ever
// reached, and the phase was re-attempted on every tick for the
// life of the deployment.
//
// The grace window exists because the container may legitimately
// not be ready on the first tick after a phase finishes. Past
// that, "cannot collect" and "collected nothing" are the same
// fact for the operator, so we fall through and let the rules
// below fail the phase and leave a marker explaining why.
let settled = completed_at
.map(|t| OffsetDateTime::now_utc() - t > COLLECT_GRACE)
.unwrap_or(true);
if !settled {
eprintln!(
"mission_outputs: could NOT collect outputs for phase {phase_id} \
of mission {mission_id} (will retry): {e}"
);
continue;
}
eprintln!(
"mission_outputs: giving up collecting phase {phase_id} of mission \
{mission_id} after {}s: {e} — treating it as having produced nothing",
COLLECT_GRACE.whole_seconds()
);
Vec::new()
}
};
for file in &captured {
let rel = match file.strip_prefix(missions_root()) {
Ok(r) => r.to_string_lossy().to_string(),
Err(_) => file.to_string_lossy().to_string(),
};
let title = file
.file_name()
.map(|n| n.to_string_lossy().to_string())
.unwrap_or_else(|| rel.clone());
if let Err(e) = cm_db::repo::missions::register_artifact(
pool,
cm_db::repo::missions::RegisterArtifact {
mission_id,
phase_id: Some(phase_id),
path: &rel,
kind: OUTPUT_KIND,
mime: Some(mime_for(file)),
title: Some(&title),
generated_by_run: None,
// No PDF. The renderer converted Markdown to HTML by
// calling an LLM — a paid API call, per document, on the
// critical path of "save my research", which promptly
// failed on depleted credits. Markdown IS the deliverable;
// it is served by `artifact_content` and styled at render
// time, which is free, offline, and cannot 429.
render_pdf: false,
metadata: Some(serde_json::json!({
"bytes": std::fs::metadata(file).map(|m| m.len()).unwrap_or(0),
"captured_from": "/mission/repo",
})),
},
)
.await
{
eprintln!("mission_outputs: registering {rel}: {e}");
}
}
if captured.is_empty() {
// Register a marker even when there is nothing to capture, or this
// phase matches the `NOT EXISTS` selection on every tick forever:
// re-running a docker copy_out each time and, because the batch is
// bounded, permanently occupying a slot so no other repo-less
// mission is ever captured again.
//
// `phase_runner::record_uncapturable` exists for exactly this
// failure on the diff path — five dead phases starved the batch
// while live work went untouched — and this code hit it again on
// its first live negative control (4 log lines, then 8, 45 seconds
// apart). Same shape, same fix: a real file behind a real row,
// because an artifact pointing at nothing turns every reader into
// an unexplained 404.
if let Err(e) = register_empty_marker(pool, mission_id, phase_id, &dest).await {
eprintln!("mission_outputs: marking phase {phase_id} as empty: {e}");
}
}
if captured.is_empty() && !allow_empty(&config) {
// The same rule `empty_delivery_is_a_failure` applies to a coding
// phase, for the only channel a repo-less phase has. Without it a
// research mission that produced nothing is indistinguishable from
// one that produced eight documents — both `completed`.
eprintln!(
"mission_outputs: phase {phase_id} ({kind}) of mission {mission_id} produced \
NO output files — failing it. Set config.allow_empty = true if this phase is \
meant to think rather than produce."
);
if let Err(e) = sqlx::query("UPDATE mission_phases SET status = 'failed' WHERE id = $1")
.bind(phase_id)
.execute(pool)
.await
{
eprintln!("mission_outputs: failing empty phase {phase_id}: {e}");
}
} else {
eprintln!(
"mission_outputs: captured {} file(s) from phase {phase_id} ({kind}) of \
mission {mission_id}",
captured.len()
);
}
}
Ok(())
}
/// Record that a phase produced nothing, so it is not reconsidered forever.
///
/// Deliberately the same `OUTPUT_KIND` the real captures use: the selection
/// query asks "has this phase been captured?", and "captured, and there was
/// nothing" is an answer to that question. `metadata.empty` is what tells the
/// two apart — the same convention `mission_delivery` uses for its "No code
/// changes" artifact.
async fn register_empty_marker(
pool: &PgPool,
mission_id: Uuid,
phase_id: Uuid,
dest: &Path,
) -> Result<(), String> {
std::fs::create_dir_all(dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
let file = dest.join(EMPTY_MARKER);
std::fs::write(
&file,
"This phase finished without leaving any files in its workspace, so there\n was nothing to publish. If the phase is meant to reason rather than\n produce, set `config.allow_empty = true` on it.\n",
)
.map_err(|e| format!("write {}: {e}", file.display()))?;
let rel = file
.strip_prefix(missions_root())
.map(|r| r.to_string_lossy().to_string())
.unwrap_or_else(|_| file.to_string_lossy().to_string());
cm_db::repo::missions::register_artifact(
pool,
cm_db::repo::missions::RegisterArtifact {
mission_id,
phase_id: Some(phase_id),
path: &rel,
kind: OUTPUT_KIND,
mime: Some("text/markdown"),
title: Some("No output produced"),
generated_by_run: None,
render_pdf: false,
metadata: Some(serde_json::json!({ "empty": true })),
},
)
.await
.map(|_| ())
.map_err(|e| format!("register empty marker: {e}"))
}
/// Gather the mission's produced files and return the ones worth keeping.
///
/// Where they come from depends on the runtime, and the difference is not
/// cosmetic: a container mission's files are still INSIDE a running container,
/// while a microVM's have already been unpacked onto the host by the collect at
/// the end of the turn (`microvm_executor` writes them over
/// `mission_workspace::checkout_path`). Asking docker for a VM mission's files
/// would query a container that never existed.
async fn collect_into(
mission_id: Uuid,
dest: &Path,
runtime_kind: &str,
) -> Result<Vec<PathBuf>, String> {
// A stale copy from an earlier attempt would be registered as this pass's
// output — the same "captured a tree nobody wrote" shape capture avoids.
let _ = std::fs::remove_dir_all(dest);
std::fs::create_dir_all(dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
if runtime_kind == "microvm" {
let src = crate::mission_workspace::checkout_path(mission_id);
if !src.is_dir() {
return Err(format!(
"{} is absent — the VM's collect did not land",
src.display()
));
}
copy_tree(&src, &dest.join("repo"))?;
return Ok(keep_files(&dest.join("repo")));
}
let container = crate::mission_runtime::container_name(mission_id);
let docker = crate::container_exec::connect()?;
crate::mission_fs::copy_out(&docker, &container, "/mission/repo", dest).await?;
Ok(keep_files(&dest.join("repo")))
}
/// Recursive file copy. Small on purpose — the alternative is a dependency or a
/// shell-out, and this runs as the server's own uid against its own directory.
fn copy_tree(src: &Path, dest: &Path) -> Result<(), String> {
std::fs::create_dir_all(dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
let entries = std::fs::read_dir(src).map_err(|e| format!("read {}: {e}", src.display()))?;
for entry in entries.flatten() {
let from = entry.path();
let to = dest.join(entry.file_name());
match entry.file_type() {
Ok(t) if t.is_dir() => copy_tree(&from, &to)?,
Ok(t) if t.is_file() => {
std::fs::copy(&from, &to).map_err(|e| format!("copy {}: {e}", from.display()))?;
}
// Symlinks and specials are skipped rather than followed: a link out
// of the tree would publish whatever it points at.
_ => {}
}
}
Ok(())
}
/// Every regular file worth keeping, recursively.
fn keep_files(root: &Path) -> Vec<PathBuf> {
let mut out = Vec::new();
let mut stack = vec![root.to_path_buf()];
while let Some(dir) = stack.pop() {
let Ok(entries) = std::fs::read_dir(&dir) else {
continue;
};
for entry in entries.flatten() {
let path = entry.path();
let name = entry.file_name().to_string_lossy().to_string();
if path.is_dir() {
if !SKIP_DIRS.contains(&name.as_str()) {
stack.push(path);
}
} else if path.is_file()
&& !name.starts_with('.')
// The agent runtime seeds its own identity files into the
// workspace root, which is pinned to the repo root. In a
// repo-backed mission `.git/info/exclude` hides them; a
// repo-less mission has no `.git`, so without this the user's
// artifact list is 7 files of agent scaffolding and 2 of their
// research. Measured exactly that way on the first live run.
&& !crate::mission_workspace::AGENT_SCAFFOLDING.contains(&name.as_str())
{
out.push(path);
}
}
}
out.sort();
out
}
/// `<missions_root>/_outputs/<mission>/<phase>` — a sibling of the mission
/// directory, so `teardown_container` reaping the mission does not take the
/// captured artifacts with it.
fn outputs_dir(mission_id: Uuid, phase_id: Uuid) -> PathBuf {
missions_root()
.join("_outputs")
.join(mission_id.to_string())
.join(phase_id.to_string())
}
/// The missions root, for callers that resolve artifact paths against it.
pub fn missions_root_dir() -> PathBuf {
missions_root()
}
/// The only directory an artifact may be read from.
pub fn outputs_root_dir() -> PathBuf {
missions_root().join("_outputs")
}
fn missions_root() -> PathBuf {
crate::mission_workspace::missions_root()
}
fn mime_for(p: &Path) -> &'static str {
match p.extension().and_then(|e| e.to_str()) {
Some("md") | Some("markdown") => "text/markdown",
Some("json") => "application/json",
Some("csv") => "text/csv",
Some("html") => "text/html",
_ => "text/plain",
}
}
fn allow_empty(config: &serde_json::Value) -> bool {
config.get("allow_empty").and_then(|v| v.as_bool()) == Some(true)
}
#[cfg(test)]
mod tests {
use super::*;
fn touch(p: &Path) {
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
std::fs::write(p, "x").unwrap();
}
/// The documents a research phase writes are what must come back — and the
/// machinery around them must not.
#[test]
fn research_documents_are_kept_and_scaffolding_is_not() {
let tmp = tempfile::tempdir().unwrap();
let repo = tmp.path().join("repo");
touch(&repo.join("research/01_repo_archaeology.md"));
touch(&repo.join("research/02_ecosystem.md"));
touch(&repo.join("notes.txt"));
// The seven the agent runtime seeds into the workspace root.
for f in crate::mission_workspace::AGENT_SCAFFOLDING {
touch(&repo.join(f));
}
touch(&repo.join(".git/HEAD"));
touch(&repo.join("node_modules/left-pad/index.js"));
touch(&repo.join("target/debug/thing"));
touch(&repo.join(".hidden"));
let kept: Vec<String> = keep_files(&repo)
.iter()
.map(|p| p.strip_prefix(&repo).unwrap().to_string_lossy().to_string())
.collect();
assert_eq!(
kept,
vec![
"notes.txt".to_string(),
"research/01_repo_archaeology.md".to_string(),
"research/02_ecosystem.md".to_string(),
],
"kept: {kept:?}"
);
}
/// Markdown is the deliverable, so it must be labelled as markdown — the
/// viewer decides how to render from the mime type.
#[test]
fn markdown_is_labelled_so_the_viewer_can_style_it() {
assert_eq!(mime_for(Path::new("/x/01_notes.md")), "text/markdown");
assert_eq!(mime_for(Path::new("/x/data.json")), "application/json");
}
/// The containment rule the content endpoint enforces: everything readable
/// lives under `_outputs`, and nothing else does.
///
/// Artifact paths are written by this server, but they are DATA in a table,
/// and a row saying `../../../etc/passwd` must be a 404 rather than a file
/// read. The endpoint canonicalises before comparing — checking the string
/// first would pass `_outputs/../../etc/passwd` straight through.
#[test]
fn everything_readable_lives_under_the_outputs_root() {
let root = outputs_root_dir();
assert!(root.ends_with("_outputs"), "{root:?}");
assert!(root.starts_with(missions_root_dir()), "{root:?}");
// A real capture is inside it...
let inside = outputs_dir(Uuid::now_v7(), Uuid::now_v7());
assert!(inside.starts_with(&root), "{inside:?}");
// ...and the traversal shape this guards against is not, once resolved.
let escaped = root.join("..").join("..").join("etc/passwd");
let normalised: PathBuf = escaped.components().fold(PathBuf::new(), |mut acc, c| {
match c {
std::path::Component::ParentDir => {
acc.pop();
}
other => acc.push(other),
}
acc
});
assert!(
!normalised.starts_with(&root),
"a traversal must not resolve back inside the outputs root: {normalised:?}"
);
}
/// A phase that produced nothing must still leave a marker, or the
/// selection query matches it on every tick forever.
///
/// Measured on the first live negative control: the guard logged "produced
/// NO output files" 4 times, then 8 times 45 seconds later — a docker
/// copy_out per tick, and with a bounded batch, five such phases would
/// starve every other repo-less mission out of capture permanently.
/// `phase_runner::record_uncapturable` was written for the identical
/// failure on the diff path.
#[test]
fn an_empty_phase_leaves_a_marker_so_it_is_not_reconsidered_forever() {
let tmp = tempfile::tempdir().unwrap();
let dest = tmp.path().join("out");
// The file-writing half of `register_empty_marker`, which is the part
// that must exist for the artifact row to point at something real.
std::fs::create_dir_all(&dest).unwrap();
let file = dest.join(EMPTY_MARKER);
std::fs::write(&file, "x").unwrap();
assert!(file.exists(), "an artifact row must not point at nothing");
assert_eq!(
file.file_name().unwrap().to_string_lossy(),
"NO-OUTPUT.md",
"the marker name is part of the contract with readers"
);
// And the marker must not itself be mistaken for captured output on a
// later pass: it is filtered like any other scaffolding would be.
assert!(keep_files(&dest).iter().any(|p| p == &file));
}
/// Artifacts must land OUTSIDE the mission directory. `teardown_container`
/// removes `<missions_root>/<mission_id>` wholesale, so a capture written
/// inside it would be destroyed by the very reap it exists to survive.
#[test]
fn captures_survive_the_mission_directory_being_reaped() {
let mission = Uuid::now_v7();
let phase = Uuid::now_v7();
let out = outputs_dir(mission, phase);
let mission_dir = missions_root().join(mission.to_string());
assert!(
!out.starts_with(&mission_dir),
"{} must not be inside {}",
out.display(),
mission_dir.display()
);
assert!(out.starts_with(missions_root().join("_outputs")), "{out:?}");
}
}