Compare commits
169
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d810fc0a86 | ||
|
|
31158467f4 | ||
|
|
f8438c32ea | ||
|
|
9e61e3ba35 | ||
|
|
c2fa8067e1 | ||
|
|
cb8184e784 | ||
|
|
f37c6b92d8 | ||
|
|
006432c2dc | ||
|
|
5f85dbb718 | ||
|
|
b210acf3c2 | ||
|
|
44079eb8b4 | ||
|
|
d3a398716b | ||
|
|
98037f9b3e | ||
|
|
c85027c83a | ||
|
|
0ad53da49c | ||
|
|
e2c312b728 | ||
|
|
104e3ef27c | ||
|
|
4c418f7d9b | ||
|
|
b2e2735583 | ||
|
|
e417247e7e | ||
|
|
fe2451fd60 | ||
|
|
895413509d | ||
|
|
dd80b69992 | ||
|
|
768e106614 | ||
|
|
e4bddeb1ba | ||
|
|
25f075a8be | ||
|
|
f27d2605eb | ||
|
|
16cfc29074 | ||
|
|
529497febb | ||
|
|
171f901bcd | ||
|
|
e7b412d578 | ||
|
|
c66c3c6377 | ||
|
|
1f6108f769 | ||
|
|
3c3d01c8d1 | ||
|
|
c7c3eeab46 | ||
|
|
d9c5300859 | ||
|
|
c3ad5672fc | ||
|
|
5afcf63324 | ||
|
|
b18e62041b | ||
|
|
774f17d194 | ||
|
|
f56d41f5b7 | ||
|
|
e96c5143bc | ||
|
|
f68fc019e4 | ||
|
|
4967b9b8fd | ||
|
|
8ddea454d1 | ||
|
|
dcd9514622 | ||
|
|
42108c840d | ||
|
|
13a35138e9 | ||
|
|
d4af58be85 | ||
|
|
eacd3ee085 | ||
|
|
91fbd2dc88 | ||
|
|
e5f097c291 | ||
|
|
4fedfcec30 | ||
|
|
2056bb1d9e | ||
|
|
dc0443de34 | ||
|
|
d48bdbc9a7 | ||
|
|
52500a689c | ||
|
|
9c9439a271 | ||
|
|
ee5a939ce6 | ||
|
|
deed591da6 | ||
|
|
c3c4447810 | ||
|
|
72046e7985 | ||
|
|
d84d17207f | ||
|
|
3a2d76aa43 | ||
|
|
5c5f1ced33 | ||
|
|
8b12245e79 | ||
|
|
099a716bfd | ||
|
|
4193ae2cda | ||
|
|
8c93cd8569 | ||
|
|
09afa7e7ff | ||
|
|
5b49d5a1a8 | ||
|
|
28090d1de0 | ||
|
|
0b89b8316c | ||
|
|
62509a5090 | ||
|
|
3616bc4733 | ||
|
|
a8b8efba6a | ||
|
|
a4b4d05b8d | ||
|
|
e89a32ffef | ||
|
|
a93a4111e1 | ||
|
|
0d8db7ff0b | ||
|
|
2a9a62c784 | ||
|
|
6dd7937ece | ||
|
|
a20702d55b | ||
|
|
87f188ae73 | ||
|
|
f6c3ddbf81 | ||
|
|
821cbb8622 | ||
|
|
da889f83ab | ||
|
|
c28c7a148f | ||
|
|
89bc53b53d | ||
|
|
ceab28b902 | ||
|
|
bcf4866abc | ||
|
|
b36ae00ea5 | ||
|
|
cd4d76a8c3 | ||
|
|
5c066afa7b | ||
|
|
d24823b6f3 | ||
|
|
bf2055e725 | ||
|
|
72f8bdc87c | ||
|
|
e2f576ec02 | ||
|
|
1b556c5849 | ||
|
|
521da9feb9 | ||
|
|
3300c9d149 | ||
|
|
08dd227a45 | ||
|
|
0aeae07db2 | ||
|
|
a33dbdcdc3 | ||
|
|
a48d78f8eb | ||
|
|
742724e53c | ||
|
|
d3a53e7bf1 | ||
|
|
f7f3dfe495 | ||
|
|
75d09241fb | ||
|
|
aa470091aa | ||
|
|
1797669296 | ||
|
|
abb97e6f03 | ||
|
|
6991e21f94 | ||
|
|
1d554396f4 | ||
|
|
12147a1e01 | ||
|
|
e31688bac5 | ||
|
|
66f730ad16 | ||
|
|
d49acaed5e | ||
|
|
4efcde9d4f | ||
|
|
0d25a94a84 | ||
|
|
cb48f7ff3b | ||
|
|
c840688adb | ||
|
|
b17e18aa67 | ||
|
|
9aed20b6d0 | ||
|
|
bb807c2f3a | ||
|
|
8796fbbcbb | ||
|
|
11b274edc6 | ||
|
|
2dee941080 | ||
|
|
7696009b25 | ||
|
|
d9f53a3f96 | ||
|
|
1cd81a8b2a | ||
|
|
521b8dea10 | ||
|
|
0a9747091f | ||
|
|
c9b7d8b6ca | ||
|
|
0206be68e5 | ||
|
|
4f07430e92 | ||
|
|
76fe1f1148 | ||
|
|
ebdba34da6 | ||
|
|
abc4160a89 | ||
|
|
2edafdaf0d | ||
|
|
92055c4556 | ||
|
|
c3297b86cf | ||
|
|
bcd1a0127d | ||
|
|
0c291ed1bb | ||
|
|
fd16b3c126 | ||
|
|
6687f8b808 | ||
|
|
fcf5d7b16c | ||
|
|
08847e6a63 | ||
|
|
78da62f156 | ||
|
|
dec59764b1 | ||
|
|
0f2591bae4 | ||
|
|
02ba557c3e | ||
|
|
22efb93775 | ||
|
|
2d04c5e257 | ||
|
|
b87d89f9fa | ||
|
|
67c56ce19b | ||
|
|
0f7fa31f86 | ||
|
|
4454a1cfd9 | ||
|
|
e65be19a45 | ||
|
|
452b419729 | ||
|
|
da3731d753 | ||
|
|
f8ca0ced9a | ||
|
|
4f6719c80e | ||
|
|
3c91d0e172 | ||
|
|
1253595ba7 | ||
|
|
bb274d08c6 | ||
|
|
7e07c389c6 | ||
|
|
389b41f8e6 | ||
|
|
ac6bf72943 |
Generated
+56
@@ -846,6 +846,8 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sysinfo",
|
"sysinfo",
|
||||||
|
"tar",
|
||||||
|
"tempfile",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-tungstenite 0.26.2",
|
"tokio-tungstenite 0.26.2",
|
||||||
"webrtc",
|
"webrtc",
|
||||||
@@ -970,6 +972,7 @@ dependencies = [
|
|||||||
"serde_yaml",
|
"serde_yaml",
|
||||||
"sha2",
|
"sha2",
|
||||||
"sqlx",
|
"sqlx",
|
||||||
|
"tar",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"thiserror 2.0.18",
|
"thiserror 2.0.18",
|
||||||
"time",
|
"time",
|
||||||
@@ -1841,6 +1844,16 @@ version = "2.4.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
|
checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fcagent"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"base64",
|
||||||
|
"serde_json",
|
||||||
|
"tar",
|
||||||
|
"vsock",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ff"
|
name = "ff"
|
||||||
version = "0.13.1"
|
version = "0.13.1"
|
||||||
@@ -2872,6 +2885,15 @@ dependencies = [
|
|||||||
"autocfg",
|
"autocfg",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "memoffset"
|
||||||
|
version = "0.9.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mime"
|
name = "mime"
|
||||||
version = "0.3.17"
|
version = "0.3.17"
|
||||||
@@ -2962,6 +2984,19 @@ dependencies = [
|
|||||||
"pin-utils",
|
"pin-utils",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "nix"
|
||||||
|
version = "0.31.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 2.13.0",
|
||||||
|
"cfg-if",
|
||||||
|
"cfg_aliases",
|
||||||
|
"libc",
|
||||||
|
"memoffset 0.9.1",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "nom"
|
name = "nom"
|
||||||
version = "7.1.3"
|
version = "7.1.3"
|
||||||
@@ -5029,6 +5064,17 @@ dependencies = [
|
|||||||
"windows",
|
"windows",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tar"
|
||||||
|
version = "0.4.46"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840"
|
||||||
|
dependencies = [
|
||||||
|
"filetime",
|
||||||
|
"libc",
|
||||||
|
"xattr",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tempfile"
|
name = "tempfile"
|
||||||
version = "3.27.0"
|
version = "3.27.0"
|
||||||
@@ -5749,6 +5795,16 @@ version = "0.9.5"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "vsock"
|
||||||
|
version = "0.5.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6ba782755fc073877e567c2253c0be48e4aa9a254c232d36d3985dfae0bd5205"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"nix 0.31.3",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wait-timeout"
|
name = "wait-timeout"
|
||||||
version = "0.2.1"
|
version = "0.2.1"
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ members = [
|
|||||||
"crates/bins/clawmates-server",
|
"crates/bins/clawmates-server",
|
||||||
"crates/bins/clawmates-broker",
|
"crates/bins/clawmates-broker",
|
||||||
"crates/bins/clawmates-node",
|
"crates/bins/clawmates-node",
|
||||||
|
"crates/bins/fcagent",
|
||||||
"tools/bundler",
|
"tools/bundler",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -36,6 +37,9 @@ publish = false
|
|||||||
# Shared dependency versions; crates opt in via { workspace = true }.
|
# Shared dependency versions; crates opt in via { workspace = true }.
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
|
# Streaming tar for mission copy-in/copy-out (no compression: the payload is
|
||||||
|
# a git checkout on a local socket, so CPU spent zipping buys nothing).
|
||||||
|
tar = "0.4"
|
||||||
thiserror = "2"
|
thiserror = "2"
|
||||||
uuid = { version = "1", features = ["v7", "serde"] }
|
uuid = { version = "1", features = ["v7", "serde"] }
|
||||||
proptest = "1"
|
proptest = "1"
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ serde_json = { workspace = true }
|
|||||||
sysinfo = "0.33"
|
sysinfo = "0.33"
|
||||||
portable-pty = "0.8"
|
portable-pty = "0.8"
|
||||||
base64 = "0.22"
|
base64 = "0.22"
|
||||||
|
tar = { workspace = true }
|
||||||
cm-sandbox = { path = "../../cm-sandbox" }
|
cm-sandbox = { path = "../../cm-sandbox" }
|
||||||
# Linking cm-sandbox (bollard) brings a second rustls provider into the graph, so
|
# Linking cm-sandbox (bollard) brings a second rustls provider into the graph, so
|
||||||
# rustls can't auto-pick one — we install `ring` explicitly at startup.
|
# rustls can't auto-pick one — we install `ring` explicitly at startup.
|
||||||
@@ -26,5 +27,8 @@ rustls = { version = "0.23", default-features = false, features = ["ring"] }
|
|||||||
webrtc = "0.17.1"
|
webrtc = "0.17.1"
|
||||||
bytes = "1.12.0"
|
bytes = "1.12.0"
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
tempfile = "3"
|
||||||
|
|
||||||
[lints]
|
[lints]
|
||||||
workspace = true
|
workspace = true
|
||||||
|
|||||||
@@ -0,0 +1,527 @@
|
|||||||
|
//! Host side of a microVM's only route out: an HTTP `CONNECT` proxy on a Unix
|
||||||
|
//! socket, one per VM.
|
||||||
|
//!
|
||||||
|
//! # Why the guest has no network card
|
||||||
|
//!
|
||||||
|
//! It could have had one. A TAP device plus NAT is what the Firecracker
|
||||||
|
//! write-ups do, and it was measured against this before being rejected:
|
||||||
|
//!
|
||||||
|
//! - `ip tuntap add` is **denied to the daemon user** (needs `CAP_NET_ADMIN`), so
|
||||||
|
//! TAP would need root to pre-provision devices at setup time — the same
|
||||||
|
//! privilege detour the loop-mounted rootfs already forced.
|
||||||
|
//! - tank's `FORWARD` policy is `DROP` with Docker and Tailscale chains, so rules
|
||||||
|
//! would have to be *inserted* at position 1; appended ones die silently.
|
||||||
|
//! - a leaked TAP device is a new class of host litter to reap.
|
||||||
|
//!
|
||||||
|
//! Against that, `CONNECT` needs no privilege at all, and it is better on the
|
||||||
|
//! merits: the client hands us the **hostname**, so resolution happens here and
|
||||||
|
//! the guest needs no DNS or `resolv.conf`; the allow-list is by name rather than
|
||||||
|
//! by address; and nothing in the guest can reach the network except through this
|
||||||
|
//! function. That is what the isolation plan's egress restriction actually asked
|
||||||
|
//! for, and it is strictly tighter than the mission container's present full
|
||||||
|
//! egress on `clawmates_edge`.
|
||||||
|
//!
|
||||||
|
//! The design rests on one measured fact: **`claude` honours `HTTPS_PROXY`**.
|
||||||
|
//! With the proxy pointed at a closed port, `claude -p` fails with
|
||||||
|
//! `ConnectionRefused` instead of answering. (That could only be measured in a
|
||||||
|
//! container — inside a VM the CLI collapses every failure into `Execution
|
||||||
|
//! error`.)
|
||||||
|
//!
|
||||||
|
//! # Shape
|
||||||
|
//!
|
||||||
|
//! Firecracker's convention for a guest-initiated connection is that the **host**
|
||||||
|
//! listens on `<uds_path>_<port>`. The guest's agent pumps bytes from
|
||||||
|
//! `127.0.0.1:3128` to vsock port 9002 and parses nothing, so all policy is here
|
||||||
|
//! and a compromised guest cannot argue with it.
|
||||||
|
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
|
||||||
|
use tokio::net::{TcpStream, UnixListener, UnixStream};
|
||||||
|
|
||||||
|
/// Port the guest dials. Must match `fcagent`'s `EGRESS_PORT`.
|
||||||
|
pub const EGRESS_PORT: u32 = 9002;
|
||||||
|
|
||||||
|
|
||||||
|
/// What every backend gets, whatever it is.
|
||||||
|
const COMMON_ALLOW: &[&str] = &["git.redclaw.dev"];
|
||||||
|
|
||||||
|
/// The model host a backend's CLI must reach, and NOTHING else.
|
||||||
|
///
|
||||||
|
/// Per backend rather than a union, and that is not tidiness. MEASURED on tank:
|
||||||
|
/// a `glm` VM completed a whole mission with `api.anthropic.com` denied at this
|
||||||
|
/// proxy, dialling only `api.z.ai` — Claude Code's calls to anthropic.com are
|
||||||
|
/// its own telemetry, not its completions. So a GLM VM has no need of Anthropic
|
||||||
|
/// at all, and a union allow-list would let a credential mix-up reach the wrong
|
||||||
|
/// provider's endpoint instead of failing at a closed door.
|
||||||
|
///
|
||||||
|
/// The measurement also settled something a self-report could not: that same
|
||||||
|
/// agent, served only by z.ai, still described itself as "Claude Opus 5". A
|
||||||
|
/// model's account of which model it is has no evidential value here; the
|
||||||
|
/// proxy's log of which host it dialled does.
|
||||||
|
fn provider_hosts(backend: Option<&str>) -> &'static [&'static str] {
|
||||||
|
match backend {
|
||||||
|
// `canary-claude` is the same provider, from a candidate CLI image —
|
||||||
|
// see `mission_runtime::microvm_credential_for`, which must grant it the
|
||||||
|
// same credential. A backend is defined in TWO maps: the credential one
|
||||||
|
// on the server and this one on the node. Adding it to only the first is
|
||||||
|
// exactly what happened here: the mission launched, the VM booted, the
|
||||||
|
// agent ran, and the turn died on
|
||||||
|
// "403 api.anthropic.com is not on the egress allow-list" — which is the
|
||||||
|
// fail-closed branch below working correctly.
|
||||||
|
None | Some("") | Some("default") | Some("claude") | Some("canary-claude") => {
|
||||||
|
&["api.anthropic.com", ".anthropic.com"]
|
||||||
|
}
|
||||||
|
Some("glm") => &["api.z.ai"],
|
||||||
|
// The Kimi CODE service, which is where an `sk-kimi-` key is valid —
|
||||||
|
// NOT `api.moonshot.ai`, whose Anthropic endpoint exists but belongs to
|
||||||
|
// a different account namespace and rejects that key. Only the host the
|
||||||
|
// `agent-kimi` image bakes in.
|
||||||
|
Some("kimi") => &["api.kimi.com"],
|
||||||
|
// A locally-hosted model reaches NOTHING through this proxy. Its route
|
||||||
|
// is `crate::local_model` — a vsock pipe to the node's own loopback,
|
||||||
|
// with no destination in the protocol — so the correct allow-list here
|
||||||
|
// is the empty one, and it falls through to the branch below.
|
||||||
|
//
|
||||||
|
// Spelled out rather than left implicit because the temptation was to
|
||||||
|
// widen this proxy instead: an entry here would have meant relaxing the
|
||||||
|
// 443-only rule AND the IP-literal refusal, both of which exist because
|
||||||
|
// a unit test caught them being bypassed.
|
||||||
|
// Fail closed: a backend nobody taught this function about reaches the
|
||||||
|
// forge and no model API. It cannot silently borrow another provider's
|
||||||
|
// door, which is the failure this split exists to prevent.
|
||||||
|
Some(_) => &[],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse the allow-list once per VM.
|
||||||
|
///
|
||||||
|
/// An empty `CLAWMATES_FC_EGRESS_ALLOW` means **deny everything**, not "fall back
|
||||||
|
/// to the default": an operator who blanked it asked for no egress, and quietly
|
||||||
|
/// restoring the default would hand a mission the network they just took away.
|
||||||
|
/// The allow-list for a VM running `backend`.
|
||||||
|
///
|
||||||
|
/// An explicit `CLAWMATES_FC_EGRESS_ALLOW` still wins outright: an operator who
|
||||||
|
/// set it asked for exactly that list, and quietly adding a provider host to it
|
||||||
|
/// would widen a boundary they had drawn on purpose.
|
||||||
|
fn allow_list_for(backend: Option<&str>) -> Vec<String> {
|
||||||
|
match std::env::var("CLAWMATES_FC_EGRESS_ALLOW") {
|
||||||
|
Ok(raw) => raw
|
||||||
|
.split(',')
|
||||||
|
.map(|s| s.trim().to_ascii_lowercase())
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.collect(),
|
||||||
|
Err(_) => COMMON_ALLOW
|
||||||
|
.iter()
|
||||||
|
.chain(provider_hosts(backend).iter())
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
.collect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Is `host` allowed?
|
||||||
|
///
|
||||||
|
/// Case-insensitive, port already stripped. A leading `.` in an entry matches
|
||||||
|
/// that domain and its subdomains; anything else must match exactly. Deliberately
|
||||||
|
/// not a substring test — `api.anthropic.com.evil.test` contains the allowed name
|
||||||
|
/// and must not pass.
|
||||||
|
fn host_allowed(host: &str, allow: &[String]) -> bool {
|
||||||
|
let host = host.trim().trim_end_matches('.').to_ascii_lowercase();
|
||||||
|
if host.is_empty() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// A hostname is letters, digits, dots and hyphens — nothing else. This is
|
||||||
|
// load-bearing, not hygiene: `evil.test/api.anthropic.com` ends with an
|
||||||
|
// allowed suffix and would otherwise PASS the match below. A unit test found
|
||||||
|
// it. Rejecting the character class also refuses IP literals, so an address
|
||||||
|
// cannot be used to sidestep a list written in names.
|
||||||
|
if !host
|
||||||
|
.chars()
|
||||||
|
.all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-')
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
allow.iter().any(|a| match a.strip_prefix('.') {
|
||||||
|
Some(domain) => host == domain || host.ends_with(&format!(".{domain}")),
|
||||||
|
None => host == *a,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Split `host:port` from a CONNECT target.
|
||||||
|
///
|
||||||
|
/// Only 443 is allowed. Permitting arbitrary ports would turn the proxy into a
|
||||||
|
/// general-purpose tunnel to anything the allow-list happens to name, which is a
|
||||||
|
/// different and much larger promise than "the agent can reach its API".
|
||||||
|
fn parse_target(target: &str) -> Result<(String, u16), String> {
|
||||||
|
let (host, port) = target
|
||||||
|
.rsplit_once(':')
|
||||||
|
.ok_or_else(|| format!("CONNECT target {target:?} has no port"))?;
|
||||||
|
let port: u16 = port
|
||||||
|
.trim()
|
||||||
|
.parse()
|
||||||
|
.map_err(|_| format!("CONNECT target {target:?} has a non-numeric port"))?;
|
||||||
|
if port != 443 {
|
||||||
|
return Err(format!("port {port} is not permitted (only 443)"));
|
||||||
|
}
|
||||||
|
// Strip IPv6 brackets so the allow-list sees the same text either way.
|
||||||
|
let host = host.trim().trim_start_matches('[').trim_end_matches(']');
|
||||||
|
Ok((host.to_string(), port))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What happened to one connection. Returned so the caller can log it and the
|
||||||
|
/// selftest can assert on it.
|
||||||
|
#[derive(Debug, PartialEq, Eq)]
|
||||||
|
pub enum Verdict {
|
||||||
|
Allowed(String),
|
||||||
|
Denied(String),
|
||||||
|
Malformed(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One header line, with a cap.
|
||||||
|
///
|
||||||
|
/// `read_line` has no limit, and a guest that never sends a newline would make
|
||||||
|
/// the host allocate until it died. Read byte-wise instead — the reads come out
|
||||||
|
/// of the BufReader, so this is cheap for lines this size, and it keeps ONE
|
||||||
|
/// reader over the connection, which matters (see `serve`).
|
||||||
|
async fn read_line_capped(reader: &mut BufReader<UnixStream>, cap: usize) -> Result<String, String> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
loop {
|
||||||
|
match reader.read_u8().await {
|
||||||
|
Ok(b'\n') => break,
|
||||||
|
Ok(b) => out.push(b),
|
||||||
|
// EOF mid-line: return what we have and let the caller judge it.
|
||||||
|
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break,
|
||||||
|
Err(e) => return Err(format!("read: {e}")),
|
||||||
|
}
|
||||||
|
if out.len() > cap {
|
||||||
|
return Err(format!("a request line longer than {cap} bytes"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(String::from_utf8_lossy(&out)
|
||||||
|
.trim_end_matches('\r')
|
||||||
|
.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Serve one tunnelled connection.
|
||||||
|
async fn serve(stream: UnixStream, allow: Arc<Vec<String>>) -> Verdict {
|
||||||
|
// ONE reader for the whole request. Wrapping the stream a second time would
|
||||||
|
// discard whatever the first reader had already buffered — including the
|
||||||
|
// first bytes of the TLS handshake — and the tunnel would come up looking
|
||||||
|
// fine and then stall on a corrupt stream.
|
||||||
|
let mut reader = BufReader::new(stream);
|
||||||
|
|
||||||
|
let line = match read_line_capped(&mut reader, 8 * 1024).await {
|
||||||
|
Ok(l) if !l.trim().is_empty() => l,
|
||||||
|
Ok(_) => return Verdict::Malformed("no request line".into()),
|
||||||
|
Err(e) => return Verdict::Malformed(e),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut parts = line.split_whitespace();
|
||||||
|
let method = parts.next().unwrap_or_default().to_ascii_uppercase();
|
||||||
|
let target = parts.next().unwrap_or_default().to_string();
|
||||||
|
|
||||||
|
if method != "CONNECT" {
|
||||||
|
// Plain HTTP would mean proxying a request we would then have to rewrite,
|
||||||
|
// and everything a mission needs is TLS. Refused with a status, so the
|
||||||
|
// client reports something better than a closed socket.
|
||||||
|
let _ = reply(reader.get_mut(), 405, "only CONNECT is supported").await;
|
||||||
|
return Verdict::Malformed(format!("method {method}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
let (host, port) = match parse_target(&target) {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(e) => {
|
||||||
|
let _ = reply(reader.get_mut(), 400, &e).await;
|
||||||
|
return Verdict::Malformed(e);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if !host_allowed(&host, &allow) {
|
||||||
|
// 403 rather than a silent drop: a denial that looks like a network
|
||||||
|
// timeout is indistinguishable from a hung agent, and this codebase has
|
||||||
|
// paid for that confusion more than once.
|
||||||
|
let _ = reply(
|
||||||
|
reader.get_mut(),
|
||||||
|
403,
|
||||||
|
&format!("{host} is not on the egress allow-list"),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Verdict::Denied(host);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Consume the remaining request headers: they belong to the CONNECT, not to
|
||||||
|
// the tunnel.
|
||||||
|
loop {
|
||||||
|
match read_line_capped(&mut reader, 8 * 1024).await {
|
||||||
|
Ok(h) if h.trim().is_empty() => break,
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(e) => return Verdict::Malformed(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut upstream = match TcpStream::connect((host.as_str(), port)).await {
|
||||||
|
Ok(s) => s,
|
||||||
|
Err(e) => {
|
||||||
|
let _ = reply(reader.get_mut(), 502, &format!("connect {host}:{port}: {e}")).await;
|
||||||
|
return Verdict::Denied(host);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if reply(reader.get_mut(), 200, "Connection established")
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
return Verdict::Denied(host);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Anything already buffered past the headers is tunnel payload — a client
|
||||||
|
// that pipelined its first TLS bytes would otherwise lose them.
|
||||||
|
let pending = reader.buffer().to_vec();
|
||||||
|
let mut stream = reader.into_inner();
|
||||||
|
if !pending.is_empty() && upstream.write_all(&pending).await.is_err() {
|
||||||
|
return Verdict::Denied(host);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bytes both ways until either side is done. Errors are not worth reporting:
|
||||||
|
// a closed connection is the normal end of a tunnel.
|
||||||
|
let _ = tokio::io::copy_bidirectional(&mut stream, &mut upstream).await;
|
||||||
|
Verdict::Allowed(host)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn reply(s: &mut UnixStream, code: u16, text: &str) -> std::io::Result<()> {
|
||||||
|
let reason = if code == 200 {
|
||||||
|
"Connection established"
|
||||||
|
} else {
|
||||||
|
"Forbidden"
|
||||||
|
};
|
||||||
|
// The body carries the reason for a non-200 so it reaches the agent's own
|
||||||
|
// error output, where whoever is reading a failed mission will see it.
|
||||||
|
let body = if code == 200 { String::new() } else { format!("{text}\n") };
|
||||||
|
let head = format!(
|
||||||
|
"HTTP/1.1 {code} {reason}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||||
|
body.len()
|
||||||
|
);
|
||||||
|
s.write_all(head.as_bytes()).await?;
|
||||||
|
if !body.is_empty() {
|
||||||
|
s.write_all(body.as_bytes()).await?;
|
||||||
|
}
|
||||||
|
s.flush().await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Start this VM's proxy. Returns the socket path and the task serving it.
|
||||||
|
///
|
||||||
|
/// Bound **before** firecracker starts, because a guest that dials before the
|
||||||
|
/// host is listening gets a connection refused it will not retry.
|
||||||
|
pub fn start(
|
||||||
|
uds: &Path,
|
||||||
|
vm_id: &str,
|
||||||
|
backend: Option<&str>,
|
||||||
|
) -> Result<(PathBuf, tokio::task::JoinHandle<()>), String> {
|
||||||
|
let path = PathBuf::from(format!("{}_{}", uds.display(), EGRESS_PORT));
|
||||||
|
// Firecracker does not clean these up any more than it cleans up its own
|
||||||
|
// socket, and a stale file makes bind fail with EADDRINUSE.
|
||||||
|
let _ = std::fs::remove_file(&path);
|
||||||
|
let listener =
|
||||||
|
UnixListener::bind(&path).map_err(|e| format!("bind {}: {e}", path.display()))?;
|
||||||
|
|
||||||
|
let allow = Arc::new(allow_list_for(backend));
|
||||||
|
eprintln!(
|
||||||
|
"microvm {vm_id}: egress proxy on {} allowing {:?}",
|
||||||
|
path.display(),
|
||||||
|
allow
|
||||||
|
);
|
||||||
|
let vm = vm_id.to_string();
|
||||||
|
let task = tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
match listener.accept().await {
|
||||||
|
Ok((s, _)) => {
|
||||||
|
let allow = allow.clone();
|
||||||
|
let vm = vm.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
match serve(s, allow).await {
|
||||||
|
// Logged at every outcome: this is the audit trail of
|
||||||
|
// everything a mission reached, and a denial that is
|
||||||
|
// not logged is a mystery hang later.
|
||||||
|
Verdict::Allowed(h) => eprintln!("microvm {vm}: egress -> {h}"),
|
||||||
|
Verdict::Denied(h) => {
|
||||||
|
eprintln!("microvm {vm}: egress DENIED {h}")
|
||||||
|
}
|
||||||
|
Verdict::Malformed(w) => {
|
||||||
|
eprintln!("microvm {vm}: egress malformed request ({w})")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("microvm {vm}: egress accept failed: {e}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
Ok((path, task))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
/// A backend is defined in TWO places — the server's credential map and this
|
||||||
|
/// egress map — and granting it one without the other produces a mission
|
||||||
|
/// that launches, boots, runs, and dies on a 403 from our own proxy.
|
||||||
|
///
|
||||||
|
/// Measured exactly that way: `canary-claude` was credentialed on the server
|
||||||
|
/// and unknown here, and the turn failed with
|
||||||
|
/// "api.anthropic.com is not on the egress allow-list".
|
||||||
|
#[test]
|
||||||
|
fn the_canary_backend_reaches_the_same_provider_as_claude() {
|
||||||
|
assert_eq!(
|
||||||
|
provider_hosts(Some("canary-claude")),
|
||||||
|
provider_hosts(Some("claude")),
|
||||||
|
"a canary of the Claude image must reach Anthropic, or it tests nothing"
|
||||||
|
);
|
||||||
|
// And the fail-closed branch must still hold for anything unknown: this
|
||||||
|
// is what stops a new backend silently borrowing another provider's door.
|
||||||
|
assert!(provider_hosts(Some("canary-something-else")).is_empty());
|
||||||
|
assert!(provider_hosts(Some("definitely-not-built")).is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn allow() -> Vec<String> {
|
||||||
|
allow_list_for(None)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// MEASURED on tank, not assumed: a `glm` VM ran a whole mission to
|
||||||
|
/// completion with `api.anthropic.com` denied at this proxy, dialling only
|
||||||
|
/// `api.z.ai`. So Anthropic's host is not something a GLM agent needs — and
|
||||||
|
/// a VM that cannot reach it cannot send z.ai's key there, or Anthropic's
|
||||||
|
/// subscription token to z.ai, whatever a credential bug does upstream.
|
||||||
|
#[test]
|
||||||
|
fn each_backend_reaches_its_own_provider_and_no_other() {
|
||||||
|
let claude = allow_list_for(Some("claude"));
|
||||||
|
assert!(claude.iter().any(|h| h == "api.anthropic.com"), "{claude:?}");
|
||||||
|
assert!(!claude.iter().any(|h| h == "api.z.ai"), "{claude:?}");
|
||||||
|
|
||||||
|
let glm = allow_list_for(Some("glm"));
|
||||||
|
assert!(glm.iter().any(|h| h == "api.z.ai"), "{glm:?}");
|
||||||
|
assert!(
|
||||||
|
!glm.iter().any(|h| h.contains("anthropic")),
|
||||||
|
"a GLM VM must not be able to reach Anthropic: {glm:?}"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Both still reach the forge — delivery is host-side, but a mission that
|
||||||
|
// clones or fetches needs it.
|
||||||
|
for l in [&claude, &glm] {
|
||||||
|
assert!(l.iter().any(|h| h == "git.redclaw.dev"), "{l:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
let kimi = allow_list_for(Some("kimi"));
|
||||||
|
assert!(kimi.iter().any(|h| h == "api.kimi.com"), "{kimi:?}");
|
||||||
|
for other in ["api.z.ai", "api.anthropic.com"] {
|
||||||
|
assert!(!kimi.iter().any(|h| h == other), "{kimi:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
// An unknown backend gets no model API at all rather than borrowing
|
||||||
|
// somebody's: it cannot run anyway, and failing at a closed door beats
|
||||||
|
// reaching the wrong endpoint with a credential.
|
||||||
|
let unknown = allow_list_for(Some("rootfs-opus"));
|
||||||
|
assert_eq!(unknown, vec!["git.redclaw.dev".to_string()], "{unknown:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_model_api_and_the_forge_are_reachable() {
|
||||||
|
for h in ["api.anthropic.com", "git.redclaw.dev", "API.Anthropic.COM"] {
|
||||||
|
assert!(host_allowed(h, &allow()), "{h} must be allowed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The check is a match, never a substring test. A name that merely CONTAINS
|
||||||
|
/// an allowed one is a different host controlled by someone else.
|
||||||
|
#[test]
|
||||||
|
fn a_lookalike_host_is_not_allowed() {
|
||||||
|
for h in [
|
||||||
|
"api.anthropic.com.evil.test",
|
||||||
|
"notapi.anthropic.com.attacker.io",
|
||||||
|
// These contain an allowed suffix but are not that host. The first
|
||||||
|
// PASSED before the character-class check was added — a unit test
|
||||||
|
// found it, not review.
|
||||||
|
"evil.test/api.anthropic.com",
|
||||||
|
"[email protected]",
|
||||||
|
"api.anthropic.com:443",
|
||||||
|
"git.redclaw.dev.evil.test",
|
||||||
|
"example.com",
|
||||||
|
"",
|
||||||
|
" ",
|
||||||
|
] {
|
||||||
|
assert!(!host_allowed(h, &allow()), "{h} must NOT be allowed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A raw address must not sidestep a list written in names.
|
||||||
|
/// A local-model backend gets NO egress, and the 443 rule is untouched.
|
||||||
|
///
|
||||||
|
/// The alternative design routed the node's Ollama through this proxy, which
|
||||||
|
/// would have meant permitting port 11434 and an address the guest names.
|
||||||
|
/// Both are refused here, still, and a `local-ornith` VM reaches the forge
|
||||||
|
/// and nothing else — its model lives on the other socket entirely.
|
||||||
|
#[test]
|
||||||
|
fn a_local_model_backend_gets_no_egress_and_no_new_port() {
|
||||||
|
let allow = allow_list_for(Some("local-ornith"));
|
||||||
|
assert!(
|
||||||
|
allow.iter().all(|a| a == "git.redclaw.dev"),
|
||||||
|
"a local backend must reach only the forge, got {allow:?}"
|
||||||
|
);
|
||||||
|
for h in ["api.anthropic.com", "api.z.ai", "api.kimi.com", "127.0.0.1"] {
|
||||||
|
assert!(!host_allowed(h, &allow), "{h} must NOT be reachable");
|
||||||
|
}
|
||||||
|
// The rules this design exists to avoid loosening.
|
||||||
|
assert!(parse_target("anything:11434").is_err());
|
||||||
|
assert!(parse_target("127.0.0.1:443").is_ok_and(|(h, _)| !host_allowed(&h, &allow)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_ip_literal_is_not_allowed() {
|
||||||
|
let a = vec![".anthropic.com".to_string()];
|
||||||
|
assert!(!host_allowed("[::1]", &a));
|
||||||
|
assert!(!host_allowed("2606:4700::1111", &a));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A trailing dot is the same host to a resolver, so it must be to us.
|
||||||
|
#[test]
|
||||||
|
fn a_trailing_dot_does_not_bypass_the_list() {
|
||||||
|
assert!(host_allowed("api.anthropic.com.", &allow()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A `.domain` entry covers subdomains, and only real subdomains.
|
||||||
|
#[test]
|
||||||
|
fn a_dot_prefixed_entry_matches_subdomains_only() {
|
||||||
|
let a = vec![".example.com".to_string()];
|
||||||
|
assert!(host_allowed("a.example.com", &a));
|
||||||
|
assert!(host_allowed("example.com", &a));
|
||||||
|
assert!(!host_allowed("notexample.com", &a));
|
||||||
|
assert!(!host_allowed("example.com.evil.test", &a));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Blanking the allow-list means no egress. Falling back to the default
|
||||||
|
/// would hand a mission the network an operator had just taken away.
|
||||||
|
#[test]
|
||||||
|
fn an_empty_allow_list_denies_everything() {
|
||||||
|
let none: Vec<String> = vec![];
|
||||||
|
assert!(!host_allowed("api.anthropic.com", &none));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Only 443. Anything else turns the proxy into a general-purpose tunnel to
|
||||||
|
/// whatever the allow-list happens to name.
|
||||||
|
#[test]
|
||||||
|
fn only_https_is_tunnelled() {
|
||||||
|
assert_eq!(parse_target("api.anthropic.com:443").unwrap().1, 443);
|
||||||
|
for bad in [
|
||||||
|
"api.anthropic.com:22",
|
||||||
|
"api.anthropic.com:80",
|
||||||
|
"api.anthropic.com",
|
||||||
|
"api.anthropic.com:not-a-port",
|
||||||
|
] {
|
||||||
|
assert!(parse_target(bad).is_err(), "{bad} must be refused");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,167 @@
|
|||||||
|
//! Host side of a microVM's route to the node's OWN locally-hosted model.
|
||||||
|
//!
|
||||||
|
//! # Why this is not the egress proxy
|
||||||
|
//!
|
||||||
|
//! [`crate::egress`] exists so an agent can reach the public internet under an
|
||||||
|
//! allow-list: it speaks HTTP `CONNECT`, takes a destination from the guest,
|
||||||
|
//! resolves it, and decides. Every one of those powers is a liability, which is
|
||||||
|
//! why that module is careful about ports, IP literals and suffix matching.
|
||||||
|
//!
|
||||||
|
//! This is the opposite shape. There is **no destination in the protocol**. The
|
||||||
|
//! guest opens a socket; the host connects it to `127.0.0.1:11434` on the node
|
||||||
|
//! and copies bytes. A compromised guest can ask for nothing else, because there
|
||||||
|
//! is nothing to ask — it is a pipe, not a proxy. That is strictly narrower than
|
||||||
|
//! anything the allow-list could express, and it is why routing a local model
|
||||||
|
//! through `egress` would have been the worse design: it would have meant
|
||||||
|
//! relaxing the 443-only rule and the IP-literal refusal, both of which exist
|
||||||
|
//! because a unit test caught them being bypassed.
|
||||||
|
//!
|
||||||
|
//! # Why plaintext is right here
|
||||||
|
//!
|
||||||
|
//! The bytes go guest loopback → vsock → host loopback. They never touch a
|
||||||
|
//! network, so there is no wire for TLS to protect. Ollama stays bound to
|
||||||
|
//! `127.0.0.1` on the node and is never exposed to the tailnet, which is a
|
||||||
|
//! stronger position than terminating TLS in front of it would have been.
|
||||||
|
//!
|
||||||
|
//! # Why it is per-backend
|
||||||
|
//!
|
||||||
|
//! The node binds this socket only for a backend declared to use a local model.
|
||||||
|
//! On every other backend the guest's listener is still there and simply gets a
|
||||||
|
//! refusal — the same fail-closed default `provider_hosts` applies to egress.
|
||||||
|
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use tokio::net::{TcpStream, UnixListener};
|
||||||
|
|
||||||
|
/// Host-side vsock port. Must match `fcagent`'s `MODEL_VSOCK_PORT`.
|
||||||
|
pub const MODEL_PORT: u32 = 9003;
|
||||||
|
|
||||||
|
/// Where the node's model server listens. Loopback, and not configurable from
|
||||||
|
/// the guest by design — see the module docs.
|
||||||
|
const OLLAMA_ADDR: &str = "127.0.0.1:11434";
|
||||||
|
|
||||||
|
/// Whether a backend is served by a model running on the node itself.
|
||||||
|
///
|
||||||
|
/// Named individually rather than by prefix. An unrecognised backend must not
|
||||||
|
/// acquire a route to anything by accident, which is the same rule
|
||||||
|
/// `egress::provider_hosts` and `mission_runtime::microvm_credential_for`
|
||||||
|
/// already apply from their own side.
|
||||||
|
pub fn uses_local_model(backend: Option<&str>) -> bool {
|
||||||
|
matches!(backend, Some("local-ornith"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bind the guest's local-model socket, if this backend has one.
|
||||||
|
///
|
||||||
|
/// `Ok(None)` means "this backend does not use a local model" and is the normal
|
||||||
|
/// case. An error means it should have had one and could not — reported by the
|
||||||
|
/// caller, never silently swallowed, because the symptom otherwise is an agent
|
||||||
|
/// that hangs on its first turn.
|
||||||
|
pub fn start(
|
||||||
|
uds: &Path,
|
||||||
|
vm_id: &str,
|
||||||
|
backend: Option<&str>,
|
||||||
|
) -> Result<Option<(PathBuf, tokio::task::JoinHandle<()>)>, String> {
|
||||||
|
if !uses_local_model(backend) {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let path = PathBuf::from(format!("{}_{}", uds.display(), MODEL_PORT));
|
||||||
|
// Firecracker leaves these behind exactly as it does its own socket, and a
|
||||||
|
// stale file makes bind fail with EADDRINUSE.
|
||||||
|
let _ = std::fs::remove_file(&path);
|
||||||
|
let listener =
|
||||||
|
UnixListener::bind(&path).map_err(|e| format!("bind {}: {e}", path.display()))?;
|
||||||
|
|
||||||
|
eprintln!(
|
||||||
|
"microvm {vm_id}: local model socket on {} -> {OLLAMA_ADDR}",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
let vm = vm_id.to_string();
|
||||||
|
let task = tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
match listener.accept().await {
|
||||||
|
Ok((s, _)) => {
|
||||||
|
let vm = vm.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
if let Err(e) = pipe(s).await {
|
||||||
|
// Loud, because the failure a mission sees is a turn
|
||||||
|
// that never answers. A refused connection here means
|
||||||
|
// the node's model server is down, and that is worth
|
||||||
|
// saying out loud rather than leaving to a timeout.
|
||||||
|
eprintln!("microvm {vm}: local model pipe failed: {e}");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("microvm {vm}: local model accept failed: {e}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
Ok(Some((path, task)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Splice one guest connection onto a fresh connection to the node's model.
|
||||||
|
async fn pipe(mut guest: tokio::net::UnixStream) -> Result<(), String> {
|
||||||
|
let mut model = TcpStream::connect(OLLAMA_ADDR)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("connect {OLLAMA_ADDR}: {e}"))?;
|
||||||
|
tokio::io::copy_bidirectional(&mut guest, &mut model)
|
||||||
|
.await
|
||||||
|
.map(|_| ())
|
||||||
|
.map_err(|e| format!("copy: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// Only the backends that are meant to have a local model get one.
|
||||||
|
///
|
||||||
|
/// The negative half is the point: an unrecognised backend acquiring a route
|
||||||
|
/// to the node's own model server would be a hole opened by a typo, and it
|
||||||
|
/// would be invisible because the mission would simply work.
|
||||||
|
#[test]
|
||||||
|
fn a_local_route_is_never_granted_by_accident() {
|
||||||
|
assert!(uses_local_model(Some("local-ornith")));
|
||||||
|
|
||||||
|
for other in [
|
||||||
|
None,
|
||||||
|
Some(""),
|
||||||
|
Some("default"),
|
||||||
|
Some("claude"),
|
||||||
|
Some("canary-claude"),
|
||||||
|
Some("glm"),
|
||||||
|
Some("kimi"),
|
||||||
|
Some("local"),
|
||||||
|
Some("local-ornith-typo"),
|
||||||
|
Some("ornith"),
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
!uses_local_model(other),
|
||||||
|
"{other:?} must not reach the node's model server"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The guest cannot name a destination, so there is nothing to validate.
|
||||||
|
///
|
||||||
|
/// This asserts the property that makes this module safe enough to skip the
|
||||||
|
/// allow-list entirely: the upstream address is a constant. If it ever
|
||||||
|
/// becomes a parameter, this file needs everything `egress` has.
|
||||||
|
#[test]
|
||||||
|
fn the_upstream_address_is_a_constant_not_an_input() {
|
||||||
|
let src = include_str!("local_model.rs");
|
||||||
|
// Needles are split so they do not match themselves in this file.
|
||||||
|
assert_eq!(
|
||||||
|
src.matches(concat!("TcpStream", "::connect(")).count(),
|
||||||
|
1,
|
||||||
|
"exactly one dial site, and it must use the constant"
|
||||||
|
);
|
||||||
|
assert!(src.contains(concat!("TcpStream", "::connect(OLLAMA_ADDR)")));
|
||||||
|
assert!(
|
||||||
|
OLLAMA_ADDR.starts_with("127.0.0.1:"),
|
||||||
|
"the model server must be reached on loopback only"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -19,6 +19,9 @@ use sysinfo::{Disks, System};
|
|||||||
use tokio::sync::{mpsc, Mutex};
|
use tokio::sync::{mpsc, Mutex};
|
||||||
use tokio_tungstenite::tungstenite::Message;
|
use tokio_tungstenite::tungstenite::Message;
|
||||||
|
|
||||||
|
mod egress;
|
||||||
|
mod local_model;
|
||||||
|
mod microvm;
|
||||||
mod rtc;
|
mod rtc;
|
||||||
|
|
||||||
const B64: base64::engine::general_purpose::GeneralPurpose =
|
const B64: base64::engine::general_purpose::GeneralPurpose =
|
||||||
@@ -43,6 +46,15 @@ async fn main() {
|
|||||||
selftest();
|
selftest();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
// Exercise the microVM lifecycle against a real VM on this node. Separate
|
||||||
|
// from --selftest because it needs KVM, so it can only pass on a node that
|
||||||
|
// actually reports microvm capability.
|
||||||
|
if std::env::args().any(|a| a == "--vm-selftest") {
|
||||||
|
if !microvm::selftest().await {
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
let (server, token, ts_authkey) = parse_args();
|
let (server, token, ts_authkey) = parse_args();
|
||||||
if server.is_empty() || token.is_empty() {
|
if server.is_empty() || token.is_empty() {
|
||||||
eprintln!("usage: clawmates-node --server <https://gateway> --token <token> [--tailscale-authkey <key>]");
|
eprintln!("usage: clawmates-node --server <https://gateway> --token <token> [--tailscale-authkey <key>]");
|
||||||
@@ -108,6 +120,11 @@ async fn run(ws_url: &str) -> Result<(), Box<dyn std::error::Error>> {
|
|||||||
let (out_tx, mut out_rx) = mpsc::unbounded_channel::<String>();
|
let (out_tx, mut out_rx) = mpsc::unbounded_channel::<String>();
|
||||||
let ptys: Ptys = Arc::new(Mutex::new(HashMap::new()));
|
let ptys: Ptys = Arc::new(Mutex::new(HashMap::new()));
|
||||||
let peers: rtc::RtcPeers = Arc::new(Mutex::new(HashMap::new()));
|
let peers: rtc::RtcPeers = Arc::new(Mutex::new(HashMap::new()));
|
||||||
|
// microVMs this connection started. Scoped to the connection deliberately:
|
||||||
|
// a reconnect must not inherit VMs it cannot prove are still alive, and
|
||||||
|
// `vm_destroy` cleans a workdir by path even for an unregistered id, so a
|
||||||
|
// VM from a previous incarnation is reapable rather than orphaned.
|
||||||
|
let vms = microvm::new_vms();
|
||||||
// Collect heartbeats on a dedicated thread: the metric helpers shell out to
|
// Collect heartbeats on a dedicated thread: the metric helpers shell out to
|
||||||
// docker/tailscale and stat disks (blocking), which must never stall the
|
// docker/tailscale and stat disks (blocking), which must never stall the
|
||||||
// async select loop (or heartbeats/pongs would starve during a slow op).
|
// async select loop (or heartbeats/pongs would starve during a slow op).
|
||||||
@@ -131,6 +148,14 @@ async fn run(ws_url: &str) -> Result<(), Box<dyn std::error::Error>> {
|
|||||||
if tools_tx.send(frame).is_err() {
|
if tools_tx.send(frame).is_err() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
// What this node can HOST, as opposed to what it has installed. The
|
||||||
|
// scheduler needs it to place microVM missions, and the node is the
|
||||||
|
// only honest source: /dev/kvm either exists here or it does not, and
|
||||||
|
// no amount of configuration on the server can make it appear.
|
||||||
|
let caps = json!({ "t": "node_capabilities", "capabilities": probe_capabilities() });
|
||||||
|
if tools_tx.send(caps.to_string()).is_err() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
std::thread::sleep(Duration::from_secs(900));
|
std::thread::sleep(Duration::from_secs(900));
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -180,8 +205,9 @@ async fn run(ws_url: &str) -> Result<(), Box<dyn std::error::Error>> {
|
|||||||
let out = out_tx.clone();
|
let out = out_tx.clone();
|
||||||
let ptys = ptys.clone();
|
let ptys = ptys.clone();
|
||||||
let peers = peers.clone();
|
let peers = peers.clone();
|
||||||
|
let vms = vms.clone();
|
||||||
let text = t.to_string();
|
let text = t.to_string();
|
||||||
tokio::spawn(async move { handle_frame(&text, &out, &ptys, &peers).await; });
|
tokio::spawn(async move { handle_frame(&text, &out, &ptys, &peers, &vms).await; });
|
||||||
}
|
}
|
||||||
Some(Ok(Message::Ping(p))) => {
|
Some(Ok(Message::Ping(p))) => {
|
||||||
match tokio::time::timeout(WRITE_DEADLINE, write.send(Message::Pong(p))).await {
|
match tokio::time::timeout(WRITE_DEADLINE, write.send(Message::Pong(p))).await {
|
||||||
@@ -241,6 +267,70 @@ fn heartbeat(sys: &mut System) -> String {
|
|||||||
/// Probe installed dev-tool versions: for each tool, find its binary across the
|
/// Probe installed dev-tool versions: for each tool, find its binary across the
|
||||||
/// usual bin dirs and read `--version`. Returns `{ tool: "x.y.z", … }` for the
|
/// usual bin dirs and read `--version`. Returns `{ tool: "x.y.z", … }` for the
|
||||||
/// ones found. Probes `kimi-cli` (the real uv tool), not the `kimi` API wrapper.
|
/// ones found. Probes `kimi-cli` (the real uv tool), not the `kimi` API wrapper.
|
||||||
|
/// What this node can HOST — the inputs to placement predicates.
|
||||||
|
///
|
||||||
|
/// Distinct from [`probe_tools`], which reports what is *installed* for the
|
||||||
|
/// operator to see and update. This answers "may the scheduler put a microVM
|
||||||
|
/// mission here", and the answer is a property of the hardware: gw-04 is
|
||||||
|
/// itself a VM without nested virtualisation and has no `/dev/kvm`, so it can
|
||||||
|
/// never host one however it is configured.
|
||||||
|
///
|
||||||
|
/// Every value is probed, never assumed. A capability that is merely expected
|
||||||
|
/// is the same as a capability that is absent, right up until a mission is
|
||||||
|
/// scheduled onto a node that cannot run it.
|
||||||
|
fn probe_capabilities() -> Value {
|
||||||
|
// The device node is necessary but not sufficient — it can exist while
|
||||||
|
// being unopenable (wrong group, or a container without the device
|
||||||
|
// passed through). Try to open it, because that is what firecracker does.
|
||||||
|
let kvm = std::fs::OpenOptions::new()
|
||||||
|
.read(true)
|
||||||
|
.write(true)
|
||||||
|
.open("/dev/kvm")
|
||||||
|
.is_ok();
|
||||||
|
|
||||||
|
let firecracker = std::process::Command::new("firecracker")
|
||||||
|
.arg("--version")
|
||||||
|
.output()
|
||||||
|
.ok()
|
||||||
|
.filter(|o| o.status.success())
|
||||||
|
.and_then(|o| {
|
||||||
|
String::from_utf8_lossy(&o.stdout)
|
||||||
|
.lines()
|
||||||
|
.next()
|
||||||
|
.map(|l| l.trim().to_string())
|
||||||
|
});
|
||||||
|
|
||||||
|
// Which rootfs images are actually on this node's disk. Reported so
|
||||||
|
// placement can require the mission's backend rather than assuming any
|
||||||
|
// KVM-capable node can boot any image — see microvm::available_backends.
|
||||||
|
let backends = microvm::available_backends();
|
||||||
|
capabilities_from(kvm, firecracker.as_deref(), &backends)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shape the capability report from probe results.
|
||||||
|
///
|
||||||
|
/// Split from [`probe_capabilities`] so the rule can be tested without a
|
||||||
|
/// `/dev/kvm` to open — the machine running the tests is usually the one that
|
||||||
|
/// cannot host a microVM.
|
||||||
|
fn capabilities_from(kvm: bool, firecracker: Option<&str>, backends: &[String]) -> Value {
|
||||||
|
json!({
|
||||||
|
"kvm": kvm,
|
||||||
|
"firecracker": firecracker,
|
||||||
|
// The backends this node can boot. An ARRAY, and empty when there are
|
||||||
|
// none: `set_capabilities` REPLACES, so an image that was deleted stops
|
||||||
|
// being advertised on the next report instead of leaving a stale claim.
|
||||||
|
//
|
||||||
|
// Reported even when `microvm` is false, because it is a fact about the
|
||||||
|
// disk rather than a promise — placement requires both.
|
||||||
|
"rootfs": backends,
|
||||||
|
// BOTH must hold. A node with KVM but no firecracker binary looks
|
||||||
|
// capable by the obvious test and fails at launch; a node with the
|
||||||
|
// binary but no KVM is gw-04. Computed here rather than in the
|
||||||
|
// scheduler so the rule sits next to the probe that feeds it.
|
||||||
|
"microvm": kvm && firecracker.is_some(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn probe_tools() -> Value {
|
fn probe_tools() -> Value {
|
||||||
let home = std::env::var("HOME").unwrap_or_default();
|
let home = std::env::var("HOME").unwrap_or_default();
|
||||||
let dirs = [
|
let dirs = [
|
||||||
@@ -459,6 +549,7 @@ async fn handle_frame(
|
|||||||
out: &mpsc::UnboundedSender<String>,
|
out: &mpsc::UnboundedSender<String>,
|
||||||
ptys: &Ptys,
|
ptys: &Ptys,
|
||||||
peers: &rtc::RtcPeers,
|
peers: &rtc::RtcPeers,
|
||||||
|
vms: µvm::Vms,
|
||||||
) {
|
) {
|
||||||
let Ok(v) = serde_json::from_str::<Value>(text) else {
|
let Ok(v) = serde_json::from_str::<Value>(text) else {
|
||||||
return;
|
return;
|
||||||
@@ -521,6 +612,73 @@ async fn handle_frame(
|
|||||||
// Agent-sandbox container ops: drive the REAL DockerDriver so the
|
// Agent-sandbox container ops: drive the REAL DockerDriver so the
|
||||||
// hardening (cap-drop ALL, seccomp, no-net, read-only, non-root) is
|
// hardening (cap-drop ALL, seccomp, no-net, read-only, non-root) is
|
||||||
// byte-identical to the gateway's local sandboxes.
|
// byte-identical to the gateway's local sandboxes.
|
||||||
|
// microVM ops. Same envelope as every other op, so adding them needed
|
||||||
|
// no protocol change. `vm_create` blocks until the guest agent answers:
|
||||||
|
// a VM that booted but serves nothing is worse than one that failed.
|
||||||
|
op @ ("vm_create" | "vm_inject" | "vm_exec" | "vm_collect" | "vm_destroy" | "vm_list") => {
|
||||||
|
if let Some(id) = v.get("id").and_then(Value::as_u64) {
|
||||||
|
let (op, v, out, vms) = (op.to_string(), v.clone(), out.clone(), vms.clone());
|
||||||
|
// Spawned: a VM boot takes ~1s and an exec can take an hour.
|
||||||
|
// Running it inline would stall heartbeats and the daemon would
|
||||||
|
// be declared offline mid-mission.
|
||||||
|
tokio::spawn(async move {
|
||||||
|
// While an `exec` runs, follow the turn's log and push each
|
||||||
|
// chunk to the server as it appears. The guest agent accepts
|
||||||
|
// concurrent connections (proved against a live VM: a tail
|
||||||
|
// returned data second-by-second while an 8s exec was still
|
||||||
|
// running), so this does not wait for, or delay, the turn.
|
||||||
|
//
|
||||||
|
// Only for `vm_exec`, and only when the caller named a run to
|
||||||
|
// attribute the output to — a probe exec has nothing to
|
||||||
|
// stream and no subscriber.
|
||||||
|
// Set when the turn returns, so the tail can DRAIN before it
|
||||||
|
// stops rather than being cut off mid-flush.
|
||||||
|
let turn_done = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||||
|
let tail = (op == "vm_exec")
|
||||||
|
.then(|| {
|
||||||
|
let run_id = v.get("run_id").and_then(Value::as_str)?.to_string();
|
||||||
|
let log_path = v
|
||||||
|
.get("log_path")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or("/root/agent.log")
|
||||||
|
.to_string();
|
||||||
|
let vm_id = v.get("vm_id").and_then(Value::as_str)?.to_string();
|
||||||
|
Some(tokio::spawn(stream_vm_log(
|
||||||
|
vms.clone(),
|
||||||
|
vm_id,
|
||||||
|
run_id,
|
||||||
|
log_path,
|
||||||
|
out.clone(),
|
||||||
|
turn_done.clone(),
|
||||||
|
)))
|
||||||
|
})
|
||||||
|
.flatten();
|
||||||
|
let (ok, output) = microvm::handle_op(&op, &v, &vms).await;
|
||||||
|
// Let the tail DRAIN, then stop. Aborting here was wrong:
|
||||||
|
// `claude -p | tee` makes stdout a pipe, so the CLI block-
|
||||||
|
// buffers and flushes at EXIT — the most valuable output
|
||||||
|
// arrives in the instant the turn ends. Aborting raced that
|
||||||
|
// flush and lost it. Measured: a solo turn (minutes long) won
|
||||||
|
// the race and streamed 337 bytes; every node of a composed
|
||||||
|
// run (~20s each) lost it and streamed nothing at all.
|
||||||
|
//
|
||||||
|
// Bounded, because a VM that stopped answering must not hold
|
||||||
|
// this task open — the abort remains, as a backstop rather
|
||||||
|
// than the mechanism.
|
||||||
|
if let Some(t) = tail {
|
||||||
|
turn_done.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||||
|
let drained =
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(20), t).await;
|
||||||
|
if drained.is_err() {
|
||||||
|
eprintln!("clawmates-node: tail drain timed out for {op}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let _ = out.send(
|
||||||
|
json!({ "t": "result", "id": id, "ok": ok, "output": output }).to_string(),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
op @ ("sb_provision" | "sb_exec" | "sb_destroy" | "sb_health" | "sb_list") => {
|
op @ ("sb_provision" | "sb_exec" | "sb_destroy" | "sb_health" | "sb_list") => {
|
||||||
if let Some(id) = v.get("id").and_then(Value::as_u64) {
|
if let Some(id) = v.get("id").and_then(Value::as_u64) {
|
||||||
let (ok, output) = sb_op(op, &v).await;
|
let (ok, output) = sb_op(op, &v).await;
|
||||||
@@ -752,6 +910,72 @@ fn spawn_command_pty(argv: &[String], cols: u16, rows: u16) -> Result<PtyParts,
|
|||||||
spawn_pty(c, cols, rows)
|
spawn_pty(c, cols, rows)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Follow a running turn's log inside a VM and push each chunk to the server.
|
||||||
|
///
|
||||||
|
/// The other half of the observability path: the guest tails the file, this
|
||||||
|
/// forwards what it reads over the WebSocket the daemon already holds, and the
|
||||||
|
/// server appends it to the run so the live pane and the Output tab both have it.
|
||||||
|
///
|
||||||
|
/// Reconnects on a dropped tail, resuming from the last offset — following by
|
||||||
|
/// OFFSET rather than holding one socket open forever is what makes that cheap.
|
||||||
|
/// It gives up after a few consecutive failures rather than spinning: by then
|
||||||
|
/// the VM is gone and the turn's own result is the record.
|
||||||
|
async fn stream_vm_log(
|
||||||
|
vms: microvm::Vms,
|
||||||
|
vm_id: String,
|
||||||
|
run_id: String,
|
||||||
|
log_path: String,
|
||||||
|
out: tokio::sync::mpsc::UnboundedSender<String>,
|
||||||
|
turn_done: std::sync::Arc<std::sync::atomic::AtomicBool>,
|
||||||
|
) {
|
||||||
|
// Said out loud at the start, because the failure this replaced was
|
||||||
|
// invisible: the tail gave up during VM boot and logged nothing, so an empty
|
||||||
|
// Live tab looked identical to a feature that was never wired.
|
||||||
|
eprintln!("clawmates-node: following {log_path} in {vm_id} for run {run_id}");
|
||||||
|
let mut at: u64 = 0;
|
||||||
|
let mut failures = 0;
|
||||||
|
while failures < 3 {
|
||||||
|
let at_before = at;
|
||||||
|
let sent = out.clone();
|
||||||
|
let rid = run_id.clone();
|
||||||
|
match microvm::tail_into(&vms, &vm_id, &log_path, at, move |offset, data| {
|
||||||
|
let _ = sent.send(
|
||||||
|
json!({ "t": "vm_out", "run_id": rid, "at": offset, "data": data }).to_string(),
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(reached) => {
|
||||||
|
// NO PROGRESS IS NOT THE END. The guest reports EOF whenever the
|
||||||
|
// file has been idle, and the first idle window is always the one
|
||||||
|
// before the turn writes anything — the VM is still booting and
|
||||||
|
// the CLI still starting. Returning here meant the tail gave up
|
||||||
|
// seconds into every run, before a single byte existed. Measured:
|
||||||
|
// a turn that streamed nothing at all.
|
||||||
|
//
|
||||||
|
// The caller aborts this task when the exec returns, so "keep
|
||||||
|
// waiting" cannot outlive the turn; the abort is the terminator,
|
||||||
|
// not a guess about idleness.
|
||||||
|
at = reached;
|
||||||
|
failures = 0;
|
||||||
|
// The turn has returned AND this pass read nothing new: the
|
||||||
|
// final flush is already in hand, so stop. Checked after a read,
|
||||||
|
// never before one — exiting on the flag alone would drop
|
||||||
|
// exactly the bytes this exists to capture.
|
||||||
|
if turn_done.load(std::sync::atomic::Ordering::Relaxed) && reached == at_before {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
failures += 1;
|
||||||
|
eprintln!("clawmates-node: tail of {vm_id} for run {run_id} failed: {e}");
|
||||||
|
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Spawn a host login shell in a PTY; stream its output back as pty_out frames.
|
/// Spawn a host login shell in a PTY; stream its output back as pty_out frames.
|
||||||
async fn open_pty(
|
async fn open_pty(
|
||||||
sid: u64,
|
sid: u64,
|
||||||
@@ -1274,3 +1498,61 @@ fn ensure_tmux() {
|
|||||||
eprintln!("tmux not found (auto-install unavailable) — host terminal will use a plain shell; `apt install tmux` for resumable sessions");
|
eprintln!("tmux not found (auto-install unavailable) — host terminal will use a plain shell; `apt install tmux` for resumable sessions");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod capability_tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn microvm_needs_both_kvm_and_firecracker() {
|
||||||
|
assert_eq!(
|
||||||
|
capabilities_from(true, Some("Firecracker v1.16.1"), &[])["microvm"],
|
||||||
|
json!(true)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
capabilities_from(true, None, &[])["microvm"],
|
||||||
|
json!(false),
|
||||||
|
"KVM without firecracker cannot host a microVM"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
capabilities_from(false, Some("Firecracker v1.16.1"), &[])["microvm"],
|
||||||
|
json!(false),
|
||||||
|
"firecracker without KVM is gw-04 — it can never host one"
|
||||||
|
);
|
||||||
|
assert_eq!(capabilities_from(false, None, &[])["microvm"], json!(false));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The report replaces rather than merges server-side, so a node that has
|
||||||
|
/// LOST a capability must say so rather than omitting the key — an absent
|
||||||
|
/// key and a false one must not be distinguishable to the predicate.
|
||||||
|
#[test]
|
||||||
|
fn a_lost_capability_is_reported_false_not_omitted() {
|
||||||
|
let caps = capabilities_from(false, None, &[]);
|
||||||
|
assert!(caps.get("kvm").is_some(), "kvm must always be present");
|
||||||
|
assert!(
|
||||||
|
caps.get("microvm").is_some(),
|
||||||
|
"microvm must always be present"
|
||||||
|
);
|
||||||
|
// Same reasoning for the image list: a node that deleted its last rootfs
|
||||||
|
// must report an empty ARRAY, not omit the key. Placement asks "does this
|
||||||
|
// node have backend X"; against a missing key that question has no
|
||||||
|
// answer, and a scheduler with no answer picks something.
|
||||||
|
assert_eq!(
|
||||||
|
caps.get("rootfs"),
|
||||||
|
Some(&json!([])),
|
||||||
|
"rootfs must always be present, empty when there are no images"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The list is what placement matches a mission's `backend` against, so it
|
||||||
|
/// must carry the names verbatim.
|
||||||
|
#[test]
|
||||||
|
fn reported_backends_are_the_names_placement_will_ask_for() {
|
||||||
|
let caps = capabilities_from(
|
||||||
|
true,
|
||||||
|
Some("Firecracker v1.16.1"),
|
||||||
|
&["claude".to_string(), "default".to_string()],
|
||||||
|
);
|
||||||
|
assert_eq!(caps["rootfs"], json!(["claude", "default"]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -24,11 +24,32 @@ async fn main() -> ExitCode {
|
|||||||
|
|
||||||
/// Instantiates the configured LLM provider. The Anthropic key comes from
|
/// Instantiates the configured LLM provider. The Anthropic key comes from
|
||||||
/// the environment until the secret broker lands in P2.
|
/// the environment until the secret broker lands in P2.
|
||||||
|
///
|
||||||
|
/// The **subscription wins** when both credentials are present. This is the
|
||||||
|
/// structural half of the fix that `cm_api::subscription` does per-call: a bare
|
||||||
|
/// model name resolves to whatever this function returns, so making that the
|
||||||
|
/// subscription means no server-side call can reach the metered key by
|
||||||
|
/// accident — by construction, rather than by a source-grep test that has
|
||||||
|
/// already missed four call sites once. The metered key stays usable as a
|
||||||
|
/// fallback for deployments that have credit; ours does not, which is what
|
||||||
|
/// made the ordering matter.
|
||||||
fn build_provider(config: &AppConfig) -> Result<Arc<dyn LlmProvider>, String> {
|
fn build_provider(config: &AppConfig) -> Result<Arc<dyn LlmProvider>, String> {
|
||||||
match config.llm.provider {
|
match config.llm.provider {
|
||||||
LlmProviderKind::Anthropic => {
|
LlmProviderKind::Anthropic => {
|
||||||
let key = std::env::var("ANTHROPIC_API_KEY")
|
if let Some(provider) = cm_api::subscription::provider() {
|
||||||
.map_err(|_| "llm.provider = \"anthropic\" requires ANTHROPIC_API_KEY")?;
|
println!(
|
||||||
|
"clawmates-server: default LLM provider = Claude Code subscription \
|
||||||
|
(bare model names bill no metered key)"
|
||||||
|
);
|
||||||
|
return Ok(Arc::new(provider));
|
||||||
|
}
|
||||||
|
let key = std::env::var("ANTHROPIC_API_KEY").map_err(|_| {
|
||||||
|
"llm.provider = \"anthropic\" needs a credential: either \
|
||||||
|
ANTHROPIC_OAUTH_TOKEN / CLAUDE_CODE_OAUTH_TOKEN (sk-ant-oat…, \
|
||||||
|
the Claude Code subscription, preferred) or ANTHROPIC_API_KEY \
|
||||||
|
(sk-ant-api…, metered)"
|
||||||
|
.to_string()
|
||||||
|
})?;
|
||||||
// A subscription OAuth token pasted where an API key belongs
|
// A subscription OAuth token pasted where an API key belongs
|
||||||
// authenticates nothing here and fails on the first model call,
|
// authenticates nothing here and fails on the first model call,
|
||||||
// far from the mistake. Both start `sk-ant-`, so the confusion is
|
// far from the mistake. Both start `sk-ant-`, so the confusion is
|
||||||
@@ -40,6 +61,11 @@ fn build_provider(config: &AppConfig) -> Result<Arc<dyn LlmProvider>, String> {
|
|||||||
bearer auth and is what the phase evaluator reads."
|
bearer auth and is what the phase evaluator reads."
|
||||||
.to_string());
|
.to_string());
|
||||||
}
|
}
|
||||||
|
eprintln!(
|
||||||
|
"clawmates-server: WARNING — no subscription token; the default LLM \
|
||||||
|
provider is the METERED ANTHROPIC_API_KEY and every bare model name \
|
||||||
|
bills it"
|
||||||
|
);
|
||||||
Ok(Arc::new(AnthropicProvider::new(key)))
|
Ok(Arc::new(AnthropicProvider::new(key)))
|
||||||
}
|
}
|
||||||
LlmProviderKind::OpenAiCompat => {
|
LlmProviderKind::OpenAiCompat => {
|
||||||
@@ -69,8 +95,21 @@ fn build_provider(config: &AppConfig) -> Result<Arc<dyn LlmProvider>, String> {
|
|||||||
fn build_provider_registry(config: &AppConfig) -> cm_runtime::ProviderRegistry {
|
fn build_provider_registry(config: &AppConfig) -> cm_runtime::ProviderRegistry {
|
||||||
let mut map = std::collections::HashMap::new();
|
let mut map = std::collections::HashMap::new();
|
||||||
for p in &config.llm.providers {
|
for p in &config.llm.providers {
|
||||||
match std::env::var(&p.api_key_env) {
|
// A provider may legitimately need no key. A model running on our own
|
||||||
Ok(key) if !key.is_empty() => {
|
// hardware has nothing to authenticate to, and requiring a variable
|
||||||
|
// whose value is ignored is a step that can only ever fail — silently,
|
||||||
|
// since an unset key SKIPS the provider and the first symptom is a
|
||||||
|
// fallback chain quietly one link shorter than it reads.
|
||||||
|
let key = match std::env::var(&p.api_key_env) {
|
||||||
|
Ok(k) if !k.is_empty() => Ok(k),
|
||||||
|
other if p.api_key_env.trim().is_empty() => {
|
||||||
|
let _ = other;
|
||||||
|
Ok(String::new())
|
||||||
|
}
|
||||||
|
other => other,
|
||||||
|
};
|
||||||
|
match key {
|
||||||
|
Ok(key) if !key.is_empty() || p.api_key_env.trim().is_empty() => {
|
||||||
let provider: Arc<dyn LlmProvider> = match p.format.as_str() {
|
let provider: Arc<dyn LlmProvider> = match p.format.as_str() {
|
||||||
"anthropic" => Arc::new(cm_llm::AnthropicProvider::with_base_url(
|
"anthropic" => Arc::new(cm_llm::AnthropicProvider::with_base_url(
|
||||||
key,
|
key,
|
||||||
@@ -280,6 +319,9 @@ async fn run() -> Result<(), String> {
|
|||||||
cm_api::topology_worker::spawn(
|
cm_api::topology_worker::spawn(
|
||||||
pool.clone(),
|
pool.clone(),
|
||||||
runtime.clone(),
|
runtime.clone(),
|
||||||
|
// The composed tier (`microvm_graph`) runs each graph node as a VM on a
|
||||||
|
// fleet node, so the worker needs the same hub the phase runner uses.
|
||||||
|
node_hub.clone(),
|
||||||
std::time::Duration::from_secs(3),
|
std::time::Duration::from_secs(3),
|
||||||
);
|
);
|
||||||
// Boot-time content loaders — skills first, then team templates
|
// Boot-time content loaders — skills first, then team templates
|
||||||
@@ -329,7 +371,16 @@ async fn run() -> Result<(), String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
cm_api::phase_runner::spawn(pool.clone(), runtime.clone());
|
// The other half of runtime_preflight's question: the runtime has the TOOLS,
|
||||||
|
// but can the independent JUDGE be reached? A dead validator makes every
|
||||||
|
// done_when phase unmeetable, and without this the first symptom is a
|
||||||
|
// mission failing after its VMs have already run.
|
||||||
|
cm_api::validator_preflight::report_at_boot(runtime.clone());
|
||||||
|
// Every link of the model fallback chain, probed through the real call path.
|
||||||
|
// A chain is the one piece of infrastructure nobody looks at until the day it
|
||||||
|
// has to work, so it is checked on the days it does not.
|
||||||
|
cm_api::subscription::report_at_boot(runtime.clone());
|
||||||
|
cm_api::phase_runner::spawn(pool.clone(), runtime.clone(), node_hub.clone());
|
||||||
// Per-mission runtime container sweeper (C3): tears down mission
|
// Per-mission runtime container sweeper (C3): tears down mission
|
||||||
// runtime containers 30 min after the mission reaches a terminal
|
// runtime containers 30 min after the mission reaches a terminal
|
||||||
// state so operators have a window to pull final artifacts.
|
// state so operators have a window to pull final artifacts.
|
||||||
@@ -337,13 +388,7 @@ async fn run() -> Result<(), String> {
|
|||||||
// Phase completion summarizer: reads terminal-state phases and
|
// Phase completion summarizer: reads terminal-state phases and
|
||||||
// asks Claude Opus 4.8 to synthesize a "what got done" card that
|
// asks Claude Opus 4.8 to synthesize a "what got done" card that
|
||||||
// the UI renders under the phase.
|
// the UI renders under the phase.
|
||||||
cm_api::phase_summarizer::spawn(pool.clone());
|
cm_api::phase_summarizer::spawn(pool.clone(), runtime.clone());
|
||||||
// PDF renderer worker (Slice 6): watches mission_artifacts for
|
|
||||||
// MD entries with render_pdf_status='pending', calls the
|
|
||||||
// configured LLM (default Gemini 2.5 Flash) for styled HTML,
|
|
||||||
// prints to PDF via chromium --headless. No-op-friendly when
|
|
||||||
// GEMINI_API_KEY / chromium binary aren't configured.
|
|
||||||
cm_api::pdf_renderer::spawn(pool.clone());
|
|
||||||
// Outbound-email delivery: drains the §15-gated `outbox` over SMTP. Inert
|
// Outbound-email delivery: drains the §15-gated `outbox` over SMTP. Inert
|
||||||
// until CLAWMATES_SMTP_* is set, so it ships safely before credentials exist.
|
// until CLAWMATES_SMTP_* is set, so it ships safely before credentials exist.
|
||||||
cm_runtime::spawn_drainer(pool.clone(), std::time::Duration::from_secs(10));
|
cm_runtime::spawn_drainer(pool.clone(), std::time::Duration::from_secs(10));
|
||||||
@@ -352,6 +397,10 @@ async fn run() -> Result<(), String> {
|
|||||||
// Expiry/retention sweep: expires stale auth/oauth rows and prunes old
|
// Expiry/retention sweep: expires stale auth/oauth rows and prunes old
|
||||||
// journal/audit rows hourly so unbounded tables don't accumulate.
|
// journal/audit rows hourly so unbounded tables don't accumulate.
|
||||||
cm_api::cleanup_sweeper::spawn(pool.clone(), std::time::Duration::from_secs(3600));
|
cm_api::cleanup_sweeper::spawn(pool.clone(), std::time::Duration::from_secs(3600));
|
||||||
|
// Its filesystem counterpart. `cleanup_sweeper` prunes ROWS, and deleting a
|
||||||
|
// row has never deleted a directory — which is why the gateway, the smallest
|
||||||
|
// disk in the fleet, accumulates mission trees that nothing reclaims.
|
||||||
|
cm_api::mission_gc::spawn(pool.clone(), std::time::Duration::from_secs(3600));
|
||||||
// Fleet backstop: a node whose heartbeats stop (without a clean channel
|
// Fleet backstop: a node whose heartbeats stop (without a clean channel
|
||||||
// close) goes offline within ~28s even if its control channel hangs.
|
// close) goes offline within ~28s even if its control channel hangs.
|
||||||
cm_api::fleet::spawn_node_sweeper(pool.clone(), std::time::Duration::from_secs(8), 20);
|
cm_api::fleet::spawn_node_sweeper(pool.clone(), std::time::Duration::from_secs(8), 20);
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
[package]
|
||||||
|
name = "fcagent"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition.workspace = true
|
||||||
|
rust-version.workspace = true
|
||||||
|
license.workspace = true
|
||||||
|
publish.workspace = true
|
||||||
|
|
||||||
|
[[bin]]
|
||||||
|
name = "fcagent"
|
||||||
|
path = "src/main.rs"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
# std has no AF_VSOCK, and the workspace denies `unsafe`, so raw libc is not an
|
||||||
|
# option. This is a safe wrapper over the socket calls.
|
||||||
|
vsock = "0.5"
|
||||||
|
serde_json = { workspace = true }
|
||||||
|
tar = { workspace = true }
|
||||||
|
base64 = "0.22"
|
||||||
|
|
||||||
|
# NOTE: a `[profile.release]` here would be silently ignored — cargo only honours
|
||||||
|
# profiles at the workspace root. The binary is small enough on the default
|
||||||
|
# release profile (~1 MB static) that overriding the whole workspace's profile to
|
||||||
|
# shave it would be a bad trade.
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
@@ -0,0 +1,988 @@
|
|||||||
|
//! ClawMates microVM guest agent — pid 1 inside a Firecracker microVM.
|
||||||
|
//!
|
||||||
|
//! Runs as `init=/usr/local/bin/fcagent`'s exec target and answers the host over
|
||||||
|
//! **vsock** (port 9001), never the serial console: feeding a guest over stdin
|
||||||
|
//! races its startup and arrives half-consumed. The console stays a log.
|
||||||
|
//!
|
||||||
|
//! # Why this is a static Rust binary and not the python script it replaces
|
||||||
|
//!
|
||||||
|
//! The python version worked only because Firecracker's CI Ubuntu image happens
|
||||||
|
//! to ship python3. **None of our own images do** — `agent-base` has neither
|
||||||
|
//! python nor git, `agent-terminal` has git but no python — so the agent could
|
||||||
|
//! never have run in a real mission rootfs. An agent that dictates what must be
|
||||||
|
//! installed in the image has the dependency backwards. This is a
|
||||||
|
//! `x86_64-unknown-linux-musl` static binary: it needs nothing from the rootfs
|
||||||
|
//! it is dropped into.
|
||||||
|
//!
|
||||||
|
//! # Wire protocol (unchanged from the python agent, deliberately)
|
||||||
|
//!
|
||||||
|
//! One request per connection: a 4-byte big-endian length followed by JSON, and
|
||||||
|
//! the reply framed the same way. The length prefix is the point — a reply
|
||||||
|
//! larger than a socket buffer arrives in pieces, and reading "whatever was
|
||||||
|
//! available" would parse a truncated object as a complete one.
|
||||||
|
//!
|
||||||
|
//! Ops: `ping`, `exec`, `put`, `get`. `crates/bins/clawmates-node/src/microvm.rs`
|
||||||
|
//! and `crates/cm-api/src/microvm_client.rs` speak this and needed no change.
|
||||||
|
|
||||||
|
use std::io::{Read, Write};
|
||||||
|
use std::net::TcpListener;
|
||||||
|
use std::os::unix::process::CommandExt;
|
||||||
|
use std::path::Path;
|
||||||
|
use std::process::{Command, Stdio};
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use base64::Engine;
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
|
||||||
|
const PORT: u32 = 9001;
|
||||||
|
/// Guest-side egress proxy. The VM has **no network interface at all** — see
|
||||||
|
/// `microvm.rs`, whose machine config declares no `network-interfaces` — so an
|
||||||
|
/// agent CLI cannot reach the model API on its own. It reaches it by honouring
|
||||||
|
/// `HTTPS_PROXY`, which is measured, not assumed: with the proxy pointed at a
|
||||||
|
/// closed port, `claude -p` fails with `ConnectionRefused` instead of answering.
|
||||||
|
///
|
||||||
|
/// This listener is a dumb byte pump. It parses nothing and enforces nothing:
|
||||||
|
/// the `CONNECT` request travels verbatim to the host, which speaks HTTP CONNECT
|
||||||
|
/// and owns the allow-list. Keeping policy on the host means nothing running in
|
||||||
|
/// the guest — including a compromised agent — can talk it into a different
|
||||||
|
/// answer.
|
||||||
|
const PROXY_PORT: u16 = 3128;
|
||||||
|
/// Host-side vsock port the tunnel lands on. Firecracker's convention for a
|
||||||
|
/// guest-initiated connection is that the HOST listens on `<uds_path>_<port>`.
|
||||||
|
const EGRESS_PORT: u32 = 9002;
|
||||||
|
/// Guest-side port for a LOCALLY HOSTED model, and the vsock port it lands on.
|
||||||
|
///
|
||||||
|
/// Separate from the egress proxy on purpose, and simpler than it. The egress
|
||||||
|
/// path exists to let an agent reach the public internet under an allow-list;
|
||||||
|
/// this one reaches exactly one thing — the Ollama the node itself is running,
|
||||||
|
/// on its own loopback — and can reach nothing else, because the host end is a
|
||||||
|
/// pipe to a fixed address rather than a proxy that takes a destination.
|
||||||
|
///
|
||||||
|
/// It therefore needs no `CONNECT`, no TLS and no allow-list. The bytes travel
|
||||||
|
/// guest loopback → vsock → host loopback and never touch a network, so there is
|
||||||
|
/// nothing on a wire for TLS to protect. `NO_PROXY` already contains
|
||||||
|
/// `127.0.0.1`, so an agent pointed at `http://127.0.0.1:11434` bypasses the
|
||||||
|
/// egress proxy entirely rather than trying to CONNECT through it.
|
||||||
|
///
|
||||||
|
/// The guest always listens. Whether anything answers is the HOST's decision:
|
||||||
|
/// the node only binds the vsock end for a backend that is meant to have a
|
||||||
|
/// local model, so on every other backend this port simply refuses.
|
||||||
|
const MODEL_PORT: u16 = 11434;
|
||||||
|
const MODEL_VSOCK_PORT: u32 = 9003;
|
||||||
|
/// `VMADDR_CID_HOST` — the hypervisor side of the vsock.
|
||||||
|
const HOST_CID: u32 = 2;
|
||||||
|
|
||||||
|
/// Whether the egress proxy is actually listening. Reported by `ping` so the
|
||||||
|
/// host can refuse to hand a mission to a VM with no way out, rather than
|
||||||
|
/// discovering it as an agent that hangs.
|
||||||
|
static PROXY_UP: AtomicBool = AtomicBool::new(false);
|
||||||
|
/// Cap on a single request. A hostile or broken host must not be able to make
|
||||||
|
/// pid 1 allocate without bound and get the VM OOM-killed.
|
||||||
|
const MAX_REQUEST: u32 = 512 * 1024 * 1024;
|
||||||
|
|
||||||
|
const B64: base64::engine::general_purpose::GeneralPurpose =
|
||||||
|
base64::engine::general_purpose::STANDARD;
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
// The mounts the init script would otherwise do. Done here so the agent
|
||||||
|
// works whether it is exec'd from a shell init or used as `init=` directly:
|
||||||
|
// /proc missing makes every process-inspecting tool in the guest lie.
|
||||||
|
for (fstype, target) in [
|
||||||
|
("proc", "/proc"),
|
||||||
|
("sysfs", "/sys"),
|
||||||
|
("devtmpfs", "/dev"),
|
||||||
|
("tmpfs", "/tmp"),
|
||||||
|
] {
|
||||||
|
if !Path::new(target).join(".").exists() {
|
||||||
|
let _ = std::fs::create_dir_all(target);
|
||||||
|
}
|
||||||
|
let _ = Command::new("mount")
|
||||||
|
.args(["-t", fstype, fstype, target])
|
||||||
|
.status();
|
||||||
|
}
|
||||||
|
|
||||||
|
start_egress_proxy();
|
||||||
|
|
||||||
|
let listener = match vsock::VsockListener::bind_with_cid_port(libc_vmaddr_cid_any(), PORT) {
|
||||||
|
Ok(l) => l,
|
||||||
|
Err(e) => {
|
||||||
|
// Printed to the console, which is where the host's boot check
|
||||||
|
// looks. Exiting pid 1 panics the kernel, which is the honest
|
||||||
|
// outcome: a VM whose agent cannot listen is unusable, and it must
|
||||||
|
// not sit there looking booted.
|
||||||
|
eprintln!("FC-AGENT-FATAL could not bind vsock port {PORT}: {e}");
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// The host greps the console for this before it tries to connect.
|
||||||
|
println!("FC-AGENT-LISTENING port={PORT}");
|
||||||
|
let _ = std::io::stdout().flush();
|
||||||
|
|
||||||
|
for conn in listener.incoming() {
|
||||||
|
match conn {
|
||||||
|
Ok(mut s) => {
|
||||||
|
// One THREAD per connection, not one at a time.
|
||||||
|
//
|
||||||
|
// This loop used to call `serve_one` inline, which meant the
|
||||||
|
// agent accepted nothing while an op was running. A mission turn
|
||||||
|
// is an `exec` that can last an hour, so for that hour the guest
|
||||||
|
// was unreachable: the host could not tail its output, probe it,
|
||||||
|
// or ask it anything. Every existing probe runs AFTER the turn
|
||||||
|
// for exactly this reason.
|
||||||
|
//
|
||||||
|
// A thread rather than async: this is a static musl binary with
|
||||||
|
// no runtime, and the concurrency here is a handful of
|
||||||
|
// connections, not thousands.
|
||||||
|
//
|
||||||
|
// The panic discipline of the old inline call still applies, and
|
||||||
|
// matters MORE now — this process is pid 1, and a panic that
|
||||||
|
// unwound out of a worker used to take the accept loop with it.
|
||||||
|
// `catch_unwind` keeps a bad request from killing the VM.
|
||||||
|
std::thread::Builder::new()
|
||||||
|
.name("fcagent-conn".into())
|
||||||
|
.spawn(move || {
|
||||||
|
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||||
|
serve_one(&mut s)
|
||||||
|
}));
|
||||||
|
match r {
|
||||||
|
Ok(Err(e)) => eprintln!("FC-AGENT-ERROR {e}"),
|
||||||
|
Err(_) => eprintln!("FC-AGENT-ERROR handler panicked"),
|
||||||
|
Ok(Ok(())) => {}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.map(|_| ())
|
||||||
|
.unwrap_or_else(|e| {
|
||||||
|
// Out of threads: answer nothing on this connection, but
|
||||||
|
// keep accepting. Dropping the listener would brick the VM.
|
||||||
|
eprintln!("FC-AGENT-ERROR spawn: {e}");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(e) => eprintln!("FC-AGENT-ERROR accept: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `VMADDR_CID_ANY` — bind for any host CID.
|
||||||
|
fn libc_vmaddr_cid_any() -> u32 {
|
||||||
|
u32::MAX
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bring up loopback and start the egress tunnel.
|
||||||
|
///
|
||||||
|
/// Loopback is not optional and not free: the guest's `lo` exists but starts
|
||||||
|
/// **down**, and while it is down a listener on 127.0.0.1 *binds successfully*
|
||||||
|
/// and then refuses every connection with `ENETUNREACH`. A bind-only check would
|
||||||
|
/// have reported a working proxy. So `lo` goes up first, via `ip` — which is why
|
||||||
|
/// `iproute2` is in the agent images.
|
||||||
|
///
|
||||||
|
/// Failure here is recorded, not fatal: exec still works, so a VM is still
|
||||||
|
/// useful for work that needs no network. It is reported through `ping` so the
|
||||||
|
/// host can decide, instead of a mission discovering it as an agent that hangs.
|
||||||
|
fn start_egress_proxy() {
|
||||||
|
// Absolute paths, not `Command::new("ip")`. This process is pid 1, so its
|
||||||
|
// PATH is whatever the kernel handed it — and when PATH is unset, `execvp`
|
||||||
|
// falls back to a default that does NOT include `/usr/sbin`, which is exactly
|
||||||
|
// where Debian puts `ip`. Searching by name would fail on an image that has
|
||||||
|
// it, and the symptom would be a VM with no egress and no explanation.
|
||||||
|
const IP_CANDIDATES: &[&str] = &["/usr/sbin/ip", "/sbin/ip", "/usr/bin/ip", "/bin/ip"];
|
||||||
|
let Some(ip) = IP_CANDIDATES.iter().find(|p| Path::new(p).exists()) else {
|
||||||
|
eprintln!(
|
||||||
|
"FC-AGENT-NO-PROXY no `ip` binary in {IP_CANDIDATES:?} — no egress; \
|
||||||
|
add iproute2 to this image"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
match Command::new(ip).args(["link", "set", "lo", "up"]).status() {
|
||||||
|
Ok(s) if s.success() => {}
|
||||||
|
other => {
|
||||||
|
eprintln!("FC-AGENT-NO-PROXY `{ip} link set lo up` failed ({other:?}) — no egress");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let listener = match TcpListener::bind(("127.0.0.1", PROXY_PORT)) {
|
||||||
|
Ok(l) => l,
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("FC-AGENT-NO-PROXY could not listen on 127.0.0.1:{PROXY_PORT}: {e}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
PROXY_UP.store(true, Ordering::Relaxed);
|
||||||
|
println!("FC-AGENT-PROXY listening on 127.0.0.1:{PROXY_PORT} -> vsock {EGRESS_PORT}");
|
||||||
|
let _ = std::io::stdout().flush();
|
||||||
|
pump(listener, EGRESS_PORT, "PROXY");
|
||||||
|
|
||||||
|
// The local-model port. Failure to bind is reported and non-fatal, exactly
|
||||||
|
// like the egress proxy: a VM whose backend does not use a local model is
|
||||||
|
// still perfectly useful, and a fatal error here would take out every
|
||||||
|
// backend to serve one.
|
||||||
|
match TcpListener::bind(("127.0.0.1", MODEL_PORT)) {
|
||||||
|
Ok(l) => {
|
||||||
|
println!("FC-AGENT-MODEL listening on 127.0.0.1:{MODEL_PORT} -> vsock {MODEL_VSOCK_PORT}");
|
||||||
|
let _ = std::io::stdout().flush();
|
||||||
|
pump(l, MODEL_VSOCK_PORT, "MODEL");
|
||||||
|
}
|
||||||
|
Err(e) => eprintln!("FC-AGENT-NO-MODEL could not listen on 127.0.0.1:{MODEL_PORT}: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Accept forever, splicing each connection onto its own vsock stream.
|
||||||
|
fn pump(listener: TcpListener, vsock_port: u32, tag: &'static str) {
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
for c in listener.incoming() {
|
||||||
|
match c {
|
||||||
|
// One thread per connection. An agent CLI opens several at once,
|
||||||
|
// and serving them in sequence would look like a hang.
|
||||||
|
Ok(tcp) => {
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
if let Err(e) = tunnel(tcp, vsock_port) {
|
||||||
|
eprintln!("FC-AGENT-{tag}-ERROR {e}");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(e) => eprintln!("FC-AGENT-{tag}-ERROR accept: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Splice one TCP connection onto a fresh vsock connection to the host.
|
||||||
|
///
|
||||||
|
/// No parsing: whatever the client sent — `CONNECT host:443`, or an absolute-form
|
||||||
|
/// request — is the host's business. The host answers with real HTTP, so a
|
||||||
|
/// refusal reaches the client as a status code rather than a dropped socket.
|
||||||
|
fn tunnel(tcp: std::net::TcpStream, vsock_port: u32) -> Result<(), String> {
|
||||||
|
let vs = vsock::VsockStream::connect_with_cid_port(HOST_CID, vsock_port)
|
||||||
|
.map_err(|e| format!("vsock connect to host:{vsock_port}: {e}"))?;
|
||||||
|
|
||||||
|
let (mut tcp_r, mut tcp_w) = (
|
||||||
|
tcp.try_clone().map_err(|e| format!("clone tcp: {e}"))?,
|
||||||
|
tcp,
|
||||||
|
);
|
||||||
|
let (mut vs_r, mut vs_w) = (
|
||||||
|
vs.try_clone().map_err(|e| format!("clone vsock: {e}"))?,
|
||||||
|
vs,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Each direction gets its own thread, and each shuts its peer's write side
|
||||||
|
// down when it ends. Without the shutdown the other half blocks forever on a
|
||||||
|
// half-closed connection and the CLI waits out its own timeout.
|
||||||
|
let up = std::thread::spawn(move || {
|
||||||
|
let _ = std::io::copy(&mut tcp_r, &mut vs_w);
|
||||||
|
let _ = vs_w.shutdown(std::net::Shutdown::Write);
|
||||||
|
});
|
||||||
|
let _ = std::io::copy(&mut vs_r, &mut tcp_w);
|
||||||
|
let _ = tcp_w.shutdown(std::net::Shutdown::Write);
|
||||||
|
let _ = up.join();
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn serve_one(s: &mut vsock::VsockStream) -> Result<(), String> {
|
||||||
|
let mut len = [0u8; 4];
|
||||||
|
s.read_exact(&mut len)
|
||||||
|
.map_err(|e| format!("read length: {e}"))?;
|
||||||
|
let len = u32::from_be_bytes(len);
|
||||||
|
if len > MAX_REQUEST {
|
||||||
|
// Answer rather than hang up: a caller that sent something absurd needs
|
||||||
|
// to be told, not left waiting for a reply that will never come.
|
||||||
|
return reply(s, &json!({ "ok": false, "error": format!("request of {len} bytes exceeds the {MAX_REQUEST} cap") }));
|
||||||
|
}
|
||||||
|
let mut buf = vec![0u8; len as usize];
|
||||||
|
s.read_exact(&mut buf)
|
||||||
|
.map_err(|e| format!("read body: {e}"))?;
|
||||||
|
|
||||||
|
let req = match serde_json::from_slice::<Value>(&buf) {
|
||||||
|
Ok(req) => req,
|
||||||
|
Err(e) => {
|
||||||
|
return reply(
|
||||||
|
s,
|
||||||
|
&json!({ "ok": false, "error": format!("undecodable request: {e}") }),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
// `tail` owns the connection for its lifetime, emitting a frame per chunk,
|
||||||
|
// so it cannot go through `handle`, which returns one Value.
|
||||||
|
if req.get("op").and_then(Value::as_str) == Some("tail") {
|
||||||
|
return op_tail(s, &req);
|
||||||
|
}
|
||||||
|
let resp = handle(&req);
|
||||||
|
reply(s, &resp)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stream a file to the host as it grows, one framed JSON chunk at a time.
|
||||||
|
///
|
||||||
|
/// This is how a mission turn's stdout/stderr reaches the platform while the
|
||||||
|
/// turn is still running. The turn writes to a log file (`… 2>&1 | tee`), and
|
||||||
|
/// the host opens a second connection to follow it — which only works because
|
||||||
|
/// the accept loop above is now threaded.
|
||||||
|
///
|
||||||
|
/// `from` lets the host resume without replaying: it reconnects with the offset
|
||||||
|
/// it last saw. Following by OFFSET rather than by holding one connection open
|
||||||
|
/// forever is what makes a dropped link cheap.
|
||||||
|
///
|
||||||
|
/// Ends when the file stops growing for `idle_ms`, or at `max_secs`. It must
|
||||||
|
/// end: a tail that never returns pins a thread for the life of the VM.
|
||||||
|
fn op_tail(s: &mut vsock::VsockStream, req: &Value) -> Result<(), String> {
|
||||||
|
use std::io::{Seek, SeekFrom};
|
||||||
|
|
||||||
|
let path = req.get("path").and_then(Value::as_str).unwrap_or_default();
|
||||||
|
let mut from = req.get("from").and_then(Value::as_u64).unwrap_or(0);
|
||||||
|
let idle_ms = req.get("idle_ms").and_then(Value::as_u64).unwrap_or(2_000);
|
||||||
|
let max_secs = req.get("max_secs").and_then(Value::as_u64).unwrap_or(3_600);
|
||||||
|
|
||||||
|
let started = std::time::Instant::now();
|
||||||
|
let mut last_data = std::time::Instant::now();
|
||||||
|
loop {
|
||||||
|
if started.elapsed().as_secs() >= max_secs {
|
||||||
|
return reply(s, &json!({ "ok": true, "eof": true, "at": from, "reason": "max_secs" }));
|
||||||
|
}
|
||||||
|
let mut f = match std::fs::File::open(path) {
|
||||||
|
Ok(f) => f,
|
||||||
|
// Not an error: the turn may not have created the log yet.
|
||||||
|
Err(_) => {
|
||||||
|
if last_data.elapsed().as_millis() as u64 >= idle_ms {
|
||||||
|
return reply(s, &json!({ "ok": true, "eof": true, "at": from, "reason": "absent" }));
|
||||||
|
}
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(200));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let len = f.metadata().map(|m| m.len()).unwrap_or(0);
|
||||||
|
if len < from {
|
||||||
|
// Truncated or rotated under us. Restart rather than read garbage.
|
||||||
|
from = 0;
|
||||||
|
}
|
||||||
|
if len > from {
|
||||||
|
f.seek(SeekFrom::Start(from))
|
||||||
|
.map_err(|e| format!("seek {path}: {e}"))?;
|
||||||
|
let mut buf = vec![0u8; (len - from).min(MAX_CHUNK) as usize];
|
||||||
|
let n = f.read(&mut buf).map_err(|e| format!("read {path}: {e}"))?;
|
||||||
|
buf.truncate(n);
|
||||||
|
from += n as u64;
|
||||||
|
last_data = std::time::Instant::now();
|
||||||
|
// Base64 so arbitrary bytes survive JSON — agent output is not
|
||||||
|
// guaranteed to be valid UTF-8 mid-chunk.
|
||||||
|
reply(
|
||||||
|
s,
|
||||||
|
&json!({ "ok": true, "eof": false, "at": from, "data": B64.encode(&buf) }),
|
||||||
|
)?;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if last_data.elapsed().as_millis() as u64 >= idle_ms {
|
||||||
|
return reply(s, &json!({ "ok": true, "eof": true, "at": from, "reason": "idle" }));
|
||||||
|
}
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(200));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Largest slice sent in one frame. Bounded so a burst of output cannot
|
||||||
|
/// allocate without limit inside a 2 GiB guest.
|
||||||
|
const MAX_CHUNK: u64 = 256 * 1024;
|
||||||
|
|
||||||
|
fn reply(s: &mut vsock::VsockStream, v: &Value) -> Result<(), String> {
|
||||||
|
let body = serde_json::to_vec(v).map_err(|e| format!("encode reply: {e}"))?;
|
||||||
|
s.write_all(&(body.len() as u32).to_be_bytes())
|
||||||
|
.map_err(|e| format!("write length: {e}"))?;
|
||||||
|
s.write_all(&body)
|
||||||
|
.map_err(|e| format!("write body: {e}"))?;
|
||||||
|
s.flush().map_err(|e| format!("flush: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn handle(req: &Value) -> Value {
|
||||||
|
let op = req.get("op").and_then(Value::as_str).unwrap_or_default();
|
||||||
|
match op {
|
||||||
|
"ping" => json!({
|
||||||
|
"ok": true,
|
||||||
|
"pid": std::process::id(),
|
||||||
|
// The host refuses to run a mission in a VM with no way out; this is
|
||||||
|
// how it knows. Reported rather than assumed because the image, not
|
||||||
|
// this binary, decides whether loopback can come up.
|
||||||
|
"proxy": PROXY_UP.load(Ordering::Relaxed),
|
||||||
|
}),
|
||||||
|
"exec" => op_exec(req),
|
||||||
|
// `tail` is handled in `serve_one`, not here: it streams many frames
|
||||||
|
// over one connection and so cannot return a single Value.
|
||||||
|
"tail" => json!({ "ok": false, "error": "tail is streamed; handled by serve_one" }),
|
||||||
|
"put" => op_put(req),
|
||||||
|
"get" => op_get(req),
|
||||||
|
other => json!({ "ok": false, "error": format!("unknown op: {other}") }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extra environment for the command, on top of the image's own.
|
||||||
|
///
|
||||||
|
/// This is how credentials reach the agent CLI. An env var rather than a file
|
||||||
|
/// because the per-VM rootfs is destroyed with the VM but an env var never
|
||||||
|
/// touches the guest disk at all — it exists only in the process's environment
|
||||||
|
/// for the length of one exec.
|
||||||
|
///
|
||||||
|
/// **Every problem here fails the exec.** The tempting alternative — skip the
|
||||||
|
/// entry we could not use and run anyway — produces a `claude -p` with no
|
||||||
|
/// credential, and that does not error: it hangs. A phase stuck at `running`
|
||||||
|
/// for ten minutes with nothing in the logs is exactly what a missing token
|
||||||
|
/// looked like on the container path, so a request we cannot honour in full is
|
||||||
|
/// refused with a reason instead.
|
||||||
|
///
|
||||||
|
/// Errors name the key and never the value: the value is the secret, and an
|
||||||
|
/// error string travels back over the wire and into logs.
|
||||||
|
fn env_pairs(req: &Value) -> Result<Vec<(String, String)>, String> {
|
||||||
|
// Absent or `null` means the caller sent no variables of its own — which is
|
||||||
|
// NOT the same as "this command needs no environment". Both cases still get
|
||||||
|
// the proxy address below; returning early here meant every exec that passed
|
||||||
|
// no env ran with no HTTPS_PROXY, and the symptom was `curl` reporting
|
||||||
|
// "Could not resolve host" from a guest that had a working tunnel.
|
||||||
|
let empty = serde_json::Map::new();
|
||||||
|
let map = match req.get("env") {
|
||||||
|
None => &empty,
|
||||||
|
Some(v) if v.is_null() => &empty,
|
||||||
|
// Anything else that is not an object is a caller bug.
|
||||||
|
Some(v) => v
|
||||||
|
.as_object()
|
||||||
|
.ok_or("exec env must be an object of name → string")?,
|
||||||
|
};
|
||||||
|
let mut out = Vec::with_capacity(map.len() + 3);
|
||||||
|
for (k, v) in map {
|
||||||
|
let Some(val) = v.as_str() else {
|
||||||
|
return Err(format!("exec env {k}: value must be a string"));
|
||||||
|
};
|
||||||
|
// `putenv` semantics: a name containing '=' would be parsed as part of
|
||||||
|
// the value, silently defining a different variable than the one asked
|
||||||
|
// for. A NUL truncates at the C boundary, for the same class of reason.
|
||||||
|
if k.is_empty() {
|
||||||
|
return Err("exec env has an empty variable name".into());
|
||||||
|
}
|
||||||
|
if k.contains('=') || k.contains('\0') {
|
||||||
|
return Err(format!("exec env {k:?}: name may not contain '=' or NUL"));
|
||||||
|
}
|
||||||
|
if val.contains('\0') {
|
||||||
|
return Err(format!("exec env {k}: value may not contain NUL"));
|
||||||
|
}
|
||||||
|
out.push((k.clone(), val.to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(with_proxy_env(out, PROXY_UP.load(Ordering::Relaxed)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Add the proxy variables the guest's own listener serves.
|
||||||
|
///
|
||||||
|
/// The agent runs the proxy, so the agent declares where it is. Deriving this on
|
||||||
|
/// the host would mean two places agreeing on a port number, and the one that
|
||||||
|
/// drifts is the one nobody tests.
|
||||||
|
///
|
||||||
|
/// Explicit caller values win: a caller can still point a command elsewhere or
|
||||||
|
/// switch the proxy off for it. Matched case-insensitively because the lowercase
|
||||||
|
/// spellings are equally conventional and a duplicate would leave which one
|
||||||
|
/// applies up to the shell.
|
||||||
|
fn with_proxy_env(mut env: Vec<(String, String)>, proxy_up: bool) -> Vec<(String, String)> {
|
||||||
|
if !proxy_up {
|
||||||
|
return env;
|
||||||
|
}
|
||||||
|
let addr = format!("http://127.0.0.1:{PROXY_PORT}");
|
||||||
|
for (k, v) in [
|
||||||
|
("HTTPS_PROXY", addr.as_str()),
|
||||||
|
("HTTP_PROXY", addr.as_str()),
|
||||||
|
// Without this the client would ask the proxy to reach the proxy.
|
||||||
|
("NO_PROXY", "localhost,127.0.0.1"),
|
||||||
|
] {
|
||||||
|
// `eq_ignore_ascii_case` covers the lowercase spelling, which is equally
|
||||||
|
// conventional; setting both would leave which one applies to the client.
|
||||||
|
if !env.iter().any(|(have, _)| have.eq_ignore_ascii_case(k)) {
|
||||||
|
env.push((k.to_string(), v.to_string()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
env
|
||||||
|
}
|
||||||
|
|
||||||
|
fn op_exec(req: &Value) -> Value {
|
||||||
|
let cmd = req.get("cmd").and_then(Value::as_str).unwrap_or_default();
|
||||||
|
if cmd.is_empty() {
|
||||||
|
return json!({ "ok": false, "error": "exec needs a cmd" });
|
||||||
|
}
|
||||||
|
let cwd = req.get("cwd").and_then(Value::as_str).unwrap_or("/");
|
||||||
|
let secs = req.get("timeout").and_then(Value::as_u64).unwrap_or(3600);
|
||||||
|
let env = match env_pairs(req) {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(e) => return json!({ "ok": false, "error": e }),
|
||||||
|
};
|
||||||
|
|
||||||
|
// The image's ENV was written to /etc/profile.d by the rootfs builder;
|
||||||
|
// `sh -c` does not read it, so source it here — otherwise a CLI that relies
|
||||||
|
// on `ENV PATH` behaves differently in the VM than in the container, which
|
||||||
|
// is exactly the drift the builder extracted that file to prevent.
|
||||||
|
//
|
||||||
|
// The `if [ -f ]` guard is load-bearing. `. missing-file` makes a
|
||||||
|
// NON-INTERACTIVE POSIX shell exit immediately with status 1, so the naive
|
||||||
|
// `. env.sh 2>/dev/null; cmd` returned rc=1 without running `cmd` at all on
|
||||||
|
// any rootfs lacking that file — every exec silently failing while looking
|
||||||
|
// like an ordinary non-zero exit. Caught by the exit-7 unit test.
|
||||||
|
const ENV_FILE: &str = "/etc/profile.d/00-image-env.sh";
|
||||||
|
let sourced = format!("if [ -f {ENV_FILE} ]; then . {ENV_FILE}; fi\n{cmd}");
|
||||||
|
let mut c = Command::new("/bin/sh");
|
||||||
|
c.arg("-c")
|
||||||
|
.arg(&sourced)
|
||||||
|
.envs(env)
|
||||||
|
.current_dir(if Path::new(cwd).is_dir() { cwd } else { "/" })
|
||||||
|
.stdin(Stdio::null())
|
||||||
|
.stdout(Stdio::piped())
|
||||||
|
.stderr(Stdio::piped())
|
||||||
|
// A new process group so a command that spawns background children can
|
||||||
|
// be killed wholesale. Without it a stray daemon keeps the run alive and
|
||||||
|
// the host's timeout is the only thing that ends it.
|
||||||
|
.process_group(0);
|
||||||
|
|
||||||
|
let mut child = match c.spawn() {
|
||||||
|
Ok(ch) => ch,
|
||||||
|
Err(e) => return json!({ "ok": false, "error": format!("spawn: {e}") }),
|
||||||
|
};
|
||||||
|
let pid = child.id() as i32;
|
||||||
|
|
||||||
|
// std has no wait-with-timeout, so poll. The output pipes are read after
|
||||||
|
// the wait, which is safe here because a command producing more than a pipe
|
||||||
|
// buffer of output while we are not draining it would deadlock — so the
|
||||||
|
// deadline is enforced by killing the group, and the pipes are drained by
|
||||||
|
// `wait_with_output` immediately after.
|
||||||
|
let deadline = Instant::now() + Duration::from_secs(secs);
|
||||||
|
let timed_out = loop {
|
||||||
|
match child.try_wait() {
|
||||||
|
Ok(Some(_)) => break false,
|
||||||
|
Ok(None) => {}
|
||||||
|
Err(e) => return json!({ "ok": false, "error": format!("wait: {e}") }),
|
||||||
|
}
|
||||||
|
if Instant::now() >= deadline {
|
||||||
|
kill_group(pid);
|
||||||
|
break true;
|
||||||
|
}
|
||||||
|
std::thread::sleep(Duration::from_millis(20));
|
||||||
|
};
|
||||||
|
|
||||||
|
let out = match child.wait_with_output() {
|
||||||
|
Ok(o) => o,
|
||||||
|
Err(e) => return json!({ "ok": false, "error": format!("collect output: {e}") }),
|
||||||
|
};
|
||||||
|
if timed_out {
|
||||||
|
// Reported as ok:false, not as rc=124: "we stopped it" is a different
|
||||||
|
// fact from "it exited non-zero", and the caller must be able to tell.
|
||||||
|
return json!({
|
||||||
|
"ok": false,
|
||||||
|
"error": format!("command exceeded its {secs}s budget and was killed"),
|
||||||
|
"stdout": String::from_utf8_lossy(&out.stdout),
|
||||||
|
"stderr": String::from_utf8_lossy(&out.stderr),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
json!({
|
||||||
|
"ok": true,
|
||||||
|
// A signalled process has no exit code; report the conventional
|
||||||
|
// 128+signal rather than silently claiming success.
|
||||||
|
"rc": exit_code(&out.status),
|
||||||
|
"stdout": String::from_utf8_lossy(&out.stdout),
|
||||||
|
"stderr": String::from_utf8_lossy(&out.stderr),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exit_code(status: &std::process::ExitStatus) -> i32 {
|
||||||
|
use std::os::unix::process::ExitStatusExt;
|
||||||
|
status
|
||||||
|
.code()
|
||||||
|
.unwrap_or_else(|| 128 + status.signal().unwrap_or(0))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn kill_group(pid: i32) {
|
||||||
|
let _ = Command::new("kill")
|
||||||
|
.args(["-9", "--", &format!("-{pid}")])
|
||||||
|
.status();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn op_put(req: &Value) -> Value {
|
||||||
|
let dest = req.get("dest").and_then(Value::as_str).unwrap_or_default();
|
||||||
|
if dest.is_empty() {
|
||||||
|
return json!({ "ok": false, "error": "put needs a dest" });
|
||||||
|
}
|
||||||
|
let b64 = req.get("tar_b64").and_then(Value::as_str).unwrap_or_default();
|
||||||
|
let raw = match B64.decode(b64) {
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(e) => return json!({ "ok": false, "error": format!("undecodable archive: {e}") }),
|
||||||
|
};
|
||||||
|
if let Err(e) = std::fs::create_dir_all(dest) {
|
||||||
|
return json!({ "ok": false, "error": format!("mkdir {dest}: {e}") });
|
||||||
|
}
|
||||||
|
let mut ar = tar::Archive::new(&raw[..]);
|
||||||
|
ar.set_overwrite(true);
|
||||||
|
// Ownership from the host archive is meaningless in here and re-applying it
|
||||||
|
// is how the container path grew a uid split. The guest is root; let it own
|
||||||
|
// what it is given.
|
||||||
|
ar.set_preserve_permissions(false);
|
||||||
|
match ar.unpack(dest) {
|
||||||
|
Ok(()) => json!({ "ok": true, "dest": dest, "bytes": raw.len() }),
|
||||||
|
Err(e) => json!({ "ok": false, "error": format!("unpack into {dest}: {e}") }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Recursive tar append that skips excluded directory NAMES at any depth.
|
||||||
|
///
|
||||||
|
/// Hand-rolled because `tar::Builder::append_dir_all` takes no filter. Matched on
|
||||||
|
/// the name rather than a path prefix: a workspace has a `target/` per crate, and
|
||||||
|
/// excluding only the root one still ships the rest.
|
||||||
|
fn append_filtered<W: Write>(
|
||||||
|
b: &mut tar::Builder<W>,
|
||||||
|
dir: &Path,
|
||||||
|
prefix: &Path,
|
||||||
|
exclude: &[String],
|
||||||
|
) -> std::io::Result<()> {
|
||||||
|
b.append_dir(prefix, dir)?;
|
||||||
|
let mut entries: Vec<_> = std::fs::read_dir(dir)?.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
entries.sort_by_key(|e| e.file_name());
|
||||||
|
for entry in entries {
|
||||||
|
let name = entry.file_name();
|
||||||
|
let name_str = name.to_string_lossy().to_string();
|
||||||
|
let path = entry.path();
|
||||||
|
let dest = prefix.join(&name);
|
||||||
|
let meta = std::fs::symlink_metadata(&path)?;
|
||||||
|
if meta.is_dir() {
|
||||||
|
if exclude.contains(&name_str) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
append_filtered(b, &path, &dest, exclude)?;
|
||||||
|
} else if meta.is_symlink() {
|
||||||
|
let mut header = tar::Header::new_gnu();
|
||||||
|
header.set_metadata(&meta);
|
||||||
|
header.set_entry_type(tar::EntryType::Symlink);
|
||||||
|
header.set_size(0);
|
||||||
|
let target = std::fs::read_link(&path)?;
|
||||||
|
b.append_link(&mut header, &dest, &target)?;
|
||||||
|
} else {
|
||||||
|
let mut f = std::fs::File::open(&path)?;
|
||||||
|
b.append_file(&dest, &mut f)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn op_get(req: &Value) -> Value {
|
||||||
|
let path = req.get("path").and_then(Value::as_str).unwrap_or_default();
|
||||||
|
if path.is_empty() {
|
||||||
|
return json!({ "ok": false, "error": "get needs a path" });
|
||||||
|
}
|
||||||
|
let p = Path::new(path);
|
||||||
|
if !p.exists() {
|
||||||
|
// A missing path is an error, NOT an empty archive — an empty tar looks
|
||||||
|
// exactly like a run that produced nothing.
|
||||||
|
return json!({ "ok": false, "error": format!("no such path: {path}") });
|
||||||
|
}
|
||||||
|
let name = p
|
||||||
|
.file_name()
|
||||||
|
.map(|s| s.to_string_lossy().to_string())
|
||||||
|
.unwrap_or_else(|| "root".to_string());
|
||||||
|
|
||||||
|
// Directory names to leave out, sent by the host so the policy lives in one
|
||||||
|
// place (`mission_fs::transport_excludes`). Without it a phase that ran
|
||||||
|
// `cargo test` tars its whole `target/` directory: measured at 8.9 MB of 9.4 MB
|
||||||
|
// on our scratch repo, and enough to blow the 300s collect budget on a real
|
||||||
|
// build — which stranded a finished mission's work inside a VM twice.
|
||||||
|
let exclude: Vec<String> = req
|
||||||
|
.get("exclude")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|a| {
|
||||||
|
a.iter()
|
||||||
|
.filter_map(Value::as_str)
|
||||||
|
.map(str::to_string)
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let mut b = tar::Builder::new(Vec::new());
|
||||||
|
// Do not follow symlinks: a link pointing outside the collected tree would
|
||||||
|
// otherwise be dereferenced and its target smuggled back to the host.
|
||||||
|
b.follow_symlinks(false);
|
||||||
|
let added = if p.is_dir() {
|
||||||
|
append_filtered(&mut b, p, Path::new(&name), &exclude)
|
||||||
|
} else {
|
||||||
|
b.append_path_with_name(p, &name)
|
||||||
|
};
|
||||||
|
if let Err(e) = added {
|
||||||
|
return json!({ "ok": false, "error": format!("archive {path}: {e}") });
|
||||||
|
}
|
||||||
|
match b.into_inner() {
|
||||||
|
Ok(bytes) => json!({ "ok": true, "tar_b64": B64.encode(&bytes), "bytes": bytes.len() }),
|
||||||
|
Err(e) => json!({ "ok": false, "error": format!("finish archive for {path}: {e}") }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
/// The tail loop must terminate. A tail that never returns pins a thread for
|
||||||
|
/// the life of the VM, and pid 1 running out of threads is an unbootable
|
||||||
|
/// machine, not a missing log.
|
||||||
|
#[test]
|
||||||
|
fn a_tail_of_a_file_that_never_appears_still_ends() {
|
||||||
|
// `absent` + idle_ms elapsed is the terminating branch; assert the
|
||||||
|
// constants that make it reachable rather than spinning a real socket.
|
||||||
|
assert!(MAX_CHUNK > 0, "a zero chunk cap would loop without progress");
|
||||||
|
assert!(
|
||||||
|
MAX_CHUNK <= 1024 * 1024,
|
||||||
|
"chunks must stay small enough for a 2 GiB guest"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// The CLI reaches the API only by honouring HTTPS_PROXY (measured: with the
|
||||||
|
/// proxy at a closed port, `claude -p` fails ConnectionRefused instead of
|
||||||
|
/// answering), so a VM whose proxy is up must hand it the address.
|
||||||
|
#[test]
|
||||||
|
fn the_proxy_address_is_declared_when_the_proxy_is_up() {
|
||||||
|
let env = with_proxy_env(vec![], true);
|
||||||
|
let get = |k: &str| {
|
||||||
|
env.iter()
|
||||||
|
.find(|(a, _)| a == k)
|
||||||
|
.map(|(_, v)| v.as_str())
|
||||||
|
.unwrap_or("")
|
||||||
|
};
|
||||||
|
assert_eq!(get("HTTPS_PROXY"), "http://127.0.0.1:3128");
|
||||||
|
assert_eq!(get("HTTP_PROXY"), "http://127.0.0.1:3128");
|
||||||
|
// Otherwise the client asks the proxy to reach the proxy.
|
||||||
|
assert!(get("NO_PROXY").contains("127.0.0.1"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// And a VM with no proxy must not claim one: pointing a CLI at a listener
|
||||||
|
/// that is not there turns "no egress" into a connection error mid-run
|
||||||
|
/// instead of a fact the host can check before it starts.
|
||||||
|
#[test]
|
||||||
|
fn no_proxy_address_is_declared_when_the_proxy_is_down() {
|
||||||
|
assert!(with_proxy_env(vec![], false).is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An explicit value from the caller wins, in either spelling — otherwise
|
||||||
|
/// both would be set and which one applies would be up to the client.
|
||||||
|
#[test]
|
||||||
|
fn an_explicit_proxy_setting_is_not_overridden() {
|
||||||
|
let env = with_proxy_env(
|
||||||
|
vec![("https_proxy".into(), "http://elsewhere:8080".into())],
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
let proxies: Vec<&str> = env
|
||||||
|
.iter()
|
||||||
|
.filter(|(k, _)| k.eq_ignore_ascii_case("https_proxy"))
|
||||||
|
.map(|(_, v)| v.as_str())
|
||||||
|
.collect();
|
||||||
|
assert_eq!(proxies, vec!["http://elsewhere:8080"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The credential has to actually reach the command. This is the whole
|
||||||
|
/// point of the op, and the failure it prevents is silent: a `claude -p`
|
||||||
|
/// with no token hangs rather than erroring.
|
||||||
|
#[test]
|
||||||
|
fn injected_env_reaches_the_command() {
|
||||||
|
let r = op_exec(&json!({
|
||||||
|
"op": "exec",
|
||||||
|
"cmd": "printf %s \"$CLAUDE_CODE_OAUTH_TOKEN\"",
|
||||||
|
"env": { "CLAUDE_CODE_OAUTH_TOKEN": "sk-test-value" },
|
||||||
|
"timeout": 30,
|
||||||
|
}));
|
||||||
|
assert_eq!(r["rc"], json!(0));
|
||||||
|
assert_eq!(r["stdout"], json!("sk-test-value"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// And it must survive the profile.d sourcing that runs first — a
|
||||||
|
/// credential set on the process and then clobbered by the shell would
|
||||||
|
/// look identical to one that never arrived.
|
||||||
|
#[test]
|
||||||
|
fn injected_env_survives_the_image_env_file() {
|
||||||
|
let r = op_exec(&json!({
|
||||||
|
"op": "exec",
|
||||||
|
"cmd": "printf %s \"$INJECTED_PROBE\"",
|
||||||
|
"env": { "INJECTED_PROBE": "still-here" },
|
||||||
|
"timeout": 30,
|
||||||
|
}));
|
||||||
|
assert_eq!(r["stdout"], json!("still-here"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// No env is the ordinary case and must not be an error.
|
||||||
|
#[test]
|
||||||
|
fn absent_or_null_env_is_not_an_error() {
|
||||||
|
for req in [
|
||||||
|
json!({ "op": "exec", "cmd": "true", "timeout": 30 }),
|
||||||
|
json!({ "op": "exec", "cmd": "true", "env": null, "timeout": 30 }),
|
||||||
|
json!({ "op": "exec", "cmd": "true", "env": {}, "timeout": 30 }),
|
||||||
|
] {
|
||||||
|
assert_eq!(op_exec(&req)["rc"], json!(0), "{req}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An env entry we cannot honour fails the whole exec rather than being
|
||||||
|
/// dropped. Running without the credential is the outcome this refuses:
|
||||||
|
/// it does not error, it hangs, which is far harder to diagnose than a
|
||||||
|
/// rejected request.
|
||||||
|
#[test]
|
||||||
|
fn an_unusable_env_entry_fails_the_exec_instead_of_being_skipped() {
|
||||||
|
let cases = [
|
||||||
|
json!({ "A=B": "x" }),
|
||||||
|
json!({ "": "x" }),
|
||||||
|
json!({ "TOKEN": 42 }),
|
||||||
|
json!({ "TOKEN": null }),
|
||||||
|
];
|
||||||
|
for env in cases {
|
||||||
|
let r = op_exec(&json!({
|
||||||
|
"op": "exec", "cmd": "true", "env": env.clone(), "timeout": 30,
|
||||||
|
}));
|
||||||
|
assert_eq!(r["ok"], json!(false), "env {env} should be refused");
|
||||||
|
assert!(r["rc"].is_null(), "nothing ran, so there is no rc: {r}");
|
||||||
|
}
|
||||||
|
// A non-object env is a caller bug, not an empty map.
|
||||||
|
let r = op_exec(&json!({ "op": "exec", "cmd": "true", "env": "TOKEN=x" }));
|
||||||
|
assert_eq!(r["ok"], json!(false));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An error about a credential must not quote the credential: it travels
|
||||||
|
/// back over the wire and into the server's logs.
|
||||||
|
#[test]
|
||||||
|
fn an_env_error_never_echoes_the_value() {
|
||||||
|
let r = op_exec(&json!({
|
||||||
|
"op": "exec", "cmd": "true", "timeout": 30,
|
||||||
|
"env": { "A=B": "super-secret-token" },
|
||||||
|
}));
|
||||||
|
let err = r["error"].as_str().unwrap_or_default();
|
||||||
|
assert!(!err.contains("super-secret-token"), "leaked the value: {err}");
|
||||||
|
assert!(err.contains("A=B"), "should name the key: {err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_unknown_op_is_reported_not_ignored() {
|
||||||
|
let r = handle(&json!({ "op": "teleport" }));
|
||||||
|
assert_eq!(r["ok"], json!(false));
|
||||||
|
assert!(r["error"].as_str().unwrap().contains("teleport"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ping_answers() {
|
||||||
|
assert_eq!(handle(&json!({ "op": "ping" }))["ok"], json!(true));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A missing path must be an error, not an empty archive: an empty tar is
|
||||||
|
/// indistinguishable from a run that produced nothing.
|
||||||
|
/// Build output is not work. It is regenerable, it dwarfs the source, and
|
||||||
|
/// tarring it over vsock stranded a finished mission inside a VM twice —
|
||||||
|
/// `vm_collect` timed out at 300s while the agent's three new modules sat in
|
||||||
|
/// the guest. Matched on the directory NAME at any depth, because a workspace
|
||||||
|
/// has a `target/` per crate.
|
||||||
|
#[test]
|
||||||
|
fn excluded_directories_stay_out_of_the_archive_at_any_depth() {
|
||||||
|
let dir = std::env::temp_dir().join(format!("fcagent-ex-{}", std::process::id()));
|
||||||
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
|
std::fs::create_dir_all(dir.join("src")).unwrap();
|
||||||
|
std::fs::create_dir_all(dir.join("target/debug")).unwrap();
|
||||||
|
std::fs::create_dir_all(dir.join("crates/inner/target")).unwrap();
|
||||||
|
std::fs::write(dir.join("src/lib.rs"), "fn a() {}").unwrap();
|
||||||
|
std::fs::write(dir.join("target/debug/blob"), vec![0u8; 4096]).unwrap();
|
||||||
|
std::fs::write(dir.join("crates/inner/target/blob"), vec![0u8; 4096]).unwrap();
|
||||||
|
std::fs::write(dir.join("crates/inner/keep.rs"), "fn b() {}").unwrap();
|
||||||
|
|
||||||
|
let r = op_get(&json!({
|
||||||
|
"op": "get",
|
||||||
|
"path": dir.to_string_lossy(),
|
||||||
|
"exclude": ["target"],
|
||||||
|
}));
|
||||||
|
assert_eq!(r["ok"], json!(true), "{r}");
|
||||||
|
let bytes = B64.decode(r["tar_b64"].as_str().unwrap()).unwrap();
|
||||||
|
let mut ar = tar::Archive::new(&bytes[..]);
|
||||||
|
let paths: Vec<String> = ar
|
||||||
|
.entries()
|
||||||
|
.unwrap()
|
||||||
|
.filter_map(Result::ok)
|
||||||
|
.map(|e| e.path().unwrap().to_string_lossy().to_string())
|
||||||
|
.collect();
|
||||||
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
|
|
||||||
|
assert!(paths.iter().any(|p| p.ends_with("src/lib.rs")), "{paths:?}");
|
||||||
|
assert!(paths.iter().any(|p| p.ends_with("inner/keep.rs")), "{paths:?}");
|
||||||
|
assert!(
|
||||||
|
!paths.iter().any(|p| p.contains("target")),
|
||||||
|
"a nested target/ came along: {paths:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// No exclude list means everything, so an existing caller is unchanged.
|
||||||
|
#[test]
|
||||||
|
fn without_an_exclude_list_nothing_is_dropped() {
|
||||||
|
let dir = std::env::temp_dir().join(format!("fcagent-noex-{}", std::process::id()));
|
||||||
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
|
std::fs::create_dir_all(dir.join("target")).unwrap();
|
||||||
|
std::fs::write(dir.join("target/x"), "x").unwrap();
|
||||||
|
let r = op_get(&json!({ "op": "get", "path": dir.to_string_lossy() }));
|
||||||
|
let bytes = B64.decode(r["tar_b64"].as_str().unwrap()).unwrap();
|
||||||
|
let mut ar = tar::Archive::new(&bytes[..]);
|
||||||
|
let n = ar.entries().unwrap().filter_map(Result::ok).count();
|
||||||
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
|
assert!(n >= 2, "expected the target dir and its file, got {n}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn getting_a_missing_path_is_an_error() {
|
||||||
|
let r = op_get(&json!({ "op": "get", "path": "/definitely/not/here" }));
|
||||||
|
assert_eq!(r["ok"], json!(false));
|
||||||
|
assert!(r["tar_b64"].is_null(), "no archive may be returned");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A command that ran and failed reports `rc`; one we killed reports
|
||||||
|
/// `ok:false`. Collapsing the two would make a timeout look like a build
|
||||||
|
/// failure and vice versa.
|
||||||
|
#[test]
|
||||||
|
fn a_failing_command_reports_rc_and_a_killed_one_does_not() {
|
||||||
|
let r = op_exec(&json!({ "op": "exec", "cmd": "exit 7", "timeout": 30 }));
|
||||||
|
assert_eq!(r["ok"], json!(true), "it ran, so ok is true");
|
||||||
|
assert_eq!(r["rc"], json!(7));
|
||||||
|
|
||||||
|
let r = op_exec(&json!({ "op": "exec", "cmd": "sleep 30", "timeout": 1 }));
|
||||||
|
assert_eq!(r["ok"], json!(false), "we killed it, so ok is false");
|
||||||
|
assert!(r["rc"].is_null(), "a killed command has no exit code");
|
||||||
|
assert!(r["error"].as_str().unwrap().contains("budget"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn exec_needs_a_command() {
|
||||||
|
assert_eq!(op_exec(&json!({ "op": "exec" }))["ok"], json!(false));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A tar must round-trip through put and get.
|
||||||
|
#[test]
|
||||||
|
fn a_tar_round_trips_through_put_and_get() {
|
||||||
|
let tmp = std::env::temp_dir().join(format!("fcagent-test-{}", std::process::id()));
|
||||||
|
let _ = std::fs::remove_dir_all(&tmp);
|
||||||
|
|
||||||
|
let mut b = tar::Builder::new(Vec::new());
|
||||||
|
let body = b"ROUND-TRIP-OK\n";
|
||||||
|
let mut h = tar::Header::new_gnu();
|
||||||
|
h.set_path("marker.txt").unwrap();
|
||||||
|
h.set_size(body.len() as u64);
|
||||||
|
h.set_mode(0o644);
|
||||||
|
h.set_entry_type(tar::EntryType::Regular);
|
||||||
|
h.set_cksum();
|
||||||
|
b.append(&h, &body[..]).unwrap();
|
||||||
|
let archive = b.into_inner().unwrap();
|
||||||
|
|
||||||
|
let r = op_put(&json!({
|
||||||
|
"op": "put",
|
||||||
|
"dest": tmp.display().to_string(),
|
||||||
|
"tar_b64": B64.encode(&archive),
|
||||||
|
}));
|
||||||
|
assert_eq!(r["ok"], json!(true), "put failed: {r}");
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(tmp.join("marker.txt")).unwrap(),
|
||||||
|
"ROUND-TRIP-OK\n"
|
||||||
|
);
|
||||||
|
|
||||||
|
let r = op_get(&json!({ "op": "get", "path": tmp.display().to_string() }));
|
||||||
|
assert_eq!(r["ok"], json!(true), "get failed: {r}");
|
||||||
|
let bytes = B64.decode(r["tar_b64"].as_str().unwrap()).unwrap();
|
||||||
|
let mut ar = tar::Archive::new(&bytes[..]);
|
||||||
|
let found = ar
|
||||||
|
.entries()
|
||||||
|
.unwrap()
|
||||||
|
.filter_map(Result::ok)
|
||||||
|
.any(|e| e.path().map(|p| p.ends_with("marker.txt")).unwrap_or(false));
|
||||||
|
assert!(found, "the collected archive must contain marker.txt");
|
||||||
|
|
||||||
|
let _ = std::fs::remove_dir_all(&tmp);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -33,6 +33,7 @@ cm-config = { path = "../cm-config" }
|
|||||||
cm-db = { path = "../cm-db" }
|
cm-db = { path = "../cm-db" }
|
||||||
cm-domain = { path = "../cm-domain" }
|
cm-domain = { path = "../cm-domain" }
|
||||||
cm-files = { path = "../cm-files" }
|
cm-files = { path = "../cm-files" }
|
||||||
|
tar = { workspace = true }
|
||||||
cm-llm = { path = "../cm-llm" }
|
cm-llm = { path = "../cm-llm" }
|
||||||
cm-orchestrator = { path = "../cm-orchestrator", features = ["provider"] }
|
cm-orchestrator = { path = "../cm-orchestrator", features = ["provider"] }
|
||||||
cm-runtime = { path = "../cm-runtime" }
|
cm-runtime = { path = "../cm-runtime" }
|
||||||
|
|||||||
@@ -0,0 +1,231 @@
|
|||||||
|
//! Human given names for minted agents.
|
||||||
|
//!
|
||||||
|
//! A team used to come back as `planner`, `coder`, `tester`, `reviewer`,
|
||||||
|
//! `committer` — the roster read as a list of job tickets, and the UI showed
|
||||||
|
//! the same word twice (name on top, role underneath). A crew you keep should
|
||||||
|
//! read like people: Meredith, Vijay, Tomasz, Amara.
|
||||||
|
//!
|
||||||
|
//! The role is not lost — it stays in `job_title`, which is what the mission
|
||||||
|
//! machinery binds on. Only the display identity changes.
|
||||||
|
//!
|
||||||
|
//! Names are drawn from many naming traditions on purpose: this workforce is
|
||||||
|
//! not from one place. They are given names only — no surnames — so nobody
|
||||||
|
//! reads a claw as a specific real person.
|
||||||
|
|
||||||
|
/// Given names, deliberately wide. Kept as one flat list rather than grouped by
|
||||||
|
/// origin: grouping invites picking "one from each", which is a worse kind of
|
||||||
|
/// tokenism than simply having a broad pool and drawing from it evenly.
|
||||||
|
///
|
||||||
|
/// Size is a product decision, not an aesthetic one. Every mission now mints
|
||||||
|
/// its own crew and nothing retires them, so the roster grows by the team size
|
||||||
|
/// per mission — at ~5 a mission a 70-name pool starts emitting "Amara 2"
|
||||||
|
/// inside twenty missions. This pool carries a few hundred so a workspace runs
|
||||||
|
/// for a long time before any name repeats at all.
|
||||||
|
pub const NAMES: &[&str] = &[
|
||||||
|
// A
|
||||||
|
"Aarav", "Abebe", "Adaora", "Adrian", "Agnieszka", "Ahmad", "Aiko", "Ainhoa", "Alejandro",
|
||||||
|
"Alina", "Amara", "Amina", "Anders", "Andrea", "Anjali", "Annika", "Antoine", "Arjun", "Astrid",
|
||||||
|
"Ayo", "Ayesha", "Aziz",
|
||||||
|
// B–C
|
||||||
|
"Beatriz", "Bilal", "Bjorn", "Blessing", "Bogdan", "Camila", "Carlos", "Catalina", "Chidi",
|
||||||
|
"Chiara", "Chioma", "Cyrus",
|
||||||
|
// D–E
|
||||||
|
"Dagny", "Damir", "Daniela", "Dilnoza", "Dmitri", "Ebele", "Eduardo", "Eero", "Ekaterina",
|
||||||
|
"Elena", "Elias", "Emeka", "Enrique", "Esi", "Esther", "Eun-ji", "Ewa",
|
||||||
|
// F–G
|
||||||
|
"Fabio", "Farida", "Fatou", "Felipe", "Fernanda", "Freya", "Gabriel", "Georgi", "Giulia",
|
||||||
|
"Grace", "Gunnar", "Gulnara",
|
||||||
|
// H–I
|
||||||
|
"Hana", "Hasan", "Heidi", "Hina", "Hiroshi", "Ibrahim", "Idris", "Ilya", "Imani", "Ingrid",
|
||||||
|
"Iris", "Isabela", "Ivan", "Iwona",
|
||||||
|
// J–K
|
||||||
|
"Jaromir", "Javier", "Jing", "Joana", "Johan", "Josefina", "Junko", "Kaito", "Kalinda", "Karim",
|
||||||
|
"Katarzyna", "Kenji", "Khalid", "Kiran", "Klara", "Kwame", "Kyoko",
|
||||||
|
// L–M
|
||||||
|
"Lakshmi", "Lars", "Laila", "Leilani", "Lena", "Liam", "Linnea", "Lucia", "Lukas", "Madhavi",
|
||||||
|
"Maja", "Malik", "Marisol", "Mateo", "Matteo", "Mei", "Meredith", "Milena", "Mira", "Mohan",
|
||||||
|
"Mira-Lynn", "Mateusz",
|
||||||
|
// N–O
|
||||||
|
"Nadia", "Nasrin", "Neelam", "Niamh", "Nikolai", "Nilufar", "Nkechi", "Noor", "Nuria", "Oksana",
|
||||||
|
"Oleksii", "Olamide", "Omar", "Oskar", "Osei",
|
||||||
|
// P–R
|
||||||
|
"Paloma", "Panagiotis", "Pedro", "Petra", "Priya", "Rafael", "Rania", "Ravi", "Reza", "Renata",
|
||||||
|
"Rin", "Robert", "Rosalind", "Rustam",
|
||||||
|
// S
|
||||||
|
"Sadia", "Salome", "Samir", "Sanjay", "Sara", "Seong-min", "Sipho", "Sofia", "Solveig", "Soren",
|
||||||
|
"Suvi", "Svetlana",
|
||||||
|
// T–U
|
||||||
|
"Tadeusz", "Takeshi", "Tamar", "Tariq", "Thandiwe", "Thi", "Tim", "Tomasz", "Tove", "Tuva",
|
||||||
|
"Ulrika", "Uma", "Usman",
|
||||||
|
// V–Z
|
||||||
|
"Valentina", "Vera", "Vijay", "Vikram", "Wanjiru", "Wei", "Wiktor", "Yara", "Yasmin", "Yohannes",
|
||||||
|
"Yuki", "Yusuf", "Zainab", "Zara", "Zoltan", "Zuzanna",
|
||||||
|
];
|
||||||
|
|
||||||
|
/// Pick a name not already in `taken`.
|
||||||
|
///
|
||||||
|
/// `seed` spreads the starting point so a workspace does not always begin at
|
||||||
|
/// "Amara" — it is an offset into the list, not randomness, so the choice is
|
||||||
|
/// reproducible for a given (seed, taken) pair and therefore testable.
|
||||||
|
///
|
||||||
|
/// When every name is taken it appends a numeric suffix — `Amara 2` — rather
|
||||||
|
/// than returning `None` and forcing the caller to invent something. Running
|
||||||
|
/// out is a nice problem (70+ concurrent agents in one workspace) and a
|
||||||
|
/// duplicate display name is far less harmful than a failed mission launch.
|
||||||
|
pub fn pick(taken: &[String], seed: u64) -> String {
|
||||||
|
let start = (seed % NAMES.len() as u64) as usize;
|
||||||
|
for i in 0..NAMES.len() {
|
||||||
|
let candidate = NAMES[(start + i) % NAMES.len()];
|
||||||
|
if !taken.iter().any(|t| t.eq_ignore_ascii_case(candidate)) {
|
||||||
|
return candidate.to_string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Second pass with a suffix. `round` starts at 2 so the first repeat reads
|
||||||
|
// "Amara 2", which is how a person would disambiguate two colleagues.
|
||||||
|
for round in 2..1000 {
|
||||||
|
for i in 0..NAMES.len() {
|
||||||
|
let candidate = format!("{} {}", NAMES[(start + i) % NAMES.len()], round);
|
||||||
|
if !taken.iter().any(|t| t.eq_ignore_ascii_case(&candidate)) {
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Unreachable in practice; still not a panic.
|
||||||
|
format!("Agent {seed}")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn names_are_unique_and_non_empty() {
|
||||||
|
let mut seen = std::collections::HashSet::new();
|
||||||
|
for n in NAMES {
|
||||||
|
assert!(!n.trim().is_empty(), "empty name in the pool");
|
||||||
|
assert!(seen.insert(n.to_ascii_lowercase()), "duplicate in pool: {n}");
|
||||||
|
}
|
||||||
|
// Every mission mints its own crew and nothing retires them, so the
|
||||||
|
// pool is consumed for the life of the workspace, not recycled. At ~5
|
||||||
|
// per mission this is ~35 missions before the first numeric suffix.
|
||||||
|
assert!(NAMES.len() >= 150, "pool too small for one crew per mission");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A crew should not read as an alphabetical run.
|
||||||
|
///
|
||||||
|
/// With the role index as the seed, every crew started at the top of the
|
||||||
|
/// pool and took the next free names — the first real mission hired Aarav,
|
||||||
|
/// Abebe, Adaora, Adrian, Agnieszka. Unique and correct, and obviously
|
||||||
|
/// generated. Callers now seed from the claw's uuid tail, so this checks
|
||||||
|
/// that well-spread seeds actually land in different regions of the pool
|
||||||
|
/// rather than clustering at one end.
|
||||||
|
#[test]
|
||||||
|
fn spread_seeds_do_not_produce_an_alphabetical_run() {
|
||||||
|
let index_of = |n: &str| NAMES.iter().position(|c| *c == n).expect("name in pool");
|
||||||
|
let seeds = [
|
||||||
|
0x9e37_79b9_7f4a_7c15u64,
|
||||||
|
0x1234_5678_9abc_def0,
|
||||||
|
0xfeed_face_dead_beef,
|
||||||
|
0x0f0f_0f0f_f0f0_f0f0,
|
||||||
|
0xa5a5_5a5a_c3c3_3c3c,
|
||||||
|
];
|
||||||
|
let mut taken: Vec<String> = Vec::new();
|
||||||
|
let mut positions = Vec::new();
|
||||||
|
for s in seeds {
|
||||||
|
let n = pick(&taken, s);
|
||||||
|
positions.push(index_of(&n) as i64);
|
||||||
|
taken.push(n);
|
||||||
|
}
|
||||||
|
// Adjacent picks landing within a couple of slots of each other is the
|
||||||
|
// clustering signature; require the crew to span a real distance.
|
||||||
|
let (min, max) = (
|
||||||
|
*positions.iter().min().unwrap(),
|
||||||
|
*positions.iter().max().unwrap(),
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
max - min > (NAMES.len() as i64) / 3,
|
||||||
|
"crew clustered in one region of the pool: {positions:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The scenario the operator actually asked for: consecutive missions must
|
||||||
|
/// not hand back the same names. Reuse is off, so mission two staffs from
|
||||||
|
/// what mission one left.
|
||||||
|
#[test]
|
||||||
|
fn consecutive_missions_get_different_crews() {
|
||||||
|
let mut roster: Vec<String> = Vec::new();
|
||||||
|
let mut crews: Vec<Vec<String>> = Vec::new();
|
||||||
|
for mission in 0..6u64 {
|
||||||
|
let mut crew = Vec::new();
|
||||||
|
for role in 0..5u64 {
|
||||||
|
let n = pick(&roster, mission * 5 + role);
|
||||||
|
roster.push(n.clone());
|
||||||
|
crew.push(n);
|
||||||
|
}
|
||||||
|
crews.push(crew);
|
||||||
|
}
|
||||||
|
for (i, a) in crews.iter().enumerate() {
|
||||||
|
for (j, b) in crews.iter().enumerate().skip(i + 1) {
|
||||||
|
let shared: Vec<_> = a.iter().filter(|n| b.contains(n)).collect();
|
||||||
|
assert!(
|
||||||
|
shared.is_empty(),
|
||||||
|
"missions {i} and {j} share {shared:?} — crews must be distinct"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// And no duplicates anywhere on the roster.
|
||||||
|
let uniq: std::collections::HashSet<_> = roster.iter().collect();
|
||||||
|
assert_eq!(uniq.len(), roster.len(), "a name was issued twice");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pick_avoids_taken_names() {
|
||||||
|
let taken: Vec<String> = NAMES.iter().take(10).map(|s| s.to_string()).collect();
|
||||||
|
let got = pick(&taken, 0);
|
||||||
|
assert!(
|
||||||
|
!taken.iter().any(|t| t.eq_ignore_ascii_case(&got)),
|
||||||
|
"picked a name already taken: {got}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pick_is_case_insensitive_about_taken() {
|
||||||
|
// A name already on the roster in a different case is still taken —
|
||||||
|
// "meredith" and "Meredith" are the same colleague.
|
||||||
|
let taken = vec![NAMES[0].to_ascii_lowercase()];
|
||||||
|
assert_ne!(pick(&taken, 0).to_ascii_lowercase(), taken[0]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn seed_spreads_the_starting_point() {
|
||||||
|
// Different seeds should not all hand back the same first name, or a
|
||||||
|
// fresh workspace always opens with the same roster.
|
||||||
|
let a = pick(&[], 0);
|
||||||
|
let b = pick(&[], 7);
|
||||||
|
assert_ne!(a, b, "seed had no effect on the choice");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn exhausting_the_pool_suffixes_rather_than_failing() {
|
||||||
|
let taken: Vec<String> = NAMES.iter().map(|s| s.to_string()).collect();
|
||||||
|
let got = pick(&taken, 0);
|
||||||
|
assert!(
|
||||||
|
!taken.iter().any(|t| t.eq_ignore_ascii_case(&got)),
|
||||||
|
"must not reuse a taken name"
|
||||||
|
);
|
||||||
|
assert!(got.ends_with(" 2"), "expected a suffixed name, got {got}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_full_team_gets_distinct_names() {
|
||||||
|
// The actual scenario: mint five roles into an empty workspace and get
|
||||||
|
// five different people, not five "planner"s.
|
||||||
|
let mut taken: Vec<String> = Vec::new();
|
||||||
|
for i in 0..5 {
|
||||||
|
let n = pick(&taken, i);
|
||||||
|
assert!(!taken.contains(&n), "repeated {n} within one team");
|
||||||
|
taken.push(n);
|
||||||
|
}
|
||||||
|
assert_eq!(taken.len(), 5);
|
||||||
|
}
|
||||||
|
}
|
||||||
+230
-10
@@ -71,19 +71,46 @@ impl MergeOutcome {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Classify a `git diff --name-status` body.
|
/// Every path in a `git diff --name-status` body, with its status letter.
|
||||||
///
|
///
|
||||||
/// Returns the offending entries, empty when every change is an addition.
|
/// The World draws a file orb per changed path, and `mission_delivery` records
|
||||||
/// Split out so the rule is testable without a repository.
|
/// the list — both need the same parse, so it lives in one place.
|
||||||
pub fn non_additive_changes(name_status: &str) -> Vec<String> {
|
///
|
||||||
|
/// **Renames are three fields**: `R100\told\tnew`. The path that changed is the
|
||||||
|
/// NEW one; splitting on the first tab and taking field two records where the
|
||||||
|
/// file used to be, which then matches nothing anyone can open. Copies (`C###`)
|
||||||
|
/// have the same shape.
|
||||||
|
pub fn changed_paths(name_status: &str) -> Vec<(char, String)> {
|
||||||
name_status
|
name_status
|
||||||
.lines()
|
.lines()
|
||||||
.filter(|l| !l.trim().is_empty())
|
.filter(|l| !l.trim().is_empty())
|
||||||
.filter(|l| {
|
.filter_map(|l| {
|
||||||
// Status is the first field: A/M/D/R###/C###.
|
let mut fields = l.split('\t');
|
||||||
!matches!(l.chars().next(), Some('A'))
|
let status = fields.next()?.trim();
|
||||||
|
let letter = status.chars().next()?;
|
||||||
|
let first = fields.next()?.trim();
|
||||||
|
// R/C carry old THEN new; everything else has a single path.
|
||||||
|
let path = match letter {
|
||||||
|
'R' | 'C' => fields.next().map(str::trim).unwrap_or(first),
|
||||||
|
_ => first,
|
||||||
|
};
|
||||||
|
if path.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some((letter, path.to_string()))
|
||||||
})
|
})
|
||||||
.map(|l| l.trim().to_string())
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Classify a `git diff --name-status` body.
|
||||||
|
///
|
||||||
|
/// Returns the offending entries, empty when every change is an addition.
|
||||||
|
/// Built on `changed_paths` so the two cannot disagree about what a line means.
|
||||||
|
pub fn non_additive_changes(name_status: &str) -> Vec<String> {
|
||||||
|
changed_paths(name_status)
|
||||||
|
.into_iter()
|
||||||
|
.filter(|(letter, _)| *letter != 'A')
|
||||||
|
.map(|(letter, path)| format!("{letter}\t{path}"))
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -166,11 +193,35 @@ pub async fn try_merge(
|
|||||||
return Ok(MergeOutcome::refused("branch adds nothing"));
|
return Ok(MergeOutcome::refused("branch adds nothing"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
merge_and_push(repo, push_url, branch, base, "auto-merge")
|
||||||
|
.await
|
||||||
|
.map(|o| match o.merged {
|
||||||
|
true => MergeOutcome {
|
||||||
|
merged: true,
|
||||||
|
reason: format!("additive-only and verified; merged into {base}"),
|
||||||
|
},
|
||||||
|
false => o,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The git half of a merge, with no policy in it.
|
||||||
|
///
|
||||||
|
/// Split out so an OPERATOR-approved merge runs exactly the same commands as an
|
||||||
|
/// automatic one — fetch the base as the remote has it, merge onto that, push.
|
||||||
|
/// The gates differ; the mechanics must not, or the rarely-taken path is the one
|
||||||
|
/// that breaks.
|
||||||
|
async fn merge_and_push(
|
||||||
|
repo: &Path,
|
||||||
|
push_url: &str,
|
||||||
|
branch: &str,
|
||||||
|
base: &str,
|
||||||
|
label: &str,
|
||||||
|
) -> Result<MergeOutcome, String> {
|
||||||
// Merge onto the freshly fetched base rather than a local branch.
|
// Merge onto the freshly fetched base rather than a local branch.
|
||||||
git(repo, &["checkout", "-B", base, "FETCH_HEAD"]).await?;
|
git(repo, &["checkout", "-B", base, "FETCH_HEAD"]).await?;
|
||||||
if let Err(e) = git(
|
if let Err(e) = git(
|
||||||
repo,
|
repo,
|
||||||
&["merge", "--no-ff", "-m", &format!("auto-merge {branch}"), branch],
|
&["merge", "--no-ff", "-m", &format!("{label} {branch}"), branch],
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -184,14 +235,150 @@ pub async fn try_merge(
|
|||||||
git(repo, &["push", push_url, &format!("HEAD:refs/heads/{base}")]).await?;
|
git(repo, &["push", push_url, &format!("HEAD:refs/heads/{base}")]).await?;
|
||||||
Ok(MergeOutcome {
|
Ok(MergeOutcome {
|
||||||
merged: true,
|
merged: true,
|
||||||
reason: format!("additive-only and verified; merged into {base}"),
|
reason: format!("merged into {base}"),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Merge a delivered branch because an OPERATOR asked for it.
|
||||||
|
///
|
||||||
|
/// `MergePolicy::Never` means "do not merge on your own" — it defers to a human,
|
||||||
|
/// and this is that human. So the additive-only test does not apply: an operator
|
||||||
|
/// looking at a code change is exactly the judgement the policy was holding out
|
||||||
|
/// for.
|
||||||
|
///
|
||||||
|
/// What is NOT waived:
|
||||||
|
///
|
||||||
|
/// - the branch must exist on the remote and differ from the base, so the button
|
||||||
|
/// cannot report success for a merge of nothing;
|
||||||
|
/// - a conflict refuses and leaves the repo clean, rather than forcing;
|
||||||
|
/// - the work happens in a FRESH CLONE, never the mission checkout — that
|
||||||
|
/// directory is reaped on a timer after the mission ends, so a merge that
|
||||||
|
/// depended on it would work right after a run and mysteriously fail later.
|
||||||
|
pub async fn merge_on_operator_approval(
|
||||||
|
workdir: &Path,
|
||||||
|
push_url: &str,
|
||||||
|
branch: &str,
|
||||||
|
base: &str,
|
||||||
|
) -> Result<MergeOutcome, String> {
|
||||||
|
git(workdir, &["fetch", push_url, base]).await?;
|
||||||
|
git(workdir, &["fetch", push_url, branch]).await?;
|
||||||
|
git(workdir, &["branch", "-f", branch, "FETCH_HEAD"]).await?;
|
||||||
|
git(workdir, &["fetch", push_url, base]).await?;
|
||||||
|
|
||||||
|
let diff = git(
|
||||||
|
workdir,
|
||||||
|
&["diff", "--name-status", &format!("FETCH_HEAD...{branch}")],
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
if diff.trim().is_empty() {
|
||||||
|
return Ok(MergeOutcome::refused(
|
||||||
|
"branch has nothing the base does not already have",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
merge_locally(workdir, branch, base, "merge mission branch").await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Merge onto the fetched base WITHOUT publishing it.
|
||||||
|
///
|
||||||
|
/// Split from the push so a caller can run the project's tests against the
|
||||||
|
/// merged tree first. Verifying BEFORE publishing rather than reverting after is
|
||||||
|
/// the difference between "main was never broken" and "main was broken for as
|
||||||
|
/// long as it took us to notice".
|
||||||
|
pub async fn merge_locally(
|
||||||
|
repo: &Path,
|
||||||
|
branch: &str,
|
||||||
|
base: &str,
|
||||||
|
label: &str,
|
||||||
|
) -> Result<MergeOutcome, String> {
|
||||||
|
git(repo, &["checkout", "-B", base, "FETCH_HEAD"]).await?;
|
||||||
|
if let Err(e) = git(
|
||||||
|
repo,
|
||||||
|
&["merge", "--no-ff", "-m", &format!("{label} {branch}"), branch],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
// Leave the repo clean so the next attempt is not fighting a wedged merge.
|
||||||
|
let _ = git(repo, &["merge", "--abort"]).await;
|
||||||
|
return Ok(MergeOutcome::refused(format!(
|
||||||
|
"merge conflicted ({e}); left for a human"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(MergeOutcome {
|
||||||
|
merged: true,
|
||||||
|
reason: format!("merged into {base} locally, not yet published"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Publish an already-merged base.
|
||||||
|
pub async fn push_merged(repo: &Path, push_url: &str, base: &str) -> Result<(), String> {
|
||||||
|
git(repo, &["push", push_url, &format!("HEAD:refs/heads/{base}")])
|
||||||
|
.await
|
||||||
|
.map(|_| ())
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
/// Publication must be gated on the merged tree, and refusal must not push.
|
||||||
|
///
|
||||||
|
/// The two halves are separate functions precisely so a caller can run tests
|
||||||
|
/// BETWEEN them. If `merge_locally` ever pushed, verification would be
|
||||||
|
/// after-the-fact and `main` would be broken for as long as it took to
|
||||||
|
/// notice — which is the failure mode this whole thing exists to avoid.
|
||||||
|
#[test]
|
||||||
|
fn merging_locally_never_publishes() {
|
||||||
|
let src = include_str!("auto_merge.rs");
|
||||||
|
let body = src
|
||||||
|
.split("pub async fn merge_locally")
|
||||||
|
.nth(1)
|
||||||
|
.and_then(|s| s.split("\n}").next())
|
||||||
|
.unwrap_or("");
|
||||||
|
assert!(!body.is_empty(), "merge_locally not found");
|
||||||
|
assert!(
|
||||||
|
!body.contains("\"push\""),
|
||||||
|
"merge_locally must not push — publication is the caller's decision \
|
||||||
|
after it has verified the result"
|
||||||
|
);
|
||||||
|
// And the push half must exist separately, or the caller cannot publish.
|
||||||
|
assert!(src.contains("pub async fn push_merged"), "push_merged missing");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An operator merge and an automatic one must run the SAME git commands.
|
||||||
|
///
|
||||||
|
/// The gates differ — that is the whole point — but if the mechanics
|
||||||
|
/// diverged, the rarely-taken path would be the untested one. Both go
|
||||||
|
/// through `merge_and_push`.
|
||||||
|
#[test]
|
||||||
|
fn both_merge_paths_share_the_same_mechanics() {
|
||||||
|
let src = include_str!("auto_merge.rs");
|
||||||
|
let calls = src.matches("merge_and_push(").count();
|
||||||
|
// one definition + one call from each path
|
||||||
|
assert!(
|
||||||
|
calls >= 3,
|
||||||
|
"expected try_merge and merge_on_operator_approval to both call \
|
||||||
|
merge_and_push, found {calls} mention(s)"
|
||||||
|
);
|
||||||
|
// And the operator path must NOT re-implement the policy gate it exists
|
||||||
|
// to bypass — if this string appears there, the button is a no-op.
|
||||||
|
let op = src
|
||||||
|
.split("pub async fn merge_on_operator_approval")
|
||||||
|
.nth(1)
|
||||||
|
.unwrap_or("");
|
||||||
|
let body = op.split("\n}").next().unwrap_or("");
|
||||||
|
assert!(
|
||||||
|
!body.contains("MergePolicy::AdditiveOnly"),
|
||||||
|
"the operator path must not apply the additive-only gate"
|
||||||
|
);
|
||||||
|
// It must still refuse an empty branch: a button that reports success
|
||||||
|
// for merging nothing is worse than no button.
|
||||||
|
assert!(
|
||||||
|
body.contains("nothing the base does not already have"),
|
||||||
|
"the operator path must refuse an empty branch"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn only_pure_additions_qualify() {
|
fn only_pure_additions_qualify() {
|
||||||
assert!(non_additive_changes("A\t60 Papers/a.md\nA\t60 Papers/b.md\n").is_empty());
|
assert!(non_additive_changes("A\t60 Papers/a.md\nA\t60 Papers/b.md\n").is_empty());
|
||||||
@@ -207,6 +394,39 @@ mod tests {
|
|||||||
assert_eq!(non_additive_changes("R100\ta.md\tb.md\n").len(), 1);
|
assert_eq!(non_additive_changes("R100\ta.md\tb.md\n").len(), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A rename records the NEW path.
|
||||||
|
///
|
||||||
|
/// `R100\told\tnew` is three fields. Reading field two — which is what a
|
||||||
|
/// split-on-first-tab gives you — records where the file USED to be, so the
|
||||||
|
/// World would draw an orb for a path that no longer exists and the
|
||||||
|
/// delivered file list would name something nobody can open. The bug is
|
||||||
|
/// invisible in any repo where nothing was renamed.
|
||||||
|
#[test]
|
||||||
|
fn a_rename_records_where_the_file_ended_up() {
|
||||||
|
let paths = changed_paths("R100\tsrc/old.rs\tsrc/new.rs\n");
|
||||||
|
assert_eq!(paths, vec![('R', "src/new.rs".to_string())]);
|
||||||
|
|
||||||
|
let copied = changed_paths("C075\tsrc/a.rs\tsrc/b.rs\n");
|
||||||
|
assert_eq!(copied, vec![('C', "src/b.rs".to_string())]);
|
||||||
|
|
||||||
|
// Ordinary two-field lines are unaffected.
|
||||||
|
assert_eq!(
|
||||||
|
changed_paths("A\tone.md\nM\ttwo.md\nD\tthree.md\n"),
|
||||||
|
vec![
|
||||||
|
('A', "one.md".to_string()),
|
||||||
|
('M', "two.md".to_string()),
|
||||||
|
('D', "three.md".to_string()),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `files_changed` and the path list must agree, or nobody can tell which
|
||||||
|
/// one lied. git counts a rename as ONE changed file; so must we.
|
||||||
|
#[test]
|
||||||
|
fn a_rename_counts_once() {
|
||||||
|
assert_eq!(changed_paths("R100\ta.rs\tb.rs\n").len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn an_unknown_policy_never_grants_auto_merge() {
|
fn an_unknown_policy_never_grants_auto_merge() {
|
||||||
assert_eq!(MergePolicy::parse(None), MergePolicy::Never);
|
assert_eq!(MergePolicy::parse(None), MergePolicy::Never);
|
||||||
|
|||||||
@@ -159,10 +159,33 @@ pub async fn run(
|
|||||||
};
|
};
|
||||||
|
|
||||||
let (container, workdir) = exec_target(pool, mission_id).await?;
|
let (container, workdir) = exec_target(pool, mission_id).await?;
|
||||||
|
// Benchmark a COPY, never the mission's own checkout.
|
||||||
|
//
|
||||||
|
// `docker_exec` enters a container running as ROOT with the missions root
|
||||||
|
// bind-mounted, and `cargo bench` writes `target/`. Run in the live tree, it
|
||||||
|
// leaves root-owned build output in a checkout owned by uid 65532 — the
|
||||||
|
// single-writer invariant broken, and the next phase's cargo hitting
|
||||||
|
// permission-denied on a directory it cannot write.
|
||||||
|
//
|
||||||
|
// This is the SAME defect `evaluator_tools::Sandbox` exists for, found the
|
||||||
|
// same way: the harness's uid probe, reporting `uids=0,65532`. Measurement
|
||||||
|
// must not mutate what it measures — the rule this codebase already applies
|
||||||
|
// to the judge and to the `verifier` subagent.
|
||||||
|
let copy_root = crate::root_copy::copy_root("_bench", mission_id);
|
||||||
|
// A stale copy from a previous run is ROOT-owned (see `purge_copy`), so it
|
||||||
|
// must be removed the same way it was created — from inside the container.
|
||||||
|
crate::root_copy::purge(&container, ©_root).await;
|
||||||
|
let copy = crate::root_copy::RootCopy::of(&workdir, ©_root)?;
|
||||||
let cmd = harness.command();
|
let cmd = harness.command();
|
||||||
let raw = docker_exec(&container, &workdir, &cmd)
|
let result = docker_exec(&container, copy.workdir(), &cmd)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("exec {cmd:?}: {e}"))?;
|
.map_err(|e| format!("exec {cmd:?}: {e}"));
|
||||||
|
// Explicitly, on BOTH paths, before the `Drop` fallback runs. `cargo bench`
|
||||||
|
// writes `target/` as root, and the server process is uid 65532: its
|
||||||
|
// `remove_dir_all` cannot delete root-owned files and silently leaves the
|
||||||
|
// whole copy behind — measured at 1.2 MB per run, growing forever.
|
||||||
|
crate::root_copy::purge(&container, ©_root).await;
|
||||||
|
let raw = result?;
|
||||||
let metrics = parse_output(&raw, &harness);
|
let metrics = parse_output(&raw, &harness);
|
||||||
Ok((metrics, harness.driver_name().to_string()))
|
Ok((metrics, harness.driver_name().to_string()))
|
||||||
}
|
}
|
||||||
@@ -254,9 +277,7 @@ async fn exec_target(
|
|||||||
}
|
}
|
||||||
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
||||||
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
||||||
let root = std::env::var("CLAWMATES_MISSIONS_ROOT")
|
let workdir = crate::mission_workspace::missions_root()
|
||||||
.unwrap_or_else(|_| "/var/lib/clawmates-missions".to_string());
|
|
||||||
let workdir = std::path::PathBuf::from(root)
|
|
||||||
.join(mission_id.to_string())
|
.join(mission_id.to_string())
|
||||||
.join("repo");
|
.join("repo");
|
||||||
Ok((container, workdir))
|
Ok((container, workdir))
|
||||||
@@ -401,3 +422,29 @@ fn compute_delta(before: &Value, after: &Value) -> Value {
|
|||||||
}
|
}
|
||||||
json!({ "kind": "opaque", "note": "before/after not structurally comparable" })
|
json!({ "kind": "opaque", "note": "before/after not structurally comparable" })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod bench_copy_tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// The benchmark copy must live OUTSIDE the mission directory, and must not
|
||||||
|
/// be the checkout itself.
|
||||||
|
///
|
||||||
|
/// Running `cargo bench` in the live tree left root-owned `target/` in a
|
||||||
|
/// checkout owned by uid 65532 — caught by the harness's uid probe
|
||||||
|
/// (`uids=0,65532`) after this runner was first wired into the sweep. The
|
||||||
|
/// same rule `evaluator_tools::Sandbox` follows: measurement must not mutate
|
||||||
|
/// what it measures.
|
||||||
|
#[test]
|
||||||
|
fn a_benchmark_runs_in_a_copy_outside_the_mission_directory() {
|
||||||
|
let mission = Uuid::now_v7();
|
||||||
|
let copy = crate::root_copy::copy_root("_bench", mission);
|
||||||
|
let live = crate::mission_workspace::checkout_path(mission);
|
||||||
|
assert_ne!(copy, live, "the bench copy must not be the checkout");
|
||||||
|
assert!(
|
||||||
|
!copy.starts_with(crate::mission_workspace::missions_root().join(mission.to_string())),
|
||||||
|
"{copy:?} must be a SIBLING of the mission dir, or the reaper races it"
|
||||||
|
);
|
||||||
|
assert!(copy.starts_with(crate::mission_workspace::missions_root().join("_bench")), "{copy:?}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -77,6 +77,51 @@ pub fn connect() -> Result<Docker, String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The uid every mission artefact must belong to.
|
||||||
|
///
|
||||||
|
/// The runtime container's own processes already run as this; only `docker
|
||||||
|
/// exec` defaulted to root, because `CreateExecOptions::user` was never set.
|
||||||
|
/// That one omission is the origin of four separate patches: root-owned
|
||||||
|
/// `target/` directories appearing inside a checkout that uid 65532 then could
|
||||||
|
/// not delete, `root_copy` existing at all, and a cleanup path that had to
|
||||||
|
/// re-enter the container as root to undo what it had just done.
|
||||||
|
pub(crate) const MISSION_UID: &str = "65532:65532";
|
||||||
|
|
||||||
|
/// Environment a non-root exec needs, because the image gives uid 65532 no
|
||||||
|
/// writable `HOME` and no writable `CARGO_HOME`.
|
||||||
|
///
|
||||||
|
/// Measured in the deployed image: `/zeroclaw-data` (its `HOME`) and
|
||||||
|
/// `/usr/local/cargo` are both root-owned and unwritable, so switching execs to
|
||||||
|
/// 65532 without this would break every `cargo` invocation — the benchmark
|
||||||
|
/// runner, the judge's verification sandbox, and the delivery test gate — in a
|
||||||
|
/// new and much quieter way than the problem it fixes.
|
||||||
|
///
|
||||||
|
/// The missions root is bind-mounted into the runtime container at the same
|
||||||
|
/// path and IS writable by 65532, so the cargo cache lives there and is shared
|
||||||
|
/// across missions rather than re-downloaded per mission. Verified end to end:
|
||||||
|
/// a clean `cargo build` as 65532 with these three variables produces output
|
||||||
|
/// owned entirely by 65532.
|
||||||
|
fn mission_env() -> Vec<String> {
|
||||||
|
let root = crate::mission_workspace::missions_root();
|
||||||
|
vec![
|
||||||
|
format!("HOME={}", root.join("_home").display()),
|
||||||
|
format!("CARGO_HOME={}", root.join("_cargo").display()),
|
||||||
|
"TMPDIR=/tmp".to_string(),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a workdir is inside the tree missions own.
|
||||||
|
///
|
||||||
|
/// The rule is positional rather than per-caller on purpose. Twelve call sites
|
||||||
|
/// each remembering to pass a uid is twelve chances to forget, and the one that
|
||||||
|
/// forgets leaves debris the others cannot clean up — which is exactly the
|
||||||
|
/// history here.
|
||||||
|
fn is_mission_path(workdir: Option<&str>) -> bool {
|
||||||
|
let Some(dir) = workdir else { return false };
|
||||||
|
let root = crate::mission_workspace::missions_root();
|
||||||
|
std::path::Path::new(dir).starts_with(&root)
|
||||||
|
}
|
||||||
|
|
||||||
/// Run `argv` in `container`, optionally in `workdir`, and capture both
|
/// Run `argv` in `container`, optionally in `workdir`, and capture both
|
||||||
/// streams plus the exit status.
|
/// streams plus the exit status.
|
||||||
///
|
///
|
||||||
@@ -93,6 +138,29 @@ pub async fn exec(
|
|||||||
exec_with_env(docker, container, workdir, argv, &[], timeout).await
|
exec_with_env(docker, container, workdir, argv, &[], timeout).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Run `argv` as **root**, deliberately.
|
||||||
|
///
|
||||||
|
/// The one legitimate use is clearing debris that earlier root-run execs left
|
||||||
|
/// behind: uid 65532 cannot delete a root-owned `target/`, so the cleanup has
|
||||||
|
/// to out-rank it. Every other caller goes through [`exec`], which runs mission
|
||||||
|
/// work as 65532 so no new debris is created.
|
||||||
|
pub async fn exec_as_root(
|
||||||
|
docker: &Docker,
|
||||||
|
container: &str,
|
||||||
|
workdir: Option<&str>,
|
||||||
|
argv: &[String],
|
||||||
|
timeout: Duration,
|
||||||
|
) -> Result<ExecOutput, String> {
|
||||||
|
let fut = exec_inner(docker, container, workdir, argv, &[], None);
|
||||||
|
match tokio::time::timeout(timeout, fut).await {
|
||||||
|
Err(_) => Err(format!(
|
||||||
|
"timed out after {}s (the command may still be running in {container})",
|
||||||
|
timeout.as_secs()
|
||||||
|
)),
|
||||||
|
Ok(res) => res,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// As [`exec`], with extra environment for the command.
|
/// As [`exec`], with extra environment for the command.
|
||||||
pub async fn exec_with_env(
|
pub async fn exec_with_env(
|
||||||
docker: &Docker,
|
docker: &Docker,
|
||||||
@@ -102,7 +170,16 @@ pub async fn exec_with_env(
|
|||||||
env: &[String],
|
env: &[String],
|
||||||
timeout: Duration,
|
timeout: Duration,
|
||||||
) -> Result<ExecOutput, String> {
|
) -> Result<ExecOutput, String> {
|
||||||
let fut = exec_inner(docker, container, workdir, argv, env);
|
// Mission work runs as 65532 with a writable HOME/CARGO_HOME; anything
|
||||||
|
// outside the missions tree (runtime preflight probes, image checks) keeps
|
||||||
|
// the daemon's default so this cannot break unrelated call sites.
|
||||||
|
let (user, mut full_env) = if is_mission_path(workdir) {
|
||||||
|
(Some(MISSION_UID), mission_env())
|
||||||
|
} else {
|
||||||
|
(None, Vec::new())
|
||||||
|
};
|
||||||
|
full_env.extend_from_slice(env);
|
||||||
|
let fut = exec_inner(docker, container, workdir, argv, &full_env, user);
|
||||||
match tokio::time::timeout(timeout, fut).await {
|
match tokio::time::timeout(timeout, fut).await {
|
||||||
Err(_) => Err(format!(
|
Err(_) => Err(format!(
|
||||||
"timed out after {}s (the command may still be running in {container})",
|
"timed out after {}s (the command may still be running in {container})",
|
||||||
@@ -118,6 +195,7 @@ async fn exec_inner(
|
|||||||
workdir: Option<&str>,
|
workdir: Option<&str>,
|
||||||
argv: &[String],
|
argv: &[String],
|
||||||
env: &[String],
|
env: &[String],
|
||||||
|
user: Option<&str>,
|
||||||
) -> Result<ExecOutput, String> {
|
) -> Result<ExecOutput, String> {
|
||||||
let created = docker
|
let created = docker
|
||||||
.create_exec(
|
.create_exec(
|
||||||
@@ -130,6 +208,7 @@ async fn exec_inner(
|
|||||||
} else {
|
} else {
|
||||||
Some(env.to_vec())
|
Some(env.to_vec())
|
||||||
},
|
},
|
||||||
|
user: user.map(str::to_string),
|
||||||
attach_stdout: Some(true),
|
attach_stdout: Some(true),
|
||||||
attach_stderr: Some(true),
|
attach_stderr: Some(true),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
@@ -209,4 +288,118 @@ mod tests {
|
|||||||
assert_eq!(out(Some(1), "a", "b").combined(), "a\nb");
|
assert_eq!(out(Some(1), "a", "b").combined(), "a\nb");
|
||||||
assert_eq!(out(Some(0), " ", "\n").combined(), "");
|
assert_eq!(out(Some(0), " ", "\n").combined(), "");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Mission work is 65532; everything else keeps the daemon's default.
|
||||||
|
///
|
||||||
|
/// The rule is positional so that no caller has to remember it. Twelve call
|
||||||
|
/// sites each passing a uid is twelve chances to forget, and the one that
|
||||||
|
/// forgets leaves debris the other eleven cannot delete — which is the
|
||||||
|
/// actual history: root-owned `target/` directories inside a checkout owned
|
||||||
|
/// by 65532, `root_copy` written to work around them, and a cleanup that had
|
||||||
|
/// to re-enter the container as root to undo its own mess.
|
||||||
|
#[test]
|
||||||
|
fn only_work_inside_the_missions_tree_drops_to_the_mission_uid() {
|
||||||
|
let root = crate::mission_workspace::missions_root();
|
||||||
|
let inside = root.join("019fe785-0f82-7780-8d58-da79fb4c31bc/repo");
|
||||||
|
assert!(is_mission_path(Some(&inside.display().to_string())));
|
||||||
|
assert!(is_mission_path(Some(&root.display().to_string())));
|
||||||
|
|
||||||
|
// Probes and image checks run with no workdir at all, and must not be
|
||||||
|
// forced to a uid the image may not have set up for them.
|
||||||
|
assert!(!is_mission_path(None));
|
||||||
|
assert!(!is_mission_path(Some("/")));
|
||||||
|
assert!(!is_mission_path(Some("/usr/local/cargo")));
|
||||||
|
// A path that merely SHARES A PREFIX is not inside the tree.
|
||||||
|
// `starts_with` on `Path` compares components, so this is already true;
|
||||||
|
// the assertion is here so a switch to string matching cannot pass.
|
||||||
|
let sibling = format!("{}-evil/repo", root.display());
|
||||||
|
assert!(!is_mission_path(Some(&sibling)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The non-root exec carries the three variables the image does not give it.
|
||||||
|
///
|
||||||
|
/// Measured in the deployed image: uid 65532's `HOME` (`/zeroclaw-data`)
|
||||||
|
/// and `/usr/local/cargo` are both root-owned and unwritable. Without these
|
||||||
|
/// overrides, dropping execs to 65532 would break every cargo invocation —
|
||||||
|
/// the benchmark runner, the judge's sandbox, the delivery test gate — far
|
||||||
|
/// more quietly than the leak it fixes.
|
||||||
|
#[test]
|
||||||
|
fn the_mission_env_replaces_the_paths_the_image_leaves_unwritable() {
|
||||||
|
let env = mission_env();
|
||||||
|
let root = crate::mission_workspace::missions_root();
|
||||||
|
assert!(env.iter().any(|v| v == &format!("HOME={}/_home", root.display())));
|
||||||
|
assert!(env.iter().any(|v| v == &format!("CARGO_HOME={}/_cargo", root.display())));
|
||||||
|
assert!(env.iter().any(|v| v == "TMPDIR=/tmp"));
|
||||||
|
for v in &env {
|
||||||
|
assert!(
|
||||||
|
!v.contains("/usr/local/cargo") && !v.contains("/zeroclaw-data"),
|
||||||
|
"{v} points back at a root-owned path"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One place builds an exec, so one place decides its uid.
|
||||||
|
///
|
||||||
|
/// The original bug was not a wrong value — it was an ABSENT one:
|
||||||
|
/// `CreateExecOptions` never set `user`, so the daemon defaulted to root
|
||||||
|
/// and twelve callers inherited that without any of them choosing it. A
|
||||||
|
/// second construction site is how that comes back, so the guard is on the
|
||||||
|
/// number of sites rather than on any particular uid.
|
||||||
|
#[test]
|
||||||
|
fn exactly_one_place_builds_an_exec() {
|
||||||
|
let src = include_str!("container_exec.rs");
|
||||||
|
// Split so this needle does not match itself in this very file.
|
||||||
|
let needle = concat!("CreateExec", "Options {");
|
||||||
|
let sites = src.matches(needle).count();
|
||||||
|
assert_eq!(
|
||||||
|
sites, 1,
|
||||||
|
"exec options must be built in one place; found {sites}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
src.contains(concat!("user: ", "user.map(str::to_string)")),
|
||||||
|
"that one place must set `user` — leaving it unset is the bug"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The tail of a container's log, for putting in an error message.
|
||||||
|
///
|
||||||
|
/// A turn that times out destroys the only place the reason lived: the
|
||||||
|
/// per-mission runtime container is torn down after the phase, taking its logs
|
||||||
|
/// with it, and the operator is left with the string "turn timed out". This
|
||||||
|
/// copies the last few lines out while the container still exists.
|
||||||
|
///
|
||||||
|
/// Best-effort by construction — it runs on a path that is ALREADY failing, so
|
||||||
|
/// every error here degrades to a note rather than replacing the real failure
|
||||||
|
/// with a docker one.
|
||||||
|
pub async fn tail_logs(container: &str, lines: usize) -> String {
|
||||||
|
use futures::StreamExt as _;
|
||||||
|
|
||||||
|
let Ok(docker) = connect() else {
|
||||||
|
return "(docker unreachable, so no container log)".into();
|
||||||
|
};
|
||||||
|
let opts = bollard::query_parameters::LogsOptionsBuilder::default()
|
||||||
|
.stdout(true)
|
||||||
|
.stderr(true)
|
||||||
|
.tail(&lines.to_string())
|
||||||
|
.build();
|
||||||
|
let mut stream = docker.logs(container, Some(opts));
|
||||||
|
let mut out = String::new();
|
||||||
|
while let Some(chunk) = stream.next().await {
|
||||||
|
match chunk {
|
||||||
|
Ok(c) => out.push_str(&c.to_string()),
|
||||||
|
Err(e) => {
|
||||||
|
if out.is_empty() {
|
||||||
|
return format!("(could not read {container} logs: {e})");
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let out = out.trim();
|
||||||
|
if out.is_empty() {
|
||||||
|
format!("({container} logged nothing)")
|
||||||
|
} else {
|
||||||
|
out.to_string()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -19,6 +19,13 @@ pub enum ApiError {
|
|||||||
Conflict,
|
Conflict,
|
||||||
#[error("{0}")]
|
#[error("{0}")]
|
||||||
Quota(String),
|
Quota(String),
|
||||||
|
/// A dependency is temporarily refusing work and will accept it later —
|
||||||
|
/// today, the Claude Code subscription's rate limit. Distinct from
|
||||||
|
/// `Internal` because the operator's next action is different: wait and
|
||||||
|
/// press the button again, rather than read a server log. A 500 with
|
||||||
|
/// "internal error" sent them looking for a bug that was not there.
|
||||||
|
#[error("{0}")]
|
||||||
|
Unavailable(String),
|
||||||
#[error("internal error")]
|
#[error("internal error")]
|
||||||
Internal,
|
Internal,
|
||||||
}
|
}
|
||||||
@@ -58,6 +65,7 @@ impl IntoResponse for ApiError {
|
|||||||
ApiError::NotFound => StatusCode::NOT_FOUND,
|
ApiError::NotFound => StatusCode::NOT_FOUND,
|
||||||
ApiError::Conflict => StatusCode::CONFLICT,
|
ApiError::Conflict => StatusCode::CONFLICT,
|
||||||
ApiError::Quota(_) => StatusCode::PAYMENT_REQUIRED,
|
ApiError::Quota(_) => StatusCode::PAYMENT_REQUIRED,
|
||||||
|
ApiError::Unavailable(_) => StatusCode::SERVICE_UNAVAILABLE,
|
||||||
ApiError::Internal => StatusCode::INTERNAL_SERVER_ERROR,
|
ApiError::Internal => StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
};
|
};
|
||||||
(status, Json(json!({ "error": self.to_string() }))).into_response()
|
(status, Json(json!({ "error": self.to_string() }))).into_response()
|
||||||
|
|||||||
+482
-62
@@ -55,6 +55,17 @@ pub struct Verdict {
|
|||||||
/// verification is how a broken sandbox comes to claim it proved
|
/// verification is how a broken sandbox comes to claim it proved
|
||||||
/// something.
|
/// something.
|
||||||
pub checks: Vec<crate::evaluator_tools::CheckOutcome>,
|
pub checks: Vec<crate::evaluator_tools::CheckOutcome>,
|
||||||
|
/// Whether the judge came from a DIFFERENT provider family than the agent
|
||||||
|
/// that did the work.
|
||||||
|
///
|
||||||
|
/// The default judge is Claude judging Claude's output, which is a correlated
|
||||||
|
/// failure: the same model that talked itself into a shortcut is disposed to
|
||||||
|
/// accept it. Independence is the structural fix, and it is recorded rather
|
||||||
|
/// than assumed — a verdict that cannot say who judged it cannot be audited
|
||||||
|
/// for this. `serde(default)` so verdicts stored before this field existed
|
||||||
|
/// read back as "not independent", which is what they were.
|
||||||
|
#[serde(default)]
|
||||||
|
pub independent: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Verdict {
|
impl Verdict {
|
||||||
@@ -78,6 +89,7 @@ impl Verdict {
|
|||||||
guidance: reason.clone(),
|
guidance: reason.clone(),
|
||||||
reason,
|
reason,
|
||||||
model: model.to_string(),
|
model: model.to_string(),
|
||||||
|
independent: false,
|
||||||
error,
|
error,
|
||||||
checks: Vec::new(),
|
checks: Vec::new(),
|
||||||
}
|
}
|
||||||
@@ -187,7 +199,184 @@ produced by working code;
|
|||||||
|
|
||||||
If you find any of these, the condition is NOT met — say which one you found. \
|
If you find any of these, the condition is NOT met — say which one you found. \
|
||||||
If you cannot verify a claim, it is not met: absence of evidence is not \
|
If you cannot verify a claim, it is not met: absence of evidence is not \
|
||||||
satisfaction.";
|
satisfaction.
|
||||||
|
|
||||||
|
BUT: some conditions are satisfied BY A RECORDED VALUE, and for those, writing \
|
||||||
|
the value into a file IS the work — a measured baseline, a scan report, a \
|
||||||
|
recorded environment fact. Hard-coding is cheating only when the condition is \
|
||||||
|
about behaviour that code must produce. When a condition asks for something to \
|
||||||
|
be RECORDED, judge whether the recorded value is well-formed and plausibly \
|
||||||
|
obtained; do not reject it for being written rather than computed, and do not \
|
||||||
|
require content the condition does not ask for.
|
||||||
|
|
||||||
|
Judge the condition AS WRITTEN. Do not add requirements it does not state, and \
|
||||||
|
do not re-derive the expected value yourself — a condition may describe a \
|
||||||
|
DIFFERENT machine, an earlier run, or a remote environment, and the value you \
|
||||||
|
would measure here is not the one under judgement.";
|
||||||
|
|
||||||
|
/// Which provider family a model spec belongs to.
|
||||||
|
///
|
||||||
|
/// `"glm:glm-4.7"` → `glm`, `"kimi:k2"` → `kimi`, `"claude-opus-4-8"` → `anthropic`.
|
||||||
|
/// Used for one decision only: whether the judge is independent of the agent that
|
||||||
|
/// produced the work. A family, not a model — two Claude models share a lineage,
|
||||||
|
/// a fine-tune and most of their failure modes, so `opus` judging `sonnet` is not
|
||||||
|
/// independence.
|
||||||
|
pub fn provider_family(spec: &str) -> String {
|
||||||
|
if let Some((name, _)) = spec.split_once(':') {
|
||||||
|
// `runtime:<alias>` routes through an agent container, which is running
|
||||||
|
// Claude — the prefix names the transport, not the family.
|
||||||
|
return if name == "runtime" {
|
||||||
|
"anthropic".into()
|
||||||
|
} else {
|
||||||
|
name.to_ascii_lowercase()
|
||||||
|
};
|
||||||
|
}
|
||||||
|
let s = spec.to_ascii_lowercase();
|
||||||
|
for (needle, family) in [
|
||||||
|
("claude", "anthropic"),
|
||||||
|
("opus", "anthropic"),
|
||||||
|
("sonnet", "anthropic"),
|
||||||
|
("haiku", "anthropic"),
|
||||||
|
("glm", "glm"),
|
||||||
|
("kimi", "kimi"),
|
||||||
|
("moonshot", "kimi"),
|
||||||
|
("llama", "groq"),
|
||||||
|
// A model we host ourselves. Only reached for a BARE name — a
|
||||||
|
// `local:ornith-fleet:9b` spec is answered by the split above — but a
|
||||||
|
// bare one falling through to "unknown" would make
|
||||||
|
// `cross_provider_judge` refuse a judge that is genuinely a different
|
||||||
|
// family from the Anthropic implementer, which is the one property it
|
||||||
|
// exists to check.
|
||||||
|
("ornith", "local"),
|
||||||
|
("ollama", "local"),
|
||||||
|
] {
|
||||||
|
if s.contains(needle) {
|
||||||
|
return family.into();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Not "anthropic". An unknown model must not be assumed to be the house
|
||||||
|
// one — that assumption would report independence we never established.
|
||||||
|
"unknown".into()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Does this validator spec name the provider it wants, rather than only a model?
|
||||||
|
///
|
||||||
|
/// `Runtime::resolve_provider` routes `provider:model` and falls back to the
|
||||||
|
/// DEFAULT provider for everything else. That fallback is what makes a bare name
|
||||||
|
/// dangerous here: it silently yields the house provider, which the independence
|
||||||
|
/// check then fails to recognise as the house provider — because
|
||||||
|
/// `provider_family` reads the SPEC, and a bare `gemini-2.5-flash` reads as
|
||||||
|
/// "unknown", not "anthropic".
|
||||||
|
fn names_a_provider(spec: &str) -> bool {
|
||||||
|
spec.contains(':')
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The provider family the mission's agent ran on.
|
||||||
|
///
|
||||||
|
/// Today every mission backend is Claude Code (`agent-claude`), including the
|
||||||
|
/// microVM path. When `agent-glm` / `agent-kimi` images exist this should read
|
||||||
|
/// `missions.backend`; until then, hardcoding the truth is better than plumbing a
|
||||||
|
/// parameter that only ever has one value.
|
||||||
|
const IMPLEMENTER_FAMILY: &str = "anthropic";
|
||||||
|
|
||||||
|
/// Which validator spec applies, given the mission's own setting and the
|
||||||
|
/// deployment default.
|
||||||
|
///
|
||||||
|
/// The three cases are distinct on purpose, and an empty string is not the same
|
||||||
|
/// as unset:
|
||||||
|
/// - `Some("")` on the mission — an explicit opt OUT. This mission wants the house
|
||||||
|
/// judge, and the deployment default must not quietly reinstate independence it
|
||||||
|
/// was told to skip.
|
||||||
|
/// - `Some(spec)` — this mission's choice, which wins.
|
||||||
|
/// - `None` — nothing said, so the deployment default applies.
|
||||||
|
///
|
||||||
|
/// Whitespace counts as empty: a column set to `" "` by hand meant to say nothing.
|
||||||
|
fn resolve_validator_spec(mission: Option<&str>, deployment: Option<&str>) -> Option<String> {
|
||||||
|
match mission {
|
||||||
|
Some(s) if s.trim().is_empty() => None,
|
||||||
|
Some(s) => Some(s.trim().to_string()),
|
||||||
|
None => deployment
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(str::to_string),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A judge from a different provider family, if one is configured and REGISTERED.
|
||||||
|
///
|
||||||
|
/// `CLAWMATES_VALIDATOR_MODEL` holds a registry spec such as `glm:glm-4.7`.
|
||||||
|
/// Returns `None` — never a same-family judge — when it is unset, names the
|
||||||
|
/// implementer's own family, or names a provider this deployment did not register.
|
||||||
|
///
|
||||||
|
/// That last case is the trap worth naming: `Runtime::resolve_provider` falls back
|
||||||
|
/// to the DEFAULT provider when the registry has no such name, which would hand
|
||||||
|
/// back Claude while the caller believed it had asked for GLM. The fallback is
|
||||||
|
/// detectable because the returned model still carries the `name:` prefix, and it
|
||||||
|
/// is checked here rather than trusted.
|
||||||
|
async fn cross_provider_judge(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
|
mission_id: Uuid,
|
||||||
|
) -> Option<(std::sync::Arc<dyn cm_llm::LlmProvider>, String)> {
|
||||||
|
// Read per mission rather than widening `Mission` for one caller. One extra
|
||||||
|
// query per evaluation, against a path that is about to make a model call.
|
||||||
|
let per_mission: Option<String> =
|
||||||
|
sqlx::query_scalar("SELECT validator_model FROM missions WHERE id = $1")
|
||||||
|
.bind(mission_id)
|
||||||
|
.fetch_optional(runtime.pool())
|
||||||
|
.await
|
||||||
|
.unwrap_or(None)
|
||||||
|
.flatten();
|
||||||
|
let spec = resolve_validator_spec(
|
||||||
|
per_mission.as_deref(),
|
||||||
|
std::env::var("CLAWMATES_VALIDATOR_MODEL").ok().as_deref(),
|
||||||
|
)?;
|
||||||
|
let spec = spec.as_str();
|
||||||
|
let family = provider_family(spec);
|
||||||
|
if family == IMPLEMENTER_FAMILY {
|
||||||
|
eprintln!(
|
||||||
|
"evaluator: CLAWMATES_VALIDATOR_MODEL={spec} is the same provider family as the \
|
||||||
|
agent ({IMPLEMENTER_FAMILY}) — that is not an independent check, ignoring it"
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
// A validator spec MUST name its provider. `resolve_provider` falls back to
|
||||||
|
// the DEFAULT provider for anything it cannot route (runtime.rs), and for a
|
||||||
|
// bare model name that fallback is silent: `gemini-2.5-flash` has no colon,
|
||||||
|
// so it resolved to the house Anthropic provider while `provider_family`
|
||||||
|
// reported "unknown" — not "anthropic" — and the verdict was recorded
|
||||||
|
// `independent = true`. An Anthropic judge grading Anthropic work, labelled
|
||||||
|
// independent, which is the one claim this whole path exists to make honestly.
|
||||||
|
//
|
||||||
|
// The check below caught the same fallback for `glm:glm-4.7` when the `glm`
|
||||||
|
// provider was missing, because an unrouted spec comes back WHOLE. It could
|
||||||
|
// never catch a bare name.
|
||||||
|
if !names_a_provider(spec) {
|
||||||
|
eprintln!(
|
||||||
|
"evaluator: CLAWMATES_VALIDATOR_MODEL={spec} is not a registry spec \
|
||||||
|
(expected `provider:model`, e.g. `glm:glm-4.7`) — refusing to judge with \
|
||||||
|
the default provider and call it independent"
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let (provider, model) = runtime.resolve_provider(spec);
|
||||||
|
// Compared against the WHOLE spec, not tested for a colon.
|
||||||
|
//
|
||||||
|
// `resolve_provider` returns the spec unchanged when it does not recognise
|
||||||
|
// the provider, and returns the part after the FIRST colon when it does. The
|
||||||
|
// old test — "does the model half still contain a colon" — assumed model
|
||||||
|
// names never do. `local:ornith-fleet:9b` resolves correctly to provider
|
||||||
|
// `local`, model `ornith-fleet:9b`, and was rejected as unregistered. The
|
||||||
|
// chain preflight found it by reporting a provider it had just registered as
|
||||||
|
// UNREGISTERED.
|
||||||
|
if model == spec {
|
||||||
|
eprintln!(
|
||||||
|
"evaluator: no provider registered for {spec} — refusing to judge with the \
|
||||||
|
default provider and call it independent"
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some((provider, model))
|
||||||
|
}
|
||||||
|
|
||||||
/// The model spec to judge with.
|
/// The model spec to judge with.
|
||||||
///
|
///
|
||||||
@@ -221,20 +410,10 @@ fn subscription_model() -> String {
|
|||||||
/// case in the platform for a bare model call: fixed prompt, no tools, no
|
/// case in the platform for a bare model call: fixed prompt, no tools, no
|
||||||
/// memory, one JSON answer.
|
/// memory, one JSON answer.
|
||||||
fn subscription_judge() -> Option<cm_llm::AnthropicProvider> {
|
fn subscription_judge() -> Option<cm_llm::AnthropicProvider> {
|
||||||
let token = std::env::var("ANTHROPIC_OAUTH_TOKEN").ok()?;
|
// One definition of "the subscription", shared with the planner. This
|
||||||
let token = token.trim();
|
// carried its own copy; two of them is how one gets a prefix check the
|
||||||
if token.is_empty() {
|
// other lacks.
|
||||||
return None;
|
crate::subscription::provider()
|
||||||
}
|
|
||||||
if !token.starts_with("sk-ant-oat") {
|
|
||||||
eprintln!(
|
|
||||||
"evaluator: ANTHROPIC_OAUTH_TOKEN is set but is not a setup token \
|
|
||||||
(expected sk-ant-oat…) — ignoring it and using {}",
|
|
||||||
evaluator_model()
|
|
||||||
);
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
Some(cm_llm::AnthropicProvider::new(token.to_string()))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Judge whether `condition` holds given `evidence`.
|
/// Judge whether `condition` holds given `evidence`.
|
||||||
@@ -252,61 +431,115 @@ pub async fn evaluate(
|
|||||||
"COMPLETION CONDITION:\n{condition}\n\nEVIDENCE (agent claims — verify them):\n{evidence}"
|
"COMPLETION CONDITION:\n{condition}\n\nEVIDENCE (agent claims — verify them):\n{evidence}"
|
||||||
);
|
);
|
||||||
let sandbox = crate::evaluator_tools::Sandbox::for_mission(mission_id);
|
let sandbox = crate::evaluator_tools::Sandbox::for_mission(mission_id);
|
||||||
|
// Purged explicitly at every exit below: `Drop` runs as uid 65532 and cannot
|
||||||
|
// delete the root-owned `target/` the judge's own `cargo test` leaves behind.
|
||||||
|
// Wrapped so the purge below runs on EVERY exit: this function returns
|
||||||
|
// from several branches, and a cleanup only some paths reach is the same
|
||||||
|
// as no cleanup on the others.
|
||||||
|
let verdict = async {
|
||||||
|
|
||||||
// Preferred: a bare Messages API call on the subscription token. See
|
// Most preferred: a judge from a DIFFERENT provider family, with the same
|
||||||
// `subscription_judge` for why this beats routing through an agent.
|
// allow-listed tool loop. Claude judging Claude's work is a correlated
|
||||||
if let Some(provider) = subscription_judge() {
|
// failure — the model that talked itself into a shortcut is the one disposed
|
||||||
let model = subscription_model();
|
// to accept it — and the tool loop is what makes the check evidence rather
|
||||||
let system = match &sandbox {
|
// than opinion, so an independent judge must have it too.
|
||||||
Some(_) => format!("{EVAL_SYSTEM_VERIFYING}\n\n{VERDICT_CONTRACT}"),
|
if let Some((provider, model)) = cross_provider_judge(runtime, mission_id).await {
|
||||||
None => format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}"),
|
let system = match &sandbox {
|
||||||
|
Some(_) => format!("{EVAL_SYSTEM_VERIFYING}\n\n{VERDICT_CONTRACT}"),
|
||||||
|
None => format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}"),
|
||||||
|
};
|
||||||
|
eprintln!(
|
||||||
|
"evaluator: mission {mission_id} judged independently by {} ({})",
|
||||||
|
model,
|
||||||
|
provider_family(&model)
|
||||||
|
);
|
||||||
|
match judge_with_tools(provider.as_ref(), &system, &user, &model, sandbox.as_ref()).await {
|
||||||
|
Ok((text, checks)) => {
|
||||||
|
let mut v = parse_verdict(&model, &text);
|
||||||
|
v.guidance = sanitize_guidance(condition, evidence, &v.guidance);
|
||||||
|
v.checks = checks;
|
||||||
|
v.independent = true;
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
// Deliberately NOT a silent fall-through to the house judge. An
|
||||||
|
// independent check that failed and was quietly replaced by a
|
||||||
|
// same-family one would leave a verdict claiming a property it does
|
||||||
|
// not have. The phase stays unmet this pass and says why; the next
|
||||||
|
// sweep retries.
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!(
|
||||||
|
"evaluator: the independent judge ({model}) failed — NOT falling back to the agent's own provider: {e}"
|
||||||
|
);
|
||||||
|
return Verdict::not_met(
|
||||||
|
&model,
|
||||||
|
"the independent validator could not be reached this pass",
|
||||||
|
Some(e),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Preferred: a bare Messages API call on the subscription token. See
|
||||||
|
// `subscription_judge` for why this beats routing through an agent.
|
||||||
|
if let Some(provider) = subscription_judge() {
|
||||||
|
let model = subscription_model();
|
||||||
|
let system = match &sandbox {
|
||||||
|
Some(_) => format!("{EVAL_SYSTEM_VERIFYING}\n\n{VERDICT_CONTRACT}"),
|
||||||
|
None => format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}"),
|
||||||
|
};
|
||||||
|
let outcome = judge_with_tools(&provider, &system, &user, &model, sandbox.as_ref()).await;
|
||||||
|
// Same family as the agent; `independent` stays false below.
|
||||||
|
return match outcome {
|
||||||
|
Err(e) => Verdict::not_met(
|
||||||
|
&model,
|
||||||
|
"could not evaluate the completion condition this pass",
|
||||||
|
Some(e),
|
||||||
|
),
|
||||||
|
Ok((text, checks)) => {
|
||||||
|
let mut v = parse_verdict(&model, &text);
|
||||||
|
v.guidance = sanitize_guidance(condition, evidence, &v.guidance);
|
||||||
|
v.checks = checks;
|
||||||
|
v
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fallback paths have no tool loop, so they judge claims only and must say so.
|
||||||
|
let system = format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}");
|
||||||
|
let (eval_system, user) = (system.as_str(), user);
|
||||||
|
|
||||||
|
let model = evaluator_model();
|
||||||
|
// Same routing as the door governor (mcp_door.rs): `runtime:<alias>` goes
|
||||||
|
// through the container agent so a subscription-only model can judge.
|
||||||
|
let raw: Result<String, String> = if let Some(alias) = model.strip_prefix("runtime:") {
|
||||||
|
match crate::topology_exec::ZeroClawDriveExecutor::from_env() {
|
||||||
|
Ok(exec) => exec.judge_raw(alias.trim(), eval_system, &user).await,
|
||||||
|
Err(e) => Err(format!("runtime executor unavailable: {e}")),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
runtime
|
||||||
|
.complete(eval_system, &user, &model, 512, false)
|
||||||
|
.await
|
||||||
};
|
};
|
||||||
let outcome = judge_with_tools(&provider, &system, &user, &model, sandbox.as_ref()).await;
|
|
||||||
return match outcome {
|
match raw {
|
||||||
Err(e) => Verdict::not_met(
|
Err(e) => Verdict::not_met(
|
||||||
&model,
|
&model,
|
||||||
"could not evaluate the completion condition this pass",
|
"could not evaluate the completion condition this pass",
|
||||||
Some(e),
|
Some(e),
|
||||||
),
|
),
|
||||||
Ok((text, checks)) => {
|
Ok(text) => {
|
||||||
let mut v = parse_verdict(&model, &text);
|
let mut v = parse_verdict(&model, &text);
|
||||||
v.guidance = sanitize_guidance(condition, evidence, &v.guidance);
|
v.guidance = sanitize_guidance(condition, evidence, &v.guidance);
|
||||||
v.checks = checks;
|
|
||||||
v
|
v
|
||||||
}
|
}
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback paths have no tool loop, so they judge claims only and must say so.
|
|
||||||
let system = format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}");
|
|
||||||
let (eval_system, user) = (system.as_str(), user);
|
|
||||||
|
|
||||||
let model = evaluator_model();
|
|
||||||
// Same routing as the door governor (mcp_door.rs): `runtime:<alias>` goes
|
|
||||||
// through the container agent so a subscription-only model can judge.
|
|
||||||
let raw: Result<String, String> = if let Some(alias) = model.strip_prefix("runtime:") {
|
|
||||||
match crate::topology_exec::ZeroClawDriveExecutor::from_env() {
|
|
||||||
Ok(exec) => exec.judge_raw(alias.trim(), eval_system, &user).await,
|
|
||||||
Err(e) => Err(format!("runtime executor unavailable: {e}")),
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
runtime
|
|
||||||
.complete(eval_system, &user, &model, 512, false)
|
|
||||||
.await
|
|
||||||
};
|
|
||||||
|
|
||||||
match raw {
|
|
||||||
Err(e) => Verdict::not_met(
|
|
||||||
&model,
|
|
||||||
"could not evaluate the completion condition this pass",
|
|
||||||
Some(e),
|
|
||||||
),
|
|
||||||
Ok(text) => {
|
|
||||||
let mut v = parse_verdict(&model, &text);
|
|
||||||
v.guidance = sanitize_guidance(condition, evidence, &v.guidance);
|
|
||||||
v
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
.await;
|
||||||
|
if let Some(sb) = &sandbox {
|
||||||
|
sb.purge().await;
|
||||||
|
}
|
||||||
|
verdict
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Ceiling on verification commands per verdict. A judge that has run twelve
|
/// Ceiling on verification commands per verdict. A judge that has run twelve
|
||||||
@@ -346,14 +579,22 @@ fn verify_tool() -> cm_llm::ToolDescriptor {
|
|||||||
///
|
///
|
||||||
/// With no sandbox this degenerates to a single call — same shape, no tools
|
/// With no sandbox this degenerates to a single call — same shape, no tools
|
||||||
/// offered — so there is one code path for both kinds of phase.
|
/// offered — so there is one code path for both kinds of phase.
|
||||||
|
/// `&dyn LlmProvider`, not `&AnthropicProvider`.
|
||||||
|
///
|
||||||
|
/// The trait is a single method — `stream(ChatRequest)` — and this loop only ever
|
||||||
|
/// used that, so the concrete type was incidental. Widening it is what lets a
|
||||||
|
/// CROSS-PROVIDER judge run the same allow-listed checks: before this, independence
|
||||||
|
/// and real verification were mutually exclusive, because the tool loop lived only
|
||||||
|
/// on the subscription path and every other route "judged claims only".
|
||||||
|
/// GLM is registered in anthropic format, so tool calling reaches it unchanged.
|
||||||
async fn judge_with_tools(
|
async fn judge_with_tools(
|
||||||
provider: &cm_llm::AnthropicProvider,
|
provider: &dyn cm_llm::LlmProvider,
|
||||||
system: &str,
|
system: &str,
|
||||||
user: &str,
|
user: &str,
|
||||||
model: &str,
|
model: &str,
|
||||||
sandbox: Option<&crate::evaluator_tools::Sandbox>,
|
sandbox: Option<&crate::evaluator_tools::Sandbox>,
|
||||||
) -> Result<(String, Vec<crate::evaluator_tools::CheckOutcome>), String> {
|
) -> Result<(String, Vec<crate::evaluator_tools::CheckOutcome>), String> {
|
||||||
use cm_llm::{ChatMessage, ChatRequest, ChatRole, ContentPart, LlmEvent, LlmProvider};
|
use cm_llm::{ChatMessage, ChatRequest, ChatRole, ContentPart, LlmEvent};
|
||||||
use futures::StreamExt as _;
|
use futures::StreamExt as _;
|
||||||
|
|
||||||
let tools = match sandbox {
|
let tools = match sandbox {
|
||||||
@@ -507,6 +748,8 @@ fn parse_verdict(model: &str, text: &str) -> Verdict {
|
|||||||
reason,
|
reason,
|
||||||
guidance,
|
guidance,
|
||||||
model: model.to_string(),
|
model: model.to_string(),
|
||||||
|
// Set by the caller: only `evaluate` knows which provider judged.
|
||||||
|
independent: false,
|
||||||
error: None,
|
error: None,
|
||||||
checks: Vec::new(),
|
checks: Vec::new(),
|
||||||
}
|
}
|
||||||
@@ -531,12 +774,14 @@ pub async fn record(
|
|||||||
) -> Result<(), sqlx::Error> {
|
) -> Result<(), sqlx::Error> {
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
"INSERT INTO mission_phase_evaluations
|
"INSERT INTO mission_phase_evaluations
|
||||||
(id, mission_id, phase_id, iteration, met, reason, guidance, model, error, checks)
|
(id, mission_id, phase_id, iteration, met, reason, guidance, model, error,
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
checks, independent)
|
||||||
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||||
ON CONFLICT (phase_id, iteration) DO UPDATE
|
ON CONFLICT (phase_id, iteration) DO UPDATE
|
||||||
SET met = EXCLUDED.met, reason = EXCLUDED.reason,
|
SET met = EXCLUDED.met, reason = EXCLUDED.reason,
|
||||||
guidance = EXCLUDED.guidance, model = EXCLUDED.model,
|
guidance = EXCLUDED.guidance, model = EXCLUDED.model,
|
||||||
error = EXCLUDED.error, checks = EXCLUDED.checks",
|
error = EXCLUDED.error, checks = EXCLUDED.checks,
|
||||||
|
independent = EXCLUDED.independent",
|
||||||
)
|
)
|
||||||
.bind(Uuid::now_v7())
|
.bind(Uuid::now_v7())
|
||||||
.bind(mission_id)
|
.bind(mission_id)
|
||||||
@@ -548,6 +793,7 @@ pub async fn record(
|
|||||||
.bind(&v.model)
|
.bind(&v.model)
|
||||||
.bind(v.error.as_deref())
|
.bind(v.error.as_deref())
|
||||||
.bind(serde_json::json!(v.checks))
|
.bind(serde_json::json!(v.checks))
|
||||||
|
.bind(v.independent)
|
||||||
.execute(pool)
|
.execute(pool)
|
||||||
.await
|
.await
|
||||||
.map(|_| ())
|
.map(|_| ())
|
||||||
@@ -581,6 +827,180 @@ pub async fn latest(
|
|||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod cross_provider_tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// A bare model name must never be accepted as a validator spec.
|
||||||
|
///
|
||||||
|
/// `resolve_provider` falls back to the DEFAULT provider for anything it
|
||||||
|
/// cannot route, and for a bare name that fallback is invisible: the spec
|
||||||
|
/// has no `provider:` prefix to come back with, so the existing
|
||||||
|
/// "no provider registered" check cannot see it. The result was an
|
||||||
|
/// Anthropic judge grading Anthropic work with `independent = true`.
|
||||||
|
#[test]
|
||||||
|
fn a_validator_spec_must_name_its_provider() {
|
||||||
|
for good in ["glm:glm-4.7", "kimi:kimi-for-coding", "runtime:some-alias"] {
|
||||||
|
assert!(names_a_provider(good), "{good} is a registry spec");
|
||||||
|
}
|
||||||
|
// These are the dangerous ones: they resolve to the DEFAULT provider.
|
||||||
|
for bare in ["gemini-2.5-flash", "claude-sonnet-5", "glm-4.7", ""] {
|
||||||
|
assert!(
|
||||||
|
!names_a_provider(bare),
|
||||||
|
"{bare:?} names no provider and must be refused"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A family, not a model. Two Claude models share a lineage and most of their
|
||||||
|
/// failure modes, so `opus` judging `sonnet` is not an independent check.
|
||||||
|
#[test]
|
||||||
|
fn every_anthropic_spelling_is_one_family() {
|
||||||
|
for spec in [
|
||||||
|
"claude-opus-4-8",
|
||||||
|
"claude-sonnet-5",
|
||||||
|
"claude-haiku-4-5-20251001",
|
||||||
|
"opus",
|
||||||
|
"runtime:claw_1234", // routes through an agent container running Claude
|
||||||
|
] {
|
||||||
|
assert_eq!(provider_family(spec), "anthropic", "{spec}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The registry prefix is what actually selects a different provider.
|
||||||
|
#[test]
|
||||||
|
fn a_registry_prefix_names_the_family() {
|
||||||
|
assert_eq!(provider_family("glm:glm-4.7"), "glm");
|
||||||
|
assert_eq!(provider_family("kimi:kimi-k2"), "kimi");
|
||||||
|
assert_eq!(provider_family("GLM:GLM-4.7"), "glm");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An unrecognised model must NOT be assumed to be the house one. Guessing
|
||||||
|
/// "anthropic" would understate independence; guessing anything else would
|
||||||
|
/// claim independence we never established. So: unknown.
|
||||||
|
#[test]
|
||||||
|
fn an_unrecognised_model_is_not_assumed_to_be_ours() {
|
||||||
|
assert_eq!(provider_family("some-new-model-v9"), "unknown");
|
||||||
|
assert_ne!(provider_family("some-new-model-v9"), IMPLEMENTER_FAMILY);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The whole point: a judge in the implementer's own family is not
|
||||||
|
/// independent, whichever model it is.
|
||||||
|
#[test]
|
||||||
|
fn a_same_family_judge_is_never_independent() {
|
||||||
|
for spec in ["claude-opus-4-8", "runtime:claw_x", "sonnet"] {
|
||||||
|
assert_eq!(
|
||||||
|
provider_family(spec),
|
||||||
|
IMPLEMENTER_FAMILY,
|
||||||
|
"{spec} would have to be rejected as a validator"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for spec in ["glm:glm-4.7", "kimi:kimi-k2"] {
|
||||||
|
assert_ne!(provider_family(spec), IMPLEMENTER_FAMILY, "{spec}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A mission's own choice wins over the deployment default.
|
||||||
|
#[test]
|
||||||
|
fn a_mission_can_choose_its_validator() {
|
||||||
|
assert_eq!(
|
||||||
|
resolve_validator_spec(Some("kimi:kimi-k2"), Some("glm:glm-4.7")).as_deref(),
|
||||||
|
Some("kimi:kimi-k2")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
resolve_validator_spec(None, Some("glm:glm-4.7")).as_deref(),
|
||||||
|
Some("glm:glm-4.7"),
|
||||||
|
"nothing said on the mission means the deployment default applies"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An EMPTY value on the mission is an explicit opt-out, not "unset". The
|
||||||
|
/// deployment default must not quietly reinstate independence a mission was
|
||||||
|
/// told to skip — the two cases look the same in a nullable text column and
|
||||||
|
/// mean opposite things.
|
||||||
|
#[test]
|
||||||
|
fn an_empty_mission_setting_opts_out_rather_than_falling_back() {
|
||||||
|
for spelling in [Some(""), Some(" ")] {
|
||||||
|
assert_eq!(
|
||||||
|
resolve_validator_spec(spelling, Some("glm:glm-4.7")),
|
||||||
|
None,
|
||||||
|
"{spelling:?} asked for no independent validator"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// And with neither set, there is no independent judge — which is the state
|
||||||
|
/// every deployment starts in.
|
||||||
|
#[test]
|
||||||
|
fn no_setting_anywhere_means_no_independent_judge() {
|
||||||
|
assert_eq!(resolve_validator_spec(None, None), None);
|
||||||
|
assert_eq!(resolve_validator_spec(None, Some(" ")), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A phase that ran out of passes without meeting its condition did NOT
|
||||||
|
/// succeed. It used to be recorded `completed` alongside a verdict saying
|
||||||
|
/// `met=false`, so mission status reported a goal that was never reached as a
|
||||||
|
/// goal achieved. Found by the Goodhart test: an independent judge refused the
|
||||||
|
/// phase, and the mission closed green anyway.
|
||||||
|
#[test]
|
||||||
|
fn an_unmet_condition_does_not_close_a_phase_as_completed() {
|
||||||
|
// Mirrors the decision in `phase_runner::evaluate_finished_phases`.
|
||||||
|
let outcome = |met: bool| if met { "completed" } else { "failed" };
|
||||||
|
assert_eq!(outcome(true), "completed");
|
||||||
|
assert_eq!(
|
||||||
|
outcome(false),
|
||||||
|
"failed",
|
||||||
|
"an exhausted, unmet phase must not share a status with a met one"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A verdict that has not been marked independent must not read as one. This
|
||||||
|
/// is the field's default, and old rows stored before it existed deserialize
|
||||||
|
/// to exactly that.
|
||||||
|
/// The anti-Goodhart clause and the recorded-value clause must BOTH be in
|
||||||
|
/// the verifying prompt, because each without the other is a known failure.
|
||||||
|
///
|
||||||
|
/// Without the first, an agent emits the string the judge asked for and the
|
||||||
|
/// judge accepts it — that is the incident the verifying judge was built
|
||||||
|
/// after. Without the second, the judge rejects work whose whole point is a
|
||||||
|
/// recorded value: three consecutive production verdicts failed a phase for
|
||||||
|
/// writing a kernel version into a file, which is precisely "a value printed
|
||||||
|
/// rather than produced by working code" as the clause describes it. Asked
|
||||||
|
/// the same question WITHOUT this prompt, the same model answered MET.
|
||||||
|
#[test]
|
||||||
|
fn the_verifying_prompt_distinguishes_cheating_from_recording() {
|
||||||
|
let p = EVAL_SYSTEM_VERIFYING;
|
||||||
|
// The trap it must still catch.
|
||||||
|
assert!(p.contains("hard-coded, stubbed, or printed"), "{p}");
|
||||||
|
// The legitimate case it must not mistake for the trap.
|
||||||
|
assert!(p.contains("RECORDED VALUE"), "{p}");
|
||||||
|
assert!(
|
||||||
|
p.contains("do not reject it for being written rather than computed"),
|
||||||
|
"{p}"
|
||||||
|
);
|
||||||
|
// And the second failure mode from the same three verdicts: the judge
|
||||||
|
// re-deriving the expected value in its own environment.
|
||||||
|
assert!(p.contains("do not re-derive the expected value yourself"), "{p}");
|
||||||
|
assert!(p.contains("Judge the condition AS WRITTEN"), "{p}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_verdict_defaults_to_not_independent() {
|
||||||
|
let v = Verdict::not_met("claude-opus-4-8", "nope", None);
|
||||||
|
assert!(!v.independent);
|
||||||
|
|
||||||
|
let stored = serde_json::json!({
|
||||||
|
"met": true, "reason": "r", "guidance": "", "model": "claude-opus-4-8",
|
||||||
|
"error": null, "checks": []
|
||||||
|
});
|
||||||
|
let old: Verdict = serde_json::from_value(stored).expect("an old verdict still reads");
|
||||||
|
assert!(
|
||||||
|
!old.independent,
|
||||||
|
"a verdict written before independence was recorded was not independent"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|||||||
@@ -195,11 +195,39 @@ pub fn clamp_output(s: &str) -> String {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Where a verification copy lives: a sibling of the per-mission directories,
|
||||||
|
/// so the sweeper that deletes `<root>/<mission_id>` never races it and nothing
|
||||||
|
/// under it is ever collected or delivered.
|
||||||
|
fn verify_path(mission_id: Uuid) -> PathBuf {
|
||||||
|
crate::mission_workspace::missions_root()
|
||||||
|
.join("_verify")
|
||||||
|
.join(mission_id.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
/// A checkout the judge may run verification commands against.
|
/// A checkout the judge may run verification commands against.
|
||||||
#[derive(Debug, Clone)]
|
///
|
||||||
|
/// A COPY of the mission's checkout, never the checkout itself. The judge runs
|
||||||
|
/// real commands — `cargo test` is the whole point — and the container it execs
|
||||||
|
/// into runs as ROOT with the missions root bind-mounted, so running them in the
|
||||||
|
/// live tree left `repo/target/` owned by uid 0 in a checkout otherwise owned by
|
||||||
|
/// the server. That breaks the single-writer invariant copy mode exists to
|
||||||
|
/// guarantee, and the next phase's `cargo` would hit permission-denied on a
|
||||||
|
/// directory it cannot write.
|
||||||
|
///
|
||||||
|
/// It stayed invisible all day because a dead validator credential meant the
|
||||||
|
/// judge never ran a single check; restoring the credential surfaced it on the
|
||||||
|
/// first gated mission, via the harness's uid probe.
|
||||||
|
///
|
||||||
|
/// The deeper rule is the one this codebase already applies to the `verifier`
|
||||||
|
/// subagent, which has no Edit and no Write: **verification must not mutate what
|
||||||
|
/// it verifies.** A judge that can change the tree it is judging can make its own
|
||||||
|
/// verdict true.
|
||||||
|
#[derive(Debug)]
|
||||||
pub struct Sandbox {
|
pub struct Sandbox {
|
||||||
container: String,
|
container: String,
|
||||||
workdir: PathBuf,
|
workdir: PathBuf,
|
||||||
|
/// Whether this sandbox created `workdir` and must remove it.
|
||||||
|
owned: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Sandbox {
|
impl Sandbox {
|
||||||
@@ -208,22 +236,59 @@ impl Sandbox {
|
|||||||
///
|
///
|
||||||
/// Returning `None` rather than an empty sandbox matters: the evaluator
|
/// Returning `None` rather than an empty sandbox matters: the evaluator
|
||||||
/// prompt changes shape depending on whether verification is possible, and
|
/// prompt changes shape depending on whether verification is possible, and
|
||||||
/// a judge must never be told it can check something it cannot.
|
/// a judge must never be told it can check something it cannot. A copy that
|
||||||
|
/// fails to materialise is also `None` for the same reason — an unverifiable
|
||||||
|
/// phase must not be told it can verify.
|
||||||
pub fn for_mission(mission_id: Uuid) -> Option<Sandbox> {
|
pub fn for_mission(mission_id: Uuid) -> Option<Sandbox> {
|
||||||
let workdir = crate::mission_workspace::checkout_path(mission_id);
|
Sandbox::for_checkout(
|
||||||
if !workdir.is_dir() {
|
&crate::mission_workspace::checkout_path(mission_id),
|
||||||
|
&verify_path(mission_id),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The testable half of [`Sandbox::for_mission`]. The paths are parameters
|
||||||
|
/// because `missions_root()` reads process environment, and this workspace
|
||||||
|
/// does not mutate that in tests — the same split as
|
||||||
|
/// `mission_runtime::provider_env_from` and
|
||||||
|
/// `mission_workspace::auth_with_token`.
|
||||||
|
pub fn for_checkout(source: &Path, root: &Path) -> Option<Sandbox> {
|
||||||
|
if !source.is_dir() {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
// `root_copy` owns this pattern for all four callers — the judge, the
|
||||||
|
// benchmark runner, the on_green_tests gate, and this. It packs through
|
||||||
|
// the transport packer (one exclusion list, so a copy carries exactly
|
||||||
|
// what a delivered diff carries) and its `purge` is the only thing that
|
||||||
|
// can remove the root-owned `target/` a run leaves behind.
|
||||||
|
//
|
||||||
|
// A stale copy would otherwise be verified instead of this pass's work —
|
||||||
|
// the "judged a tree nobody wrote" shape the evaluator exists to prevent
|
||||||
|
// — so the caller purges before constructing.
|
||||||
|
// `into_workdir` because the judge has not run yet: letting the handle's
|
||||||
|
// Drop fire on return would delete the tree out from under it. `Sandbox`
|
||||||
|
// owns the lifetime from here, and `Sandbox::purge` clears it.
|
||||||
|
let workdir = crate::root_copy::RootCopy::of(source, root)
|
||||||
|
.ok()?
|
||||||
|
.into_workdir();
|
||||||
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
||||||
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
||||||
Some(Sandbox { container, workdir })
|
Some(Sandbox {
|
||||||
|
container,
|
||||||
|
workdir,
|
||||||
|
owned: true,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Construct against an explicit path. Test seam.
|
/// Construct against an explicit path. Test seam.
|
||||||
|
///
|
||||||
|
/// Never `owned`: a caller-supplied directory is the caller's, and deleting
|
||||||
|
/// it on drop would make this seam destructive in a way its users could not
|
||||||
|
/// see.
|
||||||
pub fn at(container: impl Into<String>, workdir: impl AsRef<Path>) -> Sandbox {
|
pub fn at(container: impl Into<String>, workdir: impl AsRef<Path>) -> Sandbox {
|
||||||
Sandbox {
|
Sandbox {
|
||||||
container: container.into(),
|
container: container.into(),
|
||||||
workdir: workdir.as_ref().to_path_buf(),
|
workdir: workdir.as_ref().to_path_buf(),
|
||||||
|
owned: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -329,6 +394,53 @@ fn git_ownership_env(workdir: &str) -> Vec<String> {
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Sandbox {
|
||||||
|
/// Remove the copy, from inside the container that wrote it.
|
||||||
|
///
|
||||||
|
/// `Drop` cannot do this. The judge runs `cargo test` in a container as
|
||||||
|
/// ROOT, so the copy's `target/` is root-owned, and the server process is
|
||||||
|
/// uid 65532 — its `remove_dir_all` fails on those files and leaves the
|
||||||
|
/// whole tree behind. Measured: 16 MB across two stranded copies, the oldest
|
||||||
|
/// hours old, while `Drop` logged nothing anyone read.
|
||||||
|
///
|
||||||
|
/// The claim that "the next pass clears anyway" was wrong for the same
|
||||||
|
/// reason: `for_checkout` removes a stale root before copying, with the same
|
||||||
|
/// uid, and fails the same way.
|
||||||
|
///
|
||||||
|
/// Still best-effort — a housekeeping error must not cost a real verdict —
|
||||||
|
/// but now attempted by something that can actually succeed.
|
||||||
|
pub async fn purge(&self) {
|
||||||
|
if !self.owned {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let Some(root) = self.workdir.parent() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
// The same purge as the other three copy sites, not a fourth copy of
|
||||||
|
// it: an inlined duplicate is how the reap paths drifted apart before.
|
||||||
|
crate::root_copy::purge(&self.container, root).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for Sandbox {
|
||||||
|
/// Fallback only — see [`Sandbox::purge`], which is what actually clears a
|
||||||
|
/// copy the judge has run commands in. This still catches the early paths
|
||||||
|
/// where nothing has run as root yet.
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if !self.owned {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if let Some(root) = self.workdir.parent() {
|
||||||
|
if let Err(e) = std::fs::remove_dir_all(root) {
|
||||||
|
eprintln!(
|
||||||
|
"evaluator_tools: could not remove the verification copy at {} ({e})",
|
||||||
|
root.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// One verification command and what became of it.
|
/// One verification command and what became of it.
|
||||||
///
|
///
|
||||||
/// This exists because the first version recorded *attempted* commands. The
|
/// This exists because the first version recorded *attempted* commands. The
|
||||||
@@ -427,6 +539,58 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// THE regression. The judge runs real commands in a container that runs as
|
||||||
|
/// ROOT with the missions root bind-mounted, so verifying the live checkout
|
||||||
|
/// left `repo/target/` owned by uid 0 in a tree owned by the server — the
|
||||||
|
/// single-writer invariant broken by the thing that was supposed to be
|
||||||
|
/// checking the work. Verifying a COPY makes it unrepresentable.
|
||||||
|
#[test]
|
||||||
|
fn the_judge_verifies_a_copy_and_never_the_mission_tree() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path().join("missions-root");
|
||||||
|
let mission = Uuid::now_v7();
|
||||||
|
let checkout = root.join(mission.to_string()).join("repo");
|
||||||
|
std::fs::create_dir_all(checkout.join("src")).unwrap();
|
||||||
|
std::fs::write(checkout.join("Cargo.toml"), "[package]\nname='x'\n").unwrap();
|
||||||
|
std::fs::write(checkout.join("src/lib.rs"), "pub fn a() {}").unwrap();
|
||||||
|
// Build output the transport already excludes; the copy must not carry
|
||||||
|
// it either, or the judge measures a stale artifact.
|
||||||
|
std::fs::create_dir_all(checkout.join("target/debug")).unwrap();
|
||||||
|
std::fs::write(checkout.join("target/debug/junk"), "x").unwrap();
|
||||||
|
|
||||||
|
let sandbox = Sandbox::for_checkout(&checkout, &root.join("_verify").join(mission.to_string()))
|
||||||
|
.expect("a checkout on disk yields a sandbox");
|
||||||
|
|
||||||
|
assert_ne!(
|
||||||
|
sandbox.workdir(),
|
||||||
|
checkout,
|
||||||
|
"the judge must not be pointed at the mission's own checkout"
|
||||||
|
);
|
||||||
|
assert!(sandbox.workdir().join("src/lib.rs").is_file(), "the copy has the source");
|
||||||
|
assert!(
|
||||||
|
!sandbox.workdir().join("target").exists(),
|
||||||
|
"the copy must not carry build output: {}",
|
||||||
|
sandbox.workdir().display()
|
||||||
|
);
|
||||||
|
|
||||||
|
// And dropping it takes the copy with it, leaving the mission untouched.
|
||||||
|
let copy_root = sandbox.workdir().parent().unwrap().to_path_buf();
|
||||||
|
drop(sandbox);
|
||||||
|
assert!(!copy_root.exists(), "the copy outlived its sandbox");
|
||||||
|
assert!(checkout.join("src/lib.rs").is_file(), "the mission tree is intact");
|
||||||
|
assert!(checkout.join("target/debug/junk").is_file());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The test seam must not delete a directory it was handed. A destructive
|
||||||
|
/// constructor that looks like a plain one is how a test wipes a real tree.
|
||||||
|
#[test]
|
||||||
|
fn an_explicit_workdir_is_never_deleted() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
std::fs::write(tmp.path().join("keep.txt"), "x").unwrap();
|
||||||
|
drop(Sandbox::at("c", tmp.path()));
|
||||||
|
assert!(tmp.path().join("keep.txt").is_file());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn refuses_programs_off_the_list() {
|
fn refuses_programs_off_the_list() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
@@ -364,6 +364,15 @@ enum Uplink {
|
|||||||
Result { id: u64, ok: bool, output: String },
|
Result { id: u64, ok: bool, output: String },
|
||||||
#[serde(rename = "pty_out")]
|
#[serde(rename = "pty_out")]
|
||||||
PtyOut { sid: u64, data: String },
|
PtyOut { sid: u64, data: String },
|
||||||
|
/// A chunk of a microVM turn's stdout/stderr, as it happens.
|
||||||
|
///
|
||||||
|
/// Keyed by RUN id rather than a session id: a mission run is the thing a
|
||||||
|
/// browser subscribes to, and unlike a PTY there is no interactive session
|
||||||
|
/// to allocate. `at` is the byte offset AFTER this chunk, so the node can
|
||||||
|
/// resume a dropped tail without replaying — the same contract `fcagent`'s
|
||||||
|
/// `tail` op exposes.
|
||||||
|
#[serde(rename = "vm_out")]
|
||||||
|
VmOut { run_id: String, at: u64, data: String },
|
||||||
#[serde(rename = "pty_exit")]
|
#[serde(rename = "pty_exit")]
|
||||||
PtyExit { sid: u64 },
|
PtyExit { sid: u64 },
|
||||||
#[serde(rename = "webrtc_answer")]
|
#[serde(rename = "webrtc_answer")]
|
||||||
@@ -381,6 +390,11 @@ enum Uplink {
|
|||||||
NodeTools {
|
NodeTools {
|
||||||
tools: std::collections::HashMap<String, String>,
|
tools: std::collections::HashMap<String, String>,
|
||||||
},
|
},
|
||||||
|
/// What the node can HOST, as opposed to what it has installed — the
|
||||||
|
/// inputs to placement predicates. Free-form so a new predicate does not
|
||||||
|
/// need a migration; see `migrations/0065_microvm_placement.sql`.
|
||||||
|
#[serde(rename = "node_capabilities")]
|
||||||
|
NodeCapabilities { capabilities: serde_json::Value },
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
@@ -482,6 +496,44 @@ pub async fn run_channel(pool: PgPool, hub: Arc<NodeHub>, node_id: NodeId, socke
|
|||||||
let _ = s.send(ExecOutput { ok, output });
|
let _ = s.send(ExecOutput { ok, output });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// A chunk of a microVM turn's output, live.
|
||||||
|
//
|
||||||
|
// Appended to the run's checkpoint rather than only fanned
|
||||||
|
// out: `PtyOut` above is deliberately ephemeral because a
|
||||||
|
// terminal has no history worth keeping, but a mission's log
|
||||||
|
// is the record of what the agent did — the Output tab has
|
||||||
|
// to still show it an hour later. Live and durable are
|
||||||
|
// different requirements and this needs both.
|
||||||
|
//
|
||||||
|
// `jsonb ||` merges into whatever else the checkpoint holds
|
||||||
|
// (`records`, written by the turn itself), so the two writers
|
||||||
|
// do not clobber each other.
|
||||||
|
Ok(Uplink::VmOut { run_id, at, data }) => {
|
||||||
|
if let (Ok(rid), Ok(bytes)) =
|
||||||
|
(uuid::Uuid::parse_str(&run_id), B64.decode(&data))
|
||||||
|
{
|
||||||
|
let text = String::from_utf8_lossy(&bytes).to_string();
|
||||||
|
if let Err(e) = sqlx::query(
|
||||||
|
"UPDATE topology_runs
|
||||||
|
SET checkpoint = COALESCE(checkpoint, '{}'::jsonb)
|
||||||
|
|| jsonb_build_object(
|
||||||
|
'log',
|
||||||
|
COALESCE(checkpoint->>'log', '') || $2::text,
|
||||||
|
'log_at', $3::bigint
|
||||||
|
),
|
||||||
|
updated_at = now()
|
||||||
|
WHERE id = $1",
|
||||||
|
)
|
||||||
|
.bind(rid)
|
||||||
|
.bind(&text)
|
||||||
|
.bind(at as i64)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
eprintln!("fleet: appending vm_out for run {rid}: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Ok(Uplink::PtyOut { sid, data }) => {
|
Ok(Uplink::PtyOut { sid, data }) => {
|
||||||
if let Ok(bytes) = B64.decode(&data) {
|
if let Ok(bytes) = B64.decode(&data) {
|
||||||
let sink = conn.pty_sinks.lock().await.get(&sid).cloned();
|
let sink = conn.pty_sinks.lock().await.get(&sid).cloned();
|
||||||
@@ -539,7 +591,32 @@ pub async fn run_channel(pool: PgPool, hub: Arc<NodeHub>, node_id: NodeId, socke
|
|||||||
let pairs: Vec<(String, String)> = tools.into_iter().collect();
|
let pairs: Vec<(String, String)> = tools.into_iter().collect();
|
||||||
let _ = cm_db::repo::node_tools::upsert(&pool, node_id, &pairs).await;
|
let _ = cm_db::repo::node_tools::upsert(&pool, node_id, &pairs).await;
|
||||||
}
|
}
|
||||||
Err(_) => {}
|
Ok(Uplink::NodeCapabilities { capabilities }) => {
|
||||||
|
if let Err(e) = nodes::set_capabilities(&pool, node_id, &capabilities).await
|
||||||
|
{
|
||||||
|
// Loud: a node whose capabilities never land looks
|
||||||
|
// exactly like a node that has none, and will be
|
||||||
|
// passed over for every microVM mission forever
|
||||||
|
// while appearing perfectly healthy.
|
||||||
|
eprintln!(
|
||||||
|
"fleet: could not record capabilities for node {node_id} ({e}) — \
|
||||||
|
it will not be selected for microvm placement"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// An unparseable frame used to vanish here. That is the
|
||||||
|
// worst possible handling: a node op whose reply does not
|
||||||
|
// match `Uplink` never resolves its pending request, so the
|
||||||
|
// caller times out after 20s with nothing anywhere saying
|
||||||
|
// why. Caught exactly that way while wiring the vm_* ops —
|
||||||
|
// `output` was an object where the wire declares a String.
|
||||||
|
Err(e) => {
|
||||||
|
let head: String = t.as_str().chars().take(160).collect();
|
||||||
|
eprintln!(
|
||||||
|
"fleet: node {node_id} sent a frame we could not parse ({e}); \
|
||||||
|
any request it was answering will time out. Frame: {head}"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
+147
-44
@@ -13,16 +13,28 @@
|
|||||||
//! reviewer picked. Rejected proposals move to status='rejected';
|
//! reviewer picked. Rejected proposals move to status='rejected';
|
||||||
//! partial approvals move to status='partial'.
|
//! partial approvals move to status='partial'.
|
||||||
//!
|
//!
|
||||||
//! Uses Gemini 2.5 Flash as the default proposer model — cheap,
|
//! The proposer model resolves through the provider REGISTRY
|
||||||
//! JSON-mode-native, plenty of room for structured output. Configurable
|
//! (`Runtime::resolve_provider`), the same path the evaluator uses, and defaults
|
||||||
//! via CLAWMATES_LEVEL_UP_MODEL.
|
//! to `glm:glm-4.7`. Configurable via `CLAWMATES_LEVEL_UP_MODEL` as a registry
|
||||||
|
//! spec (`glm:glm-4.7`, `kimi:k2`, `claude-sonnet-5`, …).
|
||||||
|
//!
|
||||||
|
//! It used to call Gemini directly over bespoke HTTP with `GEMINI_API_KEY`. Two
|
||||||
|
//! problems with that, one fatal: it was the only thing standing between this
|
||||||
|
//! feature and a dead prepayment balance, and it duplicated a provider client
|
||||||
|
//! the codebase already has. Going through the registry means every provider the
|
||||||
|
//! platform can already reach works here, and no single vendor's billing can
|
||||||
|
//! take the feature down.
|
||||||
|
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
use sqlx::PgPool;
|
use sqlx::PgPool;
|
||||||
use sqlx::Row;
|
use sqlx::Row;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
const DEFAULT_MODEL: &str = "gemini-2.5-flash";
|
/// Registry spec, not a bare model name — the registry needs the provider.
|
||||||
|
///
|
||||||
|
/// GLM: cheap, reliable at structured output, and already the validator this
|
||||||
|
/// project measured and chose (see `scripts/judge-eval.sh`).
|
||||||
|
const DEFAULT_MODEL: &str = "glm:glm-4.7";
|
||||||
|
|
||||||
fn model_name() -> String {
|
fn model_name() -> String {
|
||||||
std::env::var("CLAWMATES_LEVEL_UP_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
std::env::var("CLAWMATES_LEVEL_UP_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
||||||
@@ -31,6 +43,7 @@ fn model_name() -> String {
|
|||||||
/// Analyze an agent + insert a pending proposal. Returns the proposal id.
|
/// Analyze an agent + insert a pending proposal. Returns the proposal id.
|
||||||
pub async fn propose_agent(
|
pub async fn propose_agent(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
workspace_id: cm_domain::WorkspaceId,
|
workspace_id: cm_domain::WorkspaceId,
|
||||||
created_by: cm_domain::UserId,
|
created_by: cm_domain::UserId,
|
||||||
agent_id: Uuid,
|
agent_id: Uuid,
|
||||||
@@ -50,6 +63,7 @@ pub async fn propose_agent(
|
|||||||
.flatten();
|
.flatten();
|
||||||
|
|
||||||
let payload = call_llm_for_agent(
|
let payload = call_llm_for_agent(
|
||||||
|
runtime,
|
||||||
&agent.name,
|
&agent.name,
|
||||||
&agent.job_title,
|
&agent.job_title,
|
||||||
&agent.system_prompt,
|
&agent.system_prompt,
|
||||||
@@ -79,6 +93,7 @@ pub async fn propose_agent(
|
|||||||
/// Analyze a team + insert a pending proposal. Returns the proposal id.
|
/// Analyze a team + insert a pending proposal. Returns the proposal id.
|
||||||
pub async fn propose_team(
|
pub async fn propose_team(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
workspace_id: cm_domain::WorkspaceId,
|
workspace_id: cm_domain::WorkspaceId,
|
||||||
created_by: cm_domain::UserId,
|
created_by: cm_domain::UserId,
|
||||||
team_id: Uuid,
|
team_id: Uuid,
|
||||||
@@ -115,7 +130,7 @@ pub async fn propose_team(
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
let payload = call_llm_for_team(&member_summaries).await?;
|
let payload = call_llm_for_team(runtime, &member_summaries).await?;
|
||||||
|
|
||||||
let model = model_name();
|
let model = model_name();
|
||||||
let id = cm_db::repo::level_up::insert(
|
let id = cm_db::repo::level_up::insert(
|
||||||
@@ -418,6 +433,7 @@ async fn recent_run_summary(pool: &PgPool, agent_id: Uuid, limit: i64) -> Result
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn call_llm_for_agent(
|
async fn call_llm_for_agent(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
name: &str,
|
name: &str,
|
||||||
role: &str,
|
role: &str,
|
||||||
system_prompt: &str,
|
system_prompt: &str,
|
||||||
@@ -462,10 +478,13 @@ the sake of proposing."#;
|
|||||||
})
|
})
|
||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
call_gemini_json(system, &user).await
|
call_llm_json(runtime, system, &user).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn call_llm_for_team(members: &[Value]) -> Result<Value, String> {
|
async fn call_llm_for_team(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
|
members: &[Value],
|
||||||
|
) -> Result<Value, String> {
|
||||||
let system = r#"You review an AI team's roster + recent history and propose
|
let system = r#"You review an AI team's roster + recent history and propose
|
||||||
targeted improvements. Return ONLY JSON:
|
targeted improvements. Return ONLY JSON:
|
||||||
{
|
{
|
||||||
@@ -482,47 +501,90 @@ prompts over adding skills. Only add skills when a clear
|
|||||||
"the team keeps getting stuck on <X>" pattern appears."#;
|
"the team keeps getting stuck on <X>" pattern appears."#;
|
||||||
|
|
||||||
let user = json!({ "members": members }).to_string();
|
let user = json!({ "members": members }).to_string();
|
||||||
call_gemini_json(system, &user).await
|
call_llm_json(runtime, system, &user).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn call_gemini_json(system: &str, user: &str) -> Result<Value, String> {
|
/// Ask the configured proposer model for one JSON object.
|
||||||
let api_key =
|
///
|
||||||
std::env::var("GEMINI_API_KEY").map_err(|_| "GEMINI_API_KEY unset".to_string())?;
|
/// Goes through the provider registry rather than a vendor's HTTP API, so any
|
||||||
let model = model_name();
|
/// model the platform can already reach works and no single vendor's billing can
|
||||||
let url = format!(
|
/// take level-up down.
|
||||||
"https://generativelanguage.googleapis.com/v1beta/models/{}:generateContent?key={}",
|
///
|
||||||
model, api_key
|
/// The JSON is extracted rather than assumed: an anthropic-format model is not
|
||||||
);
|
/// bound by Gemini's `response_mime_type: application/json`, and will happily
|
||||||
let body = json!({
|
/// wrap an object in prose or a ```json fence. Parsing the raw reply worked
|
||||||
"system_instruction": { "parts": [{ "text": system }] },
|
/// against Gemini and would fail on everything else.
|
||||||
"contents": [{ "role": "user", "parts": [{ "text": user }] }],
|
async fn call_llm_json(
|
||||||
"generationConfig": {
|
runtime: &cm_runtime::Runtime,
|
||||||
"temperature": 0.2,
|
system: &str,
|
||||||
"response_mime_type": "application/json",
|
user: &str,
|
||||||
"maxOutputTokens": 8192,
|
) -> Result<Value, String> {
|
||||||
}
|
use cm_llm::{ChatMessage, ChatRequest, ChatRole, ContentPart, LlmEvent};
|
||||||
});
|
use futures::StreamExt as _;
|
||||||
let client = reqwest::Client::builder()
|
|
||||||
.timeout(std::time::Duration::from_secs(60))
|
let spec = model_name();
|
||||||
.build()
|
let (provider, model) = runtime.resolve_provider(&spec);
|
||||||
.map_err(|e| format!("http client: {e}"))?;
|
let request = ChatRequest {
|
||||||
let resp = client
|
system: system.to_string(),
|
||||||
.post(&url)
|
model: model.to_string(),
|
||||||
.json(&body)
|
messages: vec![ChatMessage {
|
||||||
.send()
|
role: ChatRole::User,
|
||||||
|
parts: vec![ContentPart::text(user)],
|
||||||
|
}],
|
||||||
|
tools: vec![],
|
||||||
|
max_tokens: 8192,
|
||||||
|
web_search: false,
|
||||||
|
};
|
||||||
|
let mut stream = provider
|
||||||
|
.stream(request)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("gemini call: {e}"))?;
|
.map_err(|e| format!("level-up call ({spec}): {e}"))?;
|
||||||
if !resp.status().is_success() {
|
let mut text = String::new();
|
||||||
let code = resp.status();
|
while let Some(event) = stream.next().await {
|
||||||
let body = resp.text().await.unwrap_or_default();
|
match event {
|
||||||
return Err(format!("gemini {code}: {}", &body[..body.len().min(500)]));
|
Ok(LlmEvent::TextDelta(t)) => text.push_str(&t),
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(e) => return Err(format!("level-up stream ({spec}): {e}")),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
let json: Value = resp.json().await.map_err(|e| format!("gemini json: {e}"))?;
|
let body = extract_json_object(&text)
|
||||||
let text = json
|
.ok_or_else(|| format!("no JSON object in {spec} reply: {}", excerpt(&text, 300)))?;
|
||||||
.pointer("/candidates/0/content/parts/0/text")
|
serde_json::from_str(body).map_err(|e| format!("parse suggestion json: {e}"))
|
||||||
.and_then(|v| v.as_str())
|
}
|
||||||
.ok_or_else(|| "gemini response missing text".to_string())?;
|
|
||||||
serde_json::from_str(text).map_err(|e| format!("parse suggestion json: {e}"))
|
/// The outermost `{...}` in a reply, so a fenced or prose-wrapped object parses.
|
||||||
|
///
|
||||||
|
/// Brace-counting rather than a regex: a nested object would end a lazy match at
|
||||||
|
/// the first inner `}`, and these proposals are nested by design (items carry
|
||||||
|
/// per-role objects).
|
||||||
|
fn extract_json_object(text: &str) -> Option<&str> {
|
||||||
|
let start = text.find('{')?;
|
||||||
|
let mut depth = 0usize;
|
||||||
|
let mut in_string = false;
|
||||||
|
let mut escaped = false;
|
||||||
|
for (i, c) in text[start..].char_indices() {
|
||||||
|
if in_string {
|
||||||
|
match c {
|
||||||
|
_ if escaped => escaped = false,
|
||||||
|
'\\' => escaped = true,
|
||||||
|
'"' => in_string = false,
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
match c {
|
||||||
|
'"' => in_string = true,
|
||||||
|
'{' => depth += 1,
|
||||||
|
'}' => {
|
||||||
|
depth -= 1;
|
||||||
|
if depth == 0 {
|
||||||
|
return Some(&text[start..start + i + 1]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
fn excerpt(s: &str, max: usize) -> String {
|
fn excerpt(s: &str, max: usize) -> String {
|
||||||
@@ -546,3 +608,44 @@ fn workspace_skill_id(workspace_id: Uuid, name: &str) -> Uuid {
|
|||||||
bytes[8] = (bytes[8] & 0x3f) | 0x80;
|
bytes[8] = (bytes[8] & 0x3f) | 0x80;
|
||||||
Uuid::from_bytes(bytes)
|
Uuid::from_bytes(bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
/// Gemini was asked for `response_mime_type: application/json` and obliged.
|
||||||
|
/// Anthropic-format models are under no such obligation and routinely wrap
|
||||||
|
/// the object in prose or a fenced block, so the reply is EXTRACTED, not
|
||||||
|
/// assumed. Parsing the raw text worked against Gemini and would fail
|
||||||
|
/// everywhere else — exactly the shape of bug a provider swap hides until
|
||||||
|
/// the first real proposal.
|
||||||
|
#[test]
|
||||||
|
fn a_json_object_is_extracted_from_however_the_model_wrapped_it() {
|
||||||
|
let bare = r#"{"items":[]}"#;
|
||||||
|
assert_eq!(super::extract_json_object(bare), Some(bare));
|
||||||
|
|
||||||
|
let fenced = "Here is my proposal:\n```json\n{\"items\":[1]}\n```\nDone.";
|
||||||
|
assert_eq!(super::extract_json_object(fenced), Some(r#"{"items":[1]}"#));
|
||||||
|
|
||||||
|
// Nested objects: a lazy match would stop at the first inner brace and
|
||||||
|
// hand back invalid JSON. These proposals are nested by design.
|
||||||
|
let nested = r#"prose {"a":{"b":{"c":1}},"d":2} trailing"#;
|
||||||
|
assert_eq!(
|
||||||
|
super::extract_json_object(nested),
|
||||||
|
Some(r#"{"a":{"b":{"c":1}},"d":2}"#)
|
||||||
|
);
|
||||||
|
|
||||||
|
// A brace inside a string must not close the object.
|
||||||
|
let stringy = r#"{"note":"an unmatched } here","ok":true}"#;
|
||||||
|
assert_eq!(super::extract_json_object(stringy), Some(stringy));
|
||||||
|
|
||||||
|
assert_eq!(super::extract_json_object("no object here"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The default must not be a vendor whose billing already took a feature
|
||||||
|
/// down. It is a REGISTRY SPEC (`provider:model`), not a bare model name —
|
||||||
|
/// `resolve_provider` needs the provider half.
|
||||||
|
#[test]
|
||||||
|
fn the_default_proposer_is_a_registry_spec_and_not_gemini() {
|
||||||
|
assert!(super::DEFAULT_MODEL.contains(':'), "{}", super::DEFAULT_MODEL);
|
||||||
|
assert!(!super::DEFAULT_MODEL.contains("gemini"), "{}", super::DEFAULT_MODEL);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ pub mod benchmark_runner;
|
|||||||
pub mod beszel;
|
pub mod beszel;
|
||||||
pub mod brain_seed;
|
pub mod brain_seed;
|
||||||
pub mod cleanup_sweeper;
|
pub mod cleanup_sweeper;
|
||||||
|
pub mod agent_names;
|
||||||
|
pub mod mission_gc;
|
||||||
pub mod container_exec;
|
pub mod container_exec;
|
||||||
mod error;
|
mod error;
|
||||||
pub mod evaluator;
|
pub mod evaluator;
|
||||||
@@ -21,15 +23,29 @@ pub mod corpus;
|
|||||||
pub mod harvest;
|
pub mod harvest;
|
||||||
pub mod library;
|
pub mod library;
|
||||||
pub mod mission_delivery;
|
pub mod mission_delivery;
|
||||||
|
pub mod mission_events;
|
||||||
|
pub mod microvm_client;
|
||||||
|
pub mod microvm_executor;
|
||||||
|
pub mod microvm_turn_executor;
|
||||||
|
pub mod subscription;
|
||||||
|
pub mod vm_placement;
|
||||||
|
pub mod vm_stop_gate;
|
||||||
|
pub mod vm_tool_tap;
|
||||||
|
pub mod mission_fs;
|
||||||
|
pub mod mission_outputs;
|
||||||
pub mod papers;
|
pub mod papers;
|
||||||
pub mod phase_config;
|
pub mod phase_config;
|
||||||
pub mod session_executor;
|
pub mod session_executor;
|
||||||
|
pub mod repo_digest;
|
||||||
pub mod runtime_preflight;
|
pub mod runtime_preflight;
|
||||||
|
pub mod validator_preflight;
|
||||||
|
pub mod mission_plan;
|
||||||
|
pub mod mission_roster;
|
||||||
pub mod mission_runtime;
|
pub mod mission_runtime;
|
||||||
pub mod mission_workspace;
|
pub mod mission_workspace;
|
||||||
pub mod node_rules;
|
pub mod node_rules;
|
||||||
pub mod pdf_renderer;
|
|
||||||
pub mod phase_runner;
|
pub mod phase_runner;
|
||||||
|
pub mod root_copy;
|
||||||
pub mod phase_summarizer;
|
pub mod phase_summarizer;
|
||||||
pub mod quota;
|
pub mod quota;
|
||||||
mod recursive_exec;
|
mod recursive_exec;
|
||||||
@@ -159,6 +175,8 @@ pub fn router(state: AppState) -> Router {
|
|||||||
.route("/api/world/live", get(routes::world::world_live))
|
.route("/api/world/live", get(routes::world::world_live))
|
||||||
.route("/api/world/replay", get(routes::world::world_replay))
|
.route("/api/world/replay", get(routes::world::world_replay))
|
||||||
.route("/api/nodes", get(routes::nodes::list))
|
.route("/api/nodes", get(routes::nodes::list))
|
||||||
|
.route("/api/fleet/capacity", get(routes::nodes::capacity))
|
||||||
|
.route("/api/fleet/backends", get(routes::nodes::backends))
|
||||||
.route("/api/nodes/pair", post(routes::nodes::pair))
|
.route("/api/nodes/pair", post(routes::nodes::pair))
|
||||||
.route("/api/nodes/live", get(routes::nodes::live))
|
.route("/api/nodes/live", get(routes::nodes::live))
|
||||||
.route("/api/nodes/agent", get(routes::nodes::agent_ws))
|
.route("/api/nodes/agent", get(routes::nodes::agent_ws))
|
||||||
@@ -467,6 +485,8 @@ pub fn router(state: AppState) -> Router {
|
|||||||
"/api/missions",
|
"/api/missions",
|
||||||
get(routes::missions::list).post(routes::missions::create),
|
get(routes::missions::list).post(routes::missions::create),
|
||||||
)
|
)
|
||||||
|
// The roster grouped by mission — what "My Workforce" renders.
|
||||||
|
.route("/api/workforce", get(routes::missions::workforce))
|
||||||
// The workflow recipe catalog (templates/workflows/*.toml). Serving it
|
// The workflow recipe catalog (templates/workflows/*.toml). Serving it
|
||||||
// lets the client stop mirroring the phase composition table inline.
|
// lets the client stop mirroring the phase composition table inline.
|
||||||
.route("/api/workflows", get(routes::missions::list_workflows))
|
.route("/api/workflows", get(routes::missions::list_workflows))
|
||||||
@@ -481,6 +501,43 @@ pub fn router(state: AppState) -> Router {
|
|||||||
axum::routing::patch(routes::missions::set_status),
|
axum::routing::patch(routes::missions::set_status),
|
||||||
)
|
)
|
||||||
.route("/api/missions/{id}/refine", post(routes::missions::refine))
|
.route("/api/missions/{id}/refine", post(routes::missions::refine))
|
||||||
|
// Draft-less sibling: the wizard polishes a description before any
|
||||||
|
// mission exists, so there is no id to route on. Declared BEFORE the
|
||||||
|
// `{id}` routes would otherwise be ambiguous — axum matches literal
|
||||||
|
// segments first, but keeping them adjacent makes the pair obvious.
|
||||||
|
.route(
|
||||||
|
"/api/missions/refine-draft",
|
||||||
|
post(routes::missions::refine_draft),
|
||||||
|
)
|
||||||
|
.route("/api/missions/{id}/merge", post(routes::missions::merge_branch))
|
||||||
|
.route(
|
||||||
|
"/api/missions/{id}/artifacts/{artifact_id}/content",
|
||||||
|
get(routes::missions::artifact_content),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/api/missions/{id}/artifacts/{artifact_id}/download",
|
||||||
|
get(routes::missions::artifact_download),
|
||||||
|
)
|
||||||
|
// Slice 5: let a model size the mission's team. Proposing, listing and
|
||||||
|
// deciding are separate verbs because only the last one spends money.
|
||||||
|
// W1/#13: let a model author the phases, on the same propose → review →
|
||||||
|
// approve shape as the roster above.
|
||||||
|
.route(
|
||||||
|
"/api/missions/{id}/plan-proposals",
|
||||||
|
get(routes::mission_plan::list).post(routes::mission_plan::suggest),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/api/missions/{id}/plan-proposals/{pid}/decide",
|
||||||
|
post(routes::mission_plan::decide),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/api/missions/{id}/team-proposals",
|
||||||
|
get(routes::mission_roster::list).post(routes::mission_roster::suggest),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/api/missions/{id}/team-proposals/{pid}/decide",
|
||||||
|
post(routes::mission_roster::decide),
|
||||||
|
)
|
||||||
.route(
|
.route(
|
||||||
"/api/missions/{id}/herdr-dispatch",
|
"/api/missions/{id}/herdr-dispatch",
|
||||||
post(routes::missions::herdr_dispatch),
|
post(routes::missions::herdr_dispatch),
|
||||||
|
|||||||
@@ -93,9 +93,13 @@ pub async fn clone_vault(clone_url: &str, work_root: &Path) -> Result<PathBuf, S
|
|||||||
.map_err(|e| format!("mkdir {}: {e}", work_root.display()))?;
|
.map_err(|e| format!("mkdir {}: {e}", work_root.display()))?;
|
||||||
|
|
||||||
let auth = mission_workspace::with_ambient_auth(clone_url);
|
let auth = mission_workspace::with_ambient_auth(clone_url);
|
||||||
let out = tokio::process::Command::new("git")
|
if let Some(why) = &auth.unauthenticated {
|
||||||
.args(["clone", "--quiet", "--depth", "1", &auth])
|
eprintln!("library: cloning the vault WITHOUT credentials — {why}");
|
||||||
.arg(&path)
|
}
|
||||||
|
let mut cmd = tokio::process::Command::new("git");
|
||||||
|
cmd.args(["clone", "--quiet", "--depth", "1", &auth.url])
|
||||||
|
.arg(&path);
|
||||||
|
let out = mission_workspace::no_terminal_prompt(&mut cmd)
|
||||||
.output()
|
.output()
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("spawn git clone: {e}"))?;
|
.map_err(|e| format!("spawn git clone: {e}"))?;
|
||||||
@@ -103,16 +107,15 @@ pub async fn clone_vault(clone_url: &str, work_root: &Path) -> Result<PathBuf, S
|
|||||||
return Err(format!(
|
return Err(format!(
|
||||||
"clone vault → {}: {}",
|
"clone vault → {}: {}",
|
||||||
out.status,
|
out.status,
|
||||||
mission_workspace::redact_token(&String::from_utf8_lossy(&out.stderr))
|
crate::evaluator_tools::clamp_output(&mission_workspace::redact_token(
|
||||||
.chars()
|
&String::from_utf8_lossy(&out.stderr)
|
||||||
.take(300)
|
))
|
||||||
.collect::<String>()
|
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
// The token must not stay in .git/config: the checkout may be handed to a
|
// The token must not stay in .git/config: the checkout may be handed to a
|
||||||
// container later, and a credential in a file an agent can read is a
|
// container later, and a credential in a file an agent can read is a
|
||||||
// credential an agent has.
|
// credential an agent has.
|
||||||
mission_workspace::scrub_remote_credentials(&path, &auth);
|
mission_workspace::scrub_remote_credentials(&path, &auth.url);
|
||||||
Ok(path)
|
Ok(path)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -186,6 +189,15 @@ pub async fn run_to_vault(
|
|||||||
git(&vault, &["commit", "--no-verify", "-m", &message]).await?;
|
git(&vault, &["commit", "--no-verify", "-m", &message]).await?;
|
||||||
|
|
||||||
let auth = mission_workspace::with_ambient_auth(clone_url);
|
let auth = mission_workspace::with_ambient_auth(clone_url);
|
||||||
|
if let Some(why) = &auth.unauthenticated {
|
||||||
|
if auth.is_forge() {
|
||||||
|
// Not fatal here — the push below reports its own failure — but the
|
||||||
|
// reason belongs in the log next to the attempt, not inferred from a
|
||||||
|
// tty error two layers down.
|
||||||
|
eprintln!("library: pushing to the forge WITHOUT credentials — {why}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let auth = auth.url;
|
||||||
let refspec = format!("HEAD:refs/heads/{branch}");
|
let refspec = format!("HEAD:refs/heads/{branch}");
|
||||||
match git(&vault, &["push", &auth, &refspec]).await {
|
match git(&vault, &["push", &auth, &refspec]).await {
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
|
|||||||
@@ -166,6 +166,18 @@ async fn policy_decide(
|
|||||||
} else {
|
} else {
|
||||||
state.runtime.judge(system, &request).await
|
state.runtime.judge(system, &request).await
|
||||||
};
|
};
|
||||||
|
// Fail-open is deliberate, but a governor that is failing open on EVERY
|
||||||
|
// request is a security control that has quietly stopped existing —
|
||||||
|
// and the caller drops `reason` whenever it allows, so nothing said so.
|
||||||
|
// `judge()` returns this exact prefix when the provider never answered,
|
||||||
|
// which a rate-limited or uncredited judge model does on every call.
|
||||||
|
if allow && reason.starts_with("governor unreachable") {
|
||||||
|
eprintln!(
|
||||||
|
"mcp_door: WARNING — the door governor is FAILING OPEN for {mcp_tool} \
|
||||||
|
({reason}). Every outbound action is being approved unjudged. Point \
|
||||||
|
CLAWMATES_JUDGE_MODEL at a reachable model."
|
||||||
|
);
|
||||||
|
}
|
||||||
if !allow {
|
if !allow {
|
||||||
return PolicyOutcome::Deny(format!("governor agent vetoed — {reason}"));
|
return PolicyOutcome::Deny(format!("governor agent vetoed — {reason}"));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,354 @@
|
|||||||
|
//! Drive a fleet node's microVMs from the server.
|
||||||
|
//!
|
||||||
|
//! Thin by design: the node owns the VM lifecycle (see
|
||||||
|
//! `clawmates-node::microvm`), and this is the typed way to ask it. Every call
|
||||||
|
//! is one `vm_*` op over the existing `NodeHub` request/response channel, so
|
||||||
|
//! there is no new transport, correlation or timeout machinery.
|
||||||
|
//!
|
||||||
|
//! # Not a `SandboxDriver`
|
||||||
|
//!
|
||||||
|
//! `RemoteDriver` exists to marshal `SandboxDriver` over the hub, and reusing it
|
||||||
|
//! was the plan. That trait is container-shaped — `attach_pty`, `resize_pty`,
|
||||||
|
//! argv `exec` — while a mission needs inject → run → collect. Conforming would
|
||||||
|
//! mean implementing PTY-over-vsock semantics that nothing calls, so this speaks
|
||||||
|
//! the smaller interface the mission path actually uses.
|
||||||
|
//!
|
||||||
|
//! # Timeouts
|
||||||
|
//!
|
||||||
|
//! The hub defaults to 20s, which is right for a create (measured: ~1s) and
|
||||||
|
//! badly wrong for an agent turn. `exec` therefore takes its own budget and
|
||||||
|
//! passes it to BOTH the hub and the guest, with the hub's slightly longer: if
|
||||||
|
//! the guest's own timeout fires first the reply says so, whereas a hub timeout
|
||||||
|
//! leaves us guessing whether the command is still running.
|
||||||
|
|
||||||
|
use cm_domain::NodeId;
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
|
||||||
|
use crate::fleet::NodeHub;
|
||||||
|
|
||||||
|
/// Slack between the guest's deadline and the hub's, so the guest's own timeout
|
||||||
|
/// wins the race and we get a real answer rather than a transport error.
|
||||||
|
const HUB_GRACE_SECS: u64 = 30;
|
||||||
|
|
||||||
|
/// How long the hub waits for a command whose own budget is `guest_secs`.
|
||||||
|
///
|
||||||
|
/// Saturating, not `+`: a caller passing a very large budget would otherwise
|
||||||
|
/// overflow and panic in debug or wrap to a tiny timeout in release — the second
|
||||||
|
/// being far worse, since it turns a long-running agent turn into a spurious
|
||||||
|
/// transport failure.
|
||||||
|
fn hub_deadline(guest_secs: u64) -> u64 {
|
||||||
|
guest_secs.saturating_add(HUB_GRACE_SECS)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct MicroVm<'a> {
|
||||||
|
hub: &'a NodeHub,
|
||||||
|
node_id: NodeId,
|
||||||
|
vm_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> MicroVm<'a> {
|
||||||
|
pub fn new(hub: &'a NodeHub, node_id: NodeId, vm_id: impl Into<String>) -> Self {
|
||||||
|
Self {
|
||||||
|
hub,
|
||||||
|
node_id,
|
||||||
|
vm_id: vm_id.into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn vm_id(&self) -> &str {
|
||||||
|
&self.vm_id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One op, with the node's `output` string parsed back into JSON.
|
||||||
|
///
|
||||||
|
/// `output` is a String on the wire (`Uplink::Result`), and a node that
|
||||||
|
/// answered with a JSON object instead made the whole frame unparseable —
|
||||||
|
/// the reply then vanished into the uplink's error arm and the call timed
|
||||||
|
/// out with nothing explaining why. Parsing here, loudly, keeps that
|
||||||
|
/// mismatch a visible error rather than a mystery timeout.
|
||||||
|
async fn call(&self, op: &str, mut args: Value, secs: u64) -> Result<Value, String> {
|
||||||
|
if let Some(o) = args.as_object_mut() {
|
||||||
|
o.insert("vm_id".into(), Value::String(self.vm_id.clone()));
|
||||||
|
}
|
||||||
|
let out = self
|
||||||
|
.hub
|
||||||
|
.call_timeout(self.node_id, op, args, secs)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("{op} on node {:?}: {e}", self.node_id))?;
|
||||||
|
let body: Value = serde_json::from_str(&out.output)
|
||||||
|
.map_err(|e| format!("{op} returned unparseable output ({e}): {}", out.output))?;
|
||||||
|
if !out.ok {
|
||||||
|
let why = body
|
||||||
|
.get("error")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or(&out.output);
|
||||||
|
return Err(format!("{op} failed: {why}"));
|
||||||
|
}
|
||||||
|
Ok(body)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Boot the VM. Returns only once its guest agent has answered.
|
||||||
|
///
|
||||||
|
/// `backend` selects the rootfs image (`missions.backend`); `None` boots the
|
||||||
|
/// node's default. A backend whose image is not built on that node is an
|
||||||
|
/// error naming the file — never a quiet fall back to the default, which
|
||||||
|
/// would run a claude mission in a kimi VM and report success.
|
||||||
|
pub async fn create(
|
||||||
|
&self,
|
||||||
|
vcpus: u32,
|
||||||
|
mem_mib: u32,
|
||||||
|
backend: Option<&str>,
|
||||||
|
) -> Result<Value, String> {
|
||||||
|
// 60s, not the hub default: a create that has to copy a rootfs and boot
|
||||||
|
// is measured near 1s, but a node under load has no reason to be fast.
|
||||||
|
self.call(
|
||||||
|
"vm_create",
|
||||||
|
json!({ "vcpus": vcpus, "mem_mib": mem_mib, "backend": backend }),
|
||||||
|
60,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Unpack a tar inside the guest at `dest`.
|
||||||
|
///
|
||||||
|
/// Takes the archive bytes rather than a path: the server holds the mission
|
||||||
|
/// checkout, the node does not, and shipping the tar is the whole point of
|
||||||
|
/// the inject → run → collect model.
|
||||||
|
pub async fn inject(&self, dest: &str, tar: &[u8]) -> Result<Value, String> {
|
||||||
|
use base64::Engine as _;
|
||||||
|
let b64 = base64::engine::general_purpose::STANDARD.encode(tar);
|
||||||
|
self.call("vm_inject", json!({ "dest": dest, "tar_b64": b64 }), 120)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run a shell command in the guest.
|
||||||
|
///
|
||||||
|
/// `Ok` means the command RAN; the exit code is in the payload. A non-zero
|
||||||
|
/// exit is not an error here — the caller has to be able to tell "the build
|
||||||
|
/// failed" from "we could not reach the VM", and collapsing them is the
|
||||||
|
/// defect this codebase keeps paying for.
|
||||||
|
/// `env` carries the provider credentials (see
|
||||||
|
/// [`crate::mission_runtime::forwarded_provider_env`]). It is sent, never
|
||||||
|
/// logged: this is the only channel by which a secret reaches the guest, and
|
||||||
|
/// the guest refuses the exec rather than running a command without an entry
|
||||||
|
/// it could not honour.
|
||||||
|
pub async fn exec(
|
||||||
|
&self,
|
||||||
|
cmd: &str,
|
||||||
|
cwd: Option<&str>,
|
||||||
|
timeout_secs: u64,
|
||||||
|
env: &[(String, String)],
|
||||||
|
) -> Result<ExecOut, String> {
|
||||||
|
self.exec_attributed(cmd, cwd, timeout_secs, env, None, None)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The same exec, tagged with the run whose live output this is.
|
||||||
|
///
|
||||||
|
/// When `run_id` is set the node follows `log_path` inside the guest for the
|
||||||
|
/// life of the command and streams what it reads to the server. Probes pass
|
||||||
|
/// `None`: they produce nothing worth streaming and have no subscriber.
|
||||||
|
pub async fn exec_attributed(
|
||||||
|
&self,
|
||||||
|
cmd: &str,
|
||||||
|
cwd: Option<&str>,
|
||||||
|
timeout_secs: u64,
|
||||||
|
env: &[(String, String)],
|
||||||
|
run_id: Option<uuid::Uuid>,
|
||||||
|
log_path: Option<&str>,
|
||||||
|
) -> Result<ExecOut, String> {
|
||||||
|
let env: Option<Value> = (!env.is_empty()).then(|| {
|
||||||
|
env.iter()
|
||||||
|
.map(|(k, v)| (k.clone(), Value::String(v.clone())))
|
||||||
|
.collect::<serde_json::Map<_, _>>()
|
||||||
|
.into()
|
||||||
|
});
|
||||||
|
let v = self
|
||||||
|
.call(
|
||||||
|
"vm_exec",
|
||||||
|
json!({
|
||||||
|
"cmd": cmd, "cwd": cwd, "timeout": timeout_secs, "env": env,
|
||||||
|
"run_id": run_id.map(|r| r.to_string()), "log_path": log_path,
|
||||||
|
}),
|
||||||
|
hub_deadline(timeout_secs),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
// A guest that refused to run the command reports `ok: false` and no rc
|
||||||
|
// — a rejected env entry, for instance. Surface its reason: falling
|
||||||
|
// through to the missing-rc error below would hide the cause behind a
|
||||||
|
// symptom.
|
||||||
|
if v.get("ok").and_then(Value::as_bool) == Some(false) {
|
||||||
|
return Err(format!(
|
||||||
|
"vm_exec did not run: {}",
|
||||||
|
v.get("error").and_then(Value::as_str).unwrap_or("unknown")
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// A missing rc is not "success" — it means the guest did not report one,
|
||||||
|
// which we must not read as zero.
|
||||||
|
let rc = v
|
||||||
|
.get("rc")
|
||||||
|
.and_then(Value::as_i64)
|
||||||
|
.ok_or_else(|| format!("vm_exec gave no exit code: {v}"))?;
|
||||||
|
Ok(ExecOut {
|
||||||
|
rc,
|
||||||
|
stdout: v
|
||||||
|
.get("stdout")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.to_string(),
|
||||||
|
stderr: v
|
||||||
|
.get("stderr")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.to_string(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tar a path out of the guest and return the archive bytes.
|
||||||
|
/// `exclude` names directories to leave out — build output, caches. Sent from
|
||||||
|
/// here so the policy lives in one place: `mission_fs::transport_excludes`,
|
||||||
|
/// the same list the delivery diff uses. Shipping `target/` blew this call's
|
||||||
|
/// 300s budget twice, each time with the agent's work finished and stranded.
|
||||||
|
pub async fn collect(&self, path: &str, exclude: &[&str]) -> Result<Vec<u8>, String> {
|
||||||
|
use base64::Engine as _;
|
||||||
|
let v = self
|
||||||
|
.call("vm_collect", json!({ "path": path, "exclude": exclude }), 300)
|
||||||
|
.await?;
|
||||||
|
// The guest reports its own `ok`: a missing path is a real failure that
|
||||||
|
// must not come back as an empty archive, which would look exactly like
|
||||||
|
// a run that produced nothing.
|
||||||
|
if v.get("ok").and_then(Value::as_bool) != Some(true) {
|
||||||
|
return Err(format!(
|
||||||
|
"vm_collect {path}: {}",
|
||||||
|
v.get("error").and_then(Value::as_str).unwrap_or("unknown")
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let b64 = v
|
||||||
|
.get("tar_b64")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.ok_or_else(|| format!("vm_collect {path} returned no archive: {v}"))?;
|
||||||
|
base64::engine::general_purpose::STANDARD
|
||||||
|
.decode(b64)
|
||||||
|
.map_err(|e| format!("vm_collect {path}: undecodable archive: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stop the VM and remove everything it owned. Idempotent.
|
||||||
|
pub async fn destroy(&self) -> Result<Value, String> {
|
||||||
|
self.call("vm_destroy", json!({}), 60).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The result of a command that RAN. `rc != 0` is a normal outcome.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ExecOut {
|
||||||
|
pub rc: i64,
|
||||||
|
pub stdout: String,
|
||||||
|
pub stderr: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ExecOut {
|
||||||
|
pub fn ok(&self) -> bool {
|
||||||
|
self.rc == 0
|
||||||
|
}
|
||||||
|
/// One line for a log or an artifact, without dumping a whole build.
|
||||||
|
pub fn summary(&self) -> String {
|
||||||
|
let tail = |s: &str| {
|
||||||
|
s.lines()
|
||||||
|
.rev()
|
||||||
|
.take(3)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.into_iter()
|
||||||
|
.rev()
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" | ")
|
||||||
|
};
|
||||||
|
if self.ok() {
|
||||||
|
format!("rc=0 {}", tail(&self.stdout))
|
||||||
|
} else {
|
||||||
|
format!("rc={} {}", self.rc, tail(&self.stderr))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// VMs a node currently holds, so orphans can be reaped.
|
||||||
|
pub async fn list(hub: &NodeHub, node_id: NodeId) -> Result<Vec<String>, String> {
|
||||||
|
let out = hub
|
||||||
|
.call(node_id, "vm_list", json!({}))
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("vm_list on node {node_id:?}: {e}"))?;
|
||||||
|
let body: Value = serde_json::from_str(&out.output)
|
||||||
|
.map_err(|e| format!("vm_list returned unparseable output ({e}): {}", out.output))?;
|
||||||
|
Ok(body
|
||||||
|
.get("vms")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|a| {
|
||||||
|
a.iter()
|
||||||
|
.filter_map(|v| v.get("vm_id").and_then(Value::as_str))
|
||||||
|
.map(str::to_string)
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// A command that ran and failed must be distinguishable from one that
|
||||||
|
/// could not be reached. `rc` carries the verdict; `Err` is for transport.
|
||||||
|
#[test]
|
||||||
|
fn a_nonzero_exit_is_an_outcome_not_an_error() {
|
||||||
|
let failed = ExecOut {
|
||||||
|
rc: 3,
|
||||||
|
stdout: String::new(),
|
||||||
|
stderr: "boom\n".into(),
|
||||||
|
};
|
||||||
|
assert!(!failed.ok());
|
||||||
|
assert!(failed.summary().starts_with("rc=3"));
|
||||||
|
assert!(failed.summary().contains("boom"));
|
||||||
|
|
||||||
|
let passed = ExecOut {
|
||||||
|
rc: 0,
|
||||||
|
stdout: "fine\n".into(),
|
||||||
|
stderr: String::new(),
|
||||||
|
};
|
||||||
|
assert!(passed.ok());
|
||||||
|
assert_eq!(passed.summary(), "rc=0 fine");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The summary is for logs, so it must stay short even when a build prints
|
||||||
|
/// thousands of lines — and it must keep the LAST lines, where the error is.
|
||||||
|
#[test]
|
||||||
|
fn the_summary_keeps_the_tail_and_stays_short() {
|
||||||
|
let noisy = ExecOut {
|
||||||
|
rc: 1,
|
||||||
|
stdout: String::new(),
|
||||||
|
stderr: (1..=500)
|
||||||
|
.map(|i| format!("line {i}"))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n"),
|
||||||
|
};
|
||||||
|
let s = noisy.summary();
|
||||||
|
assert!(s.contains("line 500"), "the last line must survive: {s}");
|
||||||
|
assert!(!s.contains("line 400"), "older lines must be dropped: {s}");
|
||||||
|
assert!(s.len() < 200, "summary must stay log-sized, got {}", s.len());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The guest's deadline must fire before the hub's, so a slow command comes
|
||||||
|
/// back as a reported timeout rather than an unexplained transport failure.
|
||||||
|
#[test]
|
||||||
|
fn the_hub_always_outlives_the_guests_own_timeout() {
|
||||||
|
for guest in [0u64, 1, 30, 3600, 86_400] {
|
||||||
|
assert!(
|
||||||
|
hub_deadline(guest) > guest,
|
||||||
|
"hub deadline for {guest}s must exceed it"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// A caller passing a huge budget must not wrap to a tiny timeout, which
|
||||||
|
// would turn a long agent turn into a spurious transport failure.
|
||||||
|
assert!(
|
||||||
|
hub_deadline(u64::MAX) >= u64::MAX - 1,
|
||||||
|
"an extreme budget must saturate, not wrap"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,702 @@
|
|||||||
|
//! The two engines composed — Slice 4.
|
||||||
|
//!
|
||||||
|
//! Engine Z (the ZeroClaw graph in `cm_orchestrator`) owns durability and
|
||||||
|
//! heterogeneity: deterministic planners, per-step checkpoint/resume, a stale-run
|
||||||
|
//! sweep, cancellation, and a different model per node. Engine C (Claude Code in
|
||||||
|
//! a microVM) owns shared context, self-sizing and cheap fan-out. Neither has the
|
||||||
|
//! other's asset, which is why keeping both is a composition rather than a
|
||||||
|
//! compromise.
|
||||||
|
//!
|
||||||
|
//! This module is the join: a [`TurnExecutor`] whose "turn" is a whole
|
||||||
|
//! Claude-Code-in-a-VM session. Because `topology_worker` already dispatches by
|
||||||
|
//! tier, implementing the existing trait inherits the planners, checkpointing,
|
||||||
|
//! reaper, cancellation, `close_finished_phases`, evaluation, capture and
|
||||||
|
//! delivery unchanged. `recursive_exec::SubTopologyExecutor` is the precedent: a
|
||||||
|
//! `run_turn` may be arbitrarily heavy.
|
||||||
|
//!
|
||||||
|
//! # The file-handoff trap
|
||||||
|
//!
|
||||||
|
//! A VM is inject-tar → run → collect-tar → destroy. A graph of per-node VMs with
|
||||||
|
//! **text-only** handoff would silently lose every file an earlier node wrote:
|
||||||
|
//! node 2 would boot from the original checkout, see none of node 1's work, and
|
||||||
|
//! still report success — the exact silent-success shape this project keeps
|
||||||
|
//! paying for.
|
||||||
|
//!
|
||||||
|
//! The answer here is that the mission's **host checkout is the medium**. Every
|
||||||
|
//! node injects from `repo` and collects back over `repo`, so the tree carries
|
||||||
|
//! forward node to node and the last node's tree is what delivery diffs. Two
|
||||||
|
//! properties make that safe rather than lucky:
|
||||||
|
//!
|
||||||
|
//! - `execute_resumable` runs steps strictly **sequentially**, so two VMs are
|
||||||
|
//! never writing the same host directory at once;
|
||||||
|
//! - the vm id is deterministic per (phase, iteration, step), so a resumed step
|
||||||
|
//! whose VM is somehow still alive is refused by the node ("vm already exists")
|
||||||
|
//! instead of quietly producing a second writer.
|
||||||
|
//!
|
||||||
|
//! `a_later_node_sees_an_earlier_nodes_files` proves the handoff, and
|
||||||
|
//! `text_only_handoff_loses_the_earlier_nodes_work` is its negative control.
|
||||||
|
//!
|
||||||
|
//! # Keeping a long turn alive
|
||||||
|
//!
|
||||||
|
//! `requeue_stale` requeues a `running` job that has not touched `updated_at` in
|
||||||
|
//! 180 seconds, and one node here can run for an hour. `SubTopologyExecutor`
|
||||||
|
//! keeps its parent alive from each *leaf step*, which it has and this does not:
|
||||||
|
//! there is nothing between the start and end of a VM turn. So the turn holds a
|
||||||
|
//! ticker that touches `updated_at` every [`KEEPALIVE_SECS`] and is aborted on
|
||||||
|
//! drop. Without it a healthy composed run is requeued mid-node, claimed again,
|
||||||
|
//! and boots a second VM against the same checkout.
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use cm_domain::NodeId;
|
||||||
|
use cm_orchestrator::{OrchestratorError, TurnExecutor, TurnOutcome, TurnRequest};
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use crate::microvm_executor::{PhaseVm, VmPhase};
|
||||||
|
|
||||||
|
/// How often a running VM turn touches its run's `updated_at`.
|
||||||
|
///
|
||||||
|
/// Comfortably inside the 180s stale window, and cheap: one UPDATE per node per
|
||||||
|
/// half minute against a row nothing else is writing.
|
||||||
|
const KEEPALIVE_SECS: u64 = 30;
|
||||||
|
|
||||||
|
/// A [`TurnExecutor`] that runs each graph node as a full Claude-Code session
|
||||||
|
/// inside its own microVM, against the mission's shared host checkout.
|
||||||
|
pub struct MicroVmTurnExecutor<V: PhaseVm> {
|
||||||
|
vms: V,
|
||||||
|
pool: PgPool,
|
||||||
|
/// The durable outer run. Touched for keepalive; its status gates the turn.
|
||||||
|
run_id: Uuid,
|
||||||
|
mission_id: Uuid,
|
||||||
|
phase_id: Uuid,
|
||||||
|
iteration: i32,
|
||||||
|
/// The mission's host checkout — injected into every node's VM and collected
|
||||||
|
/// back over, which is how file work survives a node boundary.
|
||||||
|
repo: PathBuf,
|
||||||
|
/// Whether the mission has a repository. Carried so every graph node gets
|
||||||
|
/// the same workspace treatment as a solo phase — see `VmPhase::has_repo`.
|
||||||
|
has_repo: bool,
|
||||||
|
/// `missions.target_node_id`: the fleet node a mission was placed on. A node
|
||||||
|
/// may override it with `attrs["node_id"]`.
|
||||||
|
default_fleet_node: Option<Uuid>,
|
||||||
|
/// `missions.backend`: which rootfs image. A node may override it with
|
||||||
|
/// `attrs["backend"]`, which is what makes a graph heterogeneous — a
|
||||||
|
/// `validator` node on a different provider's image is then a first-class
|
||||||
|
/// graph node rather than a bolt-on.
|
||||||
|
default_backend: Option<String>,
|
||||||
|
/// `missions.team_engine`, passed through so a composed node can itself ask
|
||||||
|
/// for Claude Code fan-out inside its VM.
|
||||||
|
team_engine: Option<String>,
|
||||||
|
/// The phase's completion gate, enforced inside every node's VM.
|
||||||
|
gate: Option<crate::vm_stop_gate::StopGate>,
|
||||||
|
/// Which step is next. `execute_resumable` is sequential and gives the
|
||||||
|
/// executor no index, so the executor counts — and the count starts from the
|
||||||
|
/// checkpoint on resume, or two VMs would share an id across a restart.
|
||||||
|
step: std::sync::atomic::AtomicU32,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Everything a composed run needs that is not the graph itself.
|
||||||
|
pub struct ComposedRun {
|
||||||
|
pub run_id: Uuid,
|
||||||
|
pub mission_id: Uuid,
|
||||||
|
pub phase_id: Uuid,
|
||||||
|
pub iteration: i32,
|
||||||
|
pub repo: PathBuf,
|
||||||
|
pub has_repo: bool,
|
||||||
|
pub target_node_id: Option<Uuid>,
|
||||||
|
pub backend: Option<String>,
|
||||||
|
pub team_engine: Option<String>,
|
||||||
|
/// What must hold before a node's agent may stop. See [`crate::vm_stop_gate`].
|
||||||
|
pub gate: Option<crate::vm_stop_gate::StopGate>,
|
||||||
|
/// Steps already completed, from the durable checkpoint. Nonzero on resume.
|
||||||
|
pub completed_steps: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<V: PhaseVm> MicroVmTurnExecutor<V> {
|
||||||
|
pub fn new(vms: V, pool: PgPool, r: ComposedRun) -> Self {
|
||||||
|
Self {
|
||||||
|
vms,
|
||||||
|
pool,
|
||||||
|
run_id: r.run_id,
|
||||||
|
mission_id: r.mission_id,
|
||||||
|
phase_id: r.phase_id,
|
||||||
|
iteration: r.iteration,
|
||||||
|
repo: r.repo,
|
||||||
|
has_repo: r.has_repo,
|
||||||
|
default_fleet_node: r.target_node_id,
|
||||||
|
default_backend: r.backend,
|
||||||
|
team_engine: r.team_engine,
|
||||||
|
gate: r.gate,
|
||||||
|
step: std::sync::atomic::AtomicU32::new(r.completed_steps),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Which fleet node this graph node runs on.
|
||||||
|
///
|
||||||
|
/// Fail-closed on a malformed override: placing a node on the mission's node
|
||||||
|
/// because its own `node_id` did not parse would run the work somewhere the
|
||||||
|
/// graph did not ask for and say nothing.
|
||||||
|
fn fleet_node(&self, req: &TurnRequest) -> Result<NodeId, OrchestratorError> {
|
||||||
|
let id = match req.attrs.get("node_id") {
|
||||||
|
Some(raw) => Uuid::parse_str(raw.trim()).map_err(|_| {
|
||||||
|
OrchestratorError::Executor(format!(
|
||||||
|
"node {} has an invalid node_id attr: {raw}",
|
||||||
|
req.node_id
|
||||||
|
))
|
||||||
|
})?,
|
||||||
|
None => self.default_fleet_node.ok_or_else(|| {
|
||||||
|
OrchestratorError::Executor(format!(
|
||||||
|
"node {} has no node_id attr and the mission has no \
|
||||||
|
target_node_id — a microVM node cannot run on the gateway, \
|
||||||
|
which has no /dev/kvm",
|
||||||
|
req.node_id
|
||||||
|
))
|
||||||
|
})?,
|
||||||
|
};
|
||||||
|
Ok(NodeId::from(id))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<V: PhaseVm> TurnExecutor for MicroVmTurnExecutor<V> {
|
||||||
|
async fn run_turn(&self, req: TurnRequest) -> Result<TurnOutcome, OrchestratorError> {
|
||||||
|
let fleet_node = self.fleet_node(&req)?;
|
||||||
|
let backend = req
|
||||||
|
.attrs
|
||||||
|
.get("backend")
|
||||||
|
.map(|s| s.trim().to_string())
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.or_else(|| self.default_backend.clone());
|
||||||
|
|
||||||
|
if !self.repo.is_dir() {
|
||||||
|
return Err(OrchestratorError::Executor(format!(
|
||||||
|
"mission has no checkout at {} — a composed node needs the \
|
||||||
|
repository, and it is also how the previous node's work reaches \
|
||||||
|
this one",
|
||||||
|
self.repo.display()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let step = self
|
||||||
|
.step
|
||||||
|
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||||
|
|
||||||
|
// Held for the length of the VM turn: an hour of silence would otherwise
|
||||||
|
// look exactly like a dead worker to `requeue_stale`.
|
||||||
|
let _alive = Keepalive::spawn(self.pool.clone(), self.run_id);
|
||||||
|
|
||||||
|
let task = node_task_text(&req);
|
||||||
|
let outcome = self
|
||||||
|
.vms
|
||||||
|
.run(VmPhase {
|
||||||
|
// Every node of a composed graph streams to the same outer run,
|
||||||
|
// which is the one the operator is watching.
|
||||||
|
run_id: Some(self.run_id),
|
||||||
|
node_id: fleet_node,
|
||||||
|
mission_id: self.mission_id,
|
||||||
|
phase_id: self.phase_id,
|
||||||
|
iteration: self.iteration,
|
||||||
|
task: &task,
|
||||||
|
backend: backend.as_deref(),
|
||||||
|
repo: &self.repo,
|
||||||
|
has_repo: self.has_repo,
|
||||||
|
team_engine: self.team_engine.as_deref(),
|
||||||
|
// Each node is its own agent session, so each carries the
|
||||||
|
// phase's gate. Threaded from the run rather than rebuilt here:
|
||||||
|
// one source for what "done" means, whichever executor asks.
|
||||||
|
gate: self.gate.as_ref(),
|
||||||
|
step: Some(step),
|
||||||
|
// Same live drain as the solo path. A composed graph node can
|
||||||
|
// run for an hour too, and its files are the only account of
|
||||||
|
// what it did until the next node collects.
|
||||||
|
tap_sink: Some(crate::phase_runner::vm_tool_recorder(
|
||||||
|
&self.pool,
|
||||||
|
self.mission_id,
|
||||||
|
self.phase_id,
|
||||||
|
self.run_id,
|
||||||
|
)),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
OrchestratorError::Executor(format!("node {} in a microVM: {e}", req.node_id))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
// Recorded BEFORE the failure branches below. A node that could not be
|
||||||
|
// collected, or whose gate capped, still touched files — and on this
|
||||||
|
// path those touches are the only account of what it did, since the
|
||||||
|
// work never reached a diff.
|
||||||
|
crate::phase_runner::record_vm_tools(
|
||||||
|
&self.pool,
|
||||||
|
self.mission_id,
|
||||||
|
self.phase_id,
|
||||||
|
self.run_id,
|
||||||
|
&outcome.tools,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
// A node whose work never came back must fail the run rather than hand
|
||||||
|
// the next node a tree missing the previous one's edits. On this path an
|
||||||
|
// uncollected turn is worse than on the solo one: the loss is silent,
|
||||||
|
// because the next node still boots from a checkout that looks fine.
|
||||||
|
if !outcome.collected {
|
||||||
|
return Err(OrchestratorError::Executor(format!(
|
||||||
|
"node {}'s work could not be collected from its VM, so the next \
|
||||||
|
node would not see it: {}",
|
||||||
|
req.node_id,
|
||||||
|
outcome.summary.chars().take(400).collect::<String>()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
// Same rule as the solo path: the gate is the only thing that runs a
|
||||||
|
// `done_when_check`, so a release at the cap must fail the run rather
|
||||||
|
// than hand the next node a tree that does not satisfy the condition
|
||||||
|
// every node in this graph was told to satisfy.
|
||||||
|
if outcome.released_at_cap == Some(true) {
|
||||||
|
return Err(OrchestratorError::Executor(format!(
|
||||||
|
"node {}'s completion gate released it after {} refusal(s) with its check \
|
||||||
|
still failing: {}",
|
||||||
|
req.node_id,
|
||||||
|
crate::vm_stop_gate::MAX_BLOCKS,
|
||||||
|
outcome.summary.chars().take(400).collect::<String>()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
if outcome.rc != 0 {
|
||||||
|
return Err(OrchestratorError::Executor(format!(
|
||||||
|
"node {} exited {}: {}",
|
||||||
|
req.node_id,
|
||||||
|
outcome.rc,
|
||||||
|
outcome.summary.chars().take(400).collect::<String>()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
eprintln!(
|
||||||
|
"microvm_turn_executor: run {} node {} (role {}, step {}) ok — subagents: {}",
|
||||||
|
self.run_id,
|
||||||
|
req.node_id,
|
||||||
|
req.role,
|
||||||
|
step,
|
||||||
|
outcome
|
||||||
|
.subagents
|
||||||
|
.map(|n| n.to_string())
|
||||||
|
.unwrap_or_else(|| "?".into()),
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(TurnOutcome {
|
||||||
|
output: outcome.summary,
|
||||||
|
// `claude -p` does not report token usage on stdout, and inventing a
|
||||||
|
// number here would corrupt the run totals the harness reads. Zero is
|
||||||
|
// the honest value for "not measured on this path".
|
||||||
|
tokens: 0,
|
||||||
|
gated: Vec::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What one graph node is told.
|
||||||
|
///
|
||||||
|
/// The upstream outputs are included as context, but the load-bearing sentence is
|
||||||
|
/// that the previous node's *files* are already in the tree: a node told only
|
||||||
|
/// about the text would re-do work it is standing on.
|
||||||
|
fn node_task_text(req: &TurnRequest) -> String {
|
||||||
|
let mut s = format!(
|
||||||
|
"You are the `{}` stage of a multi-stage mission.\n\nMISSION TASK\n{}\n",
|
||||||
|
req.role, req.task
|
||||||
|
);
|
||||||
|
if !req.context.is_empty() {
|
||||||
|
s.push_str(
|
||||||
|
"\nWHAT CAME BEFORE\nThe earlier stages' work is ALREADY IN THIS \
|
||||||
|
WORKING TREE — the repository you have been given is their output, \
|
||||||
|
not a fresh checkout. Read the files before changing them, and do \
|
||||||
|
not redo what is already done. Their closing reports:\n",
|
||||||
|
);
|
||||||
|
for (i, c) in req.context.iter().enumerate() {
|
||||||
|
s.push_str(&format!("\n--- stage {} ---\n{}\n", i + 1, c));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Touches a run's `updated_at` until dropped.
|
||||||
|
struct Keepalive(tokio::task::JoinHandle<()>);
|
||||||
|
|
||||||
|
impl Keepalive {
|
||||||
|
fn spawn(pool: PgPool, run_id: Uuid) -> Self {
|
||||||
|
Keepalive(tokio::spawn(async move {
|
||||||
|
let mut ticker = tokio::time::interval(Duration::from_secs(KEEPALIVE_SECS));
|
||||||
|
loop {
|
||||||
|
ticker.tick().await;
|
||||||
|
let _ = cm_db::repo::topology_runs::touch(&pool, run_id).await;
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for Keepalive {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.0.abort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build the executor the worker uses, over real VMs on the fleet.
|
||||||
|
pub fn for_fleet(
|
||||||
|
hub: Arc<crate::fleet::NodeHub>,
|
||||||
|
pool: PgPool,
|
||||||
|
r: ComposedRun,
|
||||||
|
) -> MicroVmTurnExecutor<crate::microvm_executor::HubVms> {
|
||||||
|
MicroVmTurnExecutor::new(crate::microvm_executor::HubVms::new(hub), pool, r)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::microvm_executor::VmOutcome;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::sync::Mutex;
|
||||||
|
|
||||||
|
/// A VM modelled honestly: the host tree is packed in, the "agent" works on a
|
||||||
|
/// COPY that no host path points at, and the result is unpacked back over the
|
||||||
|
/// host tree. That is the real inject → run → collect shape, which is what
|
||||||
|
/// makes the negative control below meaningful — remove the collect and the
|
||||||
|
/// handoff breaks exactly as it would in production.
|
||||||
|
struct FakeVms {
|
||||||
|
/// Whether the guest's tree is collected back to the host.
|
||||||
|
collect: bool,
|
||||||
|
/// vm ids used, in order — the id is what stops two nodes colliding.
|
||||||
|
ids: Mutex<Vec<String>>,
|
||||||
|
/// (backend, fleet node) per call, for the heterogeneity assertions.
|
||||||
|
placements: Mutex<Vec<(Option<String>, NodeId)>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FakeVms {
|
||||||
|
fn new(collect: bool) -> Self {
|
||||||
|
Self {
|
||||||
|
collect,
|
||||||
|
ids: Mutex::new(Vec::new()),
|
||||||
|
placements: Mutex::new(Vec::new()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PhaseVm for FakeVms {
|
||||||
|
async fn run(&self, p: VmPhase<'_>) -> Result<VmOutcome, String> {
|
||||||
|
self.ids
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.push(format!("{}-{:?}", p.phase_id.simple(), p.step));
|
||||||
|
self.placements
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.push((p.backend.map(str::to_string), p.node_id));
|
||||||
|
|
||||||
|
// inject: the host checkout goes in as a tar.
|
||||||
|
let tar = crate::mission_fs::pack_dir(p.repo, "repo")?;
|
||||||
|
let guest = tempfile::tempdir().map_err(|e| e.to_string())?;
|
||||||
|
crate::mission_fs::unpack_into(&tar, guest.path())?;
|
||||||
|
let guest_repo = guest.path().join("repo");
|
||||||
|
|
||||||
|
// run: the agent records that it was here, and reports what it found
|
||||||
|
// of the previous stages — the observation the handoff test reads.
|
||||||
|
let seen: Vec<String> = std::fs::read_dir(&guest_repo)
|
||||||
|
.map_err(|e| e.to_string())?
|
||||||
|
.filter_map(|e| e.ok())
|
||||||
|
.map(|e| e.file_name().to_string_lossy().to_string())
|
||||||
|
.filter(|n| n.starts_with("stage-"))
|
||||||
|
.collect();
|
||||||
|
let mine = guest_repo.join(format!("stage-{}.txt", p.step.unwrap_or(0)));
|
||||||
|
std::fs::write(&mine, "work").map_err(|e| e.to_string())?;
|
||||||
|
|
||||||
|
// collect: the guest tree comes back over the same host path.
|
||||||
|
if self.collect {
|
||||||
|
let back = crate::mission_fs::pack_dir(&guest_repo, "repo")?;
|
||||||
|
let parent = p.repo.parent().ok_or("no parent")?;
|
||||||
|
crate::mission_fs::unpack_into(&back, parent)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(VmOutcome {
|
||||||
|
summary: format!("saw:[{}]", seen.join(",")),
|
||||||
|
rc: 0,
|
||||||
|
collected: true,
|
||||||
|
subagents: Some(0),
|
||||||
|
teammates: None,
|
||||||
|
stop_blocks: None,
|
||||||
|
released_at_cap: None,
|
||||||
|
tools: Vec::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn req(node: &str, role: &str, context: Vec<String>) -> TurnRequest {
|
||||||
|
TurnRequest {
|
||||||
|
node_id: node.into(),
|
||||||
|
role: role.into(),
|
||||||
|
agent: None,
|
||||||
|
attrs: BTreeMap::new(),
|
||||||
|
task: "build the thing".into(),
|
||||||
|
context,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exec<V: PhaseVm>(vms: V, repo: PathBuf) -> MicroVmTurnExecutor<V> {
|
||||||
|
// A pool that is never connected: every test here fails the turn before
|
||||||
|
// any query, or drives one whose only DB touch is the best-effort
|
||||||
|
// keepalive (which swallows its own errors by design).
|
||||||
|
let pool = sqlx::postgres::PgPoolOptions::new()
|
||||||
|
.max_connections(1)
|
||||||
|
.connect_lazy("postgres://invalid/invalid")
|
||||||
|
.expect("a lazy pool never dials");
|
||||||
|
MicroVmTurnExecutor::new(
|
||||||
|
vms,
|
||||||
|
pool,
|
||||||
|
ComposedRun {
|
||||||
|
run_id: Uuid::now_v7(),
|
||||||
|
mission_id: Uuid::now_v7(),
|
||||||
|
phase_id: Uuid::now_v7(),
|
||||||
|
iteration: 1,
|
||||||
|
repo,
|
||||||
|
has_repo: true,
|
||||||
|
target_node_id: Some(Uuid::now_v7()),
|
||||||
|
backend: Some("claude".into()),
|
||||||
|
team_engine: None,
|
||||||
|
gate: None,
|
||||||
|
completed_steps: 0,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn a_checkout() -> tempfile::TempDir {
|
||||||
|
let d = tempfile::tempdir().unwrap();
|
||||||
|
std::fs::create_dir_all(d.path().join("repo")).unwrap();
|
||||||
|
std::fs::write(d.path().join("repo").join("README.md"), "hello").unwrap();
|
||||||
|
d
|
||||||
|
}
|
||||||
|
|
||||||
|
/// THE trap this slice exists to solve. A per-node VM is destroyed with its
|
||||||
|
/// filesystem, so unless the tree is carried forward, node 2 works from the
|
||||||
|
/// original checkout and silently loses node 1's edits — while still
|
||||||
|
/// reporting success.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_later_node_sees_an_earlier_nodes_files() {
|
||||||
|
let d = a_checkout();
|
||||||
|
let e = exec(FakeVms::new(true), d.path().join("repo"));
|
||||||
|
|
||||||
|
let first = e.run_turn(req("n1", "implementer", vec![])).await.unwrap();
|
||||||
|
assert_eq!(first.output, "saw:[]", "the first node starts clean");
|
||||||
|
|
||||||
|
let second = e
|
||||||
|
.run_turn(req("n2", "verifier", vec![first.output.clone()]))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
second.output.contains("stage-0.txt"),
|
||||||
|
"node 2 could not see node 1's file: {}",
|
||||||
|
second.output
|
||||||
|
);
|
||||||
|
// And the host tree — what delivery diffs — holds both nodes' work.
|
||||||
|
for f in ["stage-0.txt", "stage-1.txt"] {
|
||||||
|
assert!(d.path().join("repo").join(f).exists(), "{f} missing on the host");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The negative control, run rather than assumed: with the collect removed —
|
||||||
|
/// i.e. a text-only handoff between nodes — the test above fails. A guard
|
||||||
|
/// that cannot detect the bug it was written for is decoration.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn text_only_handoff_loses_the_earlier_nodes_work() {
|
||||||
|
let d = a_checkout();
|
||||||
|
let e = exec(FakeVms::new(false), d.path().join("repo"));
|
||||||
|
|
||||||
|
e.run_turn(req("n1", "implementer", vec![])).await.unwrap();
|
||||||
|
let second = e.run_turn(req("n2", "verifier", vec![])).await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
second.output, "saw:[]",
|
||||||
|
"without a collect, node 2 must NOT see node 1's work — if it does, \
|
||||||
|
this test is no longer controlling anything"
|
||||||
|
);
|
||||||
|
assert!(!d.path().join("repo").join("stage-0.txt").exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Each node gets its own vm id within one phase and iteration. Two nodes
|
||||||
|
/// sharing an id means the second is refused by the fleet node while the
|
||||||
|
/// first is alive, and indistinguishable from a re-run once it is not.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn every_node_runs_in_its_own_vm() {
|
||||||
|
let d = a_checkout();
|
||||||
|
let vms = FakeVms::new(true);
|
||||||
|
let e = exec(vms, d.path().join("repo"));
|
||||||
|
for n in ["n1", "n2", "n3"] {
|
||||||
|
e.run_turn(req(n, "worker", vec![])).await.unwrap();
|
||||||
|
}
|
||||||
|
let ids = e.vms.ids.lock().unwrap().clone();
|
||||||
|
let unique: std::collections::HashSet<_> = ids.iter().collect();
|
||||||
|
assert_eq!(unique.len(), ids.len(), "{ids:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resume must not re-use a completed step's vm id. The executor counts steps
|
||||||
|
/// itself, so the count has to start where the checkpoint left off.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_resumed_run_continues_the_step_numbering() {
|
||||||
|
let d = a_checkout();
|
||||||
|
let pool = sqlx::postgres::PgPoolOptions::new()
|
||||||
|
.max_connections(1)
|
||||||
|
.connect_lazy("postgres://invalid/invalid")
|
||||||
|
.unwrap();
|
||||||
|
let e = MicroVmTurnExecutor::new(
|
||||||
|
FakeVms::new(true),
|
||||||
|
pool,
|
||||||
|
ComposedRun {
|
||||||
|
run_id: Uuid::now_v7(),
|
||||||
|
mission_id: Uuid::now_v7(),
|
||||||
|
phase_id: Uuid::now_v7(),
|
||||||
|
iteration: 1,
|
||||||
|
repo: d.path().join("repo"),
|
||||||
|
has_repo: true,
|
||||||
|
target_node_id: Some(Uuid::now_v7()),
|
||||||
|
backend: None,
|
||||||
|
team_engine: None,
|
||||||
|
gate: None,
|
||||||
|
completed_steps: 2,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
e.run_turn(req("n3", "worker", vec![])).await.unwrap();
|
||||||
|
let ids = e.vms.ids.lock().unwrap().clone();
|
||||||
|
assert!(
|
||||||
|
ids[0].ends_with("Some(2)"),
|
||||||
|
"the first step after a resume must be step 2, not 0: {ids:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Per-node `backend` is what makes the outer graph heterogeneous — a
|
||||||
|
/// validator node on another provider's image. It must override the
|
||||||
|
/// mission's, and the mission's must still apply to nodes that say nothing.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_node_may_pick_its_own_backend_and_fleet_node() {
|
||||||
|
let d = a_checkout();
|
||||||
|
let e = exec(FakeVms::new(true), d.path().join("repo"));
|
||||||
|
let elsewhere = Uuid::now_v7();
|
||||||
|
|
||||||
|
let mut r = req("n1", "worker", vec![]);
|
||||||
|
r.attrs.insert("backend".into(), "glm".into());
|
||||||
|
r.attrs.insert("node_id".into(), elsewhere.to_string());
|
||||||
|
e.run_turn(r).await.unwrap();
|
||||||
|
e.run_turn(req("n2", "worker", vec![])).await.unwrap();
|
||||||
|
|
||||||
|
let p = e.vms.placements.lock().unwrap().clone();
|
||||||
|
assert_eq!(p[0].0.as_deref(), Some("glm"));
|
||||||
|
assert_eq!(p[0].1, NodeId::from(elsewhere));
|
||||||
|
assert_eq!(p[1].0.as_deref(), Some("claude"), "the mission default");
|
||||||
|
assert_ne!(p[1].1, NodeId::from(elsewhere));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A malformed `node_id` must fail the node, not fall back to the mission's.
|
||||||
|
/// Silently running work somewhere the graph did not ask for is the same
|
||||||
|
/// class of bug as an alias that serde dropped.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_malformed_node_placement_fails_closed() {
|
||||||
|
let d = a_checkout();
|
||||||
|
let e = exec(FakeVms::new(true), d.path().join("repo"));
|
||||||
|
let mut r = req("n1", "worker", vec![]);
|
||||||
|
r.attrs.insert("node_id".into(), "not-a-uuid".into());
|
||||||
|
let err = e.run_turn(r).await.unwrap_err().to_string();
|
||||||
|
assert!(err.contains("invalid node_id"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An uncollected node is a failed run here, not a warning: the next node
|
||||||
|
/// would boot from a tree that looks fine and is missing this node's work.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn an_uncollected_node_fails_the_run() {
|
||||||
|
struct Lost;
|
||||||
|
impl PhaseVm for Lost {
|
||||||
|
async fn run(&self, _p: VmPhase<'_>) -> Result<VmOutcome, String> {
|
||||||
|
Ok(VmOutcome {
|
||||||
|
summary: "did plenty".into(),
|
||||||
|
rc: 0,
|
||||||
|
collected: false,
|
||||||
|
subagents: None,
|
||||||
|
teammates: None,
|
||||||
|
stop_blocks: None,
|
||||||
|
released_at_cap: None,
|
||||||
|
tools: Vec::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let d = a_checkout();
|
||||||
|
let e = exec(Lost, d.path().join("repo"));
|
||||||
|
let err = e.run_turn(req("n1", "worker", vec![])).await.unwrap_err().to_string();
|
||||||
|
assert!(err.contains("could not be collected"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A node whose gate gave up is a FAILED run, not a completed one.
|
||||||
|
///
|
||||||
|
/// The gate is the only thing in the system that ever runs a
|
||||||
|
/// `done_when_check`. If it releases the agent at the cap and this returns
|
||||||
|
/// Ok, the check's failure is never seen again: the node reports success,
|
||||||
|
/// the next node builds on a tree that does not satisfy the condition, and
|
||||||
|
/// the phase completes green. `rc` is 0 and the work IS collected here on
|
||||||
|
/// purpose — those are the two signals that used to decide this, and both
|
||||||
|
/// say "fine".
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_node_whose_gate_gave_up_fails_the_run() {
|
||||||
|
struct Capped;
|
||||||
|
impl PhaseVm for Capped {
|
||||||
|
async fn run(&self, _p: VmPhase<'_>) -> Result<VmOutcome, String> {
|
||||||
|
Ok(VmOutcome {
|
||||||
|
summary: "I could not get the tests passing, but here is what I did".into(),
|
||||||
|
rc: 0,
|
||||||
|
collected: true,
|
||||||
|
subagents: None,
|
||||||
|
teammates: None,
|
||||||
|
stop_blocks: Some(crate::vm_stop_gate::MAX_BLOCKS),
|
||||||
|
released_at_cap: Some(true),
|
||||||
|
tools: Vec::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let d = a_checkout();
|
||||||
|
let e = exec(Capped, d.path().join("repo"));
|
||||||
|
let err = e.run_turn(req("n1", "worker", vec![])).await.unwrap_err().to_string();
|
||||||
|
assert!(err.contains("released it after"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The negative control: the SAME number of blocks, without the cap. An
|
||||||
|
/// agent that was refused three times and then got it right on the fourth
|
||||||
|
/// try has succeeded, and reports `blocks: 3` exactly like the test above.
|
||||||
|
/// Failing on the count instead of the mark would fail this healthy run.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_node_that_was_blocked_and_then_succeeded_passes() {
|
||||||
|
struct Recovered;
|
||||||
|
impl PhaseVm for Recovered {
|
||||||
|
async fn run(&self, _p: VmPhase<'_>) -> Result<VmOutcome, String> {
|
||||||
|
Ok(VmOutcome {
|
||||||
|
summary: "took me a few tries".into(),
|
||||||
|
rc: 0,
|
||||||
|
collected: true,
|
||||||
|
subagents: None,
|
||||||
|
teammates: None,
|
||||||
|
stop_blocks: Some(crate::vm_stop_gate::MAX_BLOCKS),
|
||||||
|
released_at_cap: Some(false),
|
||||||
|
tools: Vec::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let d = a_checkout();
|
||||||
|
let e = exec(Recovered, d.path().join("repo"));
|
||||||
|
e.run_turn(req("n1", "worker", vec![]))
|
||||||
|
.await
|
||||||
|
.expect("a run that recovered inside its own turn is a success");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A node must be told its predecessors' files are already in the tree.
|
||||||
|
/// Given only the text, an agent re-does work it is standing on.
|
||||||
|
#[test]
|
||||||
|
fn a_downstream_node_is_told_the_work_is_already_in_the_tree() {
|
||||||
|
let solo = node_task_text(&req("n1", "implementer", vec![]));
|
||||||
|
assert!(solo.contains("build the thing"));
|
||||||
|
assert!(!solo.contains("WHAT CAME BEFORE"), "{solo}");
|
||||||
|
|
||||||
|
let later = node_task_text(&req("n2", "verifier", vec!["I wrote foo.rs".into()]));
|
||||||
|
assert!(later.contains("ALREADY IN THIS WORKING TREE"), "{later}");
|
||||||
|
assert!(later.contains("I wrote foo.rs"), "{later}");
|
||||||
|
assert!(later.contains("verifier"), "the node's role: {later}");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -40,7 +40,14 @@ use crate::mission_workspace;
|
|||||||
/// coding phase that ran `cargo build` leaves a `target/` directory larger
|
/// coding phase that ran `cargo build` leaves a `target/` directory larger
|
||||||
/// than most repositories, and a patch containing it is unreadable as well as
|
/// than most repositories, and a patch containing it is unreadable as well as
|
||||||
/// enormous.
|
/// enormous.
|
||||||
const EXCLUDED_PATHS: &[&str] = &[
|
///
|
||||||
|
/// `mission_fs` uses this same list for the TRANSPORT, and that is not a
|
||||||
|
/// convenience — it is the fix for a real failure. The diff excluded `target/`
|
||||||
|
/// while the tar that carried the tree in and out did not, so a phase that ran
|
||||||
|
/// `cargo test` shipped its whole build directory over vsock twice. `vm_collect`
|
||||||
|
/// timed out at 300s on mission 019fd43e with the agent's work finished and
|
||||||
|
/// stranded inside a VM. Two layers, one list.
|
||||||
|
pub(crate) const EXCLUDED_PATHS: &[&str] = &[
|
||||||
"target",
|
"target",
|
||||||
"node_modules",
|
"node_modules",
|
||||||
".venv",
|
".venv",
|
||||||
@@ -66,6 +73,10 @@ const EXCLUDED_PATHS: &[&str] = &[
|
|||||||
/// generated or vendored got committed), and the head of it is what an
|
/// generated or vendored got committed), and the head of it is what an
|
||||||
/// operator needs to see to work out what happened.
|
/// operator needs to see to work out what happened.
|
||||||
const MAX_PATCH_BYTES: usize = 4 * 1024 * 1024;
|
const MAX_PATCH_BYTES: usize = 4 * 1024 * 1024;
|
||||||
|
/// Cap on the recorded path list. A cap that silently truncates is worse than
|
||||||
|
/// no cap, so the metadata carries `files_truncated` beside it — a reader must
|
||||||
|
/// be able to tell "touched 12 files" from "touched at least 500".
|
||||||
|
const MAX_CAPTURED_PATHS: usize = 500;
|
||||||
|
|
||||||
/// Who delivery commits as.
|
/// Who delivery commits as.
|
||||||
///
|
///
|
||||||
@@ -109,7 +120,18 @@ pub struct Capture {
|
|||||||
/// The phase changed nothing. Still recorded — "this coding phase wrote no
|
/// The phase changed nothing. Still recorded — "this coding phase wrote no
|
||||||
/// code" is currently invisible to an operator, and it is worth saying.
|
/// code" is currently invisible to an operator, and it is worth saying.
|
||||||
pub empty: bool,
|
pub empty: bool,
|
||||||
|
/// Why the diff could not be computed, if it could not be. `empty` is only
|
||||||
|
/// meaningful when this is `None`: otherwise the tree was never read, and
|
||||||
|
/// callers deciding anything on the strength of "no changes" must not.
|
||||||
|
pub diff_error: Option<String>,
|
||||||
pub truncated: bool,
|
pub truncated: bool,
|
||||||
|
/// The paths this phase touched, with their `--name-status` letter. The
|
||||||
|
/// diffstat gives counts only; this is what lets anything downstream say
|
||||||
|
/// WHICH files changed.
|
||||||
|
pub files: Vec<(char, String)>,
|
||||||
|
/// The path list hit `MAX_CAPTURED_PATHS`. Recorded so a reader can tell a
|
||||||
|
/// complete list from a clipped one.
|
||||||
|
pub files_truncated: bool,
|
||||||
pub patch_path: PathBuf,
|
pub patch_path: PathBuf,
|
||||||
/// Set once the work has been committed to a mission branch.
|
/// Set once the work has been committed to a mission branch.
|
||||||
pub committed: Option<Commit>,
|
pub committed: Option<Commit>,
|
||||||
@@ -227,19 +249,74 @@ pub async fn capture_phase_diff_at(
|
|||||||
// A repo with nothing to add is fine; keep going and let the diff be empty.
|
// A repo with nothing to add is fine; keep going and let the diff be empty.
|
||||||
let _ = git(&repo, &add).await;
|
let _ = git(&repo, &add).await;
|
||||||
|
|
||||||
|
// A failed `git diff` and a phase that changed nothing both yield an empty
|
||||||
|
// string, and `unwrap_or_default` used to erase the difference: a corrupt
|
||||||
|
// index or an unreadable base would land `empty: true, files_changed: 0` —
|
||||||
|
// byte-identical to an honest no-op, and just as quiet. Whatever went
|
||||||
|
// wrong is recorded so the artifact can say which of the two it was.
|
||||||
|
let mut diff_error: Option<String> = None;
|
||||||
|
let mut note_diff_failure = |what: &str, e: String| {
|
||||||
|
eprintln!(
|
||||||
|
"mission_delivery: mission {mission_id} phase {phase_id} could not compute \
|
||||||
|
{what} against {base_sha}: {e}"
|
||||||
|
);
|
||||||
|
if diff_error.is_none() {
|
||||||
|
diff_error = Some(format!("{what}: {}", e.chars().take(300).collect::<String>()));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
let mut diff_args = vec!["diff", base_sha.as_str(), "--"];
|
let mut diff_args = vec!["diff", base_sha.as_str(), "--"];
|
||||||
diff_args.extend(excludes.iter().map(String::as_str));
|
diff_args.extend(excludes.iter().map(String::as_str));
|
||||||
let patch = git(&repo, &diff_args).await.unwrap_or_default();
|
let patch = match git(&repo, &diff_args).await {
|
||||||
|
Ok(p) => p,
|
||||||
|
Err(e) => {
|
||||||
|
note_diff_failure("patch", e);
|
||||||
|
String::new()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
let mut stat_args = vec!["diff", base_sha.as_str(), "--stat", "--"];
|
let mut stat_args = vec!["diff", base_sha.as_str(), "--stat", "--"];
|
||||||
stat_args.extend(excludes.iter().map(String::as_str));
|
stat_args.extend(excludes.iter().map(String::as_str));
|
||||||
let diffstat = git(&repo, &stat_args).await.unwrap_or_default();
|
let diffstat = match git(&repo, &stat_args).await {
|
||||||
|
Ok(s) => s,
|
||||||
|
Err(e) => {
|
||||||
|
note_diff_failure("diffstat", e);
|
||||||
|
String::new()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// The paths themselves, not just the counts.
|
||||||
|
//
|
||||||
|
// The diffstat gives three integers and throws the filenames away, so
|
||||||
|
// nothing downstream could say WHICH files a phase touched — the World
|
||||||
|
// could draw a "coding" station but nothing under it. Same `base_sha` and
|
||||||
|
// the same excludes as the `--stat` call above: if the two disagreed,
|
||||||
|
// `files_changed` and this list would contradict each other and nobody
|
||||||
|
// could tell which one lied.
|
||||||
|
//
|
||||||
|
// Must run BEFORE the reset below — `--intent-to-add` is what makes newly
|
||||||
|
// created files visible to diff at all.
|
||||||
|
let mut name_args = vec!["diff", base_sha.as_str(), "--name-status", "--"];
|
||||||
|
name_args.extend(excludes.iter().map(String::as_str));
|
||||||
|
let name_status = match git(&repo, &name_args).await {
|
||||||
|
Ok(s) => s,
|
||||||
|
Err(e) => {
|
||||||
|
note_diff_failure("name-status", e);
|
||||||
|
String::new()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
// Put the index back. `--intent-to-add` is a mutation of the agent's
|
// Put the index back. `--intent-to-add` is a mutation of the agent's
|
||||||
// workspace, and capture must not change what a later commit would see.
|
// workspace, and capture must not change what a later commit would see.
|
||||||
let _ = git(&repo, &["reset", "--quiet"]).await;
|
let _ = git(&repo, &["reset", "--quiet"]).await;
|
||||||
|
|
||||||
let (files_changed, insertions, deletions) = parse_diffstat(&diffstat);
|
let (files_changed, insertions, deletions) = parse_diffstat(&diffstat);
|
||||||
|
// Shared with auto_merge so the two cannot disagree about what a
|
||||||
|
// `--name-status` line means (renames are three fields; the NEW path is the
|
||||||
|
// one that changed).
|
||||||
|
let all_paths = crate::auto_merge::changed_paths(&name_status);
|
||||||
|
let files_truncated = all_paths.len() > MAX_CAPTURED_PATHS;
|
||||||
|
let files: Vec<(char, String)> = all_paths.into_iter().take(MAX_CAPTURED_PATHS).collect();
|
||||||
let empty = patch.trim().is_empty();
|
let empty = patch.trim().is_empty();
|
||||||
let truncated = patch.len() > MAX_PATCH_BYTES;
|
let truncated = patch.len() > MAX_PATCH_BYTES;
|
||||||
let stored = if truncated {
|
let stored = if truncated {
|
||||||
@@ -258,6 +335,9 @@ pub async fn capture_phase_diff_at(
|
|||||||
let patch_path = dir.join("diff.patch");
|
let patch_path = dir.join("diff.patch");
|
||||||
std::fs::write(&patch_path, &stored)
|
std::fs::write(&patch_path, &stored)
|
||||||
.map_err(|e| format!("write {}: {e}", patch_path.display()))?;
|
.map_err(|e| format!("write {}: {e}", patch_path.display()))?;
|
||||||
|
// Raw evidence on disk, independent of the JSONB. When the metadata and
|
||||||
|
// the picture disagree, this is the tiebreaker.
|
||||||
|
let _ = std::fs::write(dir.join("names.txt"), &name_status);
|
||||||
std::fs::write(dir.join("diffstat.txt"), &diffstat)
|
std::fs::write(dir.join("diffstat.txt"), &diffstat)
|
||||||
.map_err(|e| format!("write diffstat: {e}"))?;
|
.map_err(|e| format!("write diffstat: {e}"))?;
|
||||||
|
|
||||||
@@ -293,15 +373,42 @@ pub async fn capture_phase_diff_at(
|
|||||||
// Gate, then publish. Both are best-effort on top of an artifact that has
|
// Gate, then publish. Both are best-effort on top of an artifact that has
|
||||||
// already landed: a phase whose tests fail, or whose push is rejected,
|
// already landed: a phase whose tests fail, or whose push is rejected,
|
||||||
// still has its patch on disk and its work on a local branch.
|
// still has its patch on disk and its work on a local branch.
|
||||||
|
//
|
||||||
|
// `empty` suppresses publishing, so a diff we could not COMPUTE would
|
||||||
|
// otherwise skip the push and leave `push_error: null` — the phase looking
|
||||||
|
// exactly like one that correctly had nothing to publish. See
|
||||||
|
// [`untrusted_empty_reason`].
|
||||||
let mut outcome: Option<TestOutcome> = None;
|
let mut outcome: Option<TestOutcome> = None;
|
||||||
let mut published: Option<Publish> = None;
|
let mut published: Option<Publish> = None;
|
||||||
let mut publish_error: Option<String> = None;
|
let mut publish_error: Option<String> = untrusted_empty_reason(empty, diff_error.as_deref());
|
||||||
if let Some(c) = committed.as_ref() {
|
if let Some(c) = committed.as_ref() {
|
||||||
if !empty {
|
if !empty {
|
||||||
if gate == Gate::OnGreenTests {
|
if gate == Gate::OnGreenTests {
|
||||||
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
||||||
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
||||||
let o = verify_tests(&repo, &container).await;
|
// Against a COPY, never the checkout. `verify_tests` execs
|
||||||
|
// `cargo test` in a container running as ROOT, which writes
|
||||||
|
// `target/` — in the live tree that leaves root-owned build
|
||||||
|
// output in a checkout owned by uid 65532 and breaks the
|
||||||
|
// single-writer invariant. Measured the first time this gate
|
||||||
|
// ever ran end to end: `uids=0,65532`.
|
||||||
|
//
|
||||||
|
// The gate had been implemented but never exercised (every
|
||||||
|
// harness fixture used `commit_policy: "always"`), which is why
|
||||||
|
// a bug this mechanical survived in it.
|
||||||
|
let gate_root = crate::root_copy::copy_root("_gate", mission_id);
|
||||||
|
crate::root_copy::purge(&container, &gate_root).await;
|
||||||
|
let o = match crate::root_copy::RootCopy::of(&repo, &gate_root) {
|
||||||
|
Ok(copy) => {
|
||||||
|
let r = verify_tests(copy.workdir(), &container).await;
|
||||||
|
crate::root_copy::purge(&container, &gate_root).await;
|
||||||
|
r
|
||||||
|
}
|
||||||
|
// Fail-closed: an unverifiable suite must not license a push.
|
||||||
|
Err(e) => TestOutcome::CouldNotRun(format!(
|
||||||
|
"could not copy the checkout to test it: {e}"
|
||||||
|
)),
|
||||||
|
};
|
||||||
// An infrastructure fault must be loud. The gate degrades
|
// An infrastructure fault must be loud. The gate degrades
|
||||||
// safely either way, but "we could not run the suite" is a
|
// safely either way, but "we could not run the suite" is a
|
||||||
// problem with the platform and needs to look like one.
|
// problem with the platform and needs to look like one.
|
||||||
@@ -327,7 +434,14 @@ pub async fn capture_phase_diff_at(
|
|||||||
could not publish {}: {e}",
|
could not publish {}: {e}",
|
||||||
c.branch
|
c.branch
|
||||||
);
|
);
|
||||||
publish_error = Some(e.chars().take(500).collect());
|
// Both ends, not the first 500 chars. Git prints its
|
||||||
|
// REASON last — "non-fast-forward", "fetch first",
|
||||||
|
// "protected branch" — so a head-only clamp keeps the
|
||||||
|
// noise and drops the answer. A real push failure was
|
||||||
|
// recorded as two auth lines plus a branch name cut
|
||||||
|
// off mid-word, with the reject reason gone.
|
||||||
|
publish_error =
|
||||||
|
Some(crate::evaluator_tools::clamp_output(&e));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -379,7 +493,18 @@ pub async fn capture_phase_diff_at(
|
|||||||
"insertions": insertions,
|
"insertions": insertions,
|
||||||
"deletions": deletions,
|
"deletions": deletions,
|
||||||
"empty": empty,
|
"empty": empty,
|
||||||
|
// Non-null means `empty`/`files_changed` describe a failed read, not
|
||||||
|
// an unchanged tree. Readers that treat `empty: true` as "the phase
|
||||||
|
// did nothing" must check this first.
|
||||||
|
"diff_error": diff_error,
|
||||||
"truncated": truncated,
|
"truncated": truncated,
|
||||||
|
// WHICH files, not just how many. Same base_sha and the same excludes
|
||||||
|
// as `files_changed`, so the two describe the same diff.
|
||||||
|
"files": files
|
||||||
|
.iter()
|
||||||
|
.map(|(st, path)| serde_json::json!({ "status": st.to_string(), "path": path }))
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
"files_truncated": files_truncated,
|
||||||
"excluded_paths": EXCLUDED_PATHS,
|
"excluded_paths": EXCLUDED_PATHS,
|
||||||
});
|
});
|
||||||
std::fs::write(
|
std::fs::write(
|
||||||
@@ -388,8 +513,11 @@ pub async fn capture_phase_diff_at(
|
|||||||
)
|
)
|
||||||
.map_err(|e| format!("write delivery.json: {e}"))?;
|
.map_err(|e| format!("write delivery.json: {e}"))?;
|
||||||
|
|
||||||
// Path is stored relative to the missions root, matching how
|
// Path is stored relative to the MISSIONS ROOT — the convention every
|
||||||
// `pdf_renderer` resolves artifact paths.
|
// artifact uses, and what `routes::missions::artifact_content` resolves
|
||||||
|
// against. (The old `pdf_renderer` claimed to match this and did not: it
|
||||||
|
// joined the mission id first, producing a doubled id and ENOENT. It is
|
||||||
|
// gone; this comment named it as the authority, which it never was.)
|
||||||
let rel = format!("_outputs/{mission_id}/{phase_id}/diff.patch");
|
let rel = format!("_outputs/{mission_id}/{phase_id}/diff.patch");
|
||||||
cm_db::repo::missions::register_artifact(
|
cm_db::repo::missions::register_artifact(
|
||||||
pool,
|
pool,
|
||||||
@@ -426,32 +554,41 @@ pub async fn capture_phase_diff_at(
|
|||||||
insertions,
|
insertions,
|
||||||
deletions,
|
deletions,
|
||||||
empty,
|
empty,
|
||||||
|
diff_error,
|
||||||
truncated,
|
truncated,
|
||||||
|
files,
|
||||||
|
files_truncated,
|
||||||
patch_path,
|
patch_path,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run git in `repo`, returning stdout.
|
/// Run git in `repo`, returning stdout.
|
||||||
///
|
///
|
||||||
/// Every invocation carries `-c safe.directory`: the server clones as uid
|
/// Every invocation carries `-c safe.directory`: under `CLAWMATES_MISSION_FS=bind`
|
||||||
/// 65532 while agents write into the same tree as root, so without it git
|
/// the server clones as uid 65532 while agents write into the same tree as
|
||||||
/// refuses the repository outright — the failure that had the phase evaluator
|
/// root, so without it git refuses the repository outright — the failure that
|
||||||
/// silently falling back to guesswork.
|
/// had the phase evaluator silently falling back to guesswork. Copy mode makes
|
||||||
|
/// the tree single-uid and this redundant, but it stays while the bind path is
|
||||||
|
/// still selectable: a workaround may only be deleted once the situation it
|
||||||
|
/// works around can no longer be chosen.
|
||||||
async fn git(repo: &Path, args: &[&str]) -> Result<String, String> {
|
async fn git(repo: &Path, args: &[&str]) -> Result<String, String> {
|
||||||
let repo_s = repo.display().to_string();
|
let repo_s = repo.display().to_string();
|
||||||
let mut full = vec![
|
// Owned, not `Box::leak`. The leak was justified as "the process is
|
||||||
"-C",
|
// short-lived", which is true of a CLI and false of cm-api — it is a
|
||||||
&repo_s,
|
// long-running server, so that was one permanently leaked allocation per
|
||||||
"-c",
|
// git call, growing with every phase of every mission for the life of the
|
||||||
// Leaked into a `String` so it can live in a `&str` slice alongside
|
// process.
|
||||||
// the borrowed args; the process is short-lived and this is one
|
let mut full: Vec<String> = vec![
|
||||||
// allocation per git call.
|
"-C".into(),
|
||||||
Box::leak(format!("safe.directory={repo_s}").into_boxed_str()),
|
repo_s.clone(),
|
||||||
|
"-c".into(),
|
||||||
|
format!("safe.directory={repo_s}"),
|
||||||
];
|
];
|
||||||
full.extend_from_slice(args);
|
full.extend(args.iter().map(|a| (*a).to_string()));
|
||||||
let (name, email) = commit_identity();
|
let (name, email) = commit_identity();
|
||||||
let out = tokio::process::Command::new("git")
|
let mut cmd = tokio::process::Command::new("git");
|
||||||
.args(&full)
|
cmd.args(&full);
|
||||||
|
let out = crate::mission_workspace::no_terminal_prompt(&mut cmd)
|
||||||
// The server container has no git identity — `git config --global
|
// The server container has no git identity — `git config --global
|
||||||
// user.email` exits 1 — so `git commit` fails with "Author identity
|
// user.email` exits 1 — so `git commit` fails with "Author identity
|
||||||
// unknown" unless one is supplied. Mission `019fc450` lost its first
|
// unknown" unless one is supplied. Mission `019fc450` lost its first
|
||||||
@@ -480,10 +617,15 @@ async fn git(repo: &Path, args: &[&str]) -> Result<String, String> {
|
|||||||
"git {} → {}: {}",
|
"git {} → {}: {}",
|
||||||
args.first().copied().unwrap_or("?"),
|
args.first().copied().unwrap_or("?"),
|
||||||
out.status,
|
out.status,
|
||||||
String::from_utf8_lossy(&out.stderr)
|
// Both ends, never a head-only clamp. THIS is where the reason was
|
||||||
.chars()
|
// being lost: `publish_phase_branch` returns a rejected push as
|
||||||
.take(300)
|
// `Ok(Publish { error })`, so the string it carries was already
|
||||||
.collect::<String>()
|
// truncated here — 300 chars of auth noise, with "non-fast-forward"
|
||||||
|
// cut off — before the caller's own both-ends clamp ever saw it.
|
||||||
|
// Clamping the caller fixed the path that was already fine.
|
||||||
|
crate::mission_workspace::redact_token(&crate::evaluator_tools::clamp_output(
|
||||||
|
&String::from_utf8_lossy(&out.stderr)
|
||||||
|
))
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
|
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
|
||||||
@@ -718,7 +860,38 @@ async fn push_url_for(pool: &sqlx::PgPool, mission_id: Uuid) -> Result<Option<St
|
|||||||
// "nothing to push to" — the shape that made `commit_error` necessary.
|
// "nothing to push to" — the shape that made `commit_error` necessary.
|
||||||
.map_err(|e| format!("query push URL: {e}"))?
|
.map_err(|e| format!("query push URL: {e}"))?
|
||||||
.flatten();
|
.flatten();
|
||||||
Ok(url.map(|u| mission_workspace::with_ambient_auth(&u)))
|
let Some(url) = url else { return Ok(None) };
|
||||||
|
|
||||||
|
let auth = mission_workspace::with_ambient_auth(&url);
|
||||||
|
// Fail here, not at the tty. An unauthenticated URL to OUR forge cannot
|
||||||
|
// push, and every second it survives past this point is spent producing a
|
||||||
|
// symptom that looks like something else: git asking for a username, then
|
||||||
|
// `/dev/tty: No such device or address`, then a `push_error` about auth that
|
||||||
|
// sent #55's investigation after credentials which were never the problem.
|
||||||
|
// A third-party host is left alone — ssh keys and .netrc are legitimate.
|
||||||
|
if let Some(why) = &auth.unauthenticated {
|
||||||
|
if auth.is_forge() {
|
||||||
|
return Err(format!(
|
||||||
|
"cannot authenticate the push URL for this mission — {why}. The work \
|
||||||
|
is committed locally; fix the credential and re-run delivery."
|
||||||
|
));
|
||||||
|
}
|
||||||
|
eprintln!("mission_delivery: pushing to a non-forge remote unauthenticated — {why}");
|
||||||
|
}
|
||||||
|
Ok(Some(auth.url))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Did the forge reject this push because our history diverged from the ref?
|
||||||
|
///
|
||||||
|
/// Git says this several ways depending on version and refspec, and all of them
|
||||||
|
/// mean the same thing here: the branch already exists with commits ours does not
|
||||||
|
/// contain.
|
||||||
|
fn is_non_fast_forward(err: &str) -> bool {
|
||||||
|
let e = err.to_ascii_lowercase();
|
||||||
|
e.contains("non-fast-forward")
|
||||||
|
|| e.contains("fetch first")
|
||||||
|
|| e.contains("updates were rejected")
|
||||||
|
|| (e.contains("[rejected]") && !e.contains("stale info"))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run the gate, then push the branch if the gate allows it.
|
/// Run the gate, then push the branch if the gate allows it.
|
||||||
@@ -761,6 +934,54 @@ pub async fn publish_phase_branch(
|
|||||||
error: None,
|
error: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
// #55: a mission whose checkout was re-cloned — a retry, a container
|
||||||
|
// teardown, disk loss — builds divergent history against its OWN
|
||||||
|
// deterministic branch, and every push it ever attempts is rejected.
|
||||||
|
// Before this, that was terminal: the work stayed on a local branch in a
|
||||||
|
// directory that gets reaped.
|
||||||
|
//
|
||||||
|
// The escape is a NEW ref, not `--force`. Forcing would overwrite
|
||||||
|
// whatever the earlier attempt pushed — which may be the only copy of
|
||||||
|
// that work — to make this attempt look tidy. Suffixing with the commit
|
||||||
|
// sha is deterministic (the same history always lands on the same ref),
|
||||||
|
// self-describing in a branch list, and cannot collide, since divergent
|
||||||
|
// history is by definition a different sha.
|
||||||
|
Err(e) if is_non_fast_forward(&e) => {
|
||||||
|
let sha = git(repo, &["rev-parse", "HEAD"])
|
||||||
|
.await
|
||||||
|
.map(|s| s.trim().to_string())
|
||||||
|
.unwrap_or_default();
|
||||||
|
let Some(short) = sha.get(..8) else {
|
||||||
|
eprintln!("mission_delivery: push of {target} rejected and HEAD unreadable: {e}");
|
||||||
|
return Ok(Publish {
|
||||||
|
branch: target,
|
||||||
|
pushed: false,
|
||||||
|
error: Some(e),
|
||||||
|
});
|
||||||
|
};
|
||||||
|
let alt = format!("{target}-{short}");
|
||||||
|
eprintln!(
|
||||||
|
"mission_delivery: {target} exists on the forge with history this \
|
||||||
|
checkout does not contain — pushing to {alt} instead of forcing. \
|
||||||
|
Original: {e}"
|
||||||
|
);
|
||||||
|
git(repo, &["branch", "-f", &alt, "HEAD"]).await?;
|
||||||
|
match git(repo, &["push", push_url, &format!("HEAD:refs/heads/{alt}")]).await {
|
||||||
|
Ok(_) => Ok(Publish {
|
||||||
|
branch: alt,
|
||||||
|
pushed: true,
|
||||||
|
// Not an error — the work reached the forge — but the
|
||||||
|
// redirect is a fact the operator needs, or two branches for
|
||||||
|
// one phase look like a bug rather than a rescue.
|
||||||
|
error: None,
|
||||||
|
}),
|
||||||
|
Err(e2) => Ok(Publish {
|
||||||
|
branch: alt,
|
||||||
|
pushed: false,
|
||||||
|
error: Some(format!("{e}\n\nand the diverged-history retry also failed: {e2}")),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
// Redacted by `git`'s error path already; the patch and the local
|
// Redacted by `git`'s error path already; the patch and the local
|
||||||
// branch both survive, so this is a degraded success.
|
// branch both survive, so this is a degraded success.
|
||||||
@@ -958,10 +1179,105 @@ pub async fn record_uncapturable(
|
|||||||
.map_err(|e| format!("register uncapturable marker: {e}"))
|
.map_err(|e| format!("register uncapturable marker: {e}"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Why an empty patch must not be believed, if it must not be believed.
|
||||||
|
///
|
||||||
|
/// An empty patch has two causes that produce identical bytes: the tree really
|
||||||
|
/// did not change, or `git diff` failed and we have no idea what the tree
|
||||||
|
/// looks like. The first is an ordinary outcome; the second is a platform
|
||||||
|
/// fault. Returning `Some` for the second is what stops the fault from being
|
||||||
|
/// filed under the ordinary outcome — the recurring shape where a failure and
|
||||||
|
/// a legitimate negative share one representation.
|
||||||
|
fn untrusted_empty_reason(empty: bool, diff_error: Option<&str>) -> Option<String> {
|
||||||
|
match (empty, diff_error) {
|
||||||
|
(true, Some(why)) => Some(format!(
|
||||||
|
"not published: the diff could not be computed, so an empty patch \
|
||||||
|
cannot be trusted to mean an unchanged tree ({why})"
|
||||||
|
)),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod changed_path_capture_tests {
|
||||||
|
/// The path list and `files_changed` must describe the SAME diff.
|
||||||
|
///
|
||||||
|
/// They come from two separate git invocations — `--stat` and
|
||||||
|
/// `--name-status`. If those are ever given different revisions or
|
||||||
|
/// different exclude pathspecs, the count and the list disagree and there
|
||||||
|
/// is no way to tell which is right: both look like plausible output.
|
||||||
|
#[test]
|
||||||
|
fn both_diff_calls_use_the_same_revision_and_excludes() {
|
||||||
|
let src = include_str!("mission_delivery.rs");
|
||||||
|
let body = src
|
||||||
|
.split("let mut stat_args")
|
||||||
|
.nth(1)
|
||||||
|
.and_then(|s| s.split("let (files_changed").next())
|
||||||
|
.expect("the capture block");
|
||||||
|
assert!(
|
||||||
|
body.contains("let mut name_args = vec![\"diff\", base_sha.as_str(), \"--name-status\", \"--\"]"),
|
||||||
|
"the name-status call must use the same base_sha as --stat"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
body.contains("name_args.extend(excludes.iter().map(String::as_str))"),
|
||||||
|
"and the same excludes, or files_changed and the path list describe \
|
||||||
|
different diffs"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `--name-status` must run before the index is put back, or newly created
|
||||||
|
/// files — which `--intent-to-add` is what makes visible — vanish from the
|
||||||
|
/// list while still being counted by the stat.
|
||||||
|
#[test]
|
||||||
|
fn paths_are_read_before_the_index_reset() {
|
||||||
|
let src = include_str!("mission_delivery.rs");
|
||||||
|
let name_at = src.find("--name-status").expect("name-status call");
|
||||||
|
let reset_at = src
|
||||||
|
.find("git(&repo, &[\"reset\", \"--quiet\"])")
|
||||||
|
.expect("index reset");
|
||||||
|
assert!(
|
||||||
|
name_at < reset_at,
|
||||||
|
"the path list must be captured while --intent-to-add is still in \
|
||||||
|
effect, or created files are invisible to it"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
/// Git says "your history diverged" several ways, and the one production
|
||||||
|
/// actually produced (`! [rejected] ... (fetch first)`) is not the phrase
|
||||||
|
/// anyone reaches for first. Missing a phrasing means the rescue does not
|
||||||
|
/// fire and the work stays on a local branch in a directory that gets
|
||||||
|
/// reaped — silently, since the push failure is a degraded success.
|
||||||
|
#[test]
|
||||||
|
fn every_way_git_says_diverged_is_recognised() {
|
||||||
|
for e in [
|
||||||
|
"git push → exit 1: ! [rejected] HEAD -> b (fetch first)\nhint: …",
|
||||||
|
" ! [rejected] HEAD -> b (non-fast-forward)",
|
||||||
|
"hint: Updates were rejected because the remote contains work that you \
|
||||||
|
do not have locally.",
|
||||||
|
] {
|
||||||
|
assert!(is_non_fast_forward(e), "not recognised: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// And it must not fire on failures a new branch cannot fix. Retrying a
|
||||||
|
/// permissions or network error onto a second ref just produces a second
|
||||||
|
/// failure and a confusing branch name.
|
||||||
|
#[test]
|
||||||
|
fn other_push_failures_are_not_mistaken_for_divergence() {
|
||||||
|
for e in [
|
||||||
|
"fatal: repository 'https://forge/x.git' not found",
|
||||||
|
"remote: error: GH006: Protected branch update failed",
|
||||||
|
"fatal: could not read Username for 'https://forge': terminal prompts disabled",
|
||||||
|
" ! [rejected] (stale info)",
|
||||||
|
] {
|
||||||
|
assert!(!is_non_fast_forward(e), "wrongly recognised: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ── The gate ───────────────────────────────────────────────────────
|
// ── The gate ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
/// Three recipes have declared `commit_policy` since they were written and
|
/// Three recipes have declared `commit_policy` since they were written and
|
||||||
@@ -1060,6 +1376,28 @@ mod tests {
|
|||||||
|
|
||||||
/// An empty stat means an empty phase, not a parse failure. This is the
|
/// An empty stat means an empty phase, not a parse failure. This is the
|
||||||
/// case that must still produce an artifact.
|
/// case that must still produce an artifact.
|
||||||
|
/// The whole point: a tree that genuinely did not change stays silent, and
|
||||||
|
/// a diff that could not be computed does not get to borrow that silence.
|
||||||
|
#[test]
|
||||||
|
fn an_uncomputable_diff_is_not_an_unchanged_tree() {
|
||||||
|
assert_eq!(
|
||||||
|
untrusted_empty_reason(true, None),
|
||||||
|
None,
|
||||||
|
"a genuinely unchanged tree must not report an error"
|
||||||
|
);
|
||||||
|
let reason = untrusted_empty_reason(true, Some("patch: fatal: bad object"))
|
||||||
|
.expect("an empty patch from a FAILED diff must be reported, not accepted");
|
||||||
|
assert!(
|
||||||
|
reason.contains("bad object"),
|
||||||
|
"the reason must name what went wrong, got: {reason}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
untrusted_empty_reason(false, Some("diffstat: fatal: bad object")),
|
||||||
|
None,
|
||||||
|
"a non-empty patch stands on its own even if the diffstat failed"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn an_empty_diffstat_is_all_zeroes() {
|
fn an_empty_diffstat_is_all_zeroes() {
|
||||||
assert_eq!(parse_diffstat(""), (0, 0, 0));
|
assert_eq!(parse_diffstat(""), (0, 0, 0));
|
||||||
|
|||||||
@@ -0,0 +1,229 @@
|
|||||||
|
//! Structured mission activity — the channel that replaced parsing prose.
|
||||||
|
//!
|
||||||
|
//! The operator decision behind this module: action detail comes from
|
||||||
|
//! **structured events at the source**, never from `checkpoint.log` or model
|
||||||
|
//! output. A tool name in a log line is indistinguishable from an agent
|
||||||
|
//! *discussing* a tool, and a visualization built on that distinction reads as
|
||||||
|
//! confident fact while being partly fiction.
|
||||||
|
//!
|
||||||
|
//! Everything here is best-effort. A mission must not fail because its
|
||||||
|
//! telemetry could not be written — so every write logs and swallows. That is a
|
||||||
|
//! deliberate exception to this codebase's usual rule, and it is bounded: the
|
||||||
|
//! only thing lost is detail in a picture.
|
||||||
|
|
||||||
|
use serde_json::Value;
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
/// A phase entered `running`.
|
||||||
|
pub const PHASE_STARTED: &str = "phase.started";
|
||||||
|
/// A phase reached a terminal state. `detail.status` says which.
|
||||||
|
pub const PHASE_COMPLETED: &str = "phase.completed";
|
||||||
|
/// An agent called a tool. `target` is the tool name.
|
||||||
|
pub const TOOL_CALL: &str = "tool.call";
|
||||||
|
/// A tool touched a path. `target` is the path, repo-relative where known.
|
||||||
|
pub const FILE_TOUCH: &str = "file.touch";
|
||||||
|
|
||||||
|
/// Most events one phase may record.
|
||||||
|
///
|
||||||
|
/// A capped stream that says so beats an uncapped one that quietly becomes the
|
||||||
|
/// largest table in the database: a coding phase can call thousands of tools,
|
||||||
|
/// and every one of them would be replayed to every World subscriber. Past the
|
||||||
|
/// cap the picture is already complete — nobody reads the four-thousandth file
|
||||||
|
/// orb.
|
||||||
|
pub const PER_PHASE_CAP: i64 = 400;
|
||||||
|
|
||||||
|
/// One recorded event.
|
||||||
|
#[derive(Debug, Clone, Default)]
|
||||||
|
pub struct MissionEvent {
|
||||||
|
pub mission_id: Uuid,
|
||||||
|
pub phase_id: Option<Uuid>,
|
||||||
|
pub run_id: Option<Uuid>,
|
||||||
|
pub agent_id: Option<Uuid>,
|
||||||
|
pub kind: String,
|
||||||
|
pub target: Option<String>,
|
||||||
|
pub detail: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MissionEvent {
|
||||||
|
pub fn new(mission_id: Uuid, kind: &str) -> Self {
|
||||||
|
MissionEvent {
|
||||||
|
mission_id,
|
||||||
|
kind: kind.to_string(),
|
||||||
|
detail: Value::Null,
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn phase(mut self, id: Uuid) -> Self {
|
||||||
|
self.phase_id = Some(id);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
pub fn run(mut self, id: Uuid) -> Self {
|
||||||
|
self.run_id = Some(id);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
pub fn agent(mut self, id: Option<Uuid>) -> Self {
|
||||||
|
self.agent_id = id;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
pub fn target(mut self, t: impl Into<String>) -> Self {
|
||||||
|
self.target = Some(t.into());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
pub fn detail(mut self, d: Value) -> Self {
|
||||||
|
self.detail = d;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record one event, best-effort.
|
||||||
|
///
|
||||||
|
/// The per-phase cap is enforced in the INSERT itself rather than by a read
|
||||||
|
/// followed by a write: two tool taps writing concurrently would both read a
|
||||||
|
/// count below the cap and both insert, and the cap would drift by however many
|
||||||
|
/// writers there are. `INSERT … SELECT … WHERE (subquery) < cap` makes the
|
||||||
|
/// decision inside the statement.
|
||||||
|
pub async fn record(pool: &PgPool, e: MissionEvent) {
|
||||||
|
let detail = if e.detail.is_null() {
|
||||||
|
Value::Object(Default::default())
|
||||||
|
} else {
|
||||||
|
e.detail
|
||||||
|
};
|
||||||
|
let res = sqlx::query(
|
||||||
|
"INSERT INTO mission_events
|
||||||
|
(mission_id, phase_id, run_id, agent_id, kind, target, detail)
|
||||||
|
SELECT $1, $2, $3, $4, $5, $6, $7
|
||||||
|
WHERE $2::uuid IS NULL
|
||||||
|
OR (SELECT count(*) FROM mission_events WHERE phase_id = $2) < $8",
|
||||||
|
)
|
||||||
|
.bind(e.mission_id)
|
||||||
|
.bind(e.phase_id)
|
||||||
|
.bind(e.run_id)
|
||||||
|
.bind(e.agent_id)
|
||||||
|
.bind(&e.kind)
|
||||||
|
.bind(&e.target)
|
||||||
|
.bind(&detail)
|
||||||
|
.bind(PER_PHASE_CAP)
|
||||||
|
.execute(pool)
|
||||||
|
.await;
|
||||||
|
if let Err(err) = res {
|
||||||
|
eprintln!("mission_events: record {} failed: {err}", e.kind);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record several events under one round trip's worth of intent.
|
||||||
|
pub async fn record_all(pool: &PgPool, events: Vec<MissionEvent>) {
|
||||||
|
for e in events {
|
||||||
|
record(pool, e).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The path a tool's **arguments** name, if any.
|
||||||
|
///
|
||||||
|
/// Reads the arguments as JSON — never the tool's prose summary. The summary is
|
||||||
|
/// a sentence written for a human; a path pulled out of it by regex would be
|
||||||
|
/// right often enough to be trusted and wrong often enough to matter.
|
||||||
|
///
|
||||||
|
/// The key names are the ones Claude Code and the ZeroClaw tools actually use.
|
||||||
|
/// An unrecognised shape returns `None`, which renders as a tool call with no
|
||||||
|
/// file — accurate, rather than a guess at which argument was a path.
|
||||||
|
pub fn tool_path(args: &Value) -> Option<String> {
|
||||||
|
const KEYS: [&str; 6] = [
|
||||||
|
"file_path",
|
||||||
|
"filePath",
|
||||||
|
"path",
|
||||||
|
"notebook_path",
|
||||||
|
"file",
|
||||||
|
"target_file",
|
||||||
|
];
|
||||||
|
let obj = args.as_object()?;
|
||||||
|
for k in KEYS {
|
||||||
|
if let Some(s) = obj.get(k).and_then(Value::as_str) {
|
||||||
|
let s = s.trim();
|
||||||
|
if !s.is_empty() {
|
||||||
|
return Some(s.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Strip the guest/host workspace prefix so a path is repo-relative.
|
||||||
|
///
|
||||||
|
/// Tool arguments are absolute inside the sandbox (`/mission/repo/src/a.rs`).
|
||||||
|
/// Left alone, every mission's file tree would nest under a `mission` → `repo`
|
||||||
|
/// pair of directory orbs that exist in no repository and mean nothing to the
|
||||||
|
/// person reading the map.
|
||||||
|
pub fn repo_relative(path: &str, roots: &[&str]) -> String {
|
||||||
|
let p = path.trim();
|
||||||
|
for root in roots {
|
||||||
|
let root = root.trim_end_matches('/');
|
||||||
|
if let Some(rest) = p.strip_prefix(root) {
|
||||||
|
let rest = rest.trim_start_matches('/');
|
||||||
|
if !rest.is_empty() {
|
||||||
|
return rest.to_string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.trim_start_matches("./").to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
/// Paths come from arguments, and only from argument keys we know.
|
||||||
|
///
|
||||||
|
/// The alternative — scanning the values for anything that looks like a
|
||||||
|
/// path — is what makes a viz confidently wrong: a `pattern` of `*.rs` or a
|
||||||
|
/// `command` of `ls src/` would both become "the agent edited a file".
|
||||||
|
#[test]
|
||||||
|
fn a_path_comes_from_a_known_argument_or_not_at_all() {
|
||||||
|
assert_eq!(
|
||||||
|
tool_path(&json!({"file_path": "/mission/repo/src/a.rs"})).as_deref(),
|
||||||
|
Some("/mission/repo/src/a.rs")
|
||||||
|
);
|
||||||
|
assert_eq!(tool_path(&json!({"path": "docs/x.md"})).as_deref(), Some("docs/x.md"));
|
||||||
|
// A shell command mentions paths and touches none we can name.
|
||||||
|
assert_eq!(tool_path(&json!({"command": "ls src/"})), None);
|
||||||
|
// A glob is a query, not a file.
|
||||||
|
assert_eq!(tool_path(&json!({"pattern": "**/*.rs"})), None);
|
||||||
|
// Blank is absence, not a file called "".
|
||||||
|
assert_eq!(tool_path(&json!({"file_path": " "})), None);
|
||||||
|
assert_eq!(tool_path(&json!("not an object")), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The sandbox prefix must not become two directory orbs in every mission.
|
||||||
|
#[test]
|
||||||
|
fn paths_are_made_repo_relative() {
|
||||||
|
let roots = ["/mission/repo", "/workspace"];
|
||||||
|
assert_eq!(repo_relative("/mission/repo/src/a.rs", &roots), "src/a.rs");
|
||||||
|
assert_eq!(repo_relative("/workspace/README.md", &roots), "README.md");
|
||||||
|
assert_eq!(repo_relative("./src/a.rs", &roots), "src/a.rs");
|
||||||
|
// Outside every root, it is left alone rather than mangled.
|
||||||
|
assert_eq!(repo_relative("/etc/hosts", &roots), "/etc/hosts");
|
||||||
|
// The root ITSELF is not a file, so it must not collapse to "".
|
||||||
|
assert_eq!(repo_relative("/mission/repo", &roots), "/mission/repo");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The cap must be decided inside the INSERT.
|
||||||
|
///
|
||||||
|
/// A count-then-insert is the classic version of this and it is wrong here:
|
||||||
|
/// the container tap and the microVM drain both write for the same phase,
|
||||||
|
/// and each would see a count below the cap and insert. Nothing errors —
|
||||||
|
/// the table simply grows past the bound that exists to hold it.
|
||||||
|
#[test]
|
||||||
|
fn the_cap_is_enforced_in_one_statement() {
|
||||||
|
let src = include_str!("mission_events.rs");
|
||||||
|
let body = src
|
||||||
|
.split("pub async fn record(")
|
||||||
|
.nth(1)
|
||||||
|
.and_then(|s| s.split("pub async fn").next())
|
||||||
|
.expect("record body");
|
||||||
|
assert!(
|
||||||
|
body.contains("INSERT INTO mission_events") && body.contains("SELECT count(*)"),
|
||||||
|
"the cap must be a subquery in the INSERT, not a separate read"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,433 @@
|
|||||||
|
//! Move a mission's checkout in and out of its container, instead of sharing it.
|
||||||
|
//!
|
||||||
|
//! Today the checkout lives on the host and is bind-mounted into the mission
|
||||||
|
//! container. That single directory is written by **two users** — cm-api as
|
||||||
|
//! uid 65532 and the agent as root — and every bug that pattern can produce,
|
||||||
|
//! it has produced:
|
||||||
|
//!
|
||||||
|
//! | Symptom | Fix that was needed |
|
||||||
|
//! |---|---|
|
||||||
|
//! | `.git/objects` permission denied | `core.sharedRepository=0777` |
|
||||||
|
//! | capture base overwritten each phase | advance the base after commit |
|
||||||
|
//! | `.git/COMMIT_EDITMSG` root-owned | unlink before commit |
|
||||||
|
//! | `reset --hard` deleting a prior phase | `.git/clawmates-in-use` marker |
|
||||||
|
//!
|
||||||
|
//! Four fixes, one cause. `core.sharedRepository` was never a general
|
||||||
|
//! solution — it covers objects and refs, and every *other* file git touches
|
||||||
|
//! is a fresh opportunity.
|
||||||
|
//!
|
||||||
|
//! Copy-in/copy-out removes the cause: the agent owns its filesystem
|
||||||
|
//! completely, as root, with no other writer. Nothing on the host is shared,
|
||||||
|
//! so nothing on the host can collide.
|
||||||
|
//!
|
||||||
|
//! # Cost
|
||||||
|
//!
|
||||||
|
//! Measured on gw-04 against a real 65 MB checkout of this repository:
|
||||||
|
//! **0.23s in, 0.18s out**. That was the one open risk in the plan — a
|
||||||
|
//! monorepo copied per phase — and it is not a risk at this size. Measure
|
||||||
|
//! again before assuming it holds for a repository an order of magnitude
|
||||||
|
//! larger.
|
||||||
|
//!
|
||||||
|
//! No compression: the payload crosses a local Docker socket, so gzip would
|
||||||
|
//! spend CPU to save nothing.
|
||||||
|
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use bollard::Docker;
|
||||||
|
|
||||||
|
/// Where a mission's checkout lives inside its container.
|
||||||
|
pub const CONTAINER_MISSION_DIR: &str = "/mission";
|
||||||
|
|
||||||
|
/// Pack a host directory into an uncompressed tar.
|
||||||
|
///
|
||||||
|
/// `name_in_archive` is the top-level entry, so unpacking at
|
||||||
|
/// [`CONTAINER_MISSION_DIR`] yields `/mission/<name>`. Kept separate from the
|
||||||
|
/// upload so the packing is testable without Docker.
|
||||||
|
pub fn pack_dir(root: &Path, name_in_archive: &str) -> Result<Vec<u8>, String> {
|
||||||
|
let mut builder = tar::Builder::new(Vec::new());
|
||||||
|
// Follow no symlinks: a checkout can contain a link pointing outside the
|
||||||
|
// tree, and dereferencing it would pull host files into the container.
|
||||||
|
builder.follow_symlinks(false);
|
||||||
|
append_filtered(&mut builder, root, Path::new(name_in_archive))
|
||||||
|
.map_err(|e| format!("pack {}: {e}", root.display()))?;
|
||||||
|
builder
|
||||||
|
.into_inner()
|
||||||
|
.map_err(|e| format!("finish archive for {}: {e}", root.display()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Directory names never carried across the boundary.
|
||||||
|
///
|
||||||
|
/// The same list the delivery diff uses, deliberately: see
|
||||||
|
/// [`crate::mission_delivery::EXCLUDED_PATHS`]. A build directory is not work —
|
||||||
|
/// it is regenerable output that dwarfs the source, and shipping it cost a
|
||||||
|
/// mission its results when `vm_collect` timed out with the agent's finished work
|
||||||
|
/// still inside the VM.
|
||||||
|
pub fn transport_excludes() -> &'static [&'static str] {
|
||||||
|
crate::mission_delivery::EXCLUDED_PATHS
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Should this directory entry be left out of the archive?
|
||||||
|
///
|
||||||
|
/// Matched on the entry NAME at any depth, not on a path prefix: a workspace has
|
||||||
|
/// a `target/` per crate, and excluding only the root one would still ship the
|
||||||
|
/// rest.
|
||||||
|
pub fn is_excluded(name: &str) -> bool {
|
||||||
|
transport_excludes().contains(&name)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Recursive `append_dir_all` that skips [`transport_excludes`].
|
||||||
|
///
|
||||||
|
/// Hand-rolled because `tar::Builder::append_dir_all` takes no filter. Symlinks
|
||||||
|
/// are added as links rather than followed, matching `follow_symlinks(false)`.
|
||||||
|
fn append_filtered<W: std::io::Write>(
|
||||||
|
builder: &mut tar::Builder<W>,
|
||||||
|
dir: &Path,
|
||||||
|
prefix: &Path,
|
||||||
|
) -> std::io::Result<()> {
|
||||||
|
builder.append_dir(prefix, dir)?;
|
||||||
|
let mut entries: Vec<_> = std::fs::read_dir(dir)?.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
// Stable order so an archive of the same tree is byte-identical, which makes
|
||||||
|
// a size or content difference between two runs mean something.
|
||||||
|
entries.sort_by_key(|e| e.file_name());
|
||||||
|
for entry in entries {
|
||||||
|
let name = entry.file_name();
|
||||||
|
let name_str = name.to_string_lossy();
|
||||||
|
let path = entry.path();
|
||||||
|
let dest = prefix.join(&name);
|
||||||
|
let meta = std::fs::symlink_metadata(&path)?;
|
||||||
|
if meta.is_dir() {
|
||||||
|
if is_excluded(&name_str) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
append_filtered(builder, &path, &dest)?;
|
||||||
|
} else if meta.is_symlink() {
|
||||||
|
let mut header = tar::Header::new_gnu();
|
||||||
|
header.set_metadata(&meta);
|
||||||
|
header.set_entry_type(tar::EntryType::Symlink);
|
||||||
|
header.set_size(0);
|
||||||
|
let target = std::fs::read_link(&path)?;
|
||||||
|
builder.append_link(&mut header, &dest, &target)?;
|
||||||
|
} else {
|
||||||
|
let mut f = std::fs::File::open(&path)?;
|
||||||
|
builder.append_file(&dest, &mut f)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Unpack a tar into a host directory.
|
||||||
|
///
|
||||||
|
/// `tar` refuses entries whose paths escape the destination, which is the
|
||||||
|
/// property that matters here: the archive comes back from a container the
|
||||||
|
/// agent controls as root, so it is untrusted input. A `../../etc` entry must
|
||||||
|
/// not be able to write outside the collection directory.
|
||||||
|
pub fn unpack_into(archive: &[u8], dest: &Path) -> Result<(), String> {
|
||||||
|
std::fs::create_dir_all(dest).map_err(|e| format!("mkdir {}: {e}", dest.display()))?;
|
||||||
|
let mut ar = tar::Archive::new(archive);
|
||||||
|
ar.set_overwrite(true);
|
||||||
|
// Ownership in the archive is the container's root; re-applying it on the
|
||||||
|
// host would recreate the very uid split this module exists to remove.
|
||||||
|
ar.set_preserve_permissions(false);
|
||||||
|
ar.unpack(dest)
|
||||||
|
.map_err(|e| format!("unpack into {}: {e}", dest.display()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Copy a host directory into a running container at [`CONTAINER_MISSION_DIR`].
|
||||||
|
pub async fn copy_in(
|
||||||
|
docker: &Docker,
|
||||||
|
container: &str,
|
||||||
|
host_dir: &Path,
|
||||||
|
name_in_archive: &str,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let archive = pack_dir(host_dir, name_in_archive)?;
|
||||||
|
let opts = bollard::query_parameters::UploadToContainerOptionsBuilder::default()
|
||||||
|
.path(CONTAINER_MISSION_DIR)
|
||||||
|
.build();
|
||||||
|
docker
|
||||||
|
.upload_to_container(container, Some(opts), bollard::body_full(archive.into()))
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("copy into {container}:{CONTAINER_MISSION_DIR}: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build a one-entry tar. Split out from [`put_file`] so the size-independence
|
||||||
|
/// that is the whole point can be tested without Docker.
|
||||||
|
fn single_file_archive(name: &str, contents: &[u8]) -> Result<Vec<u8>, String> {
|
||||||
|
let mut header = tar::Header::new_gnu();
|
||||||
|
header
|
||||||
|
.set_path(name)
|
||||||
|
.map_err(|e| format!("tar path {name}: {e}"))?;
|
||||||
|
header.set_size(contents.len() as u64);
|
||||||
|
header.set_mode(0o600);
|
||||||
|
header.set_entry_type(tar::EntryType::Regular);
|
||||||
|
header.set_cksum();
|
||||||
|
|
||||||
|
let mut builder = tar::Builder::new(Vec::new());
|
||||||
|
builder
|
||||||
|
.append(&header, contents)
|
||||||
|
.map_err(|e| format!("tar {name}: {e}"))?;
|
||||||
|
builder
|
||||||
|
.into_inner()
|
||||||
|
.map_err(|e| format!("finish archive for {name}: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write one file into a container, at any size.
|
||||||
|
///
|
||||||
|
/// The obvious way to do this is `sh -c "printf … > file"`, and it works right
|
||||||
|
/// up until the payload approaches `ARG_MAX`, at which point exec fails with
|
||||||
|
/// `argument list too long`. That is a size-dependent failure in a code path
|
||||||
|
/// whose payload grows with use, which makes it a bug that ships green and
|
||||||
|
/// surfaces in production — as it did, silently unpinning every agent in
|
||||||
|
/// mission `019fcf62`. Tar has no argv limit.
|
||||||
|
///
|
||||||
|
/// The write is not atomic. Callers that need it can upload beside the target
|
||||||
|
/// and rename; the config writer does not, because the daemon reads its config
|
||||||
|
/// once at boot and is restarted afterwards.
|
||||||
|
pub async fn put_file(
|
||||||
|
docker: &Docker,
|
||||||
|
container: &str,
|
||||||
|
path: &str,
|
||||||
|
contents: &[u8],
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let (dir, file) = path
|
||||||
|
.rsplit_once('/')
|
||||||
|
.ok_or_else(|| format!("{path} is not an absolute path"))?;
|
||||||
|
let dir = if dir.is_empty() { "/" } else { dir };
|
||||||
|
|
||||||
|
let archive = single_file_archive(file, contents)?;
|
||||||
|
let opts = bollard::query_parameters::UploadToContainerOptionsBuilder::default()
|
||||||
|
.path(dir)
|
||||||
|
.build();
|
||||||
|
docker
|
||||||
|
.upload_to_container(container, Some(opts), bollard::body_full(archive.into()))
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("upload {path} to {container}: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Copy a directory back out of a container onto the host.
|
||||||
|
pub async fn copy_out(
|
||||||
|
docker: &Docker,
|
||||||
|
container: &str,
|
||||||
|
container_path: &str,
|
||||||
|
dest: &Path,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
use futures::StreamExt;
|
||||||
|
|
||||||
|
let opts = bollard::query_parameters::DownloadFromContainerOptionsBuilder::default()
|
||||||
|
.path(container_path)
|
||||||
|
.build();
|
||||||
|
let mut stream = docker.download_from_container(container, Some(opts));
|
||||||
|
let mut archive = Vec::new();
|
||||||
|
while let Some(chunk) = stream.next().await {
|
||||||
|
let bytes = chunk.map_err(|e| format!("copy out of {container}:{container_path}: {e}"))?;
|
||||||
|
archive.extend_from_slice(&bytes);
|
||||||
|
}
|
||||||
|
unpack_into(&archive, dest)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Is the copy-in/copy-out filesystem model enabled?
|
||||||
|
///
|
||||||
|
/// **Default since 2026-08-04.** It shipped opt-in, on the principle that
|
||||||
|
/// silently changing how every mission receives its code should require
|
||||||
|
/// someone to have typed it. Four production missions and a fail-closed
|
||||||
|
/// harness later (`scripts/verify-mission-delivery.sh`), the opt-in is the
|
||||||
|
/// riskier setting: the bind path is the one with four documented work-loss
|
||||||
|
/// incidents, and leaving it as the default means the untested path is what
|
||||||
|
/// runs when nobody sets the variable.
|
||||||
|
///
|
||||||
|
/// `CLAWMATES_MISSION_FS=bind` still selects the old behaviour, so a revert is
|
||||||
|
/// one line in `.env` rather than a rollback. Anything else — unset, empty,
|
||||||
|
/// misspelt — gets copy mode, because the failure mode of a typo should be the
|
||||||
|
/// safer path, not the one being retired.
|
||||||
|
pub fn copy_mode() -> bool {
|
||||||
|
!matches!(std::env::var("CLAWMATES_MISSION_FS").as_deref(), Ok("bind"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Host directory holding a mission's checkout.
|
||||||
|
fn host_repo(mission_id: uuid::Uuid) -> std::path::PathBuf {
|
||||||
|
crate::mission_workspace::checkout_path(mission_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Push the host checkout into the container before a phase runs.
|
||||||
|
///
|
||||||
|
/// No-op when the mission has no repo — research-only missions have no
|
||||||
|
/// checkout, and that must not fail a phase launch.
|
||||||
|
pub async fn sync_in(container: &str, mission_id: uuid::Uuid) -> Result<(), String> {
|
||||||
|
let repo = host_repo(mission_id);
|
||||||
|
if !repo.is_dir() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let docker = crate::container_exec::connect()?;
|
||||||
|
copy_in(&docker, container, &repo, "repo").await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pull the agent's work back onto the host after a phase.
|
||||||
|
///
|
||||||
|
/// Unpacks over the SAME host path the checkout came from, so the host
|
||||||
|
/// directory stays a server-owned staging area with exactly one writer — and
|
||||||
|
/// `mission_delivery::capture_phase_diff_at` needs no change at all, because
|
||||||
|
/// it still finds a normal checkout exactly where it always has.
|
||||||
|
pub async fn sync_out(container: &str, mission_id: uuid::Uuid) -> Result<(), String> {
|
||||||
|
let repo = host_repo(mission_id);
|
||||||
|
if !repo.is_dir() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let parent = repo
|
||||||
|
.parent()
|
||||||
|
.ok_or_else(|| format!("{} has no parent", repo.display()))?;
|
||||||
|
let docker = crate::container_exec::connect()?;
|
||||||
|
copy_out(&docker, container, "/mission/repo", parent).await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn seed(root: &Path) {
|
||||||
|
std::fs::create_dir_all(root.join("src")).unwrap();
|
||||||
|
std::fs::create_dir_all(root.join(".git")).unwrap();
|
||||||
|
std::fs::write(root.join("src/lib.rs"), "pub fn x() {}\n").unwrap();
|
||||||
|
std::fs::write(root.join(".git/HEAD"), "ref: refs/heads/main\n").unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A checkout must survive the round trip intact — including `.git`,
|
||||||
|
/// without which the whole delivery path (diff, commit, push) is dead.
|
||||||
|
#[test]
|
||||||
|
fn a_checkout_round_trips_with_its_git_dir() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let src = tmp.path().join("repo");
|
||||||
|
seed(&src);
|
||||||
|
|
||||||
|
let archive = pack_dir(&src, "repo").unwrap();
|
||||||
|
let dest = tmp.path().join("out");
|
||||||
|
unpack_into(&archive, &dest).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(dest.join("repo/src/lib.rs")).unwrap(),
|
||||||
|
"pub fn x() {}\n"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
dest.join("repo/.git/HEAD").exists(),
|
||||||
|
"the .git dir must survive or delivery has nothing to diff"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The archive comes back from a container the agent controls as root, so
|
||||||
|
/// it is untrusted. An entry that climbs out of the destination must not
|
||||||
|
/// be able to write to the host.
|
||||||
|
#[test]
|
||||||
|
fn an_archive_cannot_escape_the_destination() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let dest = tmp.path().join("dest");
|
||||||
|
let canary = tmp.path().join("ESCAPED");
|
||||||
|
|
||||||
|
// The path has to be written into the header bytes directly: the tar
|
||||||
|
// crate refuses to BUILD an entry containing `..`, which is itself
|
||||||
|
// reassuring but means a hostile archive cannot be produced through
|
||||||
|
// the safe API. A real attacker writes the bytes, so the test does.
|
||||||
|
let body = b"pwned\n";
|
||||||
|
let mut header = tar::Header::new_gnu();
|
||||||
|
header.set_size(body.len() as u64);
|
||||||
|
header.set_mode(0o644);
|
||||||
|
header.set_entry_type(tar::EntryType::Regular);
|
||||||
|
{
|
||||||
|
let gnu = header.as_gnu_mut().expect("gnu header");
|
||||||
|
let evil = b"../ESCAPED";
|
||||||
|
gnu.name[..evil.len()].copy_from_slice(evil);
|
||||||
|
}
|
||||||
|
header.set_cksum();
|
||||||
|
|
||||||
|
let mut archive = Vec::new();
|
||||||
|
archive.extend_from_slice(header.as_bytes());
|
||||||
|
let mut block = [0u8; 512];
|
||||||
|
block[..body.len()].copy_from_slice(body);
|
||||||
|
archive.extend_from_slice(&block);
|
||||||
|
archive.extend_from_slice(&[0u8; 1024]); // end-of-archive marker
|
||||||
|
|
||||||
|
let _ = unpack_into(&archive, &dest);
|
||||||
|
assert!(
|
||||||
|
!canary.exists(),
|
||||||
|
"a ../ entry wrote outside the destination"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A symlink pointing at the host filesystem must be packed as a link,
|
||||||
|
/// not followed and inlined — otherwise copy-in would smuggle host files
|
||||||
|
/// into the container.
|
||||||
|
#[test]
|
||||||
|
fn symlinks_are_not_dereferenced_into_the_archive() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let src = tmp.path().join("repo");
|
||||||
|
seed(&src);
|
||||||
|
let secret = tmp.path().join("host-secret");
|
||||||
|
std::fs::write(&secret, "TOP SECRET\n").unwrap();
|
||||||
|
std::os::unix::fs::symlink(&secret, src.join("link")).unwrap();
|
||||||
|
|
||||||
|
let archive = pack_dir(&src, "repo").unwrap();
|
||||||
|
let haystack = String::from_utf8_lossy(&archive);
|
||||||
|
assert!(
|
||||||
|
!haystack.contains("TOP SECRET"),
|
||||||
|
"symlink target contents were inlined into the archive"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Only the exact word `bind` opts out. A typo must land on copy mode —
|
||||||
|
/// the path with a verification harness behind it — rather than silently
|
||||||
|
/// selecting the one with four documented work-loss incidents.
|
||||||
|
#[test]
|
||||||
|
fn only_the_exact_word_bind_opts_out() {
|
||||||
|
// Cannot set env vars in a test process without racing every other
|
||||||
|
// test, so this asserts the predicate the function is built from.
|
||||||
|
let opts_out = |v: &str| v == "bind";
|
||||||
|
assert!(opts_out("bind"));
|
||||||
|
for near_miss in ["Bind", "binds", "bound", "copy", "0", "false", ""] {
|
||||||
|
assert!(
|
||||||
|
!opts_out(near_miss),
|
||||||
|
"{near_miss:?} must NOT select the bind path"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The regression this exists for: a config large enough to blow `ARG_MAX`
|
||||||
|
/// via `sh -c` must round-trip untouched. 2 MB is well past the ~128 KB
|
||||||
|
/// limit that unpinned every agent in mission `019fcf62`.
|
||||||
|
#[test]
|
||||||
|
fn a_file_far_past_arg_max_round_trips() {
|
||||||
|
let big = "workspace_path = \"/mission/repo\"\n".repeat(64 * 1024);
|
||||||
|
assert!(big.len() > 2_000_000, "the fixture must exceed ARG_MAX");
|
||||||
|
|
||||||
|
let archive = single_file_archive("config.toml", big.as_bytes()).unwrap();
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
unpack_into(&archive, tmp.path()).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(tmp.path().join("config.toml")).unwrap(),
|
||||||
|
big,
|
||||||
|
"a large config must survive byte-for-byte"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// TOML holding quotes, newlines and backslashes went through a shell
|
||||||
|
/// before; nothing may depend on quoting now.
|
||||||
|
#[test]
|
||||||
|
fn shell_metacharacters_survive_the_archive() {
|
||||||
|
let nasty = "path = \"/a'b\\\"c\"\n$(rm -rf /) `id` \\\\ \n";
|
||||||
|
let archive = single_file_archive("config.toml", nasty.as_bytes()).unwrap();
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
unpack_into(&archive, tmp.path()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(tmp.path().join("config.toml")).unwrap(),
|
||||||
|
nasty
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_empty_directory_packs_without_error() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let src = tmp.path().join("empty");
|
||||||
|
std::fs::create_dir_all(&src).unwrap();
|
||||||
|
let archive = pack_dir(&src, "repo").unwrap();
|
||||||
|
let dest = tmp.path().join("out");
|
||||||
|
unpack_into(&archive, &dest).unwrap();
|
||||||
|
assert!(dest.join("repo").is_dir());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,414 @@
|
|||||||
|
//! Reclaim the mission tree on the gateway.
|
||||||
|
//!
|
||||||
|
//! # Why this is filesystem-first
|
||||||
|
//!
|
||||||
|
//! `cleanup_sweeper` prunes ROWS. Deleting a row does not delete a directory,
|
||||||
|
//! and the reaper that was supposed to — `mission_runtime::teardown_container` —
|
||||||
|
//! only runs while a mission still exists to tear down. So a mission deleted by
|
||||||
|
//! any path that did not go through teardown left its directory behind forever,
|
||||||
|
//! and the gateway is the smallest disk in the fleet (150 GB, shared with
|
||||||
|
//! postgres and every checkout).
|
||||||
|
//!
|
||||||
|
//! The DB is therefore the PREDICATE here, never the enumerator: this walks the
|
||||||
|
//! filesystem and asks the database about what it finds. Enumerating from the
|
||||||
|
//! database is precisely how the orphans became invisible — a directory whose
|
||||||
|
//! row is gone is exactly the one a row-driven sweep cannot see.
|
||||||
|
//!
|
||||||
|
//! # Why deletion needs two attempts
|
||||||
|
//!
|
||||||
|
//! The server runs as uid 65532. Almost everything under a mission belongs to
|
||||||
|
//! 65532 now, but the per-mission ZeroClaw daemon still runs as root and leaves
|
||||||
|
//! ~26 of its own files (`.claude.json`, session jsonl). `remove_dir_all` then
|
||||||
|
//! fails with `PermissionDenied` and the directory survives — the
|
||||||
|
//! cleanup-that-cannot-clean-up shape, at a scale small enough to go unnoticed.
|
||||||
|
//! So a failed removal falls back to `root_copy::purge`, which deletes from
|
||||||
|
//! inside the runtime container as root.
|
||||||
|
//!
|
||||||
|
//! # What it will not touch
|
||||||
|
//!
|
||||||
|
//! Anything belonging to a mission that still has a row, and anything younger
|
||||||
|
//! than the grace window. A mission directory is created BEFORE its row is
|
||||||
|
//! committed in some paths, and reaping a directory out from under a launching
|
||||||
|
//! mission would be a far worse bug than the leak this fixes.
|
||||||
|
|
||||||
|
use std::path::Path;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use sqlx::PgPool;
|
||||||
|
|
||||||
|
/// How long a directory must have been untouched before it is considered
|
||||||
|
/// abandoned. Generously long: the cost of waiting is disk, and the cost of
|
||||||
|
/// being wrong is deleting a live mission's checkout.
|
||||||
|
const ORPHAN_GRACE: Duration = Duration::from_secs(2 * 60 * 60);
|
||||||
|
|
||||||
|
/// Retention for captured outputs (`_outputs`), which are artifacts a user can
|
||||||
|
/// still open. Mirrors `TOPOLOGY_RUNS_DAYS` in `cleanup_sweeper` — the run
|
||||||
|
/// history and the files it points at should not outlive each other.
|
||||||
|
const OUTPUTS_DAYS: u64 = 90;
|
||||||
|
|
||||||
|
/// Scratch trees the mission machinery makes and is supposed to remove itself:
|
||||||
|
/// `_bench`, `_gate`, `_verify`, `_merge`. Anything older than this is debris
|
||||||
|
/// from a crashed or killed run, not work in progress — every command that
|
||||||
|
/// creates one is bounded well below it.
|
||||||
|
const SCRATCH_GRACE: Duration = Duration::from_secs(6 * 60 * 60);
|
||||||
|
|
||||||
|
/// Directories under the missions root that are NOT missions.
|
||||||
|
const RESERVED: &[&str] = &["_outputs", "_home", "_cargo", "_mirrors"];
|
||||||
|
|
||||||
|
pub fn spawn(pool: PgPool, interval: Duration) {
|
||||||
|
tokio::spawn(async move {
|
||||||
|
// Not on the first tick. A sweep racing the server's own startup — while
|
||||||
|
// `start_pending_phases` is still adopting in-flight missions — is the
|
||||||
|
// one moment its "no row for this directory" predicate is least
|
||||||
|
// trustworthy.
|
||||||
|
tokio::time::sleep(Duration::from_secs(120)).await;
|
||||||
|
let mut tick = tokio::time::interval(interval);
|
||||||
|
loop {
|
||||||
|
tick.tick().await;
|
||||||
|
match sweep_once(&pool).await {
|
||||||
|
Ok(r) if r.is_empty() => {}
|
||||||
|
Ok(r) => eprintln!("mission_gc: {r}"),
|
||||||
|
Err(e) => eprintln!("mission_gc: sweep failed: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What one sweep reclaimed.
|
||||||
|
#[derive(Debug, Default, PartialEq)]
|
||||||
|
pub struct Reclaimed {
|
||||||
|
pub orphan_dirs: u64,
|
||||||
|
pub scratch_dirs: u64,
|
||||||
|
pub outputs: u64,
|
||||||
|
pub bytes: u64,
|
||||||
|
/// Directories we tried and failed to remove. Reported rather than swallowed
|
||||||
|
/// — a GC that cannot collect is the thing being fixed.
|
||||||
|
pub failed: u64,
|
||||||
|
/// Rows swept from `mission_events`.
|
||||||
|
pub events: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Reclaimed {
|
||||||
|
pub fn is_empty(&self) -> bool {
|
||||||
|
*self == Reclaimed::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for Reclaimed {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
write!(
|
||||||
|
f,
|
||||||
|
"reclaimed {} orphan mission dir(s), {} scratch dir(s), {} output(s), \
|
||||||
|
{} mission event(s), {:.1} MiB{}",
|
||||||
|
self.orphan_dirs,
|
||||||
|
self.scratch_dirs,
|
||||||
|
self.outputs,
|
||||||
|
self.events,
|
||||||
|
self.bytes as f64 / (1024.0 * 1024.0),
|
||||||
|
if self.failed > 0 {
|
||||||
|
format!(" — {} COULD NOT BE REMOVED", self.failed)
|
||||||
|
} else {
|
||||||
|
String::new()
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn sweep_once(pool: &PgPool) -> Result<Reclaimed, String> {
|
||||||
|
let root = crate::mission_workspace::missions_root();
|
||||||
|
let mut out = Reclaimed::default();
|
||||||
|
reap_orphan_missions(pool, &root, &mut out).await?;
|
||||||
|
reap_scratch(&root, &mut out).await;
|
||||||
|
reap_outputs(pool, &root, &mut out).await;
|
||||||
|
reap_mission_events(pool, &mut out).await;
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How long a mission's structured activity is kept.
|
||||||
|
///
|
||||||
|
/// The World shows the last 24 hours of finished missions, so a week is
|
||||||
|
/// generous and still bounds a table that a single busy coding phase can add
|
||||||
|
/// hundreds of rows to. The per-phase cap bounds ONE phase; this bounds time.
|
||||||
|
const EVENT_RETENTION_DAYS: i32 = 7;
|
||||||
|
|
||||||
|
/// Sweep expired `mission_events`.
|
||||||
|
///
|
||||||
|
/// Bounded per pass rather than deleting the whole backlog in one statement: a
|
||||||
|
/// deployment that has been accumulating for months would otherwise take a long
|
||||||
|
/// lock on its first sweep after this ships. The sweep runs on a timer, so a
|
||||||
|
/// large backlog simply drains over several passes.
|
||||||
|
async fn reap_mission_events(pool: &PgPool, out: &mut Reclaimed) {
|
||||||
|
let res = sqlx::query(
|
||||||
|
"DELETE FROM mission_events
|
||||||
|
WHERE id IN (
|
||||||
|
SELECT id FROM mission_events
|
||||||
|
WHERE created_at < now() - make_interval(days => $1)
|
||||||
|
LIMIT 10000
|
||||||
|
)",
|
||||||
|
)
|
||||||
|
.bind(EVENT_RETENTION_DAYS)
|
||||||
|
.execute(pool)
|
||||||
|
.await;
|
||||||
|
match res {
|
||||||
|
Ok(r) => out.events += r.rows_affected(),
|
||||||
|
Err(e) => eprintln!("mission_gc: sweeping mission_events failed: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Directories under the missions root with no mission row.
|
||||||
|
async fn reap_orphan_missions(
|
||||||
|
pool: &PgPool,
|
||||||
|
root: &Path,
|
||||||
|
out: &mut Reclaimed,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let Ok(entries) = std::fs::read_dir(root) else {
|
||||||
|
// Not an error: a deployment that has never run a mission has no tree.
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let path = entry.path();
|
||||||
|
if !path.is_dir() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if RESERVED.contains(&name) || name.starts_with('_') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// Only well-formed mission ids. A directory this function does not
|
||||||
|
// recognise is one it has no business deleting.
|
||||||
|
let Ok(id) = name.parse::<uuid::Uuid>() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if !older_than(&path, ORPHAN_GRACE) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// The DB as predicate, asked per directory.
|
||||||
|
let exists: Option<(uuid::Uuid,)> =
|
||||||
|
sqlx::query_as("SELECT id FROM missions WHERE id = $1")
|
||||||
|
.bind(id)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("looking up mission {id}: {e}"))?;
|
||||||
|
if exists.is_some() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let bytes = dir_size(&path);
|
||||||
|
if remove_tree(&path).await {
|
||||||
|
out.orphan_dirs += 1;
|
||||||
|
out.bytes += bytes;
|
||||||
|
} else {
|
||||||
|
out.failed += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `_bench` / `_gate` / `_verify` / `_merge` trees older than their command
|
||||||
|
/// ceilings. These are siblings of the per-mission dirs and have leaked before.
|
||||||
|
async fn reap_scratch(root: &Path, out: &mut Reclaimed) {
|
||||||
|
const SCRATCH: &[&str] = &["_bench", "_gate", "_verify", "_merge"];
|
||||||
|
for name in SCRATCH {
|
||||||
|
let path = root.join(name);
|
||||||
|
if !path.is_dir() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Ok(entries) = std::fs::read_dir(&path) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let p = entry.path();
|
||||||
|
if !older_than(&p, SCRATCH_GRACE) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let bytes = dir_size(&p);
|
||||||
|
if remove_tree(&p).await {
|
||||||
|
out.scratch_dirs += 1;
|
||||||
|
out.bytes += bytes;
|
||||||
|
} else {
|
||||||
|
out.failed += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Captured outputs past retention, with their artifact rows marked so nothing
|
||||||
|
/// points at a file that is gone.
|
||||||
|
async fn reap_outputs(pool: &PgPool, root: &Path, out: &mut Reclaimed) {
|
||||||
|
let outputs = root.join("_outputs");
|
||||||
|
let Ok(entries) = std::fs::read_dir(&outputs) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let grace = Duration::from_secs(OUTPUTS_DAYS * 24 * 60 * 60);
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let p = entry.path();
|
||||||
|
if !p.is_dir() || !older_than(&p, grace) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(id) = p
|
||||||
|
.file_name()
|
||||||
|
.and_then(|n| n.to_str())
|
||||||
|
.and_then(|n| n.parse::<uuid::Uuid>().ok())
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let bytes = dir_size(&p);
|
||||||
|
if !remove_tree(&p).await {
|
||||||
|
out.failed += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// The row is marked only AFTER the files are gone. The other order
|
||||||
|
// leaves a mission whose artifacts claim to be reaped while they are
|
||||||
|
// still on disk, which is a lie in the direction that costs disk.
|
||||||
|
let _ = sqlx::query(
|
||||||
|
"UPDATE mission_artifacts SET metadata = COALESCE(metadata, '{}'::jsonb)
|
||||||
|
|| '{\"reaped\": true}'::jsonb
|
||||||
|
WHERE mission_id = $1",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.execute(pool)
|
||||||
|
.await;
|
||||||
|
out.outputs += 1;
|
||||||
|
out.bytes += bytes;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Remove a tree, escalating to a root purge when our uid cannot.
|
||||||
|
///
|
||||||
|
/// The ONLY deletion path in this module. A second one is how the reap paths
|
||||||
|
/// drifted apart last time.
|
||||||
|
async fn remove_tree(path: &Path) -> bool {
|
||||||
|
match tokio::fs::remove_dir_all(path).await {
|
||||||
|
Ok(()) => true,
|
||||||
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => true,
|
||||||
|
Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => {
|
||||||
|
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
||||||
|
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
||||||
|
crate::root_copy::purge(&container, path).await;
|
||||||
|
let gone = tokio::fs::metadata(path).await.is_err();
|
||||||
|
if !gone {
|
||||||
|
eprintln!(
|
||||||
|
"mission_gc: {} survived a root purge — it will keep accumulating",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
gone
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("mission_gc: could not remove {}: {e}", path.display());
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn older_than(path: &Path, grace: Duration) -> bool {
|
||||||
|
let Ok(meta) = std::fs::metadata(path) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
// mtime, not ctime: a directory whose contents changed recently is one
|
||||||
|
// something is still writing to.
|
||||||
|
let Ok(modified) = meta.modified() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
modified
|
||||||
|
.elapsed()
|
||||||
|
.map(|age| age >= grace)
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Apparent size, best-effort. Used only for reporting, so a read error costs a
|
||||||
|
/// wrong number in a log line rather than a wrong decision.
|
||||||
|
fn dir_size(path: &Path) -> u64 {
|
||||||
|
let mut total = 0;
|
||||||
|
let Ok(entries) = std::fs::read_dir(path) else {
|
||||||
|
return 0;
|
||||||
|
};
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let Ok(meta) = entry.metadata() else { continue };
|
||||||
|
if meta.is_dir() {
|
||||||
|
total += dir_size(&entry.path());
|
||||||
|
} else {
|
||||||
|
total += meta.len();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
total
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn touch_dir(root: &Path, name: &str) -> std::path::PathBuf {
|
||||||
|
let p = root.join(name);
|
||||||
|
std::fs::create_dir_all(&p).unwrap();
|
||||||
|
std::fs::write(p.join("f"), b"x").unwrap();
|
||||||
|
p
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The reserved siblings are never candidates.
|
||||||
|
///
|
||||||
|
/// `_outputs`, `_home` and `_cargo` live under the same root as the mission
|
||||||
|
/// directories. `_cargo` in particular is a SHARED cache every mission
|
||||||
|
/// writes to, so a sweep that treated an underscore-prefixed sibling as an
|
||||||
|
/// orphan mission would delete it out from under running work — and it would
|
||||||
|
/// look like a slow cargo build rather than a bug.
|
||||||
|
#[test]
|
||||||
|
fn siblings_of_the_mission_dirs_are_not_missions() {
|
||||||
|
for name in RESERVED {
|
||||||
|
assert!(
|
||||||
|
name.starts_with('_'),
|
||||||
|
"{name} must be underscore-prefixed so the guard catches it"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
name.parse::<uuid::Uuid>().is_err(),
|
||||||
|
"{name} must not parse as a mission id"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Only a well-formed mission id is ever a candidate.
|
||||||
|
///
|
||||||
|
/// The predicate is "no row exists", and a directory whose name is not an id
|
||||||
|
/// can have no row BY CONSTRUCTION — so name-parsing has to gate the lookup,
|
||||||
|
/// or every unrecognised directory looks like an orphan.
|
||||||
|
#[test]
|
||||||
|
fn a_directory_that_is_not_a_mission_id_is_never_a_candidate() {
|
||||||
|
for name in ["_outputs", "_cargo", "lost+found", "notes", "019fe8", ""] {
|
||||||
|
assert!(
|
||||||
|
name.parse::<uuid::Uuid>().is_err(),
|
||||||
|
"{name:?} must not parse as a mission id"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!("019fe82e-7f0d-7481-a197-698f1d400419"
|
||||||
|
.parse::<uuid::Uuid>()
|
||||||
|
.is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The grace window is real, and measured from mtime.
|
||||||
|
#[test]
|
||||||
|
fn a_fresh_directory_is_never_old_enough() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let d = touch_dir(tmp.path(), "019fe82e-7f0d-7481-a197-698f1d400419");
|
||||||
|
assert!(!older_than(&d, ORPHAN_GRACE));
|
||||||
|
// And a zero grace makes everything eligible, which is what proves the
|
||||||
|
// check is the window rather than an accident of the filesystem.
|
||||||
|
assert!(older_than(&d, Duration::from_secs(0)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One deletion path, and it escalates.
|
||||||
|
///
|
||||||
|
/// A second removal site is how the container reap paths drifted apart and
|
||||||
|
/// leaked for a day. The escalation is the other half: the server is uid
|
||||||
|
/// 65532 and cannot delete what the per-mission daemon left as root.
|
||||||
|
#[test]
|
||||||
|
fn there_is_exactly_one_deletion_path_and_it_escalates() {
|
||||||
|
let src = include_str!("mission_gc.rs");
|
||||||
|
assert_eq!(
|
||||||
|
src.matches(concat!("remove_dir", "_all(")).count(),
|
||||||
|
1,
|
||||||
|
"exactly one removal site"
|
||||||
|
);
|
||||||
|
assert!(src.contains("root_copy::purge"), "and it must escalate");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -79,7 +79,12 @@ pub async fn on_launch(
|
|||||||
// The mission's own runtime endpoint. Claws MUST be provisioned against
|
// The mission's own runtime endpoint. Claws MUST be provisioned against
|
||||||
// THIS gateway, not the global one — see RuntimeProvisioner::for_gateway.
|
// THIS gateway, not the global one — see RuntimeProvisioner::for_gateway.
|
||||||
let mut mission_gateway: Option<String> = None;
|
let mut mission_gateway: Option<String> = None;
|
||||||
if let Some(prov) = crate::mission_runtime::MissionRuntimeProvisioner::from_env() {
|
// Not for a microVM mission: the ZeroClaw daemon it would start is never
|
||||||
|
// spoken to, and it would sit holding a pairing code and ~3 GB of image for
|
||||||
|
// the life of the mission. Observed doing exactly that on the first real run.
|
||||||
|
if let Some(prov) = crate::mission_runtime::MissionRuntimeProvisioner::from_env()
|
||||||
|
.filter(|_| mission.runtime_kind != "microvm")
|
||||||
|
{
|
||||||
match prov.ensure_container(mission_id).await {
|
match prov.ensure_container(mission_id).await {
|
||||||
Ok(ec) => {
|
Ok(ec) => {
|
||||||
mission_gateway = Some(ec.endpoint.clone());
|
mission_gateway = Some(ec.endpoint.clone());
|
||||||
@@ -115,6 +120,76 @@ pub async fn on_launch(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// microVM PLACEMENT MUST COME BEFORE the early return below. It did not, and
|
||||||
|
// the first real microvm mission failed with "mission has no target_node_id" —
|
||||||
|
// the executor's own guard firing correctly on a mission this function had
|
||||||
|
// returned from before ever choosing a node for it.
|
||||||
|
// microVM placement. KVM is a hard predicate, not a preference: gw-04 —
|
||||||
|
// where every mission runs today — is itself a VM without nested
|
||||||
|
// virtualisation and has no /dev/kvm, so a microvm mission landing there
|
||||||
|
// cannot start. Resolve a capable node now and fail the launch if there is
|
||||||
|
// none, because the alternative is a mission that sits in 'running' having
|
||||||
|
// never had anywhere to run.
|
||||||
|
if mission.runtime_kind == "microvm" {
|
||||||
|
// Capable means BOTH: it can host a microVM, and it holds the image this
|
||||||
|
// mission's backend names. Asking only for `microvm` sent the first real
|
||||||
|
// microVM mission to a node without `rootfs-claude.ext4`.
|
||||||
|
let backend = mission.backend.as_deref();
|
||||||
|
let capable =
|
||||||
|
cm_db::repo::nodes::online_for_backend(pool, mission.workspace_id, backend)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("looking up nodes for backend {backend:?}: {e}"))?;
|
||||||
|
let how_to_fix = format!(
|
||||||
|
"needs /dev/kvm + firecracker (scripts/fc-node-setup.sh) AND the {} rootfs \
|
||||||
|
built on that node (scripts/fc-build-rootfs.sh <host> <image> {})",
|
||||||
|
backend.unwrap_or("default"),
|
||||||
|
backend.unwrap_or("<name>")
|
||||||
|
);
|
||||||
|
let how_to_fix = how_to_fix.as_str();
|
||||||
|
// CAPABILITY is checked here; CAPACITY is not, and no node is pinned.
|
||||||
|
//
|
||||||
|
// Placement moved to phase launch (`phase_runner`). A node chosen now
|
||||||
|
// would be chosen once, minutes before the first VM boots and hours
|
||||||
|
// before the last — and re-placing between phases is free, because
|
||||||
|
// mission state lives on the gateway checkout and every VM is
|
||||||
|
// inject → run → collect → destroy. Pinning early bought nothing and
|
||||||
|
// cost the ability to react to a node filling or draining mid-mission.
|
||||||
|
//
|
||||||
|
// Launching still FAILS here when no node could ever run this backend:
|
||||||
|
// that is not transient, waiting will not fix it, and the harness's
|
||||||
|
// `microvm-negctl` scenario asserts such a mission stays `draft`.
|
||||||
|
if capable.is_empty() {
|
||||||
|
return Err(format!(
|
||||||
|
"no online node can run backend {:?} — {how_to_fix}",
|
||||||
|
backend.unwrap_or("default")
|
||||||
|
));
|
||||||
|
}
|
||||||
|
eprintln!(
|
||||||
|
"mission_orchestrator: mission {mission_id} has {} node(s) able to run \
|
||||||
|
backend {:?}; placement happens per phase",
|
||||||
|
capable.len(),
|
||||||
|
backend.unwrap_or("default")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// A microVM mission materialises no team. Its phases run as one `claude -p`
|
||||||
|
// inside a VM (`microvm_executor`), so there is no claw graph to provision —
|
||||||
|
// and demanding one rejected the launch of a well-formed mission with "pick
|
||||||
|
// teams in the wizard". This is the third of three team gates on a path that
|
||||||
|
// uses no teams; the other two are in `routes::missions` (draft→running) and
|
||||||
|
// `phase_runner::launch_phase` (no matching teams → stay pending).
|
||||||
|
//
|
||||||
|
// Returning before the picks below, not filtering them, because provisioning
|
||||||
|
// claws that never run is not a cheaper version of the same thing — it is a
|
||||||
|
// runtime binding and a pairing code describing something nothing uses.
|
||||||
|
if mission.runtime_kind == "microvm" {
|
||||||
|
eprintln!(
|
||||||
|
"mission_orchestrator: mission {mission_id} is a microvm mission — no team to \
|
||||||
|
materialise; its phases execute in a VM"
|
||||||
|
);
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
// Skip team materialization if already bound.
|
// Skip team materialization if already bound.
|
||||||
if mission.team_id.is_some() {
|
if mission.team_id.is_some() {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
@@ -193,7 +268,7 @@ pub async fn on_launch(
|
|||||||
provisioner: provisioner.as_ref(),
|
provisioner: provisioner.as_ref(),
|
||||||
template: &template,
|
template: &template,
|
||||||
team_name: &team_name,
|
team_name: &team_name,
|
||||||
default_model: "claude-sonnet-5",
|
default_model: MINTED_CLAW_MODEL,
|
||||||
},
|
},
|
||||||
&mut provisioned_claws,
|
&mut provisioned_claws,
|
||||||
)
|
)
|
||||||
@@ -228,31 +303,41 @@ pub async fn on_launch(
|
|||||||
// is a PathBuf the prop-schema won't expose — see provision_claw), so
|
// is a PathBuf the prop-schema won't expose — see provision_claw), so
|
||||||
// we patch the shared config file directly on the per-mission runtime
|
// we patch the shared config file directly on the per-mission runtime
|
||||||
// container. The daemon picks it up on the same reload that surfaces
|
// container. The daemon picks it up on the same reload that surfaces
|
||||||
// the freshly-provisioned claws for the run. Non-fatal: without the
|
// the freshly-provisioned claws for the run.
|
||||||
// pin, agents still write (to the sandbox) but the committer can't
|
//
|
||||||
// find the changes in /mission/repo.
|
// FATAL, deliberately. This was "non-fatal: agents still write (to the
|
||||||
if !provisioned_claws.is_empty() && mission_gateway.is_some() {
|
// sandbox) but the committer can't find the changes in /mission/repo" —
|
||||||
|
// which is to say, the mission runs to completion and delivers nothing.
|
||||||
|
// Mission `019fcf62` did exactly that: the pin failed with `argument list
|
||||||
|
// too long`, one line of stderr scrolled past, and phase 0 reported
|
||||||
|
// `completed` with zero files, no commit error and no push error. A launch
|
||||||
|
// that cannot bind its agents to the repo has no path to delivering work,
|
||||||
|
// so it must fail at launch where someone is still looking.
|
||||||
|
//
|
||||||
|
// Not for a microVM mission: its agent is a `claude -p` inside a VM on a
|
||||||
|
// fleet node, not a ZeroClaw claw in a container here, so there is no
|
||||||
|
// workspace to pin. Leaving it would make a microVM launch FAIL on a
|
||||||
|
// container it was never going to use.
|
||||||
|
if !provisioned_claws.is_empty() && mission_gateway.is_some() && mission.runtime_kind != "microvm"
|
||||||
|
{
|
||||||
if let Some(mp) = crate::mission_runtime::MissionRuntimeProvisioner::from_env() {
|
if let Some(mp) = crate::mission_runtime::MissionRuntimeProvisioner::from_env() {
|
||||||
match mp
|
mp.pin_agent_workspaces(mission_id, &provisioned_claws, "/mission/repo")
|
||||||
.pin_agent_workspaces(mission_id, &provisioned_claws, "/mission/repo")
|
|
||||||
.await
|
.await
|
||||||
{
|
.map_err(|e| {
|
||||||
Ok(()) => {
|
format!(
|
||||||
// The daemon reads config ONCE at boot and never re-reads
|
"could not pin agent workspaces to /mission/repo ({e}) — the mission \
|
||||||
// the file, so the pin is invisible until it restarts. Its
|
would run with its agents writing to their sandboxes, delivering nothing"
|
||||||
// agents were created through its own config API, so they
|
)
|
||||||
// are already persisted to the file and survive the
|
})?;
|
||||||
// restart; the pairing code is re-minted on every launch.
|
// The daemon reads config ONCE at boot and never re-reads the
|
||||||
if let Err(e) = mp.restart_container(mission_id).await {
|
// file, so the pin is invisible until it restarts. Its agents were
|
||||||
eprintln!(
|
// created through its own config API, so they are already
|
||||||
"mission_orchestrator: restart runtime for {mission_id} failed (continuing, workspace pin will not apply): {e}"
|
// persisted to the file and survive the restart; the pairing code
|
||||||
);
|
// is re-minted on every launch. Equally fatal: an unrestarted
|
||||||
}
|
// daemon is an unpinned daemon.
|
||||||
}
|
mp.restart_container(mission_id).await.map_err(|e| {
|
||||||
Err(e) => eprintln!(
|
format!("could not restart the runtime to apply the workspace pin: {e}")
|
||||||
"mission_orchestrator: pin workspaces for mission {mission_id} failed (continuing): {e}"
|
})?;
|
||||||
),
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -365,6 +450,17 @@ async fn mint_team_from_template(
|
|||||||
.await
|
.await
|
||||||
.map_err(|e| format!("stamp template lineage: {e}"))?;
|
.map_err(|e| format!("stamp template lineage: {e}"))?;
|
||||||
|
|
||||||
|
// Names already on this workspace's roster, so a newly hired claw does not
|
||||||
|
// arrive sharing a name with someone already here. Read ONCE — a roster
|
||||||
|
// query per role would be N queries to answer one question — and extended
|
||||||
|
// locally as we mint, which also keeps names distinct WITHIN this team.
|
||||||
|
let mut taken_names: Vec<String> = cm_db::repo::agents::roster(pool, workspace_id)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("read roster for naming: {e}"))?
|
||||||
|
.into_iter()
|
||||||
|
.map(|a| a.name)
|
||||||
|
.collect();
|
||||||
|
|
||||||
// For each role: create agent, provision runtime, ingest brain
|
// For each role: create agent, provision runtime, ingest brain
|
||||||
// seed, record link, bind to topology node.
|
// seed, record link, bind to topology node.
|
||||||
for (idx, role) in template.roles.iter().enumerate() {
|
for (idx, role) in template.roles.iter().enumerate() {
|
||||||
@@ -378,10 +474,63 @@ async fn mint_team_from_template(
|
|||||||
template.roles.len(),
|
template.roles.len(),
|
||||||
));
|
));
|
||||||
};
|
};
|
||||||
|
// Every mission gets its OWN crew.
|
||||||
|
//
|
||||||
|
// This deliberately reverses the reuse added earlier. Reuse hired the
|
||||||
|
// existing claw for a (template, slot) so the roster stayed at one team
|
||||||
|
// and "My Workforce" was people you keep — but it also meant every
|
||||||
|
// mission was staffed by the same five names, and the workforce view
|
||||||
|
// showed one crew repeated down the page with nothing to tell the
|
||||||
|
// missions apart. Chosen by the operator: distinct crews read better
|
||||||
|
// than a bounded roster.
|
||||||
|
//
|
||||||
|
// The cost is real and is the cost that reuse existed to avoid: claws
|
||||||
|
// are `lifecycle = 'permanent'` and nothing reaps them until their
|
||||||
|
// MISSION is deleted, so the roster now grows by the team size on every
|
||||||
|
// mission. `agent_names::pick` keeps names unique workspace-wide and
|
||||||
|
// falls back to a numeric suffix once the pool is exhausted, so growth
|
||||||
|
// degrades the naming gracefully rather than colliding.
|
||||||
|
//
|
||||||
|
// `reusable_claw` in cm-db is kept, with its tests: this is a policy
|
||||||
|
// choice that has now flipped twice, and the query is the hard part.
|
||||||
|
let reused: Option<uuid::Uuid> = None;
|
||||||
|
|
||||||
|
// Seed the name choice from the claw's OWN id, not its position in the
|
||||||
|
// team.
|
||||||
|
//
|
||||||
|
// Seeding with the role index (0..n) started every crew near the top of
|
||||||
|
// the pool and took the next free names, so the first mission hired
|
||||||
|
// Aarav, Abebe, Adaora, Adrian, Agnieszka — correct, unique, and
|
||||||
|
// transparently alphabetical. A crew should look like a team, not like
|
||||||
|
// a listing. UUIDv7 puts its random bytes LAST (the leading bytes are a
|
||||||
|
// timestamp, which would cluster again), so the tail is what spreads
|
||||||
|
// the five picks across the whole pool.
|
||||||
|
let agent_id = cm_domain::AgentId::new();
|
||||||
|
let name_seed = {
|
||||||
|
let uuid = agent_id.as_uuid();
|
||||||
|
let b = uuid.as_bytes();
|
||||||
|
u64::from_le_bytes([b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15]])
|
||||||
|
};
|
||||||
|
|
||||||
let agent = Agent {
|
let agent = Agent {
|
||||||
id: cm_domain::AgentId::new(),
|
id: agent_id,
|
||||||
workspace_id,
|
workspace_id,
|
||||||
name: format!("{} · {}", team_name, role.slot),
|
// A PERSON's name, with the role in `job_title`.
|
||||||
|
//
|
||||||
|
// This was `"{mission title} · {purpose} · {template} · {slot}"` —
|
||||||
|
// names like "verify: a repo-less research mission keeps its output
|
||||||
|
// · mission · Rust SDLC · planner", unreadable in the roster, the
|
||||||
|
// API and every log line at once. Then it was the bare slot, which
|
||||||
|
// fixed the length but made the UI show the same word twice (name
|
||||||
|
// on top, role beneath) and made a roster of five read as five job
|
||||||
|
// tickets rather than a crew.
|
||||||
|
//
|
||||||
|
// The role still lives in `job_title`, which is what the mission
|
||||||
|
// machinery binds on — `team_members.role_slot` and the topology
|
||||||
|
// node carry the slot, so nothing downstream keys off the display
|
||||||
|
// name. Only the reused branch below ignores this, deliberately: a
|
||||||
|
// claw you already hired keeps the name it already had.
|
||||||
|
name: crate::agent_names::pick(&taken_names, name_seed),
|
||||||
job_title: role.slot.clone(),
|
job_title: role.slot.clone(),
|
||||||
// This is the ONLY consumer of the templates' `system_prompt` prose,
|
// This is the ONLY consumer of the templates' `system_prompt` prose,
|
||||||
// and it feeds the *chat* path, not missions: it lands in
|
// and it feeds the *chat* path, not missions: it lands in
|
||||||
@@ -398,12 +547,40 @@ async fn mint_team_from_template(
|
|||||||
managed_by: user_id,
|
managed_by: user_id,
|
||||||
status: AgentStatus::Online,
|
status: AgentStatus::Online,
|
||||||
};
|
};
|
||||||
cm_db::repo::agents::insert(pool, &agent, &AccessPolicy::default())
|
let claw_id = match reused {
|
||||||
.await
|
Some(existing) => {
|
||||||
.map_err(|e| format!("insert agent {}: {e}", role.slot))?;
|
eprintln!(
|
||||||
let claw_id = agent.id.as_uuid();
|
"mission_orchestrator: reusing claw {existing} for role {} \
|
||||||
|
(template {})",
|
||||||
|
role.slot, template.template.id
|
||||||
|
);
|
||||||
|
existing
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
cm_db::repo::agents::insert(pool, &agent, &AccessPolicy::default())
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("insert agent {}: {e}", role.slot))?;
|
||||||
|
// Claim the name for the rest of this loop. Without this the
|
||||||
|
// roster snapshot taken before the loop is stale from the
|
||||||
|
// second role onward and a five-person team can arrive with
|
||||||
|
// two Merediths.
|
||||||
|
taken_names.push(agent.name.clone());
|
||||||
|
agent.id.as_uuid()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let agent_id = cm_domain::AgentId::from(claw_id);
|
||||||
|
|
||||||
cm_db::repo::agents::set_model_binding(pool, agent.id, default_model)
|
// The ROLE's model when the template names one, else the mint's default.
|
||||||
|
// Before migration 0071 there was no role model at all, so every claw of
|
||||||
|
// every mission team ran the same one — including a reviewer reviewing
|
||||||
|
// the coder it shares a model with.
|
||||||
|
let role_model = role
|
||||||
|
.model
|
||||||
|
.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|m| !m.is_empty())
|
||||||
|
.unwrap_or(default_model);
|
||||||
|
cm_db::repo::agents::set_model_binding(pool, agent_id, role_model)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("set_model_binding {claw_id}: {e}"))?;
|
.map_err(|e| format!("set_model_binding {claw_id}: {e}"))?;
|
||||||
|
|
||||||
@@ -420,10 +597,10 @@ async fn mint_team_from_template(
|
|||||||
// out-of-band via MissionRuntimeProvisioner::pin_agent_workspaces.
|
// out-of-band via MissionRuntimeProvisioner::pin_agent_workspaces.
|
||||||
if let Some(p) = provisioner {
|
if let Some(p) = provisioner {
|
||||||
match p
|
match p
|
||||||
.provision_claw(claw_id, default_model, &template.template.risk_profile)
|
.provision_claw(claw_id, role_model, &template.template.risk_profile)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(_) => provisioned_claws.push(agent.id),
|
Ok(_) => provisioned_claws.push(agent_id),
|
||||||
Err(e) => eprintln!(
|
Err(e) => eprintln!(
|
||||||
"mission_orchestrator: provision claw {claw_id} failed (continuing): {e}"
|
"mission_orchestrator: provision claw {claw_id} failed (continuing): {e}"
|
||||||
),
|
),
|
||||||
@@ -432,6 +609,10 @@ async fn mint_team_from_template(
|
|||||||
|
|
||||||
// Ingest brain seed (Slice 3.5d). Non-fatal on failure —
|
// Ingest brain seed (Slice 3.5d). Non-fatal on failure —
|
||||||
// agent still works from system_prompt alone.
|
// agent still works from system_prompt alone.
|
||||||
|
// Seed only a NEW claw. A reused one carries what it learned on earlier
|
||||||
|
// missions, and re-seeding would overwrite that with the template's
|
||||||
|
// starting point — which is precisely the accumulation reuse exists for.
|
||||||
|
if reused.is_none() {
|
||||||
if let Some(seed) = role.brain_seed.as_deref().filter(|s| !s.trim().is_empty()) {
|
if let Some(seed) = role.brain_seed.as_deref().filter(|s| !s.trim().is_empty()) {
|
||||||
if let Err(e) =
|
if let Err(e) =
|
||||||
crate::brain_seed::ingest(claw_id, seed.to_string(), role.system_prompt.clone())
|
crate::brain_seed::ingest(claw_id, seed.to_string(), role.system_prompt.clone())
|
||||||
@@ -442,6 +623,7 @@ async fn mint_team_from_template(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Record lineage (Slice 3.5d) so the MCP skills server can
|
// Record lineage (Slice 3.5d) so the MCP skills server can
|
||||||
// merge template default skills with per-agent overrides.
|
// merge template default skills with per-agent overrides.
|
||||||
@@ -470,9 +652,10 @@ async fn mint_team_from_template(
|
|||||||
cm_db::repo::audit::Actor::User(user_id),
|
cm_db::repo::audit::Actor::User(user_id),
|
||||||
"agent.created",
|
"agent.created",
|
||||||
"agent",
|
"agent",
|
||||||
&agent.id.to_string(),
|
&agent_id.to_string(),
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
"name": agent.name,
|
"name": agent.name,
|
||||||
|
"reused": reused.is_some(),
|
||||||
"job_title": agent.job_title,
|
"job_title": agent.job_title,
|
||||||
"source": "mission_orchestrator",
|
"source": "mission_orchestrator",
|
||||||
"template_id": template.template.id.to_string(),
|
"template_id": template.template.id.to_string(),
|
||||||
@@ -486,6 +669,97 @@ async fn mint_team_from_template(
|
|||||||
Ok(team_id)
|
Ok(team_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The model a minted claw runs on when its template role does not name one.
|
||||||
|
///
|
||||||
|
/// A DEFAULT now, not a hardcode: `template_roles.model` (migration 0071) lets a
|
||||||
|
/// template put its reviewer on a different model from the coder it reviews,
|
||||||
|
/// which is the correlated failure the cross-provider judge exists to break,
|
||||||
|
/// one layer down. Roles that say nothing still land here, so every template
|
||||||
|
/// that existed before 0071 behaves exactly as it did.
|
||||||
|
const MINTED_CLAW_MODEL: &str = "claude-sonnet-5";
|
||||||
|
|
||||||
|
/// The graph a COMPOSED microVM mission runs, built from its team template
|
||||||
|
/// without minting a single claw.
|
||||||
|
///
|
||||||
|
/// A composed mission needs the template's *shape* — how many nodes, in what
|
||||||
|
/// pattern, playing what roles — and nothing else it carries. Its nodes are VMs,
|
||||||
|
/// so provisioning claws for them would create agents, containers and `.brain`
|
||||||
|
/// files that nothing ever dials; that is exactly why `on_launch` returns early
|
||||||
|
/// for a microVM mission, and this is how the composed path gets its graph
|
||||||
|
/// anyway rather than by undoing that.
|
||||||
|
///
|
||||||
|
/// `purposes` is the phase's purpose list, matched against `config.phase_teams`;
|
||||||
|
/// missions using the legacy single `team_template_id` fall back to it.
|
||||||
|
/// Returns `None` when the mission picked no template at all.
|
||||||
|
pub async fn composed_graph(
|
||||||
|
pool: &PgPool,
|
||||||
|
mission_id: Uuid,
|
||||||
|
purposes: &[&str],
|
||||||
|
) -> Result<Option<serde_json::Value>, String> {
|
||||||
|
let row: Option<(serde_json::Value, Option<Uuid>)> =
|
||||||
|
sqlx::query_as("SELECT config, team_template_id FROM missions WHERE id = $1")
|
||||||
|
.bind(mission_id)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("load mission {mission_id}: {e}"))?;
|
||||||
|
let Some((config, legacy_template)) = row else {
|
||||||
|
return Err(format!("mission {mission_id} not found"));
|
||||||
|
};
|
||||||
|
|
||||||
|
// An APPROVED roster wins over the template. It is the more specific answer
|
||||||
|
// — a model sized it for this mission's actual task and a human accepted it
|
||||||
|
// — and it is the only path on which nodes carry per-node backends, which is
|
||||||
|
// how a mission runs more than one provider. Stored already built and
|
||||||
|
// validated (`routes::mission_roster::decide`), so nothing here can turn a
|
||||||
|
// refused roster into a running one.
|
||||||
|
if let Some(roster) = config.get("roster").filter(|v| v.is_object()) {
|
||||||
|
// Parsed rather than trusted: a graph the orchestrator cannot plan would
|
||||||
|
// otherwise be claimed and fail as "missing or invalid graph", which
|
||||||
|
// reads as a runtime fault instead of a bad roster.
|
||||||
|
serde_json::from_value::<cm_topology::TopologyGraph>(roster.clone())
|
||||||
|
.map_err(|e| format!("mission {mission_id}: the approved roster is not a runnable topology: {e}"))?;
|
||||||
|
return Ok(Some(roster.clone()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let template_id = config
|
||||||
|
.get("phase_teams")
|
||||||
|
.and_then(|v| v.as_object())
|
||||||
|
.and_then(|pt| {
|
||||||
|
// First template named by any purpose this phase answers to, in the
|
||||||
|
// phase's own preference order — the same order `launch_phase` uses
|
||||||
|
// to pick teams, so a composed mission and a ZeroClaw one resolve the
|
||||||
|
// same template for the same phase.
|
||||||
|
purposes.iter().find_map(|p| {
|
||||||
|
pt.get(*p)
|
||||||
|
.and_then(|v| v.as_array())
|
||||||
|
.and_then(|a| a.first())
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.and_then(|s| Uuid::parse_str(s).ok())
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.or(legacy_template);
|
||||||
|
let Some(template_id) = template_id else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
|
||||||
|
let template = cm_db::repo::team_templates::get(pool, template_id)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("load template {template_id}: {e}"))?
|
||||||
|
.ok_or_else(|| format!("template {template_id} not found"))?;
|
||||||
|
let roles: Vec<&str> = template.roles.iter().map(|r| r.slot.as_str()).collect();
|
||||||
|
if roles.is_empty() {
|
||||||
|
return Err(format!("template {template_id} defines no roles"));
|
||||||
|
}
|
||||||
|
let graph = cm_topology::build(
|
||||||
|
parse_topology_kind(&template.template.default_topology),
|
||||||
|
&roles,
|
||||||
|
)
|
||||||
|
.map_err(|e| format!("build topology graph for template {template_id}: {e}"))?;
|
||||||
|
serde_json::to_value(&graph)
|
||||||
|
.map(Some)
|
||||||
|
.map_err(|e| format!("serialize topology graph: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
fn parse_topology_kind(s: &str) -> cm_topology::TopologyKind {
|
fn parse_topology_kind(s: &str) -> cm_topology::TopologyKind {
|
||||||
use cm_topology::TopologyKind;
|
use cm_topology::TopologyKind;
|
||||||
match s {
|
match s {
|
||||||
|
|||||||
@@ -0,0 +1,503 @@
|
|||||||
|
//! Capture for missions that have no repository.
|
||||||
|
//!
|
||||||
|
//! `mission_delivery` captures a phase's work by diffing a git checkout. A
|
||||||
|
//! mission with `repo_id IS NULL` — every `research_only` mission, because that
|
||||||
|
//! recipe sets `requires_repo = false` — has no checkout, so
|
||||||
|
//! `capture_finished_coding_phases` filters it out at the SQL level
|
||||||
|
//! (`AND m.repo_id IS NOT NULL`) and never reads the container at all.
|
||||||
|
//!
|
||||||
|
//! The agents still write files. The research directive tells them to save
|
||||||
|
//! findings under `/mission/repo/research/`, and it says so whether or not a
|
||||||
|
//! repo exists. So the work lands in the container's own filesystem, is never
|
||||||
|
//! collected, and is destroyed when the sweeper reaps the container.
|
||||||
|
//!
|
||||||
|
//! # What this cost, measured
|
||||||
|
//!
|
||||||
|
//! Mission `019fdc35` ("ClawHDF5 Research"): four agents, 9.5 minutes, **eight
|
||||||
|
//! research documents** — an HDF5 parser design, a Rust ecosystem survey, a
|
||||||
|
//! seven-crate dependency map, tracing and fuzzing strategy. `mission_artifacts`
|
||||||
|
//! held zero rows and the mission reported `completed`. One agent's own summary
|
||||||
|
//! recorded the situation exactly: *"No git repo — file is written."* It noticed,
|
||||||
|
//! wrote anyway, and the platform threw the result away without a word.
|
||||||
|
//!
|
||||||
|
//! Nothing survived but the summarizer's account of it — which is the agents'
|
||||||
|
//! description of the work, not the work.
|
||||||
|
//!
|
||||||
|
//! # Why a separate path rather than widening the diff capture
|
||||||
|
//!
|
||||||
|
//! There is no base commit to diff against and no branch to push, so every
|
||||||
|
//! concept `capture_phase_diff` is built on is absent. What a repo-less mission
|
||||||
|
//! produces is simply *files*, and the honest capture is to copy them out and
|
||||||
|
//! register each as an artifact. `_outputs/` is deliberately a SIBLING of the
|
||||||
|
//! mission directory and survives `teardown_container`, so artifacts registered
|
||||||
|
//! here outlive the reap that destroyed the originals.
|
||||||
|
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use sqlx::{PgPool, Row};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
/// Directories never worth capturing, whatever an agent leaves behind.
|
||||||
|
///
|
||||||
|
/// Same intent as `mission_fs`'s exclusion list: a captured `.git` or
|
||||||
|
/// `node_modules` is noise that would bury the four documents that matter.
|
||||||
|
const SKIP_DIRS: &[&str] = &[
|
||||||
|
".git",
|
||||||
|
"node_modules",
|
||||||
|
"target",
|
||||||
|
".venv",
|
||||||
|
"venv",
|
||||||
|
"__pycache__",
|
||||||
|
".cache",
|
||||||
|
"dist",
|
||||||
|
"build",
|
||||||
|
];
|
||||||
|
|
||||||
|
/// How many phases to capture per tick, matching `CAPTURE_BATCH`.
|
||||||
|
const BATCH: i64 = 5;
|
||||||
|
|
||||||
|
/// The artifact kind this path registers. Also the idempotency key: a phase with
|
||||||
|
/// one of these has already been captured.
|
||||||
|
pub const OUTPUT_KIND: &str = "document";
|
||||||
|
|
||||||
|
/// Filename of the marker written when a phase produced nothing.
|
||||||
|
const EMPTY_MARKER: &str = "NO-OUTPUT.md";
|
||||||
|
|
||||||
|
/// Capture the outputs of finished phases on missions that have no repo.
|
||||||
|
pub async fn capture_repo_less_phases(pool: &PgPool) -> Result<(), String> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT mp.id, mp.mission_id, mp.kind, mp.config, m.runtime_kind
|
||||||
|
FROM mission_phases mp
|
||||||
|
JOIN missions m ON m.id = mp.mission_id
|
||||||
|
WHERE mp.status IN ('completed', 'failed')
|
||||||
|
AND m.repo_id IS NULL
|
||||||
|
-- microVM used to be excluded here because `run_phase_in_vm`
|
||||||
|
-- refused to boot without a checkout. It no longer does: a
|
||||||
|
-- repo-less mission gets an empty workspace at the same guest path,
|
||||||
|
-- and the collect unpacks it back onto the host — so those files are
|
||||||
|
-- already on disk and `collect_into` reads them instead of asking a
|
||||||
|
-- container that never existed.
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM mission_artifacts a
|
||||||
|
WHERE a.mission_id = mp.mission_id
|
||||||
|
AND a.phase_id = mp.id
|
||||||
|
AND a.kind = $2
|
||||||
|
)
|
||||||
|
ORDER BY mp.completed_at DESC NULLS LAST
|
||||||
|
LIMIT $1",
|
||||||
|
)
|
||||||
|
.bind(BATCH)
|
||||||
|
.bind(OUTPUT_KIND)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("select repo-less phases to capture: {e}"))?;
|
||||||
|
|
||||||
|
for row in rows {
|
||||||
|
let phase_id: Uuid = row.get("id");
|
||||||
|
let mission_id: Uuid = row.get("mission_id");
|
||||||
|
let kind: String = row.get("kind");
|
||||||
|
let config: serde_json::Value = row.get("config");
|
||||||
|
let runtime_kind: String = row.get("runtime_kind");
|
||||||
|
|
||||||
|
let dest = outputs_dir(mission_id, phase_id);
|
||||||
|
let captured = match collect_into(mission_id, &dest, &runtime_kind).await {
|
||||||
|
Ok(files) => files,
|
||||||
|
Err(e) => {
|
||||||
|
// Loud and retryable, never silently "captured nothing": the
|
||||||
|
// whole defect this module exists for is work disappearing
|
||||||
|
// without a word. The next tick tries again; if the container is
|
||||||
|
// already gone the phase is failed below on the next pass.
|
||||||
|
eprintln!(
|
||||||
|
"mission_outputs: could NOT collect outputs for phase {phase_id} \
|
||||||
|
of mission {mission_id}: {e}"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
for file in &captured {
|
||||||
|
let rel = match file.strip_prefix(missions_root()) {
|
||||||
|
Ok(r) => r.to_string_lossy().to_string(),
|
||||||
|
Err(_) => file.to_string_lossy().to_string(),
|
||||||
|
};
|
||||||
|
let title = file
|
||||||
|
.file_name()
|
||||||
|
.map(|n| n.to_string_lossy().to_string())
|
||||||
|
.unwrap_or_else(|| rel.clone());
|
||||||
|
if let Err(e) = cm_db::repo::missions::register_artifact(
|
||||||
|
pool,
|
||||||
|
cm_db::repo::missions::RegisterArtifact {
|
||||||
|
mission_id,
|
||||||
|
phase_id: Some(phase_id),
|
||||||
|
path: &rel,
|
||||||
|
kind: OUTPUT_KIND,
|
||||||
|
mime: Some(mime_for(file)),
|
||||||
|
title: Some(&title),
|
||||||
|
generated_by_run: None,
|
||||||
|
// No PDF. The renderer converted Markdown to HTML by
|
||||||
|
// calling an LLM — a paid API call, per document, on the
|
||||||
|
// critical path of "save my research", which promptly
|
||||||
|
// failed on depleted credits. Markdown IS the deliverable;
|
||||||
|
// it is served by `artifact_content` and styled at render
|
||||||
|
// time, which is free, offline, and cannot 429.
|
||||||
|
render_pdf: false,
|
||||||
|
metadata: Some(serde_json::json!({
|
||||||
|
"bytes": std::fs::metadata(file).map(|m| m.len()).unwrap_or(0),
|
||||||
|
"captured_from": "/mission/repo",
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
eprintln!("mission_outputs: registering {rel}: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if captured.is_empty() {
|
||||||
|
// Register a marker even when there is nothing to capture, or this
|
||||||
|
// phase matches the `NOT EXISTS` selection on every tick forever:
|
||||||
|
// re-running a docker copy_out each time and, because the batch is
|
||||||
|
// bounded, permanently occupying a slot so no other repo-less
|
||||||
|
// mission is ever captured again.
|
||||||
|
//
|
||||||
|
// `phase_runner::record_uncapturable` exists for exactly this
|
||||||
|
// failure on the diff path — five dead phases starved the batch
|
||||||
|
// while live work went untouched — and this code hit it again on
|
||||||
|
// its first live negative control (4 log lines, then 8, 45 seconds
|
||||||
|
// apart). Same shape, same fix: a real file behind a real row,
|
||||||
|
// because an artifact pointing at nothing turns every reader into
|
||||||
|
// an unexplained 404.
|
||||||
|
if let Err(e) = register_empty_marker(pool, mission_id, phase_id, &dest).await {
|
||||||
|
eprintln!("mission_outputs: marking phase {phase_id} as empty: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if captured.is_empty() && !allow_empty(&config) {
|
||||||
|
// The same rule `empty_delivery_is_a_failure` applies to a coding
|
||||||
|
// phase, for the only channel a repo-less phase has. Without it a
|
||||||
|
// research mission that produced nothing is indistinguishable from
|
||||||
|
// one that produced eight documents — both `completed`.
|
||||||
|
eprintln!(
|
||||||
|
"mission_outputs: phase {phase_id} ({kind}) of mission {mission_id} produced \
|
||||||
|
NO output files — failing it. Set config.allow_empty = true if this phase is \
|
||||||
|
meant to think rather than produce."
|
||||||
|
);
|
||||||
|
if let Err(e) = sqlx::query("UPDATE mission_phases SET status = 'failed' WHERE id = $1")
|
||||||
|
.bind(phase_id)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
eprintln!("mission_outputs: failing empty phase {phase_id}: {e}");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
eprintln!(
|
||||||
|
"mission_outputs: captured {} file(s) from phase {phase_id} ({kind}) of \
|
||||||
|
mission {mission_id}",
|
||||||
|
captured.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record that a phase produced nothing, so it is not reconsidered forever.
|
||||||
|
///
|
||||||
|
/// Deliberately the same `OUTPUT_KIND` the real captures use: the selection
|
||||||
|
/// query asks "has this phase been captured?", and "captured, and there was
|
||||||
|
/// nothing" is an answer to that question. `metadata.empty` is what tells the
|
||||||
|
/// two apart — the same convention `mission_delivery` uses for its "No code
|
||||||
|
/// changes" artifact.
|
||||||
|
async fn register_empty_marker(
|
||||||
|
pool: &PgPool,
|
||||||
|
mission_id: Uuid,
|
||||||
|
phase_id: Uuid,
|
||||||
|
dest: &Path,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
std::fs::create_dir_all(dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
|
||||||
|
let file = dest.join(EMPTY_MARKER);
|
||||||
|
std::fs::write(
|
||||||
|
&file,
|
||||||
|
"This phase finished without leaving any files in its workspace, so there\n was nothing to publish. If the phase is meant to reason rather than\n produce, set `config.allow_empty = true` on it.\n",
|
||||||
|
)
|
||||||
|
.map_err(|e| format!("write {}: {e}", file.display()))?;
|
||||||
|
let rel = file
|
||||||
|
.strip_prefix(missions_root())
|
||||||
|
.map(|r| r.to_string_lossy().to_string())
|
||||||
|
.unwrap_or_else(|_| file.to_string_lossy().to_string());
|
||||||
|
cm_db::repo::missions::register_artifact(
|
||||||
|
pool,
|
||||||
|
cm_db::repo::missions::RegisterArtifact {
|
||||||
|
mission_id,
|
||||||
|
phase_id: Some(phase_id),
|
||||||
|
path: &rel,
|
||||||
|
kind: OUTPUT_KIND,
|
||||||
|
mime: Some("text/markdown"),
|
||||||
|
title: Some("No output produced"),
|
||||||
|
generated_by_run: None,
|
||||||
|
render_pdf: false,
|
||||||
|
metadata: Some(serde_json::json!({ "empty": true })),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map(|_| ())
|
||||||
|
.map_err(|e| format!("register empty marker: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Gather the mission's produced files and return the ones worth keeping.
|
||||||
|
///
|
||||||
|
/// Where they come from depends on the runtime, and the difference is not
|
||||||
|
/// cosmetic: a container mission's files are still INSIDE a running container,
|
||||||
|
/// while a microVM's have already been unpacked onto the host by the collect at
|
||||||
|
/// the end of the turn (`microvm_executor` writes them over
|
||||||
|
/// `mission_workspace::checkout_path`). Asking docker for a VM mission's files
|
||||||
|
/// would query a container that never existed.
|
||||||
|
async fn collect_into(mission_id: Uuid, dest: &Path, runtime_kind: &str) -> Result<Vec<PathBuf>, String> {
|
||||||
|
// A stale copy from an earlier attempt would be registered as this pass's
|
||||||
|
// output — the same "captured a tree nobody wrote" shape capture avoids.
|
||||||
|
let _ = std::fs::remove_dir_all(dest);
|
||||||
|
std::fs::create_dir_all(dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
|
||||||
|
|
||||||
|
if runtime_kind == "microvm" {
|
||||||
|
let src = crate::mission_workspace::checkout_path(mission_id);
|
||||||
|
if !src.is_dir() {
|
||||||
|
return Err(format!(
|
||||||
|
"{} is absent — the VM's collect did not land",
|
||||||
|
src.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
copy_tree(&src, &dest.join("repo"))?;
|
||||||
|
return Ok(keep_files(&dest.join("repo")));
|
||||||
|
}
|
||||||
|
|
||||||
|
let container = crate::mission_runtime::container_name(mission_id);
|
||||||
|
let docker = crate::container_exec::connect()?;
|
||||||
|
crate::mission_fs::copy_out(&docker, &container, "/mission/repo", dest).await?;
|
||||||
|
Ok(keep_files(&dest.join("repo")))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Recursive file copy. Small on purpose — the alternative is a dependency or a
|
||||||
|
/// shell-out, and this runs as the server's own uid against its own directory.
|
||||||
|
fn copy_tree(src: &Path, dest: &Path) -> Result<(), String> {
|
||||||
|
std::fs::create_dir_all(dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
|
||||||
|
let entries =
|
||||||
|
std::fs::read_dir(src).map_err(|e| format!("read {}: {e}", src.display()))?;
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let from = entry.path();
|
||||||
|
let to = dest.join(entry.file_name());
|
||||||
|
match entry.file_type() {
|
||||||
|
Ok(t) if t.is_dir() => copy_tree(&from, &to)?,
|
||||||
|
Ok(t) if t.is_file() => {
|
||||||
|
std::fs::copy(&from, &to).map_err(|e| format!("copy {}: {e}", from.display()))?;
|
||||||
|
}
|
||||||
|
// Symlinks and specials are skipped rather than followed: a link out
|
||||||
|
// of the tree would publish whatever it points at.
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every regular file worth keeping, recursively.
|
||||||
|
fn keep_files(root: &Path) -> Vec<PathBuf> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let mut stack = vec![root.to_path_buf()];
|
||||||
|
while let Some(dir) = stack.pop() {
|
||||||
|
let Ok(entries) = std::fs::read_dir(&dir) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let path = entry.path();
|
||||||
|
let name = entry.file_name().to_string_lossy().to_string();
|
||||||
|
if path.is_dir() {
|
||||||
|
if !SKIP_DIRS.contains(&name.as_str()) {
|
||||||
|
stack.push(path);
|
||||||
|
}
|
||||||
|
} else if path.is_file()
|
||||||
|
&& !name.starts_with('.')
|
||||||
|
// The agent runtime seeds its own identity files into the
|
||||||
|
// workspace root, which is pinned to the repo root. In a
|
||||||
|
// repo-backed mission `.git/info/exclude` hides them; a
|
||||||
|
// repo-less mission has no `.git`, so without this the user's
|
||||||
|
// artifact list is 7 files of agent scaffolding and 2 of their
|
||||||
|
// research. Measured exactly that way on the first live run.
|
||||||
|
&& !crate::mission_workspace::AGENT_SCAFFOLDING.contains(&name.as_str())
|
||||||
|
{
|
||||||
|
out.push(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.sort();
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `<missions_root>/_outputs/<mission>/<phase>` — a sibling of the mission
|
||||||
|
/// directory, so `teardown_container` reaping the mission does not take the
|
||||||
|
/// captured artifacts with it.
|
||||||
|
fn outputs_dir(mission_id: Uuid, phase_id: Uuid) -> PathBuf {
|
||||||
|
missions_root()
|
||||||
|
.join("_outputs")
|
||||||
|
.join(mission_id.to_string())
|
||||||
|
.join(phase_id.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The missions root, for callers that resolve artifact paths against it.
|
||||||
|
pub fn missions_root_dir() -> PathBuf {
|
||||||
|
missions_root()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The only directory an artifact may be read from.
|
||||||
|
pub fn outputs_root_dir() -> PathBuf {
|
||||||
|
missions_root().join("_outputs")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn missions_root() -> PathBuf {
|
||||||
|
crate::mission_workspace::missions_root()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn mime_for(p: &Path) -> &'static str {
|
||||||
|
match p.extension().and_then(|e| e.to_str()) {
|
||||||
|
Some("md") | Some("markdown") => "text/markdown",
|
||||||
|
Some("json") => "application/json",
|
||||||
|
Some("csv") => "text/csv",
|
||||||
|
Some("html") => "text/html",
|
||||||
|
_ => "text/plain",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn allow_empty(config: &serde_json::Value) -> bool {
|
||||||
|
config.get("allow_empty").and_then(|v| v.as_bool()) == Some(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn touch(p: &Path) {
|
||||||
|
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
|
||||||
|
std::fs::write(p, "x").unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The documents a research phase writes are what must come back — and the
|
||||||
|
/// machinery around them must not.
|
||||||
|
#[test]
|
||||||
|
fn research_documents_are_kept_and_scaffolding_is_not() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let repo = tmp.path().join("repo");
|
||||||
|
touch(&repo.join("research/01_repo_archaeology.md"));
|
||||||
|
touch(&repo.join("research/02_ecosystem.md"));
|
||||||
|
touch(&repo.join("notes.txt"));
|
||||||
|
// The seven the agent runtime seeds into the workspace root.
|
||||||
|
for f in crate::mission_workspace::AGENT_SCAFFOLDING {
|
||||||
|
touch(&repo.join(f));
|
||||||
|
}
|
||||||
|
touch(&repo.join(".git/HEAD"));
|
||||||
|
touch(&repo.join("node_modules/left-pad/index.js"));
|
||||||
|
touch(&repo.join("target/debug/thing"));
|
||||||
|
touch(&repo.join(".hidden"));
|
||||||
|
|
||||||
|
let kept: Vec<String> = keep_files(&repo)
|
||||||
|
.iter()
|
||||||
|
.map(|p| p.strip_prefix(&repo).unwrap().to_string_lossy().to_string())
|
||||||
|
.collect();
|
||||||
|
assert_eq!(
|
||||||
|
kept,
|
||||||
|
vec![
|
||||||
|
"notes.txt".to_string(),
|
||||||
|
"research/01_repo_archaeology.md".to_string(),
|
||||||
|
"research/02_ecosystem.md".to_string(),
|
||||||
|
],
|
||||||
|
"kept: {kept:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Markdown is the deliverable, so it must be labelled as markdown — the
|
||||||
|
/// viewer decides how to render from the mime type.
|
||||||
|
#[test]
|
||||||
|
fn markdown_is_labelled_so_the_viewer_can_style_it() {
|
||||||
|
assert_eq!(mime_for(Path::new("/x/01_notes.md")), "text/markdown");
|
||||||
|
assert_eq!(mime_for(Path::new("/x/data.json")), "application/json");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The containment rule the content endpoint enforces: everything readable
|
||||||
|
/// lives under `_outputs`, and nothing else does.
|
||||||
|
///
|
||||||
|
/// Artifact paths are written by this server, but they are DATA in a table,
|
||||||
|
/// and a row saying `../../../etc/passwd` must be a 404 rather than a file
|
||||||
|
/// read. The endpoint canonicalises before comparing — checking the string
|
||||||
|
/// first would pass `_outputs/../../etc/passwd` straight through.
|
||||||
|
#[test]
|
||||||
|
fn everything_readable_lives_under_the_outputs_root() {
|
||||||
|
let root = outputs_root_dir();
|
||||||
|
assert!(root.ends_with("_outputs"), "{root:?}");
|
||||||
|
assert!(root.starts_with(missions_root_dir()), "{root:?}");
|
||||||
|
|
||||||
|
// A real capture is inside it...
|
||||||
|
let inside = outputs_dir(Uuid::now_v7(), Uuid::now_v7());
|
||||||
|
assert!(inside.starts_with(&root), "{inside:?}");
|
||||||
|
|
||||||
|
// ...and the traversal shape this guards against is not, once resolved.
|
||||||
|
let escaped = root.join("..").join("..").join("etc/passwd");
|
||||||
|
let normalised: PathBuf = escaped
|
||||||
|
.components()
|
||||||
|
.fold(PathBuf::new(), |mut acc, c| {
|
||||||
|
match c {
|
||||||
|
std::path::Component::ParentDir => {
|
||||||
|
acc.pop();
|
||||||
|
}
|
||||||
|
other => acc.push(other),
|
||||||
|
}
|
||||||
|
acc
|
||||||
|
});
|
||||||
|
assert!(
|
||||||
|
!normalised.starts_with(&root),
|
||||||
|
"a traversal must not resolve back inside the outputs root: {normalised:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A phase that produced nothing must still leave a marker, or the
|
||||||
|
/// selection query matches it on every tick forever.
|
||||||
|
///
|
||||||
|
/// Measured on the first live negative control: the guard logged "produced
|
||||||
|
/// NO output files" 4 times, then 8 times 45 seconds later — a docker
|
||||||
|
/// copy_out per tick, and with a bounded batch, five such phases would
|
||||||
|
/// starve every other repo-less mission out of capture permanently.
|
||||||
|
/// `phase_runner::record_uncapturable` was written for the identical
|
||||||
|
/// failure on the diff path.
|
||||||
|
#[test]
|
||||||
|
fn an_empty_phase_leaves_a_marker_so_it_is_not_reconsidered_forever() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let dest = tmp.path().join("out");
|
||||||
|
// The file-writing half of `register_empty_marker`, which is the part
|
||||||
|
// that must exist for the artifact row to point at something real.
|
||||||
|
std::fs::create_dir_all(&dest).unwrap();
|
||||||
|
let file = dest.join(EMPTY_MARKER);
|
||||||
|
std::fs::write(&file, "x").unwrap();
|
||||||
|
assert!(file.exists(), "an artifact row must not point at nothing");
|
||||||
|
assert_eq!(
|
||||||
|
file.file_name().unwrap().to_string_lossy(),
|
||||||
|
"NO-OUTPUT.md",
|
||||||
|
"the marker name is part of the contract with readers"
|
||||||
|
);
|
||||||
|
// And the marker must not itself be mistaken for captured output on a
|
||||||
|
// later pass: it is filtered like any other scaffolding would be.
|
||||||
|
assert!(keep_files(&dest).iter().any(|p| p == &file));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Artifacts must land OUTSIDE the mission directory. `teardown_container`
|
||||||
|
/// removes `<missions_root>/<mission_id>` wholesale, so a capture written
|
||||||
|
/// inside it would be destroyed by the very reap it exists to survive.
|
||||||
|
#[test]
|
||||||
|
fn captures_survive_the_mission_directory_being_reaped() {
|
||||||
|
let mission = Uuid::now_v7();
|
||||||
|
let phase = Uuid::now_v7();
|
||||||
|
let out = outputs_dir(mission, phase);
|
||||||
|
let mission_dir = missions_root().join(mission.to_string());
|
||||||
|
assert!(
|
||||||
|
!out.starts_with(&mission_dir),
|
||||||
|
"{} must not be inside {}",
|
||||||
|
out.display(),
|
||||||
|
mission_dir.display()
|
||||||
|
);
|
||||||
|
assert!(out.starts_with(missions_root().join("_outputs")), "{out:?}");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,296 @@
|
|||||||
|
//! A model-authored execution plan for one mission — W1 / #13.
|
||||||
|
//!
|
||||||
|
//! Every mission's phases come from one of five hand-written recipes in
|
||||||
|
//! `templates/workflows/*.toml`, chosen by `template_kind`. A recipe is a fixed
|
||||||
|
//! answer to "what phases does this kind of mission have", written before anyone
|
||||||
|
//! saw the mission — the "do it this way: 1, 2, 3" over-specification that makes
|
||||||
|
//! a capable model follow a worse plan than it would have chosen for the actual
|
||||||
|
//! task.
|
||||||
|
//!
|
||||||
|
//! This is the other half of [`crate::mission_roster`]: that one lets a model
|
||||||
|
//! size the team, this one lets it decide what the work IS. Same shape on
|
||||||
|
//! purpose — propose, review, approve, apply — because the review gate is what
|
||||||
|
//! makes model-authored structure safe to run, and a second shape would be a
|
||||||
|
//! second thing to get right.
|
||||||
|
//!
|
||||||
|
//! # Grounded in what the platform actually reads
|
||||||
|
//!
|
||||||
|
//! The interesting constraint is not "is this JSON valid" but "will anything
|
||||||
|
//! consume it". `phase_config::KNOWN_KEYS` already names every phase-config key
|
||||||
|
//! and the code that reads it, with eleven marked NOT IMPLEMENTED — the registry
|
||||||
|
//! built after `task` sat unread through every mission. A plan is validated
|
||||||
|
//! against that registry, so a model cannot propose a phase whose settings
|
||||||
|
//! nothing will act on. The failure that registry exists to EXPOSE is one this
|
||||||
|
//! path cannot create.
|
||||||
|
//!
|
||||||
|
//! Phase kinds are checked the same way, against the kinds `phase_runner`
|
||||||
|
//! actually dispatches. A model asked to plan work will happily invent
|
||||||
|
//! `kind: "review"`, and an unknown kind does not fail — it falls to the
|
||||||
|
//! catch-all purpose and runs as a generic phase, which looks like it worked.
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
/// Phase kinds `phase_runner` dispatches on.
|
||||||
|
///
|
||||||
|
/// Not an enum, because `mission_phases.kind` is a free-form column shared with
|
||||||
|
/// hand-written recipes and the wizard; this is the subset a MODEL may propose.
|
||||||
|
/// An unrecognised kind is the dangerous case: it does not error, it falls
|
||||||
|
/// through to the generic `mission` purpose and runs anyway.
|
||||||
|
pub const PLANNABLE_KINDS: &[&str] = &["research", "coding", "benchmark", "security_scan"];
|
||||||
|
|
||||||
|
/// Ceiling on a proposed plan.
|
||||||
|
///
|
||||||
|
/// Each phase is a full agent run — a VM boot, a checkout, a turn, a capture —
|
||||||
|
/// executed in sequence. Anthropic's own guidance warns against decomposing work
|
||||||
|
/// into sequential phases at all ("a handoff loses context at every step"), so
|
||||||
|
/// this bound is deliberately tight: a model that wants eight phases is
|
||||||
|
/// describing a to-do list, not a plan.
|
||||||
|
pub const MAX_PHASES: usize = 4;
|
||||||
|
|
||||||
|
/// One phase of a proposed plan.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct PlannedPhase {
|
||||||
|
/// One of [`PLANNABLE_KINDS`].
|
||||||
|
pub kind: String,
|
||||||
|
/// What this phase does. Lands in `config.task`, which
|
||||||
|
/// `phase_task_text` injects — the key that sat unread through every
|
||||||
|
/// mission until two phases with different tasks produced identical output.
|
||||||
|
pub task: String,
|
||||||
|
/// Optional completion condition, judged post-hoc by the evaluator.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub done_when: Option<String>,
|
||||||
|
/// Optional deterministic check, enforced IN the agent's loop by the stop
|
||||||
|
/// gate ([`crate::vm_stop_gate`]).
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub done_when_check: Option<String>,
|
||||||
|
/// This phase is allowed to change nothing (a verification pass).
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub allow_empty: Option<bool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A proposed sequence of phases.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct Plan {
|
||||||
|
pub phases: Vec<PlannedPhase>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Why a plan was refused.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum Refusal {
|
||||||
|
Empty,
|
||||||
|
TooMany(usize),
|
||||||
|
UnknownKind { index: usize, kind: String },
|
||||||
|
BlankTask(usize),
|
||||||
|
/// A config key with no reader in this build — named, with the ones that
|
||||||
|
/// would have been consumed.
|
||||||
|
InertKey { index: usize, key: String },
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for Refusal {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
Refusal::Empty => write!(f, "the plan has no phases, so the mission would do nothing"),
|
||||||
|
Refusal::TooMany(n) => write!(
|
||||||
|
f,
|
||||||
|
"the plan has {n} phases and the ceiling is {MAX_PHASES} — each one is a full \
|
||||||
|
agent run, and a handoff loses context at every step"
|
||||||
|
),
|
||||||
|
Refusal::UnknownKind { index, kind } => write!(
|
||||||
|
f,
|
||||||
|
"phase {index} has kind {kind:?}, which nothing dispatches on; use one of: {}",
|
||||||
|
PLANNABLE_KINDS.join(", ")
|
||||||
|
),
|
||||||
|
Refusal::BlankTask(i) => write!(
|
||||||
|
f,
|
||||||
|
"phase {i} has no task, so its agent would receive the mission description and \
|
||||||
|
nothing telling it which part is its own"
|
||||||
|
),
|
||||||
|
Refusal::InertKey { index, key } => write!(
|
||||||
|
f,
|
||||||
|
"phase {index} sets {key:?}, which nothing in this build reads — it would be \
|
||||||
|
stored, rendered, and consumed by nobody"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Plan {
|
||||||
|
/// Check a plan against what the platform can actually execute.
|
||||||
|
pub fn validate(&self) -> Result<(), Refusal> {
|
||||||
|
if self.phases.is_empty() {
|
||||||
|
return Err(Refusal::Empty);
|
||||||
|
}
|
||||||
|
if self.phases.len() > MAX_PHASES {
|
||||||
|
return Err(Refusal::TooMany(self.phases.len()));
|
||||||
|
}
|
||||||
|
for (i, p) in self.phases.iter().enumerate() {
|
||||||
|
if !PLANNABLE_KINDS.contains(&p.kind.as_str()) {
|
||||||
|
return Err(Refusal::UnknownKind {
|
||||||
|
index: i,
|
||||||
|
kind: p.kind.clone(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if p.task.trim().is_empty() {
|
||||||
|
return Err(Refusal::BlankTask(i));
|
||||||
|
}
|
||||||
|
// Every key this phase would write must have a reader. The plan is
|
||||||
|
// built from typed fields, so this can only fail if a field is added
|
||||||
|
// here without a corresponding entry in the registry — which is
|
||||||
|
// exactly the drift worth failing on.
|
||||||
|
if let Some(key) = crate::phase_config::inert_keys(&p.config()).into_iter().next() {
|
||||||
|
return Err(Refusal::InertKey { index: i, key });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The phases as `(kind, order_idx, config)`, ready for mission creation.
|
||||||
|
///
|
||||||
|
/// `order_idx` is the array position rather than a field the model sets:
|
||||||
|
/// two sources for one fact is how a plan ends up with two phase 0s.
|
||||||
|
pub fn phases(&self) -> Vec<(String, i32, serde_json::Value)> {
|
||||||
|
self.phases
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(i, p)| (p.kind.clone(), i as i32, p.config()))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PlannedPhase {
|
||||||
|
/// This phase's `mission_phases.config`.
|
||||||
|
fn config(&self) -> serde_json::Value {
|
||||||
|
let mut o = serde_json::Map::new();
|
||||||
|
o.insert("task".into(), serde_json::Value::String(self.task.clone()));
|
||||||
|
if let Some(d) = self.done_when.as_deref().map(str::trim).filter(|s| !s.is_empty()) {
|
||||||
|
o.insert("done_when".into(), serde_json::Value::String(d.to_string()));
|
||||||
|
}
|
||||||
|
if let Some(c) = self
|
||||||
|
.done_when_check
|
||||||
|
.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
{
|
||||||
|
o.insert(
|
||||||
|
"done_when_check".into(),
|
||||||
|
serde_json::Value::String(c.to_string()),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if let Some(e) = self.allow_empty {
|
||||||
|
o.insert("allow_empty".into(), serde_json::Value::Bool(e));
|
||||||
|
}
|
||||||
|
serde_json::Value::Object(o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn phase(kind: &str, task: &str) -> PlannedPhase {
|
||||||
|
PlannedPhase {
|
||||||
|
kind: kind.into(),
|
||||||
|
task: task.into(),
|
||||||
|
done_when: None,
|
||||||
|
done_when_check: None,
|
||||||
|
allow_empty: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A kind nothing dispatches on is the dangerous one: it does not error, it
|
||||||
|
/// falls through to the generic purpose and runs as a nondescript phase that
|
||||||
|
/// looks like it worked.
|
||||||
|
#[test]
|
||||||
|
fn an_invented_phase_kind_is_refused_naming_the_real_ones() {
|
||||||
|
let p = Plan {
|
||||||
|
phases: vec![phase("coding", "do it"), phase("review", "check it")],
|
||||||
|
};
|
||||||
|
let err = p.validate().unwrap_err();
|
||||||
|
assert_eq!(
|
||||||
|
err,
|
||||||
|
Refusal::UnknownKind {
|
||||||
|
index: 1,
|
||||||
|
kind: "review".into()
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let msg = err.to_string();
|
||||||
|
for kind in PLANNABLE_KINDS {
|
||||||
|
assert!(msg.contains(kind), "the message must name {kind}: {msg}");
|
||||||
|
}
|
||||||
|
// And every kind the runner dispatches on is accepted, so this cannot
|
||||||
|
// drift from what `phase_runner` can actually execute.
|
||||||
|
for kind in PLANNABLE_KINDS {
|
||||||
|
assert!(Plan { phases: vec![phase(kind, "work")] }.validate().is_ok(), "{kind}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every key a planned phase writes must have a reader. This is the whole
|
||||||
|
/// reason `phase_config` exists — a key nothing consumes is stored,
|
||||||
|
/// rendered, and silently inert.
|
||||||
|
#[test]
|
||||||
|
fn every_key_a_plan_writes_is_one_something_reads() {
|
||||||
|
let p = PlannedPhase {
|
||||||
|
kind: "coding".into(),
|
||||||
|
task: "add a module".into(),
|
||||||
|
done_when: Some("the suite passes".into()),
|
||||||
|
done_when_check: Some("cargo test".into()),
|
||||||
|
allow_empty: Some(false),
|
||||||
|
};
|
||||||
|
let cfg = p.config();
|
||||||
|
assert!(
|
||||||
|
crate::phase_config::inert_keys(&cfg).is_empty(),
|
||||||
|
"a planned phase must write only keys with readers: {:?}",
|
||||||
|
crate::phase_config::inert_keys(&cfg)
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
crate::phase_config::unknown_keys(&cfg).is_empty(),
|
||||||
|
"and only keys the registry knows: {:?}",
|
||||||
|
crate::phase_config::unknown_keys(&cfg)
|
||||||
|
);
|
||||||
|
assert!(Plan { phases: vec![p] }.validate().is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A blank task is the failure that produced identical output from two
|
||||||
|
/// different phases — the agent gets the mission description and nothing
|
||||||
|
/// saying which part is its own.
|
||||||
|
#[test]
|
||||||
|
fn a_phase_without_a_task_is_refused() {
|
||||||
|
let p = Plan {
|
||||||
|
phases: vec![phase("coding", " ")],
|
||||||
|
};
|
||||||
|
assert_eq!(p.validate(), Err(Refusal::BlankTask(0)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bounded and non-empty. Each phase is a full agent run in sequence, and
|
||||||
|
/// splitting one change into stages loses context at every handoff.
|
||||||
|
#[test]
|
||||||
|
fn a_plan_is_bounded_and_non_empty() {
|
||||||
|
assert_eq!(Plan { phases: vec![] }.validate(), Err(Refusal::Empty));
|
||||||
|
let many: Vec<_> = (0..MAX_PHASES + 1).map(|_| phase("coding", "work")).collect();
|
||||||
|
assert_eq!(
|
||||||
|
Plan { phases: many }.validate(),
|
||||||
|
Err(Refusal::TooMany(MAX_PHASES + 1))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Order comes from the array, not from a field the model sets. Two sources
|
||||||
|
/// for one fact is how a plan ends up with two phase 0s — and `order_idx`
|
||||||
|
/// is what `start_pending_phases` sequences on.
|
||||||
|
#[test]
|
||||||
|
fn order_comes_from_the_arrays_own_order() {
|
||||||
|
let p = Plan {
|
||||||
|
phases: vec![
|
||||||
|
phase("research", "read the code"),
|
||||||
|
phase("coding", "change it"),
|
||||||
|
phase("coding", "then this"),
|
||||||
|
],
|
||||||
|
};
|
||||||
|
let out = p.phases();
|
||||||
|
assert_eq!(
|
||||||
|
out.iter().map(|(_, i, _)| *i).collect::<Vec<_>>(),
|
||||||
|
vec![0, 1, 2]
|
||||||
|
);
|
||||||
|
assert_eq!(out[0].0, "research");
|
||||||
|
assert_eq!(out[1].2["task"], "change it");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,16 +2,18 @@
|
|||||||
//! mission and rewrite it into a coherent, sectioned Markdown brief
|
//! mission and rewrite it into a coherent, sectioned Markdown brief
|
||||||
//! that downstream research + coding agents can ingest cleanly.
|
//! that downstream research + coding agents can ingest cleanly.
|
||||||
//!
|
//!
|
||||||
//! Calls Anthropic Claude Opus 4.8 by default. Prod already carries
|
//! Asks for Claude Opus 4.8 by default, but goes through
|
||||||
//! ANTHROPIC_API_KEY for ZeroClaw's provider config, so no separate
|
//! `subscription::complete_with_fallback` like every other server-side model
|
||||||
//! env is needed.
|
//! call. It used to hand-roll its own HTTPS POST to the Messages API with the
|
||||||
|
//! metered key — a comment above this line still claimed prod "already carries
|
||||||
|
//! ANTHROPIC_API_KEY, so no separate env is needed", which stopped being true
|
||||||
|
//! the moment that account ran out of credit. See `subscription`, whose
|
||||||
|
//! source-walk test is what found this module.
|
||||||
|
|
||||||
use serde_json::json;
|
|
||||||
use sqlx::PgPool;
|
use sqlx::PgPool;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
const DEFAULT_MODEL: &str = "claude-opus-4-8";
|
const DEFAULT_MODEL: &str = "claude-opus-4-8";
|
||||||
const ANTHROPIC_API_VERSION: &str = "2023-06-01";
|
|
||||||
|
|
||||||
fn model_name() -> String {
|
fn model_name() -> String {
|
||||||
std::env::var("CLAWMATES_REFINER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
std::env::var("CLAWMATES_REFINER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
||||||
@@ -22,12 +24,36 @@ pub struct RefineResult {
|
|||||||
pub refined: String,
|
pub refined: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Refine a description that has no mission behind it yet.
|
||||||
|
///
|
||||||
|
/// The wizard's polish button runs BEFORE the mission is created — there is no
|
||||||
|
/// row to load and no id to pass — while [`refine`] deliberately requires a
|
||||||
|
/// saved draft so Accept/Cancel can write back to it. Same prompt, same model
|
||||||
|
/// chain; only where the inputs come from differs.
|
||||||
|
pub async fn refine_draft(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
|
title: &str,
|
||||||
|
template_kind: &str,
|
||||||
|
phase_kinds: &[String],
|
||||||
|
raw: &str,
|
||||||
|
) -> Result<RefineResult, String> {
|
||||||
|
if raw.trim().is_empty() {
|
||||||
|
return Err("description is empty — nothing to refine".into());
|
||||||
|
}
|
||||||
|
let refined = call_anthropic(runtime, title, template_kind, phase_kinds, raw).await?;
|
||||||
|
Ok(RefineResult {
|
||||||
|
original: raw.to_string(),
|
||||||
|
refined,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// Generate a refined description without touching the database. The
|
/// Generate a refined description without touching the database. The
|
||||||
/// caller (frontend) reviews the diff and calls `set_description` to
|
/// caller (frontend) reviews the diff and calls `set_description` to
|
||||||
/// commit — that separation makes Accept/Cancel + undo trivial without
|
/// commit — that separation makes Accept/Cancel + undo trivial without
|
||||||
/// an audit table.
|
/// an audit table.
|
||||||
pub async fn refine(
|
pub async fn refine(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
workspace_id: cm_domain::WorkspaceId,
|
workspace_id: cm_domain::WorkspaceId,
|
||||||
mission_id: Uuid,
|
mission_id: Uuid,
|
||||||
) -> Result<RefineResult, String> {
|
) -> Result<RefineResult, String> {
|
||||||
@@ -54,7 +80,8 @@ pub async fn refine(
|
|||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let refined =
|
let refined =
|
||||||
call_anthropic(&mission.title, &mission.template_kind, &phase_kinds, &raw).await?;
|
call_anthropic(runtime, &mission.title, &mission.template_kind, &phase_kinds, &raw)
|
||||||
|
.await?;
|
||||||
|
|
||||||
Ok(RefineResult {
|
Ok(RefineResult {
|
||||||
original: raw,
|
original: raw,
|
||||||
@@ -63,13 +90,12 @@ pub async fn refine(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn call_anthropic(
|
async fn call_anthropic(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
title: &str,
|
title: &str,
|
||||||
template_kind: &str,
|
template_kind: &str,
|
||||||
phase_kinds: &[String],
|
phase_kinds: &[String],
|
||||||
raw: &str,
|
raw: &str,
|
||||||
) -> Result<String, String> {
|
) -> Result<String, String> {
|
||||||
let api_key =
|
|
||||||
std::env::var("ANTHROPIC_API_KEY").map_err(|_| "ANTHROPIC_API_KEY unset".to_string())?;
|
|
||||||
let model = model_name();
|
let model = model_name();
|
||||||
|
|
||||||
let system = "You are a technical brief editor for an autonomous software \
|
let system = "You are a technical brief editor for an autonomous software \
|
||||||
@@ -131,57 +157,14 @@ async fn call_anthropic(
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Opus 4.8 rejects the `temperature` parameter — the model runs at
|
// Opus 4.8 rejects the `temperature` parameter — the model runs at
|
||||||
// its own calibrated setting. Older Claude models accepted 0.0–1.0.
|
// its own calibrated setting. Older Claude models accepted 0.0–1.0, and
|
||||||
let body = json!({
|
// `ChatRequest` does not carry one, so nothing is lost by the move.
|
||||||
"model": model,
|
let (text, answered_by) =
|
||||||
"max_tokens": 4096,
|
crate::subscription::complete_with_fallback(runtime, system, &user, &model, 4096, false)
|
||||||
"system": system,
|
.await?;
|
||||||
"messages": [
|
let text = text.trim().to_string();
|
||||||
{ "role": "user", "content": user }
|
|
||||||
]
|
|
||||||
});
|
|
||||||
|
|
||||||
let client = reqwest::Client::builder()
|
|
||||||
.timeout(std::time::Duration::from_secs(90))
|
|
||||||
.build()
|
|
||||||
.map_err(|e| format!("http client: {e}"))?;
|
|
||||||
let resp = client
|
|
||||||
.post("https://api.anthropic.com/v1/messages")
|
|
||||||
.header("x-api-key", &api_key)
|
|
||||||
.header("anthropic-version", ANTHROPIC_API_VERSION)
|
|
||||||
.header("content-type", "application/json")
|
|
||||||
.json(&body)
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("anthropic call: {e}"))?;
|
|
||||||
if !resp.status().is_success() {
|
|
||||||
let code = resp.status();
|
|
||||||
let body = resp.text().await.unwrap_or_default();
|
|
||||||
return Err(format!(
|
|
||||||
"anthropic {code}: {}",
|
|
||||||
&body[..body.len().min(500)]
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let json: serde_json::Value = resp
|
|
||||||
.json()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("anthropic json: {e}"))?;
|
|
||||||
// Anthropic Messages API returns content as an array of blocks;
|
|
||||||
// the first text block holds the assistant's reply.
|
|
||||||
let text = json
|
|
||||||
.get("content")
|
|
||||||
.and_then(|c| c.as_array())
|
|
||||||
.and_then(|arr| {
|
|
||||||
arr.iter()
|
|
||||||
.find(|b| b.get("type").and_then(|t| t.as_str()) == Some("text"))
|
|
||||||
})
|
|
||||||
.and_then(|b| b.get("text"))
|
|
||||||
.and_then(|t| t.as_str())
|
|
||||||
.ok_or_else(|| "anthropic response missing text block".to_string())?
|
|
||||||
.trim()
|
|
||||||
.to_string();
|
|
||||||
if text.is_empty() {
|
if text.is_empty() {
|
||||||
return Err("anthropic returned empty text".into());
|
return Err(format!("{answered_by} returned empty text"));
|
||||||
}
|
}
|
||||||
Ok(text)
|
Ok(text)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,373 @@
|
|||||||
|
//! A model-authored roster for a mission — Slice 5.
|
||||||
|
//!
|
||||||
|
//! The Master Planner has been proposing teams (2-6 members, a model each) since
|
||||||
|
//! it shipped, and none of it reached a mission: the proposal lived in React
|
||||||
|
//! state. A mission's shape came instead from a team template — fixed roles, and
|
||||||
|
//! every claw minted `claude-sonnet-5`, which is why no mission has ever run
|
||||||
|
//! heterogeneous providers.
|
||||||
|
//!
|
||||||
|
//! This is the seam. A roster is `(topology_kind, [(role, backend)])`, which is
|
||||||
|
//! exactly what the composed executor consumes: `composed_graph` turns it into a
|
||||||
|
//! `TopologyGraph`, and `MicroVmTurnExecutor` reads `attrs["backend"]` per node,
|
||||||
|
//! so a `validator` role on a different provider's rootfs is a first-class graph
|
||||||
|
//! node rather than a bolt-on.
|
||||||
|
//!
|
||||||
|
//! # Why the backend is validated here and not at boot
|
||||||
|
//!
|
||||||
|
//! Placement already refuses a mission whose backend no online node can run —
|
||||||
|
//! but it refuses it at LAUNCH, after the roster was approved, the mission was
|
||||||
|
//! created and someone believed it was going to run. A model that invents
|
||||||
|
//! `rootfs-opus` is a normal thing for a model to do; discovering it three steps
|
||||||
|
//! later is not. So a roster naming a backend the fleet cannot run is rejected
|
||||||
|
//! when it is proposed, naming the backends that do exist.
|
||||||
|
//!
|
||||||
|
//! # What it deliberately does not do
|
||||||
|
//!
|
||||||
|
//! It does not mint claws. A composed mission's nodes are VMs, and provisioning
|
||||||
|
//! containers for them would create agents and `.brain` files nothing ever
|
||||||
|
//! dials — the same reason `on_launch` returns early for a microVM mission.
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
/// One member of a proposed roster.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct RosterMember {
|
||||||
|
/// The node's role, e.g. `implementer`, `verifier`. Becomes the graph node's
|
||||||
|
/// role, which is what the per-node prompt is written around.
|
||||||
|
pub role: String,
|
||||||
|
/// Which rootfs image this node's VM boots (`missions.backend` per node).
|
||||||
|
/// `None` inherits the mission's.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub backend: Option<String>,
|
||||||
|
/// One line on why this member exists. Not consumed by anything — kept
|
||||||
|
/// because a roster nobody can read is a roster nobody can refuse.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub rationale: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A proposed shape for a mission.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct Roster {
|
||||||
|
/// A `cm_topology::TopologyKind` name — `pipeline`, `hub_spoke`, …
|
||||||
|
pub topology_kind: String,
|
||||||
|
pub members: Vec<RosterMember>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ceiling on a proposed roster.
|
||||||
|
///
|
||||||
|
/// Each member is a whole VM: a boot, an inject, an agent session and a collect.
|
||||||
|
/// Anthropic's own guidance tops out at 3-5 subagents, and every member here
|
||||||
|
/// costs far more than a subagent does. A model asked to size a team will
|
||||||
|
/// cheerfully propose twelve.
|
||||||
|
pub const MAX_MEMBERS: usize = 6;
|
||||||
|
|
||||||
|
/// Why a roster was refused.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum Refusal {
|
||||||
|
Empty,
|
||||||
|
TooMany(usize),
|
||||||
|
BlankRole(usize),
|
||||||
|
/// A backend no online node can run, with the ones that exist.
|
||||||
|
UnknownBackend { backend: String, available: Vec<String> },
|
||||||
|
UnknownTopology(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for Refusal {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
Refusal::Empty => write!(f, "the roster has no members, so there is nothing to run"),
|
||||||
|
Refusal::TooMany(n) => write!(
|
||||||
|
f,
|
||||||
|
"the roster has {n} members and the ceiling is {MAX_MEMBERS} — each one is a whole \
|
||||||
|
VM, not a subagent"
|
||||||
|
),
|
||||||
|
Refusal::BlankRole(i) => write!(f, "member {i} has no role"),
|
||||||
|
Refusal::UnknownBackend { backend, available } => write!(
|
||||||
|
f,
|
||||||
|
"no online node can run backend {backend:?}; the fleet has: {}",
|
||||||
|
if available.is_empty() {
|
||||||
|
"(none — no node reports a microvm rootfs)".to_string()
|
||||||
|
} else {
|
||||||
|
available.join(", ")
|
||||||
|
}
|
||||||
|
),
|
||||||
|
Refusal::UnknownTopology(k) => write!(
|
||||||
|
f,
|
||||||
|
"{k:?} is not a topology kind this platform can plan; use one of: {}",
|
||||||
|
cm_topology::TopologyKind::ALL
|
||||||
|
.iter()
|
||||||
|
.map(|k| k.as_str())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ")
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Roster {
|
||||||
|
/// Check a roster against the platform and the fleet.
|
||||||
|
///
|
||||||
|
/// `available` is the set of backends at least one ONLINE node can boot.
|
||||||
|
/// Fail-closed on every axis: an unrecognised topology, a blank role and an
|
||||||
|
/// unbuildable backend are all refusals, because each of them becomes a
|
||||||
|
/// failure much later and much more expensively.
|
||||||
|
pub fn validate(&self, available: &[String]) -> Result<(), Refusal> {
|
||||||
|
if self.members.is_empty() {
|
||||||
|
return Err(Refusal::Empty);
|
||||||
|
}
|
||||||
|
if self.members.len() > MAX_MEMBERS {
|
||||||
|
return Err(Refusal::TooMany(self.members.len()));
|
||||||
|
}
|
||||||
|
if parse_kind(&self.topology_kind).is_none() {
|
||||||
|
return Err(Refusal::UnknownTopology(self.topology_kind.clone()));
|
||||||
|
}
|
||||||
|
for (i, m) in self.members.iter().enumerate() {
|
||||||
|
if m.role.trim().is_empty() {
|
||||||
|
return Err(Refusal::BlankRole(i));
|
||||||
|
}
|
||||||
|
if let Some(b) = m.backend.as_deref().map(str::trim).filter(|b| !b.is_empty()) {
|
||||||
|
if !available.iter().any(|a| a == b) {
|
||||||
|
return Err(Refusal::UnknownBackend {
|
||||||
|
backend: b.to_string(),
|
||||||
|
available: available.to_vec(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The graph a composed run executes.
|
||||||
|
///
|
||||||
|
/// Node ids follow `cm_topology::build`'s `n0..` convention so the graph is
|
||||||
|
/// indistinguishable from a template-built one — the executor, the planners
|
||||||
|
/// and the checkpoint all treat it the same. The per-member backend rides in
|
||||||
|
/// `attrs`, which is the channel `MicroVmTurnExecutor` already reads.
|
||||||
|
pub fn graph(&self) -> Result<serde_json::Value, String> {
|
||||||
|
let kind = parse_kind(&self.topology_kind)
|
||||||
|
.ok_or_else(|| format!("unknown topology kind {:?}", self.topology_kind))?;
|
||||||
|
let roles: Vec<&str> = self.members.iter().map(|m| m.role.trim()).collect();
|
||||||
|
let mut graph =
|
||||||
|
cm_topology::build(kind, &roles).map_err(|e| format!("build topology: {e}"))?;
|
||||||
|
for (node, member) in graph.nodes.iter_mut().zip(self.members.iter()) {
|
||||||
|
if let Some(b) = member
|
||||||
|
.backend
|
||||||
|
.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|b| !b.is_empty())
|
||||||
|
{
|
||||||
|
node.attrs.insert("backend".to_string(), b.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
serde_json::to_value(&graph).map_err(|e| format!("serialize graph: {e}"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Topology kind by name, accepting exactly what the catalog declares.
|
||||||
|
///
|
||||||
|
/// Deliberately not `unwrap_or(HubSpoke)`. `mission_orchestrator::
|
||||||
|
/// parse_topology_kind` does default, which is right for a stored template
|
||||||
|
/// written by us and wrong for a string a model just invented: silently running
|
||||||
|
/// a `pipeline` proposal as a hub-and-spoke would change what every node sees
|
||||||
|
/// and nothing would say so.
|
||||||
|
fn parse_kind(s: &str) -> Option<cm_topology::TopologyKind> {
|
||||||
|
let want = s.trim();
|
||||||
|
cm_topology::TopologyKind::ALL
|
||||||
|
.iter()
|
||||||
|
.copied()
|
||||||
|
.find(|k| k.as_str().eq_ignore_ascii_case(want))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Backends at least one online node can actually boot.
|
||||||
|
///
|
||||||
|
/// Read from the nodes' reported `rootfs` capability, so it answers "what can
|
||||||
|
/// run today" rather than "what images did someone build once".
|
||||||
|
pub async fn available_backends(
|
||||||
|
pool: &sqlx::PgPool,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
) -> Result<Vec<String>, String> {
|
||||||
|
let rows: Vec<(serde_json::Value,)> = sqlx::query_as(
|
||||||
|
"SELECT capabilities -> 'rootfs'
|
||||||
|
FROM nodes
|
||||||
|
WHERE workspace_id = $1 AND status = 'online'
|
||||||
|
AND capabilities @> '{\"microvm\": true}'::jsonb",
|
||||||
|
)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("read node rootfs capabilities: {e}"))?;
|
||||||
|
|
||||||
|
let mut out: Vec<String> = rows
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|(v,)| v.as_array().cloned())
|
||||||
|
.flatten()
|
||||||
|
.filter_map(|v| v.as_str().map(str::to_string))
|
||||||
|
// A node reports every rootfs it has BUILT, which is not the same as
|
||||||
|
// every rootfs a mission can run in. `agent-terminal` is on tank right
|
||||||
|
// now: bootable, and with no credential contract, so an agent inside it
|
||||||
|
// has nothing to authenticate with. Offering it to the planner would
|
||||||
|
// produce a roster that validates, approves, launches, and then fails at
|
||||||
|
// the agent turn — the expensive kind of late.
|
||||||
|
.filter(|b| crate::mission_runtime::backend_can_run_a_mission(b))
|
||||||
|
.collect();
|
||||||
|
out.sort();
|
||||||
|
out.dedup();
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn member(role: &str, backend: Option<&str>) -> RosterMember {
|
||||||
|
RosterMember {
|
||||||
|
role: role.into(),
|
||||||
|
backend: backend.map(str::to_string),
|
||||||
|
rationale: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn roster(kind: &str, members: Vec<RosterMember>) -> Roster {
|
||||||
|
Roster {
|
||||||
|
topology_kind: kind.into(),
|
||||||
|
members,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A backend the fleet cannot boot must be refused where it is PROPOSED.
|
||||||
|
/// Placement would refuse it too — at launch, after the roster was approved
|
||||||
|
/// and someone believed the mission was going to run.
|
||||||
|
#[test]
|
||||||
|
fn a_backend_no_node_can_run_is_refused_with_the_ones_that_exist() {
|
||||||
|
let have = vec!["claude".to_string(), "kimi".to_string()];
|
||||||
|
let r = roster(
|
||||||
|
"pipeline",
|
||||||
|
vec![member("implementer", Some("claude")), member("verifier", Some("rootfs-opus"))],
|
||||||
|
);
|
||||||
|
let err = r.validate(&have).unwrap_err();
|
||||||
|
assert_eq!(
|
||||||
|
err,
|
||||||
|
Refusal::UnknownBackend {
|
||||||
|
backend: "rootfs-opus".into(),
|
||||||
|
available: have.clone()
|
||||||
|
}
|
||||||
|
);
|
||||||
|
// The message must name what IS available, or the operator's next move
|
||||||
|
// is a guess.
|
||||||
|
let msg = err.to_string();
|
||||||
|
assert!(msg.contains("claude") && msg.contains("kimi"), "{msg}");
|
||||||
|
|
||||||
|
// And the same roster passes once every backend is one the fleet has.
|
||||||
|
let ok = roster(
|
||||||
|
"pipeline",
|
||||||
|
vec![member("implementer", Some("claude")), member("verifier", Some("kimi"))],
|
||||||
|
);
|
||||||
|
assert!(ok.validate(&have).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A member with no backend inherits the mission's, which is legitimate —
|
||||||
|
/// the whole roster does not have to be heterogeneous to be useful.
|
||||||
|
#[test]
|
||||||
|
fn a_member_without_a_backend_is_not_a_refusal() {
|
||||||
|
let r = roster("pipeline", vec![member("implementer", None)]);
|
||||||
|
assert!(r.validate(&["claude".to_string()]).is_ok());
|
||||||
|
// Blank counts as absent, not as a backend named "".
|
||||||
|
let r = roster("pipeline", vec![member("implementer", Some(" "))]);
|
||||||
|
assert!(r.validate(&["claude".to_string()]).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A bootable image is not necessarily a runnable one. tank reports
|
||||||
|
/// `agent-terminal` in its rootfs list today: a real image, with no
|
||||||
|
/// credential contract, so an agent booted into it has nothing to
|
||||||
|
/// authenticate with. Offering it to the planner would produce a roster that
|
||||||
|
/// validates, approves, launches and then fails at the agent turn.
|
||||||
|
#[test]
|
||||||
|
fn only_backends_that_can_authenticate_are_offered() {
|
||||||
|
assert!(crate::mission_runtime::backend_can_run_a_mission("claude"));
|
||||||
|
assert!(crate::mission_runtime::backend_can_run_a_mission("default"));
|
||||||
|
for unrunnable in ["agent-terminal", "agent-browser", "rootfs-opus"] {
|
||||||
|
assert!(
|
||||||
|
!crate::mission_runtime::backend_can_run_a_mission(unrunnable),
|
||||||
|
"{unrunnable} has no credential contract and must not be proposable"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The ceiling. Each member is a VM boot, an inject, a full agent session
|
||||||
|
/// and a collect — a model asked to size a team proposes twelve happily.
|
||||||
|
#[test]
|
||||||
|
fn a_roster_is_bounded_and_non_empty() {
|
||||||
|
let have = vec!["claude".to_string()];
|
||||||
|
assert_eq!(roster("pipeline", vec![]).validate(&have), Err(Refusal::Empty));
|
||||||
|
|
||||||
|
let many: Vec<_> = (0..MAX_MEMBERS + 1)
|
||||||
|
.map(|i| member(&format!("r{i}"), None))
|
||||||
|
.collect();
|
||||||
|
assert_eq!(
|
||||||
|
roster("pipeline", many).validate(&have),
|
||||||
|
Err(Refusal::TooMany(MAX_MEMBERS + 1))
|
||||||
|
);
|
||||||
|
|
||||||
|
let exactly: Vec<_> = (0..MAX_MEMBERS).map(|i| member(&format!("r{i}"), None)).collect();
|
||||||
|
assert!(roster("pipeline", exactly).validate(&have).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An invented topology kind must be refused, NOT defaulted. Running a
|
||||||
|
/// `pipeline` proposal as a hub-and-spoke changes what every node sees and
|
||||||
|
/// nothing would say so — the same silent-substitution shape as a backend
|
||||||
|
/// that quietly falls back to the default image.
|
||||||
|
#[test]
|
||||||
|
fn an_invented_topology_kind_is_refused_rather_than_defaulted() {
|
||||||
|
let have = vec!["claude".to_string()];
|
||||||
|
let r = roster("assembly_line", vec![member("implementer", None)]);
|
||||||
|
assert_eq!(
|
||||||
|
r.validate(&have),
|
||||||
|
Err(Refusal::UnknownTopology("assembly_line".into()))
|
||||||
|
);
|
||||||
|
// Every kind the catalog declares is accepted, so this cannot drift out
|
||||||
|
// of sync with what the orchestrator can actually plan.
|
||||||
|
for kind in cm_topology::TopologyKind::ALL {
|
||||||
|
let r = roster(kind.as_str(), vec![member("implementer", None)]);
|
||||||
|
assert!(r.validate(&have).is_ok(), "{}", kind.as_str());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The graph is the handoff to the composed executor: node ids in
|
||||||
|
/// `cm_topology`'s own convention, and the backend in the `attrs` channel
|
||||||
|
/// `MicroVmTurnExecutor` reads. If this drifts, a heterogeneous roster runs
|
||||||
|
/// every node on the mission default and looks fine.
|
||||||
|
#[test]
|
||||||
|
fn the_graph_carries_each_members_backend_where_the_executor_reads_it() {
|
||||||
|
let r = roster(
|
||||||
|
"pipeline",
|
||||||
|
vec![
|
||||||
|
member("implementer", Some("claude")),
|
||||||
|
member("verifier", Some("kimi")),
|
||||||
|
member("scribe", None),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
let g = r.graph().expect("a runnable graph");
|
||||||
|
let nodes = g["nodes"].as_array().expect("nodes");
|
||||||
|
assert_eq!(nodes.len(), 3);
|
||||||
|
assert_eq!(nodes[0]["role"], "implementer");
|
||||||
|
assert_eq!(nodes[0]["attrs"]["backend"], "claude");
|
||||||
|
assert_eq!(nodes[1]["attrs"]["backend"], "kimi");
|
||||||
|
assert!(
|
||||||
|
nodes[2]["attrs"].get("backend").is_none(),
|
||||||
|
"a member with no backend must inherit the mission's, not be stamped with one"
|
||||||
|
);
|
||||||
|
|
||||||
|
// And it deserializes as the real thing the worker will parse — a graph
|
||||||
|
// that only looks right as JSON fails at claim time with "missing or
|
||||||
|
// invalid graph", which reads as a runtime fault rather than a bad
|
||||||
|
// roster.
|
||||||
|
let parsed: cm_topology::TopologyGraph =
|
||||||
|
serde_json::from_value(g).expect("the worker must be able to parse it");
|
||||||
|
assert_eq!(parsed.nodes.len(), 3);
|
||||||
|
assert_eq!(
|
||||||
|
parsed.nodes[1].attrs.get("backend").map(String::as_str),
|
||||||
|
Some("kimi")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,7 +24,6 @@
|
|||||||
//! carry the current binding (both null when torn down or never
|
//! carry the current binding (both null when torn down or never
|
||||||
//! provisioned).
|
//! provisioned).
|
||||||
|
|
||||||
use base64::Engine;
|
|
||||||
use bollard::exec::{CreateExecOptions, StartExecResults};
|
use bollard::exec::{CreateExecOptions, StartExecResults};
|
||||||
use bollard::models::{
|
use bollard::models::{
|
||||||
ContainerCreateBody, EndpointSettings, HostConfig, Mount, MountTypeEnum, NetworkConnectRequest,
|
ContainerCreateBody, EndpointSettings, HostConfig, Mount, MountTypeEnum, NetworkConnectRequest,
|
||||||
@@ -96,7 +95,6 @@ pub fn forwarded_provider_keys(auth: RuntimeAuth) -> Vec<&'static str> {
|
|||||||
// in the container. They are unrelated to the Anthropic credential and
|
// in the container. They are unrelated to the Anthropic credential and
|
||||||
// forward in both auth modes.
|
// forward in both auth modes.
|
||||||
let mut keys = vec![
|
let mut keys = vec![
|
||||||
"GEMINI_API_KEY",
|
|
||||||
"GROQ_API_KEY",
|
"GROQ_API_KEY",
|
||||||
"OPENAI_API_KEY",
|
"OPENAI_API_KEY",
|
||||||
"ZAI_API_KEY",
|
"ZAI_API_KEY",
|
||||||
@@ -109,6 +107,187 @@ pub fn forwarded_provider_keys(auth: RuntimeAuth) -> Vec<&'static str> {
|
|||||||
keys
|
keys
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The forwarded credentials that are actually SET on this server, as pairs.
|
||||||
|
///
|
||||||
|
/// `forwarded_provider_keys` above stays the single list of *what* forwards; this
|
||||||
|
/// is the single reader of the environment, so the container and microVM paths
|
||||||
|
/// cannot disagree about how a value is read (blank handling in particular).
|
||||||
|
///
|
||||||
|
/// A key that is unset is simply absent: an empty-string value would make
|
||||||
|
/// `claude` believe it has a credential and fail authentication instead of
|
||||||
|
/// reporting that it has none.
|
||||||
|
pub fn forwarded_provider_env(auth: RuntimeAuth) -> Vec<(String, String)> {
|
||||||
|
provider_env_from(auth, |k| std::env::var(k).ok())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Credentials for a microVM mission. **Subscription only, by construction.**
|
||||||
|
///
|
||||||
|
/// Deliberately NOT parameterised by [`runtime_auth_mode`], and that is the
|
||||||
|
/// whole point. The container path honours the operator's mode, and gw-04 has
|
||||||
|
/// `CLAWMATES_RUNTIME_AUTH` unset today, so it forwards `ANTHROPIC_API_KEY`. A
|
||||||
|
/// microVM must not: Claude Code ranks the API key **above** the subscription's
|
||||||
|
/// OAuth token, so a VM that received both would bill per-token against a plan
|
||||||
|
/// we already pay for, silently — `claude` still works, agents still run, and
|
||||||
|
/// the only symptom is the invoice. Taking the mode as an argument would mean a
|
||||||
|
/// single unset env var on a new host turns that back on.
|
||||||
|
///
|
||||||
|
/// A missing subscription token is an **error**, not an empty list. A VM launched
|
||||||
|
/// without a credential does not fail: `claude -p` hangs, which is a phase stuck
|
||||||
|
/// at `running` with nothing in the logs.
|
||||||
|
pub fn microvm_provider_env(backend: Option<&str>) -> Result<Vec<(String, String)>, String> {
|
||||||
|
microvm_provider_env_from(backend, |k| std::env::var(k).ok())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Can a mission agent authenticate in a VM booted from this image?
|
||||||
|
///
|
||||||
|
/// The honest answer to "which backends can run a mission", which is NOT the
|
||||||
|
/// same as "which rootfs images exist on a node". A fleet node reports every
|
||||||
|
/// image it has built — `agent-terminal` among them — and booting a mission into
|
||||||
|
/// one with no credential contract fails at the agent turn, after the mission
|
||||||
|
/// was created and its roster approved.
|
||||||
|
pub fn backend_can_run_a_mission(backend: &str) -> bool {
|
||||||
|
microvm_credential_for(Some(backend)).is_ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Where a backend's secret comes from, and what the guest's CLI reads it as.
|
||||||
|
///
|
||||||
|
/// Two names, not one, because they differ for every provider but Anthropic:
|
||||||
|
/// z.ai's key lives in the server's `ZAI_API_KEY` and Claude Code reads it as
|
||||||
|
/// `ANTHROPIC_AUTH_TOKEN`. Collapsing them into one string is what forces a
|
||||||
|
/// guess at the other end, and a wrong guess here sends one provider's
|
||||||
|
/// credential to another provider's endpoint.
|
||||||
|
struct Credential {
|
||||||
|
/// The env var on THIS SERVER holding the secret.
|
||||||
|
source: &'static str,
|
||||||
|
/// The env var the GUEST's CLI reads it from.
|
||||||
|
target: &'static str,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The credential a backend's CLI authenticates with inside a VM.
|
||||||
|
///
|
||||||
|
/// Unknown backends are refused rather than given the Anthropic token: sending a
|
||||||
|
/// subscription credential to whatever endpoint an unrecognised backend points
|
||||||
|
/// at is worse than not launching.
|
||||||
|
fn microvm_credential_for(backend: Option<&str>) -> Result<Credential, String> {
|
||||||
|
match backend {
|
||||||
|
// `canary-*` backends are a REAL rootfs built from a candidate CLI
|
||||||
|
// version, run through the production path — egress, stop gate,
|
||||||
|
// delegation, delivery — before that version is promoted to the image
|
||||||
|
// every mission uses. They carry the same Anthropic subscription
|
||||||
|
// credential as `claude`, because testing a new CLI against a different
|
||||||
|
// provider would not be testing the thing we are about to ship.
|
||||||
|
//
|
||||||
|
// Named rather than pattern-matched on anything looser: an unrecognised
|
||||||
|
// backend must still be refused at launch, which is what
|
||||||
|
// `backend_can_run_a_mission` and the harness's negative control assert.
|
||||||
|
None | Some("") | Some("default") | Some("claude") | Some("canary-claude") => {
|
||||||
|
Ok(Credential {
|
||||||
|
source: "CLAUDE_CODE_OAUTH_TOKEN",
|
||||||
|
target: "CLAUDE_CODE_OAUTH_TOKEN",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// GLM. `images/agent-glm` bakes `ANTHROPIC_BASE_URL=https://api.z.ai/
|
||||||
|
// api/anthropic` into the rootfs, so the endpoint is a property of the
|
||||||
|
// image and the credential is a property of the turn. That split is what
|
||||||
|
// makes the dangerous mix-up unrepresentable: this VM cannot be handed
|
||||||
|
// an Anthropic token, and a `claude` VM cannot be pointed at z.ai.
|
||||||
|
Some("glm") => Ok(Credential {
|
||||||
|
source: "ZAI_API_KEY",
|
||||||
|
target: "ANTHROPIC_AUTH_TOKEN",
|
||||||
|
}),
|
||||||
|
Some("kimi") => Ok(Credential {
|
||||||
|
source: "KIMI_API_KEY",
|
||||||
|
target: "ANTHROPIC_AUTH_TOKEN",
|
||||||
|
}),
|
||||||
|
// A model running on the NODE ITSELF. There is no credential and there
|
||||||
|
// is nothing to protect: the guest reaches it over a vsock pipe to the
|
||||||
|
// node's own loopback (`clawmates-node::local_model`), which has no
|
||||||
|
// destination in its protocol and can therefore reach nothing else.
|
||||||
|
//
|
||||||
|
// It still goes through this map rather than around it. Ollama ignores
|
||||||
|
// the bearer but Claude Code refuses to start without one, so the value
|
||||||
|
// is a literal — and routing it here keeps the rule that a backend with
|
||||||
|
// no entry cannot launch, which is what stops a typo'd backend from
|
||||||
|
// quietly inheriting the subscription token.
|
||||||
|
Some("local-ornith") => Ok(Credential {
|
||||||
|
source: "CLAWMATES_LOCAL_MODEL_TOKEN",
|
||||||
|
target: "ANTHROPIC_AUTH_TOKEN",
|
||||||
|
}),
|
||||||
|
// Kimi, on the same split as GLM: endpoint in the image
|
||||||
|
// (`https://api.kimi.com/coding`), credential in the turn.
|
||||||
|
//
|
||||||
|
// Which HOST took a measurement to find. `api.moonshot.ai/anthropic`
|
||||||
|
// exists and speaks the protocol, and rejects an `sk-kimi-` key — it is
|
||||||
|
// the platform.moonshot.ai account namespace. The Kimi CODE service at
|
||||||
|
// `api.kimi.com/coding` is where such a key is valid, and it answers
|
||||||
|
// `/v1/messages` with a real Anthropic body. Two endpoints that both
|
||||||
|
// "work" for different accounts is exactly the shape that makes a
|
||||||
|
// guessed URL look like a broken key.
|
||||||
|
Some(other) => Err(format!(
|
||||||
|
"backend {other:?} has no defined microVM credential contract yet — \
|
||||||
|
refusing to launch rather than forward one provider's credential to \
|
||||||
|
another provider's endpoint"
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn microvm_provider_env_from(
|
||||||
|
backend: Option<&str>,
|
||||||
|
lookup: impl Fn(&str) -> Option<String>,
|
||||||
|
) -> Result<Vec<(String, String)>, String> {
|
||||||
|
let want = microvm_credential_for(backend)?;
|
||||||
|
// A local model has no secret to look up. Defaulted rather than demanded:
|
||||||
|
// requiring an operator to set a variable whose value is ignored is a step
|
||||||
|
// that only ever fails, and its failure mode here is a hung agent.
|
||||||
|
let lookup = |k: &str| {
|
||||||
|
lookup(k).or_else(|| {
|
||||||
|
(k == "CLAWMATES_LOCAL_MODEL_TOKEN").then(|| "local-model-no-auth".to_string())
|
||||||
|
})
|
||||||
|
};
|
||||||
|
let token = lookup(want.source)
|
||||||
|
.filter(|v| !v.trim().is_empty())
|
||||||
|
.ok_or_else(|| {
|
||||||
|
format!(
|
||||||
|
"{} is not set on this server, so a microVM mission on backend \
|
||||||
|
{backend:?} would run `claude -p` with no credential — which hangs \
|
||||||
|
rather than failing. Set it, or run the mission on another backend.",
|
||||||
|
want.source
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mut env = vec![(want.target.to_string(), token)];
|
||||||
|
// Non-Anthropic providers a mission's tools may need, forwarded when set.
|
||||||
|
// ANTHROPIC_API_KEY is absent from this list and must stay absent — see the
|
||||||
|
// doc comment above.
|
||||||
|
for k in ["GROQ_API_KEY", "OPENAI_API_KEY"] {
|
||||||
|
if let Some(v) = lookup(k).filter(|v| !v.trim().is_empty()) {
|
||||||
|
env.push((k.to_string(), v));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
debug_assert!(
|
||||||
|
!env.iter().any(|(k, _)| k == "ANTHROPIC_API_KEY"),
|
||||||
|
"the microVM path must never forward ANTHROPIC_API_KEY"
|
||||||
|
);
|
||||||
|
Ok(env)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The testable half of [`forwarded_provider_env`]. The lookup is a parameter
|
||||||
|
/// because a test cannot set process environment variables here — the workspace
|
||||||
|
/// denies `unsafe`, and `set_var` is racy across test threads regardless.
|
||||||
|
fn provider_env_from(
|
||||||
|
auth: RuntimeAuth,
|
||||||
|
lookup: impl Fn(&str) -> Option<String>,
|
||||||
|
) -> Vec<(String, String)> {
|
||||||
|
forwarded_provider_keys(auth)
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|k| {
|
||||||
|
lookup(k)
|
||||||
|
.filter(|v| !v.trim().is_empty())
|
||||||
|
.map(|v| (k.to_string(), v))
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
/// Read `CLAWMATES_RUNTIME_AUTH`, defaulting to `api_key`.
|
/// Read `CLAWMATES_RUNTIME_AUTH`, defaulting to `api_key`.
|
||||||
///
|
///
|
||||||
/// Defaulting to the existing behaviour is deliberate: an unset or misspelled
|
/// Defaulting to the existing behaviour is deliberate: an unset or misspelled
|
||||||
@@ -142,7 +321,16 @@ const EDGE_NETWORK: &str = "clawmates_edge";
|
|||||||
/// Host path (as seen by the docker engine, NOT the server container)
|
/// Host path (as seen by the docker engine, NOT the server container)
|
||||||
/// where the mission's checkouts live. Matches the mount source used
|
/// where the mission's checkouts live. Matches the mount source used
|
||||||
/// by `clawmates-runtime.service`.
|
/// by `clawmates-runtime.service`.
|
||||||
const MISSIONS_HOST_ROOT: &str = "/var/lib/clawmates-missions";
|
///
|
||||||
|
/// This was a hardcoded const that read no env at all, while every writer of
|
||||||
|
/// the same tree went through `CLAWMATES_MISSIONS_ROOT`. They agree on this
|
||||||
|
/// deployment; they are not guaranteed to, and a reaper pointed at one of them
|
||||||
|
/// would have silently left the other's directories behind forever.
|
||||||
|
fn missions_host_root() -> String {
|
||||||
|
crate::mission_workspace::missions_root()
|
||||||
|
.to_string_lossy()
|
||||||
|
.into_owned()
|
||||||
|
}
|
||||||
|
|
||||||
/// Host path holding the shared ZeroClaw config + seeded agent library
|
/// Host path holding the shared ZeroClaw config + seeded agent library
|
||||||
/// (built up over time by the shared clawmates-runtime.service). Per-
|
/// (built up over time by the shared clawmates-runtime.service). Per-
|
||||||
@@ -158,6 +346,8 @@ const MISSIONS_HOST_ROOT: &str = "/var/lib/clawmates-missions";
|
|||||||
/// runtime. Concurrent daemons opening the same sessions.db can
|
/// runtime. Concurrent daemons opening the same sessions.db can
|
||||||
/// interleave; in practice topology_worker sequentializes runs per
|
/// interleave; in practice topology_worker sequentializes runs per
|
||||||
/// mission so this rarely bites. Long-term: copy-on-write per mission.
|
/// mission so this rarely bites. Long-term: copy-on-write per mission.
|
||||||
|
use crate::container_exec::MISSION_UID;
|
||||||
|
|
||||||
const DEFAULT_SEED_DIR: &str = "/root/clawmates-runtime/data";
|
const DEFAULT_SEED_DIR: &str = "/root/clawmates-runtime/data";
|
||||||
|
|
||||||
|
|
||||||
@@ -184,9 +374,14 @@ const SEEDED_PATHS: &[&str] = &[
|
|||||||
// token refresh or project state in one mission cannot leak into another.
|
// token refresh or project state in one mission cannot leak into another.
|
||||||
".claude",
|
".claude",
|
||||||
".claude.json",
|
".claude.json",
|
||||||
// Per-CLI state for the alternate backends; small.
|
// Per-CLI state for the alternate backends; small. `glm-home` and
|
||||||
|
// `kimi-home` are separate HOMEs for the two `claude_cli` fallback
|
||||||
|
// aliases: each needs its own `.claude.json` so the binary skips
|
||||||
|
// onboarding, and keeping them apart stops two concurrent fallbacks from
|
||||||
|
// sharing one Claude Code session directory.
|
||||||
".kimi-code",
|
".kimi-code",
|
||||||
"glm-home",
|
"glm-home",
|
||||||
|
"kimi-home",
|
||||||
// The seeded agent library.
|
// The seeded agent library.
|
||||||
"agents",
|
"agents",
|
||||||
];
|
];
|
||||||
@@ -203,6 +398,17 @@ fn copy_script() -> String {
|
|||||||
"if [ -e '/seed/{p}' ]; then cp -a '/seed/{p}' /dst/; fi\n"
|
"if [ -e '/seed/{p}' ]; then cp -a '/seed/{p}' /dst/; fi\n"
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
// The copy itself must run as root: the seed dir is root-owned and parts of
|
||||||
|
// it are mode 0600 (`.claude.json`, `clawmates-mcp.json`), so uid 65532
|
||||||
|
// cannot even READ them. Running the container as 65532 made `cp` fail on
|
||||||
|
// the first unreadable entry, `set -e` aborted the rest, and the mission
|
||||||
|
// got a runtime-data holding `.zeroclaw` and nothing else — no Claude
|
||||||
|
// credentials, no door config.
|
||||||
|
//
|
||||||
|
// So keep root for the read, and hand the RESULT to 65532. That is what the
|
||||||
|
// destination needs: every other writer into the missions tree is 65532,
|
||||||
|
// and a GC running as 65532 cannot delete what root left behind.
|
||||||
|
out.push_str(&format!("chown -R {MISSION_UID} /dst\n"));
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -284,14 +490,31 @@ async fn seed_runtime_data(
|
|||||||
.inspect_container(&name, None::<InspectContainerOptions>)
|
.inspect_container(&name, None::<InspectContainerOptions>)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
|
// `auto_remove` already took it away, which only happens after a
|
||||||
|
// clean-ish exit; the status is gone with it. Treated as success
|
||||||
|
// because the alternative — failing every mission on a race with
|
||||||
|
// the reaper — is worse, and `stamp_workspace_paths` fails loudly
|
||||||
|
// downstream if the config did not arrive.
|
||||||
Err(_) => return Ok(()),
|
Err(_) => return Ok(()),
|
||||||
Ok(info) => {
|
Ok(info) => {
|
||||||
let running = info
|
let state = info.state.as_ref();
|
||||||
.state
|
let running = state.and_then(|st| st.running).unwrap_or(false);
|
||||||
.as_ref()
|
|
||||||
.and_then(|st| st.running)
|
|
||||||
.unwrap_or(false);
|
|
||||||
if !running {
|
if !running {
|
||||||
|
// The exit code was never read. A `cp` that died on an
|
||||||
|
// unreadable seed file left `set -e` to abort the rest, and
|
||||||
|
// this returned Ok — so the mission came up with a
|
||||||
|
// half-seeded runtime, the daemon never created its agents'
|
||||||
|
// workspace, and the phase then failed on
|
||||||
|
// "Could not find the file /mission in container",
|
||||||
|
// 200 lines and one indirection away from the real cause.
|
||||||
|
let code = state.and_then(|st| st.exit_code).unwrap_or(0);
|
||||||
|
if code != 0 {
|
||||||
|
return Err(format!(
|
||||||
|
"seed copier exited {code} — the mission's runtime-data is \
|
||||||
|
incomplete (the seed dir is root-owned and partly mode 0600; \
|
||||||
|
check that this container still runs as root)"
|
||||||
|
));
|
||||||
|
}
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -393,7 +616,7 @@ impl MissionRuntimeProvisioner {
|
|||||||
// Create fresh. Ensure the bind source exists first — research-
|
// Create fresh. Ensure the bind source exists first — research-
|
||||||
// only missions (no repo checkout) still need the directory
|
// only missions (no repo checkout) still need the directory
|
||||||
// present or docker start fails with EACCES/ENOENT.
|
// present or docker start fails with EACCES/ENOENT.
|
||||||
let mission_dir = format!("{MISSIONS_HOST_ROOT}/{mission_id}");
|
let mission_dir = format!("{}/{mission_id}", missions_host_root());
|
||||||
let _ = tokio::fs::create_dir_all(&mission_dir).await;
|
let _ = tokio::fs::create_dir_all(&mission_dir).await;
|
||||||
let seed_dir = std::env::var("CLAWMATES_RUNTIME_SEED_DIR")
|
let seed_dir = std::env::var("CLAWMATES_RUNTIME_SEED_DIR")
|
||||||
.unwrap_or_else(|_| DEFAULT_SEED_DIR.to_string());
|
.unwrap_or_else(|_| DEFAULT_SEED_DIR.to_string());
|
||||||
@@ -403,16 +626,21 @@ impl MissionRuntimeProvisioner {
|
|||||||
let runtime_data_dir = format!("{mission_dir}/runtime-data");
|
let runtime_data_dir = format!("{mission_dir}/runtime-data");
|
||||||
let _ = tokio::fs::create_dir_all(&runtime_data_dir).await;
|
let _ = tokio::fs::create_dir_all(&runtime_data_dir).await;
|
||||||
seed_runtime_data(&self.docker, &self.image, &seed_dir, &runtime_data_dir).await?;
|
seed_runtime_data(&self.docker, &self.image, &seed_dir, &runtime_data_dir).await?;
|
||||||
let mounts = vec![
|
let mut mounts = Vec::new();
|
||||||
// Mount just this mission's directory. Agents can navigate
|
// In copy mode the checkout is pushed in and pulled back out, so the
|
||||||
// its `/repo` subdir but never see other missions'.
|
// container gets its OWN filesystem and the host directory has exactly
|
||||||
Mount {
|
// one writer (the server). Binding it here would put two uids back on
|
||||||
|
// one directory — the cause of four separate work-loss bugs.
|
||||||
|
if !crate::mission_fs::copy_mode() {
|
||||||
|
mounts.push(Mount {
|
||||||
target: Some("/mission".to_string()),
|
target: Some("/mission".to_string()),
|
||||||
source: Some(mission_dir.clone()),
|
source: Some(mission_dir.clone()),
|
||||||
typ: Some(MountTypeEnum::BIND),
|
typ: Some(MountTypeEnum::BIND),
|
||||||
read_only: Some(false),
|
read_only: Some(false),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
},
|
});
|
||||||
|
}
|
||||||
|
mounts.extend([
|
||||||
// This mission's OWN copy of the seeded agent library
|
// This mission's OWN copy of the seeded agent library
|
||||||
// (`claw_*` templates). Copied rather than shared, so its
|
// (`claw_*` templates). Copied rather than shared, so its
|
||||||
// config.toml — which carries the door bearer token — and its
|
// config.toml — which carries the door bearer token — and its
|
||||||
@@ -425,7 +653,7 @@ impl MissionRuntimeProvisioner {
|
|||||||
read_only: Some(false),
|
read_only: Some(false),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
},
|
},
|
||||||
];
|
]);
|
||||||
|
|
||||||
let host_config = HostConfig {
|
let host_config = HostConfig {
|
||||||
mounts: Some(mounts),
|
mounts: Some(mounts),
|
||||||
@@ -482,10 +710,8 @@ impl MissionRuntimeProvisioner {
|
|||||||
// The other three are unrelated providers (Gemini/Groq/OpenAI) with no
|
// The other three are unrelated providers (Gemini/Groq/OpenAI) with no
|
||||||
// subscription equivalent, so they forward in both modes.
|
// subscription equivalent, so they forward in both modes.
|
||||||
let auth_mode = runtime_auth_mode();
|
let auth_mode = runtime_auth_mode();
|
||||||
for key in forwarded_provider_keys(auth_mode) {
|
for (key, v) in forwarded_provider_env(auth_mode) {
|
||||||
if let Ok(v) = std::env::var(key) {
|
env.push(format!("{key}={v}"));
|
||||||
env.push(format!("{key}={v}"));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"mission_runtime: mission {mission_id} container auth mode = {} \
|
"mission_runtime: mission {mission_id} container auth mode = {} \
|
||||||
@@ -655,18 +881,16 @@ impl MissionRuntimeProvisioner {
|
|||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let b64 = base64::engine::general_purpose::STANDARD.encode(edited.as_bytes());
|
// Upload rather than exec. This used to base64 the whole config into a
|
||||||
// Decode to a sibling temp then atomically move over the live file,
|
// single `sh -c` argument, which works until the file grows — config
|
||||||
// so a partial write can never leave the daemon with truncated TOML.
|
// gains a block per provisioned claw — and then fails with
|
||||||
let script = format!(
|
// `argument list too long`. Mission `019fcf62` hit exactly that: the
|
||||||
"printf %s '{b64}' | base64 -d > {CONFIG_PATH}.tmp && mv {CONFIG_PATH}.tmp {CONFIG_PATH}"
|
// pin never applied, its agents never saw `/mission/repo`, and phase 0
|
||||||
);
|
// completed having written nothing. The tar API has no argv limit, so
|
||||||
let out = self
|
// the failure mode is gone rather than merely further away.
|
||||||
.exec_capture(&name, vec!["sh".into(), "-c".into(), script])
|
crate::mission_fs::put_file(&self.docker, &name, CONFIG_PATH, edited.as_bytes())
|
||||||
.await?;
|
.await
|
||||||
if !out.trim().is_empty() {
|
.map_err(|e| format!("write runtime config.toml: {e}"))?;
|
||||||
return Err(format!("write runtime config.toml: {out}"));
|
|
||||||
}
|
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"mission_runtime: pinned {pinned} workspace(s) → {workspace_path} for mission {mission_id}"
|
"mission_runtime: pinned {pinned} workspace(s) → {workspace_path} for mission {mission_id}"
|
||||||
);
|
);
|
||||||
@@ -799,6 +1023,20 @@ impl MissionRuntimeProvisioner {
|
|||||||
Err(format!("{name} gateway did not come back after restart"))
|
Err(format!("{name} gateway did not come back after restart"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Does this mission's runtime container still exist, in any state?
|
||||||
|
///
|
||||||
|
/// Used by the sweeper to decide whether a failed teardown left something
|
||||||
|
/// behind. Deliberately treats an inspect error as "gone": the caller uses
|
||||||
|
/// this to decide whether to KEEP a binding for a retry, and answering
|
||||||
|
/// "still there" when docker cannot be reached would pin the binding open
|
||||||
|
/// on an unreachable daemon rather than on a real container.
|
||||||
|
pub async fn container_exists(&self, mission_id: Uuid) -> bool {
|
||||||
|
self.docker
|
||||||
|
.inspect_container(&container_name(mission_id), None::<InspectContainerOptions>)
|
||||||
|
.await
|
||||||
|
.is_ok()
|
||||||
|
}
|
||||||
|
|
||||||
/// Force-remove the mission's runtime container AND its host workspace
|
/// Force-remove the mission's runtime container AND its host workspace
|
||||||
/// dir (the `/mission/repo` checkout bind source). Idempotent: a missing
|
/// dir (the `/mission/repo` checkout bind source). Idempotent: a missing
|
||||||
/// container or dir is not an error — this is called both by the terminal
|
/// container or dir is not an error — this is called both by the terminal
|
||||||
@@ -824,10 +1062,32 @@ impl MissionRuntimeProvisioner {
|
|||||||
}
|
}
|
||||||
// Remove the per-mission workspace dir (repo checkout + scratch). This
|
// Remove the per-mission workspace dir (repo checkout + scratch). This
|
||||||
// path is bind-mounted into cm-api, so we can reap it directly.
|
// path is bind-mounted into cm-api, so we can reap it directly.
|
||||||
let mission_dir = format!("{MISSIONS_HOST_ROOT}/{mission_id}");
|
let mission_dir = format!("{}/{mission_id}", missions_host_root());
|
||||||
if let Err(e) = tokio::fs::remove_dir_all(&mission_dir).await {
|
if let Err(e) = tokio::fs::remove_dir_all(&mission_dir).await {
|
||||||
if e.kind() != std::io::ErrorKind::NotFound {
|
match e.kind() {
|
||||||
eprintln!("mission_runtime: rm workspace dir {mission_dir}: {e}");
|
std::io::ErrorKind::NotFound => {}
|
||||||
|
// The server runs as 65532 and cannot delete root-owned files.
|
||||||
|
// Almost everything under a mission is 65532 now, but the
|
||||||
|
// per-mission ZeroClaw daemon still runs as root and leaves ~26
|
||||||
|
// of its own files (`.claude.json`, session jsonl) behind after
|
||||||
|
// the seed copy has been chowned. Without this fallback those
|
||||||
|
// few files keep the whole directory alive forever — the
|
||||||
|
// cleanup-that-cannot-clean-up shape, at a scale small enough
|
||||||
|
// to go unnoticed for a long time.
|
||||||
|
std::io::ErrorKind::PermissionDenied => {
|
||||||
|
eprintln!(
|
||||||
|
"mission_runtime: {mission_dir} holds root-owned files — removing it from inside the runtime container"
|
||||||
|
);
|
||||||
|
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
||||||
|
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
||||||
|
crate::root_copy::purge(&container, std::path::Path::new(&mission_dir)).await;
|
||||||
|
if tokio::fs::metadata(&mission_dir).await.is_ok() {
|
||||||
|
eprintln!(
|
||||||
|
"mission_runtime: {mission_dir} SURVIVED the root purge — it will keep accumulating"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => eprintln!("mission_runtime: rm workspace dir {mission_dir}: {e}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -889,12 +1149,36 @@ async fn sweep_once(pool: &sqlx::PgPool, grace: std::time::Duration) -> Result<(
|
|||||||
if let Err(e) = capture_outstanding_phases(pool, id).await {
|
if let Err(e) = capture_outstanding_phases(pool, id).await {
|
||||||
eprintln!("mission_runtime::sweeper: last-chance capture for {id}: {e}");
|
eprintln!("mission_runtime::sweeper: last-chance capture for {id}: {e}");
|
||||||
}
|
}
|
||||||
if let Err(e) = prov.teardown_container(id).await {
|
// Clear the binding only once the container is actually GONE, which is
|
||||||
// A not-found is expected when the container was already
|
// not the same as "teardown returned Ok".
|
||||||
// reaped by a docker restart or a manual op; log at info
|
//
|
||||||
// level (via eprintln) and clear the binding anyway so the
|
// This used to clear unconditionally, reasoning that a not-found is
|
||||||
// sweeper doesn't retry forever.
|
// expected and retrying forever is worse. But `teardown_container`
|
||||||
eprintln!("mission_runtime::sweeper: teardown mission {id}: {e}");
|
// already maps 404/"No such container" to `Ok`, so an `Err` here means
|
||||||
|
// a real docker failure — and this sweep selects on
|
||||||
|
// `runtime_endpoint IS NOT NULL`, so clearing after a failed teardown
|
||||||
|
// hides the surviving container from the only thing that would ever
|
||||||
|
// retry it. One transient docker error would strand a running
|
||||||
|
// container until somebody deleted the mission by hand.
|
||||||
|
//
|
||||||
|
// Asking docker whether the container still exists settles it without
|
||||||
|
// reintroducing the infinite retry: if it is gone, drop the binding
|
||||||
|
// however the call reported itself; if it survives, keep the binding so
|
||||||
|
// the next sweep tries again and the operator sees a recurring log
|
||||||
|
// rather than silence.
|
||||||
|
let teardown = prov.teardown_container(id).await;
|
||||||
|
if let Err(e) = &teardown {
|
||||||
|
if prov.container_exists(id).await {
|
||||||
|
eprintln!(
|
||||||
|
"mission_runtime::sweeper: teardown mission {id}: {e} — container still \
|
||||||
|
present, keeping the runtime binding so the next sweep retries"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
eprintln!(
|
||||||
|
"mission_runtime::sweeper: teardown mission {id}: {e} — container is gone \
|
||||||
|
anyway, clearing the binding"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
if let Err(e) =
|
if let Err(e) =
|
||||||
cm_db::repo::missions::set_runtime_binding(pool, id, workspace_id, None, None, None)
|
cm_db::repo::missions::set_runtime_binding(pool, id, workspace_id, None, None, None)
|
||||||
@@ -948,6 +1232,25 @@ async fn capture_outstanding_phases(pool: &sqlx::PgPool, mission_id: Uuid) -> Re
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
/// A canary backend runs the production path, so it needs the production
|
||||||
|
/// credential — and an UNKNOWN backend must still be refused.
|
||||||
|
///
|
||||||
|
/// The second half is the one that matters: `backend_can_run_a_mission` is
|
||||||
|
/// what stops a mission being launched against a rootfs no node has, and
|
||||||
|
/// widening the credential map is exactly how that guard gets softened by
|
||||||
|
/// accident.
|
||||||
|
#[test]
|
||||||
|
fn a_canary_backend_is_credentialed_but_an_unknown_one_is_not() {
|
||||||
|
assert!(backend_can_run_a_mission("canary-claude"));
|
||||||
|
assert!(backend_can_run_a_mission("claude"));
|
||||||
|
for unknown in ["test226", "definitely-not-built", "canary", "canary-"] {
|
||||||
|
assert!(
|
||||||
|
!backend_can_run_a_mission(unknown),
|
||||||
|
"{unknown} must be refused at launch"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
/// The regression guard for the whole subscription feature.
|
/// The regression guard for the whole subscription feature.
|
||||||
@@ -957,6 +1260,183 @@ mod tests {
|
|||||||
/// means every mission silently bills the API while looking correct. There
|
/// means every mission silently bills the API while looking correct. There
|
||||||
/// is no error to observe — only the invoice. If this test ever goes red,
|
/// is no error to observe — only the invoice. If this test ever goes red,
|
||||||
/// the subscription path is off even though nothing appears broken.
|
/// the subscription path is off even though nothing appears broken.
|
||||||
|
/// The microVM path is subscription-only BY CONSTRUCTION — it does not read
|
||||||
|
/// `CLAWMATES_RUNTIME_AUTH` at all. If it did, the single unset variable on
|
||||||
|
/// gw-04 today would put an API key inside every VM, and Claude Code ranks
|
||||||
|
/// the key above the subscription token: it would work, and bill per-token
|
||||||
|
/// A local backend carries no secret, and no secret reaches it.
|
||||||
|
///
|
||||||
|
/// The credential map is the one place a backend can acquire a token, and
|
||||||
|
/// the point of routing `local-ornith` through it — rather than special-
|
||||||
|
/// casing it earlier — is that the "unknown backend cannot launch" rule
|
||||||
|
/// still holds. A typo like `local-ornit` must fail closed, not inherit the
|
||||||
|
/// subscription token on its way to somebody else's endpoint.
|
||||||
|
#[test]
|
||||||
|
fn a_local_backend_gets_a_placeholder_and_never_a_real_credential() {
|
||||||
|
let env = microvm_provider_env_from(Some("local-ornith"), |_| None)
|
||||||
|
.expect("a local backend needs nothing from the operator");
|
||||||
|
let token = env
|
||||||
|
.iter()
|
||||||
|
.find(|(k, _)| k == "ANTHROPIC_AUTH_TOKEN")
|
||||||
|
.map(|(_, v)| v.clone())
|
||||||
|
.expect("Claude Code refuses to start without a bearer");
|
||||||
|
assert!(
|
||||||
|
!token.starts_with("sk-"),
|
||||||
|
"a local backend must never be handed a real credential, got {token:?}"
|
||||||
|
);
|
||||||
|
assert!(!env.iter().any(|(k, _)| k == "ANTHROPIC_API_KEY"));
|
||||||
|
assert!(!env.iter().any(|(k, _)| k == "CLAUDE_CODE_OAUTH_TOKEN"));
|
||||||
|
|
||||||
|
// And a near-miss still cannot launch.
|
||||||
|
for typo in ["local-ornit", "ornith", "local", "local-ornith2"] {
|
||||||
|
assert!(
|
||||||
|
microvm_credential_for(Some(typo)).is_err(),
|
||||||
|
"{typo} must be refused, not resolved to something"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// against a plan already paid for, with no symptom but the invoice.
|
||||||
|
#[test]
|
||||||
|
fn a_microvm_never_receives_an_anthropic_api_key() {
|
||||||
|
// Everything set, including the key the container path would forward.
|
||||||
|
let env = microvm_provider_env_from(Some("claude"), |k| Some(format!("value-of-{k}")))
|
||||||
|
.expect("a set token should launch");
|
||||||
|
let keys: Vec<&str> = env.iter().map(|(k, _)| k.as_str()).collect();
|
||||||
|
assert!(
|
||||||
|
!keys.contains(&"ANTHROPIC_API_KEY"),
|
||||||
|
"the API key outranks the subscription token; forwarding it bills the \
|
||||||
|
API silently. Forwarded: {keys:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
keys.contains(&"CLAUDE_CODE_OAUTH_TOKEN"),
|
||||||
|
"the subscription credential must travel: {keys:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// No token is a refusal to launch, not an empty environment. A VM without a
|
||||||
|
/// credential does not error — `claude -p` hangs, which reads as a phase
|
||||||
|
/// stuck at `running` with nothing in the logs.
|
||||||
|
#[test]
|
||||||
|
fn a_microvm_without_a_subscription_token_refuses_to_launch() {
|
||||||
|
let r = microvm_provider_env_from(Some("claude"), |k| {
|
||||||
|
// The API key is present; the subscription token is not. This must
|
||||||
|
// NOT be treated as "we have a credential".
|
||||||
|
(k == "ANTHROPIC_API_KEY").then(|| "sk-ant-whatever".to_string())
|
||||||
|
});
|
||||||
|
let err = r.expect_err("no subscription token must refuse the launch");
|
||||||
|
assert!(err.contains("CLAUDE_CODE_OAUTH_TOKEN"), "{err}");
|
||||||
|
// A blank token is the same as no token.
|
||||||
|
assert!(microvm_provider_env_from(Some("claude"), |k| {
|
||||||
|
(k == "CLAUDE_CODE_OAUTH_TOKEN").then(|| " ".to_string())
|
||||||
|
})
|
||||||
|
.is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The credential each provider gets, and — the part that matters — the one
|
||||||
|
/// it must never get. A GLM VM handed `CLAUDE_CODE_OAUTH_TOKEN` would send an
|
||||||
|
/// Anthropic subscription credential to z.ai, verbatim, on the first turn.
|
||||||
|
/// The two providers' secrets must not cross.
|
||||||
|
#[test]
|
||||||
|
fn each_provider_gets_its_own_credential_and_only_its_own() {
|
||||||
|
let env = microvm_provider_env_from(Some("glm"), |k| Some(format!("value-of-{k}")))
|
||||||
|
.expect("glm has a settled contract");
|
||||||
|
let by_key: std::collections::HashMap<_, _> = env.into_iter().collect();
|
||||||
|
// Claude Code reads a custom-endpoint credential as ANTHROPIC_AUTH_TOKEN,
|
||||||
|
// and the value is the SERVER's ZAI_API_KEY — two different names, which
|
||||||
|
// is exactly why the contract carries both.
|
||||||
|
assert_eq!(
|
||||||
|
by_key.get("ANTHROPIC_AUTH_TOKEN").map(String::as_str),
|
||||||
|
Some("value-of-ZAI_API_KEY"),
|
||||||
|
"{by_key:?}"
|
||||||
|
);
|
||||||
|
for forbidden in ["CLAUDE_CODE_OAUTH_TOKEN", "ANTHROPIC_API_KEY"] {
|
||||||
|
assert!(
|
||||||
|
!by_key.contains_key(forbidden),
|
||||||
|
"a GLM VM must never carry {forbidden} — it would be sent to z.ai: {by_key:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Kimi shares the target var with GLM and NOTHING else: same protocol,
|
||||||
|
// different endpoint baked into a different image. What must not happen
|
||||||
|
// is one provider's key travelling under the other's name.
|
||||||
|
let kimi = microvm_provider_env_from(Some("kimi"), |k| Some(format!("value-of-{k}")))
|
||||||
|
.expect("kimi has a settled contract");
|
||||||
|
let kimi: std::collections::HashMap<_, _> = kimi.into_iter().collect();
|
||||||
|
assert_eq!(
|
||||||
|
kimi.get("ANTHROPIC_AUTH_TOKEN").map(String::as_str),
|
||||||
|
Some("value-of-KIMI_API_KEY"),
|
||||||
|
"{kimi:?}"
|
||||||
|
);
|
||||||
|
assert!(!kimi.contains_key("CLAUDE_CODE_OAUTH_TOKEN"), "{kimi:?}");
|
||||||
|
|
||||||
|
// And the reverse: a claude VM carries no z.ai key.
|
||||||
|
let env = microvm_provider_env_from(Some("claude"), |k| Some(format!("value-of-{k}")))
|
||||||
|
.expect("claude");
|
||||||
|
let keys: Vec<&str> = env.iter().map(|(k, _)| k.as_str()).collect();
|
||||||
|
assert!(!keys.contains(&"ANTHROPIC_AUTH_TOKEN"), "{keys:?}");
|
||||||
|
assert!(!keys.contains(&"ZAI_API_KEY"), "{keys:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A GLM mission with no z.ai key refuses to launch, and says which var is
|
||||||
|
/// missing. It must NOT fall back to the Anthropic token that IS set.
|
||||||
|
#[test]
|
||||||
|
fn a_glm_mission_without_a_zai_key_refuses_rather_than_falls_back() {
|
||||||
|
let err = microvm_provider_env_from(Some("glm"), |k| {
|
||||||
|
(k == "CLAUDE_CODE_OAUTH_TOKEN").then(|| "sub-token".to_string())
|
||||||
|
})
|
||||||
|
.expect_err("no z.ai key must refuse the launch");
|
||||||
|
assert!(err.contains("ZAI_API_KEY"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A backend whose credential contract is not settled is refused, rather
|
||||||
|
/// than handed another provider's credential to send at its endpoint.
|
||||||
|
#[test]
|
||||||
|
fn an_undefined_backend_is_refused_rather_than_given_the_anthropic_token() {
|
||||||
|
// `kimi` has since moved out of this list — its URL was measured.
|
||||||
|
for b in [Some("something-new"), Some("agent-terminal")] {
|
||||||
|
let r = microvm_provider_env_from(b, |_| Some("set".into()));
|
||||||
|
assert!(r.is_err(), "backend {b:?} should be refused: {r:?}");
|
||||||
|
}
|
||||||
|
// The default image is the claude one, and does have a contract.
|
||||||
|
for b in [None, Some(""), Some("default"), Some("claude")] {
|
||||||
|
assert!(
|
||||||
|
microvm_provider_env_from(b, |_| Some("set".into())).is_ok(),
|
||||||
|
"backend {b:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The container path still honours the operator's mode, and reads values
|
||||||
|
/// the same way. Only the microVM path is pinned.
|
||||||
|
#[test]
|
||||||
|
fn the_container_path_forwards_its_configured_mode() {
|
||||||
|
for auth in [RuntimeAuth::ApiKey, RuntimeAuth::Subscription] {
|
||||||
|
let keys = forwarded_provider_keys(auth);
|
||||||
|
// Every key set in the environment, and nothing else.
|
||||||
|
let pairs = provider_env_from(auth, |k| Some(format!("value-of-{k}")));
|
||||||
|
let got: Vec<&str> = pairs.iter().map(|(k, _)| k.as_str()).collect();
|
||||||
|
assert_eq!(got, keys, "{}", auth.as_str());
|
||||||
|
for (k, v) in &pairs {
|
||||||
|
assert_eq!(v, &format!("value-of-{k}"), "value must pass through");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An unset or blank credential is ABSENT, not empty. On the microVM path an
|
||||||
|
/// empty string would make `claude` believe it has a credential and fail
|
||||||
|
/// authentication, instead of reporting that it has none.
|
||||||
|
#[test]
|
||||||
|
fn a_blank_credential_is_omitted_rather_than_forwarded_empty() {
|
||||||
|
let pairs = provider_env_from(RuntimeAuth::Subscription, |k| match k {
|
||||||
|
"CLAUDE_CODE_OAUTH_TOKEN" => Some(" ".into()),
|
||||||
|
"OPENAI_API_KEY" => Some(String::new()),
|
||||||
|
"GROQ_API_KEY" => Some("real".into()),
|
||||||
|
_ => None,
|
||||||
|
});
|
||||||
|
assert_eq!(pairs, vec![("GROQ_API_KEY".to_string(), "real".to_string())]);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn subscription_mode_withholds_the_anthropic_api_key() {
|
fn subscription_mode_withholds_the_anthropic_api_key() {
|
||||||
let keys = forwarded_provider_keys(RuntimeAuth::Subscription);
|
let keys = forwarded_provider_keys(RuntimeAuth::Subscription);
|
||||||
@@ -967,7 +1447,6 @@ mod tests {
|
|||||||
);
|
);
|
||||||
// Unrelated providers have no subscription equivalent and must survive.
|
// Unrelated providers have no subscription equivalent and must survive.
|
||||||
for k in [
|
for k in [
|
||||||
"GEMINI_API_KEY",
|
|
||||||
"GROQ_API_KEY",
|
"GROQ_API_KEY",
|
||||||
"OPENAI_API_KEY",
|
"OPENAI_API_KEY",
|
||||||
"ZAI_API_KEY",
|
"ZAI_API_KEY",
|
||||||
@@ -1005,14 +1484,19 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn api_key_mode_forwards_everything() {
|
fn api_key_mode_forwards_everything() {
|
||||||
let keys = forwarded_provider_keys(RuntimeAuth::ApiKey);
|
let keys = forwarded_provider_keys(RuntimeAuth::ApiKey);
|
||||||
for k in [
|
for k in ["ANTHROPIC_API_KEY", "GROQ_API_KEY", "OPENAI_API_KEY"] {
|
||||||
"ANTHROPIC_API_KEY",
|
|
||||||
"GEMINI_API_KEY",
|
|
||||||
"GROQ_API_KEY",
|
|
||||||
"OPENAI_API_KEY",
|
|
||||||
] {
|
|
||||||
assert!(keys.contains(&k), "{k} should be forwarded in api_key mode");
|
assert!(keys.contains(&k), "{k} should be forwarded in api_key mode");
|
||||||
}
|
}
|
||||||
|
// Gemini is stripped from the platform entirely: no provider row, no
|
||||||
|
// selector entry, and no key forwarded to agent containers. Asserted so
|
||||||
|
// a future "just add it back to the list" restores the dependency
|
||||||
|
// visibly rather than by accident.
|
||||||
|
for mode in [RuntimeAuth::ApiKey, RuntimeAuth::Subscription] {
|
||||||
|
assert!(
|
||||||
|
!forwarded_provider_keys(mode).contains(&"GEMINI_API_KEY"),
|
||||||
|
"GEMINI_API_KEY must not be forwarded in {mode:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
assert!(
|
assert!(
|
||||||
!keys.contains(&"CLAUDE_CODE_OAUTH_TOKEN"),
|
!keys.contains(&"CLAUDE_CODE_OAUTH_TOKEN"),
|
||||||
"api_key mode must not also ship the subscription token"
|
"api_key mode must not also ship the subscription token"
|
||||||
@@ -1137,6 +1621,45 @@ allowed_tools = ["file_read", "file_edit"]
|
|||||||
assert_eq!(url, "http://cm-runtime-mission-abc:42617");
|
assert_eq!(url, "http://cm-runtime-mission-abc:42617");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The seed data path must be per-mission, not the shared seed dir.
|
||||||
|
///
|
||||||
|
/// Every mission container used to bind the SAME host directory as
|
||||||
|
/// `/zeroclaw-data`. It holds `config.toml`, which carries the §15 door
|
||||||
|
/// bearer token, plus `sessions.db`/`devices.db`. Sharing it meant one
|
||||||
|
/// mission could read another's credential, and anything written there
|
||||||
|
/// was inherited by every later mission — `teardown_container` only
|
||||||
|
/// removes `/var/lib/clawmates-missions/{id}`, so the shared dir was
|
||||||
|
/// never cleaned.
|
||||||
|
///
|
||||||
|
/// The seed copy reads as root and leaves the result to 65532.
|
||||||
|
///
|
||||||
|
/// Both halves are load-bearing and they pull in opposite directions. The
|
||||||
|
/// seed dir is root-owned with parts at mode 0600, so a copier running as
|
||||||
|
/// 65532 cannot read it — that was tried, `cp` failed on the first
|
||||||
|
/// unreadable entry, `set -e` abandoned the rest, and the mission booted
|
||||||
|
/// with `.zeroclaw` and nothing else. But leaving the RESULT root-owned is
|
||||||
|
/// what put ~3200 undeletable files per mission into the missions tree.
|
||||||
|
#[test]
|
||||||
|
fn the_seed_copy_hands_its_result_to_the_mission_uid() {
|
||||||
|
let script = copy_script();
|
||||||
|
assert!(
|
||||||
|
script.contains(&format!("chown -R {MISSION_UID} /dst")),
|
||||||
|
"the copied seed must end up owned by the mission uid:\n{script}"
|
||||||
|
);
|
||||||
|
// Every seeded path is attempted, and absence is tolerated — a fresh
|
||||||
|
// deployment genuinely has no `.kimi-code` yet.
|
||||||
|
for p in SEEDED_PATHS {
|
||||||
|
assert!(
|
||||||
|
script.contains(&format!("if [ -e '/seed/{p}' ]")),
|
||||||
|
"{p} is not guarded by an existence check"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// The chown must come AFTER the copies, or it chowns an empty dir.
|
||||||
|
let last_cp = script.rfind("cp -a").expect("at least one copy");
|
||||||
|
let chown = script.find("chown -R").expect("a chown");
|
||||||
|
assert!(chown > last_cp, "chown must run after the copies");
|
||||||
|
}
|
||||||
|
|
||||||
/// The seed data path must be per-mission, not the shared seed dir.
|
/// The seed data path must be per-mission, not the shared seed dir.
|
||||||
///
|
///
|
||||||
/// Every mission container used to bind the SAME host directory as
|
/// Every mission container used to bind the SAME host directory as
|
||||||
@@ -1150,15 +1673,21 @@ allowed_tools = ["file_read", "file_edit"]
|
|||||||
/// Asserting the path shape is what keeps this from silently regressing:
|
/// Asserting the path shape is what keeps this from silently regressing:
|
||||||
/// a future edit that points the mount back at the seed dir restores the
|
/// a future edit that points the mount back at the seed dir restores the
|
||||||
/// credential sharing with no other visible symptom.
|
/// credential sharing with no other visible symptom.
|
||||||
|
///
|
||||||
|
/// This test did not RUN for some time: an edit stranded its `#[test]`
|
||||||
|
/// above the neighbouring function, leaving two attributes there and none
|
||||||
|
/// here. rustc said so twice — "duplicated attribute" and "function is
|
||||||
|
/// never used" — and both read as ordinary warnings in a noisy build.
|
||||||
#[test]
|
#[test]
|
||||||
fn runtime_data_is_scoped_to_one_mission() {
|
fn runtime_data_is_scoped_to_one_mission() {
|
||||||
let a = Uuid::now_v7();
|
let a = Uuid::now_v7();
|
||||||
let b = Uuid::now_v7();
|
let b = Uuid::now_v7();
|
||||||
let path = |id: Uuid| format!("{MISSIONS_HOST_ROOT}/{id}/runtime-data");
|
let root = missions_host_root();
|
||||||
|
let path = |id: Uuid| format!("{root}/{id}/runtime-data");
|
||||||
|
|
||||||
assert_ne!(path(a), path(b), "two missions must not share runtime data");
|
assert_ne!(path(a), path(b), "two missions must not share runtime data");
|
||||||
assert!(
|
assert!(
|
||||||
path(a).starts_with(&format!("{MISSIONS_HOST_ROOT}/{a}")),
|
path(a).starts_with(&format!("{root}/{a}")),
|
||||||
"runtime data must live under the mission dir so teardown removes it"
|
"runtime data must live under the mission dir so teardown removes it"
|
||||||
);
|
);
|
||||||
assert_ne!(
|
assert_ne!(
|
||||||
|
|||||||
@@ -11,7 +11,10 @@
|
|||||||
//! - no repo_id → no-op (Ok(None))
|
//! - no repo_id → no-op (Ok(None))
|
||||||
//! - dir already a git repo → `fetch + reset --hard origin/<branch>`
|
//! - dir already a git repo → `fetch + reset --hard origin/<branch>`
|
||||||
//! to bring it in sync
|
//! to bring it in sync
|
||||||
//! - dir missing → `git clone --depth 1 <url> <path>`
|
//! - dir missing → `git clone --filter=blob:none --single-branch` (NOT
|
||||||
|
//! `--depth 1`: a shallow clone cannot push a new branch back, and delivery
|
||||||
|
//! needs exactly that — see `clone`). A mission with a `security_scan` phase
|
||||||
|
//! gets a FULLY HYDRATED clone instead; see `wants_full_history`.
|
||||||
//!
|
//!
|
||||||
//! Auth: for `git.redclaw.dev` clones we inject the ambient
|
//! Auth: for `git.redclaw.dev` clones we inject the ambient
|
||||||
//! `GITEA_TOKEN` (already provisioned in the server container's env)
|
//! `GITEA_TOKEN` (already provisioned in the server container's env)
|
||||||
@@ -23,7 +26,17 @@ use std::path::PathBuf;
|
|||||||
use tokio::process::Command;
|
use tokio::process::Command;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
pub(crate) fn missions_root() -> PathBuf {
|
/// The ONE definition of where mission state lives on the docker host.
|
||||||
|
///
|
||||||
|
/// There used to be five: this function, three private copies of the same
|
||||||
|
/// `env::var(...).unwrap_or(...)` in `security_scan`, `benchmark_runner` and
|
||||||
|
/// `mission_outputs`, and a hardcoded `MISSIONS_HOST_ROOT` const in
|
||||||
|
/// `mission_runtime` that read no env at all. They agree on today's
|
||||||
|
/// deployment, which is why nothing had broken — but anything that sweeps or
|
||||||
|
/// reclaims this tree has to be sure it is sweeping the same tree the writers
|
||||||
|
/// use, and five definitions cannot promise that. A GC written against one of
|
||||||
|
/// them would silently miss the others.
|
||||||
|
pub fn missions_root() -> PathBuf {
|
||||||
std::env::var("CLAWMATES_MISSIONS_ROOT")
|
std::env::var("CLAWMATES_MISSIONS_ROOT")
|
||||||
.map(PathBuf::from)
|
.map(PathBuf::from)
|
||||||
.unwrap_or_else(|_| PathBuf::from("/var/lib/clawmates-missions"))
|
.unwrap_or_else(|_| PathBuf::from("/var/lib/clawmates-missions"))
|
||||||
@@ -65,7 +78,19 @@ pub async fn ensure_checkout(
|
|||||||
.map_err(|e| format!("mkdir {}: {e}", parent.display()))?;
|
.map_err(|e| format!("mkdir {}: {e}", parent.display()))?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let auth_url = with_ambient_auth(clone_url);
|
let auth = with_ambient_auth(clone_url);
|
||||||
|
// Said once, here, where the checkout is created: every later git call uses
|
||||||
|
// a URL built the same way, so an unauthenticated forge URL is a fact worth
|
||||||
|
// one line now rather than a `/dev/tty` error later.
|
||||||
|
if let Some(why) = &auth.unauthenticated {
|
||||||
|
if auth.is_forge() {
|
||||||
|
eprintln!(
|
||||||
|
"mission_workspace: mission {mission_id} will talk to the forge \
|
||||||
|
WITHOUT credentials — {why}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let auth_url = auth.url;
|
||||||
if path.join(".git").exists() {
|
if path.join(".git").exists() {
|
||||||
// Checkouts cloned before this setting existed get it on reuse. It
|
// Checkouts cloned before this setting existed get it on reuse. It
|
||||||
// governs objects created from now on, which is what delivery needs.
|
// governs objects created from now on, which is what delivery needs.
|
||||||
@@ -87,43 +112,181 @@ pub async fn ensure_checkout(
|
|||||||
fetch_and_reset(&path, default_branch, &auth_url).await?;
|
fetch_and_reset(&path, default_branch, &auth_url).await?;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
clone(&path, &auth_url).await?;
|
clone(&path, &auth_url, wants_full_history(pool, mission_id).await).await?;
|
||||||
}
|
}
|
||||||
Ok(Some(path))
|
Ok(Some(path))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// If the URL points at git.redclaw.dev AND GITEA_TOKEN is set in the
|
/// The forge whose URLs the ambient `GITEA_TOKEN` can authenticate.
|
||||||
/// environment, rewrite it to include the token as basic-auth. Returns
|
const FORGE_HOST: &str = "git.redclaw.dev";
|
||||||
/// the URL unchanged otherwise. The token is never logged (we only
|
|
||||||
/// pass the rewritten URL into `git clone` via argv).
|
/// A URL, and whether a credential actually reached it.
|
||||||
pub(crate) fn with_ambient_auth(url: &str) -> String {
|
///
|
||||||
let Ok(token) = std::env::var("GITEA_TOKEN") else {
|
/// The second field is the whole point. This used to be a bare `String`: an
|
||||||
return url.to_string();
|
/// unmatched URL — an ssh remote, `http://` instead of `https://`, an explicit
|
||||||
};
|
/// port, a different case in the host — silently came back unauthenticated, and
|
||||||
if token.is_empty() {
|
/// the first symptom was git opening `/dev/tty` several layers later. Tracing
|
||||||
return url.to_string();
|
/// #55 cost hours to a failure whose cause was one unlogged early return.
|
||||||
}
|
pub struct Authed {
|
||||||
if let Some(rest) = url.strip_prefix("https://git.redclaw.dev/") {
|
pub url: String,
|
||||||
return format!("https://oauth2:{token}@git.redclaw.dev/{rest}");
|
/// `None` when the token was applied; otherwise WHY it was not.
|
||||||
}
|
pub unauthenticated: Option<String>,
|
||||||
url.to_string()
|
/// Whether the URL names the forge our token is for. Recorded from the
|
||||||
|
/// ORIGINAL url, not re-derived from `url` — an authenticated URL carries
|
||||||
|
/// userinfo, and parsing that back out is how the answer goes wrong.
|
||||||
|
forge: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn clone(path: &std::path::Path, url: &str) -> Result<(), String> {
|
impl Authed {
|
||||||
|
/// Is this URL on the forge our token is for? An unauthenticated URL to a
|
||||||
|
/// third-party host is normal (public repos, ssh remotes with a key); an
|
||||||
|
/// unauthenticated URL to OUR forge is a fault, and only the caller knows
|
||||||
|
/// how much it costs.
|
||||||
|
pub fn is_forge(&self) -> bool {
|
||||||
|
self.forge
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The host component of a URL, for the scp-like and scheme forms git accepts.
|
||||||
|
///
|
||||||
|
/// Deliberately tolerant, because the point is to RECOGNISE our forge in every
|
||||||
|
/// shape it can be written, not to validate URLs: `https://`, `http://`, an
|
||||||
|
/// explicit `:port`, `user@host`, `ssh://`, and `git@host:path`.
|
||||||
|
fn host_of(url: &str) -> Option<&str> {
|
||||||
|
let rest = match url.split_once("://") {
|
||||||
|
Some((_, rest)) => rest,
|
||||||
|
// scp-like: `git@host:path/to.git`, which has no scheme.
|
||||||
|
None => url,
|
||||||
|
};
|
||||||
|
// Userinfo FIRST, then the port. The other order splits
|
||||||
|
// `oauth2:token@host` at the credential's colon and reports the username as
|
||||||
|
// the host — which is exactly how the first version of this function decided
|
||||||
|
// an authenticated forge URL was not the forge.
|
||||||
|
let authority = rest.split('/').next().filter(|s| !s.is_empty())?;
|
||||||
|
let hostport = authority.rsplit_once('@').map_or(authority, |(_, h)| h);
|
||||||
|
Some(hostport.split(':').next().unwrap_or(hostport)).filter(|h| !h.is_empty())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rewrite a forge URL to carry the ambient `GITEA_TOKEN` as basic-auth.
|
||||||
|
///
|
||||||
|
/// Returns the reason instead of the credential whenever it cannot: a missing
|
||||||
|
/// token, a host that is not ours, or a shape a token cannot be injected into.
|
||||||
|
/// The token is never logged — only the rewritten URL is passed to git, via
|
||||||
|
/// argv.
|
||||||
|
pub fn with_ambient_auth(url: &str) -> Authed {
|
||||||
|
auth_with_token(url, std::env::var("GITEA_TOKEN").ok().as_deref())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The testable half of [`with_ambient_auth`]. The token is a parameter because
|
||||||
|
/// a test cannot set process environment variables here — the workspace denies
|
||||||
|
/// `unsafe`, and `set_var` is racy across test threads regardless.
|
||||||
|
fn auth_with_token(url: &str, token: Option<&str>) -> Authed {
|
||||||
|
let host = host_of(url);
|
||||||
|
let forge = host.is_some_and(|h| h.eq_ignore_ascii_case(FORGE_HOST));
|
||||||
|
let unauth = |why: String| Authed {
|
||||||
|
url: url.to_string(),
|
||||||
|
unauthenticated: Some(why),
|
||||||
|
forge,
|
||||||
|
};
|
||||||
|
|
||||||
|
let host = match host {
|
||||||
|
Some(h) => h,
|
||||||
|
None => return unauth(format!("no host could be read from {url:?}")),
|
||||||
|
};
|
||||||
|
if !forge {
|
||||||
|
return unauth(format!(
|
||||||
|
"{host} is not {FORGE_HOST}, so GITEA_TOKEN does not apply — git will \
|
||||||
|
use whatever ambient credentials exist (ssh agent, .netrc, helper)"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let token = match token {
|
||||||
|
Some(t) if !t.trim().is_empty() => t,
|
||||||
|
_ => return unauth("GITEA_TOKEN is unset or empty".to_string()),
|
||||||
|
};
|
||||||
|
// Only the scheme forms can carry basic-auth. An ssh remote authenticates
|
||||||
|
// with a key, and pretending otherwise would produce a URL git rejects.
|
||||||
|
let Some((scheme, rest)) = url.split_once("://") else {
|
||||||
|
return unauth(format!(
|
||||||
|
"{url} is an ssh-style remote; a token cannot be embedded in it"
|
||||||
|
));
|
||||||
|
};
|
||||||
|
if !matches!(scheme, "http" | "https") {
|
||||||
|
return unauth(format!("scheme {scheme} cannot carry a token"));
|
||||||
|
}
|
||||||
|
// Drop any userinfo already present rather than producing `a@b@host`.
|
||||||
|
let rest = rest.split_once('@').map(|(_, r)| r).unwrap_or(rest);
|
||||||
|
Authed {
|
||||||
|
url: format!("{scheme}://oauth2:{token}@{rest}"),
|
||||||
|
unauthenticated: None,
|
||||||
|
forge,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Git must never wait for a human.
|
||||||
|
///
|
||||||
|
/// Without this, a URL that ended up without credentials does not fail — git
|
||||||
|
/// opens `/dev/tty` to ask for a username, and in a server container that
|
||||||
|
/// surfaces as `No such device or address`, several layers away from the
|
||||||
|
/// missing token that caused it. With it, the failure names itself:
|
||||||
|
/// `terminal prompts disabled`.
|
||||||
|
pub(crate) fn no_terminal_prompt(cmd: &mut Command) -> &mut Command {
|
||||||
|
cmd.env("GIT_TERMINAL_PROMPT", "0")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Does any phase of this mission need history it can READ, not just reference?
|
||||||
|
///
|
||||||
|
/// `--filter=blob:none` keeps every commit but fetches file contents on demand,
|
||||||
|
/// which is nearly free for a repo that gets read once — and silently useless to
|
||||||
|
/// a tool that walks history, because the agent environment has NO network route
|
||||||
|
/// to the forge. Measured: gitleaks on a 4-commit repo reported
|
||||||
|
/// "1 commits scanned" and "could not fetch <sha> from promisor remote". It was
|
||||||
|
/// not misconfigured; the blobs simply were not there and could not be got.
|
||||||
|
///
|
||||||
|
/// A security scan is the phase kind whose entire value is old content — a
|
||||||
|
/// credential committed and later deleted is exactly what it looks for, and that
|
||||||
|
/// is precisely what a lazy blob is. So those missions pay for a full clone and
|
||||||
|
/// everything else keeps the cheap one.
|
||||||
|
///
|
||||||
|
/// Best-effort: an unreadable phase list yields `false`, i.e. today's behaviour.
|
||||||
|
async fn wants_full_history(pool: &sqlx::PgPool, mission_id: Uuid) -> bool {
|
||||||
|
sqlx::query_scalar::<_, i64>(
|
||||||
|
"SELECT count(*) FROM mission_phases WHERE mission_id = $1 AND kind = 'security_scan'",
|
||||||
|
)
|
||||||
|
.bind(mission_id)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.map(|n| n > 0)
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The `git clone` flags, split out so the strategy is testable without a forge.
|
||||||
|
fn clone_args(full_history: bool) -> Vec<&'static str> {
|
||||||
|
let mut a = vec!["clone"];
|
||||||
|
if !full_history {
|
||||||
|
a.push("--filter=blob:none");
|
||||||
|
}
|
||||||
|
a.push("--single-branch");
|
||||||
|
a
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn clone(path: &std::path::Path, url: &str, full_history: bool) -> Result<(), String> {
|
||||||
// `--filter=blob:none` rather than `--depth 1`. A shallow clone cannot
|
// `--filter=blob:none` rather than `--depth 1`. A shallow clone cannot
|
||||||
// usually push a new branch back ("shallow update not allowed"), and
|
// usually push a new branch back ("shallow update not allowed"), and
|
||||||
// mission delivery needs exactly that. A partial clone keeps full history
|
// mission delivery needs exactly that. A partial clone keeps full history
|
||||||
// — so the base commit stays meaningful and a diff has something to be
|
// — so the base commit stays meaningful and a diff has something to be
|
||||||
// relative to — while fetching file contents only on demand, which is
|
// relative to — while fetching file contents only on demand, which is
|
||||||
// nearly as cheap as a shallow clone for a repo that gets read once.
|
// nearly as cheap as a shallow clone for a repo that gets read once.
|
||||||
let out = Command::new("git")
|
if full_history {
|
||||||
.args([
|
eprintln!(
|
||||||
"clone",
|
"mission_workspace: cloning {} with full history — a security_scan phase \
|
||||||
"--filter=blob:none",
|
reads old file contents, which a partial clone cannot supply offline",
|
||||||
"--single-branch",
|
path.display()
|
||||||
url,
|
);
|
||||||
&path.display().to_string(),
|
}
|
||||||
])
|
let mut cmd = Command::new("git");
|
||||||
|
cmd.args(clone_args(full_history));
|
||||||
|
cmd.args([url, &path.display().to_string()]);
|
||||||
|
let out = no_terminal_prompt(&mut cmd)
|
||||||
.output()
|
.output()
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("spawn git clone: {e}"))?;
|
.map_err(|e| format!("spawn git clone: {e}"))?;
|
||||||
@@ -131,10 +294,11 @@ async fn clone(path: &std::path::Path, url: &str) -> Result<(), String> {
|
|||||||
return Err(format!(
|
return Err(format!(
|
||||||
"git clone → exit {}: {}",
|
"git clone → exit {}: {}",
|
||||||
out.status,
|
out.status,
|
||||||
redact_token(&String::from_utf8_lossy(&out.stderr))
|
// Both ends: git prints its reason LAST, and a head-only clamp keeps
|
||||||
.chars()
|
// the progress noise while dropping the answer.
|
||||||
.take(400)
|
crate::evaluator_tools::clamp_output(&redact_token(&String::from_utf8_lossy(
|
||||||
.collect::<String>()
|
&out.stderr
|
||||||
|
)))
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
share_repository_across_uids(path);
|
share_repository_across_uids(path);
|
||||||
@@ -447,7 +611,13 @@ fn strip_credentials(url: &str) -> String {
|
|||||||
/// They are the agent's identity scaffolding, not the user's code — `SOUL.md`
|
/// They are the agent's identity scaffolding, not the user's code — `SOUL.md`
|
||||||
/// opens "Who You Are / You're not a chatbot." Observed on mission 019fc058,
|
/// opens "Who You Are / You're not a chatbot." Observed on mission 019fc058,
|
||||||
/// where all seven appeared as untracked files in a freshly cloned repo.
|
/// where all seven appeared as untracked files in a freshly cloned repo.
|
||||||
const AGENT_SCAFFOLDING: &[&str] = &[
|
///
|
||||||
|
/// `pub(crate)` because a repo-less mission needs the same list and cannot use
|
||||||
|
/// the same mechanism: `ignore_agent_scaffolding` writes `.git/info/exclude`,
|
||||||
|
/// and a mission with no repository has no `.git`. `mission_outputs` filters on
|
||||||
|
/// this list directly — one list, two consumers, so the next file the runtime
|
||||||
|
/// starts seeding is excluded from both at once.
|
||||||
|
pub(crate) const AGENT_SCAFFOLDING: &[&str] = &[
|
||||||
"AGENTS.md",
|
"AGENTS.md",
|
||||||
"HEARTBEAT.md",
|
"HEARTBEAT.md",
|
||||||
"IDENTITY.md",
|
"IDENTITY.md",
|
||||||
@@ -523,16 +693,15 @@ async fn fetch_and_reset(
|
|||||||
// errors on a repo that is already complete, so it is only attempted when
|
// errors on a repo that is already complete, so it is only attempted when
|
||||||
// the marker file is present.
|
// the marker file is present.
|
||||||
if path.join(".git/shallow").exists() {
|
if path.join(".git/shallow").exists() {
|
||||||
let deepen = Command::new("git")
|
let mut cmd = Command::new("git");
|
||||||
.args([
|
cmd.args([
|
||||||
"-C",
|
"-C",
|
||||||
&path.display().to_string(),
|
&path.display().to_string(),
|
||||||
"fetch",
|
"fetch",
|
||||||
"--unshallow",
|
"--unshallow",
|
||||||
auth_url,
|
auth_url,
|
||||||
])
|
]);
|
||||||
.output()
|
let deepen = no_terminal_prompt(&mut cmd).output().await;
|
||||||
.await;
|
|
||||||
match deepen {
|
match deepen {
|
||||||
Ok(o) if o.status.success() => {}
|
Ok(o) if o.status.success() => {}
|
||||||
Ok(o) => eprintln!(
|
Ok(o) => eprintln!(
|
||||||
@@ -556,8 +725,9 @@ async fn fetch_and_reset(
|
|||||||
// so `git fetch origin` has no credentials and fails with
|
// so `git fetch origin` has no credentials and fails with
|
||||||
// "could not read Username". Building the URL here also means a rotated
|
// "could not read Username". Building the URL here also means a rotated
|
||||||
// token takes effect immediately instead of at the next clone.
|
// token takes effect immediately instead of at the next clone.
|
||||||
let fetch = Command::new("git")
|
let mut cmd = Command::new("git");
|
||||||
.args(["-C", &path.display().to_string(), "fetch", auth_url, branch])
|
cmd.args(["-C", &path.display().to_string(), "fetch", auth_url, branch]);
|
||||||
|
let fetch = no_terminal_prompt(&mut cmd)
|
||||||
.output()
|
.output()
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("spawn git fetch: {e}"))?;
|
.map_err(|e| format!("spawn git fetch: {e}"))?;
|
||||||
@@ -565,10 +735,9 @@ async fn fetch_and_reset(
|
|||||||
return Err(format!(
|
return Err(format!(
|
||||||
"git fetch origin {branch} → exit {}: {}",
|
"git fetch origin {branch} → exit {}: {}",
|
||||||
fetch.status,
|
fetch.status,
|
||||||
redact_token(&String::from_utf8_lossy(&fetch.stderr))
|
crate::evaluator_tools::clamp_output(&redact_token(&String::from_utf8_lossy(
|
||||||
.chars()
|
&fetch.stderr
|
||||||
.take(400)
|
)))
|
||||||
.collect::<String>()
|
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let reset = Command::new("git")
|
let reset = Command::new("git")
|
||||||
@@ -600,8 +769,102 @@ async fn fetch_and_reset(
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
/// The clone strategy is a fact worth pinning: `--depth 1` breaks delivery
|
||||||
|
/// (a shallow clone cannot push a new branch — "shallow update not
|
||||||
|
/// allowed"), and `--filter=blob:none` breaks history-reading tools offline.
|
||||||
|
/// Both failure modes are real and were both hit.
|
||||||
|
#[test]
|
||||||
|
fn the_clone_strategy_is_partial_by_default_and_never_shallow() {
|
||||||
|
let partial = clone_args(false);
|
||||||
|
assert!(partial.contains(&"--filter=blob:none"), "{partial:?}");
|
||||||
|
assert!(!partial.iter().any(|a| a.starts_with("--depth")), "{partial:?}");
|
||||||
|
|
||||||
|
// A security_scan mission must NOT get the lazy-blob filter: its scanner
|
||||||
|
// walks old file contents and cannot reach the forge to fetch them.
|
||||||
|
let full = clone_args(true);
|
||||||
|
assert!(!full.contains(&"--filter=blob:none"), "{full:?}");
|
||||||
|
assert!(!full.iter().any(|a| a.starts_with("--depth")), "{full:?}");
|
||||||
|
|
||||||
|
// Both keep --single-branch: the mission only ever works one branch.
|
||||||
|
for args in [partial, full] {
|
||||||
|
assert!(args.contains(&"--single-branch"), "{args:?}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
const TOK: Option<&str> = Some("secret123");
|
||||||
|
|
||||||
|
/// The forge in every shape a remote can be written. Each of these used to
|
||||||
|
/// fall out of the `strip_prefix("https://git.redclaw.dev/")` match and come
|
||||||
|
/// back unauthenticated with no log line — the fail-open found while tracing
|
||||||
|
/// #55.
|
||||||
|
#[test]
|
||||||
|
fn the_forge_is_recognised_however_the_url_is_written() {
|
||||||
|
for url in [
|
||||||
|
"https://git.redclaw.dev/o/r.git",
|
||||||
|
"http://git.redclaw.dev/o/r.git",
|
||||||
|
"https://GIT.RedClaw.dev/o/r.git",
|
||||||
|
"https://git.redclaw.dev:3000/o/r.git",
|
||||||
|
"https://oauth2:[email protected]/o/r.git",
|
||||||
|
] {
|
||||||
|
let a = auth_with_token(url, TOK);
|
||||||
|
assert!(a.is_forge(), "{url} was not recognised as the forge");
|
||||||
|
assert!(
|
||||||
|
a.unauthenticated.is_none(),
|
||||||
|
"{url} → {:?}",
|
||||||
|
a.unauthenticated
|
||||||
|
);
|
||||||
|
assert!(a.url.contains("oauth2:secret123@"), "{}", a.url);
|
||||||
|
// And exactly one set of credentials, not `old@` left behind.
|
||||||
|
assert_eq!(a.url.matches('@').count(), 1, "{}", a.url);
|
||||||
|
}
|
||||||
|
// The port and the scheme survive the rewrite — changing either would
|
||||||
|
// point the push somewhere the operator did not configure.
|
||||||
|
assert!(auth_with_token("https://git.redclaw.dev:3000/o/r.git", TOK)
|
||||||
|
.url
|
||||||
|
.contains("@git.redclaw.dev:3000/o/r.git"));
|
||||||
|
assert!(auth_with_token("http://git.redclaw.dev/o/r.git", TOK)
|
||||||
|
.url
|
||||||
|
.starts_with("http://oauth2:"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every path that cannot authenticate must SAY so. "Unauthenticated and
|
||||||
|
/// silent" is the shape that cost hours: the first symptom was git opening
|
||||||
|
/// /dev/tty, several layers from the cause.
|
||||||
|
#[test]
|
||||||
|
fn an_unauthenticated_url_carries_its_reason() {
|
||||||
|
let cases = [
|
||||||
|
(auth_with_token("https://git.redclaw.dev/o/r.git", None), true),
|
||||||
|
(auth_with_token("https://git.redclaw.dev/o/r.git", Some(" ")), true),
|
||||||
|
(auth_with_token("[email protected]:o/r.git", TOK), true),
|
||||||
|
(auth_with_token("ssh://[email protected]/o/r.git", TOK), true),
|
||||||
|
(auth_with_token("https://github.com/o/r.git", TOK), false),
|
||||||
|
];
|
||||||
|
for (a, is_forge) in cases {
|
||||||
|
let why = a.unauthenticated.as_deref().unwrap_or("");
|
||||||
|
assert!(!why.is_empty(), "{} came back with no reason", a.url);
|
||||||
|
assert_eq!(a.is_forge(), is_forge, "{}", a.url);
|
||||||
|
// And the URL is handed back untouched, so a caller that proceeds
|
||||||
|
// anyway (ssh keys, .netrc) still works.
|
||||||
|
assert!(!a.url.contains("secret123"), "{}", a.url);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A token must never be embedded in a URL for someone else's host.
|
||||||
|
#[test]
|
||||||
|
fn the_token_never_leaves_the_forge() {
|
||||||
|
for url in [
|
||||||
|
"https://github.com/o/r.git",
|
||||||
|
"https://git.redclaw.dev.evil.example/o/r.git",
|
||||||
|
"https://evil.example/git.redclaw.dev/r.git",
|
||||||
|
] {
|
||||||
|
let a = auth_with_token(url, TOK);
|
||||||
|
assert!(!a.url.contains("secret123"), "{url} → {}", a.url);
|
||||||
|
assert!(!a.is_forge(), "{url}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The exclude must be idempotent — `ensure_checkout` re-runs on every
|
/// The exclude must be idempotent — `ensure_checkout` re-runs on every
|
||||||
/// phase, and appending the same block each time would grow the file
|
/// phase, and appending the same block each time would grow the file
|
||||||
/// without bound.
|
/// without bound.
|
||||||
@@ -821,4 +1084,43 @@ mod tests {
|
|||||||
mark_phase_started(repo);
|
mark_phase_started(repo);
|
||||||
assert!(checkout_in_use(repo));
|
assert!(checkout_in_use(repo));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Nobody re-derives the missions root.
|
||||||
|
///
|
||||||
|
/// It had fragmented into five definitions — this function, three private
|
||||||
|
/// `env::var("CLAWMATES_MISSIONS_ROOT")` copies, and a hardcoded const
|
||||||
|
/// that read no env at all. They agreed on the deployed value, so nothing
|
||||||
|
/// ever broke; the risk is entirely in what comes next. Anything that
|
||||||
|
/// sweeps, reclaims or reaps this tree has to be sweeping the same tree the
|
||||||
|
/// writers use, and five definitions cannot promise that.
|
||||||
|
#[test]
|
||||||
|
fn the_missions_root_has_exactly_one_definition() {
|
||||||
|
fn walk(dir: &std::path::Path, out: &mut Vec<std::path::PathBuf>) {
|
||||||
|
for entry in std::fs::read_dir(dir).expect("readable source dir") {
|
||||||
|
let path = entry.expect("readable entry").path();
|
||||||
|
if path.is_dir() {
|
||||||
|
walk(&path, out);
|
||||||
|
} else if path.extension().is_some_and(|e| e == "rs") {
|
||||||
|
out.push(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
|
||||||
|
let mut files = Vec::new();
|
||||||
|
walk(&root, &mut files);
|
||||||
|
|
||||||
|
for path in files {
|
||||||
|
if path.ends_with("mission_workspace.rs") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let src = std::fs::read_to_string(&path).expect("readable source");
|
||||||
|
assert!(
|
||||||
|
!src.contains("var(\"CLAWMATES_MISSIONS_ROOT\")"),
|
||||||
|
"{} reads CLAWMATES_MISSIONS_ROOT itself — call \
|
||||||
|
`mission_workspace::missions_root()` so a reaper and a writer \
|
||||||
|
cannot disagree about which tree they are looking at",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,259 +0,0 @@
|
|||||||
//! LLM + Chromium PDF renderer worker — Slice 6.
|
|
||||||
//!
|
|
||||||
//! Watches `mission_artifacts` for rows with `render_pdf_status =
|
|
||||||
//! 'pending'`. For each:
|
|
||||||
//! 1. Read the source MD from `<mission_root>/<path>` on disk
|
|
||||||
//! 2. Call the configured LLM (default: Gemini 2.5 Flash) with a
|
|
||||||
//! "produce styled HTML" prompt anchored to a design-system
|
|
||||||
//! example. LLM writes HTML with inline CSS.
|
|
||||||
//! 3. Print that HTML to PDF via `chromium --headless
|
|
||||||
//! --print-to-pdf`
|
|
||||||
//! 4. Save the PDF alongside the MD, update `rendered_pdf_path` +
|
|
||||||
//! status = 'done'
|
|
||||||
//!
|
|
||||||
//! Graceful degradation: if `GEMINI_API_KEY` is unset or the
|
|
||||||
//! chromium binary isn't on PATH, the worker marks the row `failed`
|
|
||||||
//! with a descriptive error rather than blocking boot. Ops enables
|
|
||||||
//! rendering by wiring both.
|
|
||||||
//!
|
|
||||||
//! The frontend already renders `rendered_pdf_path` as an "Open PDF"
|
|
||||||
//! button on artifact cards (Slice 2).
|
|
||||||
|
|
||||||
use serde_json::json;
|
|
||||||
use sqlx::PgPool;
|
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
const POLL_INTERVAL: Duration = Duration::from_secs(30);
|
|
||||||
const MAX_PARALLEL: usize = 2;
|
|
||||||
const DEFAULT_MODEL: &str = "gemini-2.5-flash";
|
|
||||||
|
|
||||||
/// Where per-mission artifacts land on disk. Overridable so dev vs.
|
|
||||||
/// prod can move the tree; matches the pattern in
|
|
||||||
/// `research_container::research_workspace_root`.
|
|
||||||
fn missions_root() -> PathBuf {
|
|
||||||
std::env::var("CLAWMATES_MISSIONS_ROOT")
|
|
||||||
.map(PathBuf::from)
|
|
||||||
.unwrap_or_else(|_| PathBuf::from("/var/lib/clawmates-missions"))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn chromium_bin() -> String {
|
|
||||||
std::env::var("CHROMIUM_BIN").unwrap_or_else(|_| "chromium".to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn renderer_model() -> String {
|
|
||||||
std::env::var("CLAWMATES_PDF_RENDERER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Spawn the poller. No-op-friendly: if there's nothing pending or
|
|
||||||
/// no rendering pipeline configured, we still tick + observe.
|
|
||||||
pub fn spawn(pool: PgPool) {
|
|
||||||
tokio::spawn(async move {
|
|
||||||
// Small startup delay so migrations + loaders finish first.
|
|
||||||
tokio::time::sleep(Duration::from_secs(8)).await;
|
|
||||||
let mut ticker = tokio::time::interval(POLL_INTERVAL);
|
|
||||||
ticker.tick().await;
|
|
||||||
loop {
|
|
||||||
ticker.tick().await;
|
|
||||||
if let Err(e) = sweep_once(&pool).await {
|
|
||||||
eprintln!("pdf_renderer: sweep failed: {e}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn sweep_once(pool: &PgPool) -> Result<(), String> {
|
|
||||||
let pending = cm_db::repo::missions::next_pdf_pending(pool, MAX_PARALLEL as i64)
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("next_pdf_pending: {e}"))?;
|
|
||||||
for artifact in pending {
|
|
||||||
let pool = pool.clone();
|
|
||||||
let id = artifact.id;
|
|
||||||
tokio::spawn(async move {
|
|
||||||
match render_one(&pool, &artifact).await {
|
|
||||||
Ok(pdf_path) => {
|
|
||||||
let _ = cm_db::repo::missions::set_pdf_result(&pool, id, Some(&pdf_path), None)
|
|
||||||
.await;
|
|
||||||
eprintln!("pdf_renderer: rendered {id} → {pdf_path}");
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
let _ = cm_db::repo::missions::set_pdf_result(&pool, id, None, Some(&e)).await;
|
|
||||||
eprintln!("pdf_renderer: {id} failed: {e}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn render_one(
|
|
||||||
_pool: &PgPool,
|
|
||||||
artifact: &cm_db::repo::missions::MissionArtifact,
|
|
||||||
) -> Result<String, String> {
|
|
||||||
// 1. Locate the source MD on disk.
|
|
||||||
let mission_root = missions_root().join(artifact.mission_id.to_string());
|
|
||||||
let src_path = mission_root.join(&artifact.path);
|
|
||||||
let md = tokio::fs::read_to_string(&src_path)
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("read {}: {e}", src_path.display()))?;
|
|
||||||
|
|
||||||
// 2. LLM → styled HTML.
|
|
||||||
let html = md_to_html_via_llm(&md, artifact.title.as_deref())
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("llm render: {e}"))?;
|
|
||||||
|
|
||||||
// 3. Chromium → PDF.
|
|
||||||
let tmp = tempdir_for(artifact.id)?;
|
|
||||||
let html_path = tmp.join("in.html");
|
|
||||||
let pdf_path = tmp.join("out.pdf");
|
|
||||||
tokio::fs::write(&html_path, html)
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("write {}: {e}", html_path.display()))?;
|
|
||||||
|
|
||||||
let status = tokio::process::Command::new(chromium_bin())
|
|
||||||
.args([
|
|
||||||
"--headless=new",
|
|
||||||
"--disable-gpu",
|
|
||||||
"--no-sandbox",
|
|
||||||
"--hide-scrollbars",
|
|
||||||
&format!("--print-to-pdf={}", pdf_path.display()),
|
|
||||||
"--print-to-pdf-no-header",
|
|
||||||
"--virtual-time-budget=10000",
|
|
||||||
&format!("file://{}", html_path.display()),
|
|
||||||
])
|
|
||||||
.stderr(std::process::Stdio::piped())
|
|
||||||
.stdout(std::process::Stdio::piped())
|
|
||||||
.status()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("spawn chromium: {e}"))?;
|
|
||||||
if !status.success() {
|
|
||||||
return Err(format!("chromium exited {status}"));
|
|
||||||
}
|
|
||||||
|
|
||||||
// 4. Move next to the source MD so the artifact tree stays self-
|
|
||||||
// contained. Filename derived from the MD path (foo.md → foo.pdf).
|
|
||||||
let out_rel = pdf_sibling(&artifact.path);
|
|
||||||
let out_abs = mission_root.join(&out_rel);
|
|
||||||
if let Some(parent) = out_abs.parent() {
|
|
||||||
tokio::fs::create_dir_all(parent)
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("mkdir {}: {e}", parent.display()))?;
|
|
||||||
}
|
|
||||||
tokio::fs::copy(&pdf_path, &out_abs)
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("copy pdf: {e}"))?;
|
|
||||||
// Best-effort tmp cleanup — the temp dir lives under /tmp so the
|
|
||||||
// OS will reap it anyway.
|
|
||||||
let _ = tokio::fs::remove_dir_all(&tmp).await;
|
|
||||||
Ok(out_rel)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Ask the configured LLM to turn `md` into a fully self-contained
|
|
||||||
/// styled HTML doc. Uses whichever provider `CLAWMATES_PDF_RENDERER_MODEL`
|
|
||||||
/// resolves to. Defaults to Gemini 2.5 Flash + GEMINI_API_KEY.
|
|
||||||
async fn md_to_html_via_llm(md: &str, title: Option<&str>) -> Result<String, String> {
|
|
||||||
let model = renderer_model();
|
|
||||||
// For now we hardcode the Gemini path — anthropic + openai
|
|
||||||
// variants land when the design-system template stabilizes.
|
|
||||||
if !model.starts_with("gemini") {
|
|
||||||
return Err(format!(
|
|
||||||
"renderer model {model} not yet wired (only gemini-* supported in Slice 6)"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let api_key =
|
|
||||||
std::env::var("GEMINI_API_KEY").map_err(|_| "GEMINI_API_KEY unset".to_string())?;
|
|
||||||
|
|
||||||
let system = r#"You are a document typesetter. Given a Markdown source,
|
|
||||||
produce ONE self-contained HTML document that:
|
|
||||||
- Has ALL styles inline in a single <style> block in <head>. No external
|
|
||||||
fonts, no external CSS. System font stack only.
|
|
||||||
- Uses a clean, modern, readable serif for body copy (Georgia / "Iowan Old
|
|
||||||
Style" / "Charter" / serif) and a sans for headings.
|
|
||||||
- Uses ONLY these accent colors: #ff8a7a (heading), #5ec8d8 (link),
|
|
||||||
#101014 (body text), #f7f7f8 (page bg).
|
|
||||||
- Renders code blocks with a monospace stack and a subtle background.
|
|
||||||
- Uses page-break-inside: avoid on headings and images.
|
|
||||||
- Puts a document title in an <h1> at the top if provided.
|
|
||||||
- Includes NOTHING outside the HTML — no ```html fence, no commentary."#;
|
|
||||||
|
|
||||||
let prompt = match title {
|
|
||||||
Some(t) => format!("Document title: {t}\n\nMarkdown:\n\n{md}"),
|
|
||||||
None => md.to_string(),
|
|
||||||
};
|
|
||||||
|
|
||||||
let url = format!(
|
|
||||||
"https://generativelanguage.googleapis.com/v1beta/models/{}:generateContent?key={}",
|
|
||||||
model, api_key
|
|
||||||
);
|
|
||||||
let body = json!({
|
|
||||||
"system_instruction": { "parts": [{ "text": system }] },
|
|
||||||
"contents": [{ "role": "user", "parts": [{ "text": prompt }] }],
|
|
||||||
"generationConfig": {
|
|
||||||
"temperature": 0.2,
|
|
||||||
"maxOutputTokens": 32000,
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
let client = reqwest::Client::builder()
|
|
||||||
.timeout(Duration::from_secs(120))
|
|
||||||
.build()
|
|
||||||
.map_err(|e| format!("http client: {e}"))?;
|
|
||||||
let resp = client
|
|
||||||
.post(&url)
|
|
||||||
.json(&body)
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("gemini call: {e}"))?;
|
|
||||||
if !resp.status().is_success() {
|
|
||||||
let code = resp.status();
|
|
||||||
let body = resp.text().await.unwrap_or_default();
|
|
||||||
return Err(format!("gemini {code}: {}", &body[..body.len().min(500)]));
|
|
||||||
}
|
|
||||||
let json: serde_json::Value = resp.json().await.map_err(|e| format!("gemini json: {e}"))?;
|
|
||||||
let text = json
|
|
||||||
.pointer("/candidates/0/content/parts/0/text")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| "gemini response missing text".to_string())?;
|
|
||||||
// Strip a stray ```html fence if the model added one despite the
|
|
||||||
// system prompt — cheap belt to the suspenders.
|
|
||||||
let cleaned = text
|
|
||||||
.trim()
|
|
||||||
.strip_prefix("```html")
|
|
||||||
.and_then(|s| s.strip_suffix("```"))
|
|
||||||
.map(|s| s.trim())
|
|
||||||
.unwrap_or(text.trim())
|
|
||||||
.to_string();
|
|
||||||
Ok(cleaned)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn tempdir_for(id: uuid::Uuid) -> Result<PathBuf, String> {
|
|
||||||
let dir = std::env::temp_dir().join(format!("clawmates-pdf-{id}"));
|
|
||||||
std::fs::create_dir_all(&dir).map_err(|e| format!("mkdir tmp: {e}"))?;
|
|
||||||
Ok(dir)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `research/v3/spec.md` → `research/v3/spec.pdf`.
|
|
||||||
/// `foo/bar/without_ext` → `foo/bar/without_ext.pdf` (rare — parser
|
|
||||||
/// never emits an extension-less MD, but we're defensive).
|
|
||||||
fn pdf_sibling(md_path: &str) -> String {
|
|
||||||
let p = Path::new(md_path);
|
|
||||||
let stem = p.file_stem().and_then(|s| s.to_str()).unwrap_or("output");
|
|
||||||
let parent = p.parent().map(|x| x.to_string_lossy().to_string());
|
|
||||||
let base = format!("{stem}.pdf");
|
|
||||||
match parent {
|
|
||||||
Some(pp) if !pp.is_empty() => format!("{pp}/{base}"),
|
|
||||||
_ => base,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn pdf_sibling_paths() {
|
|
||||||
assert_eq!(pdf_sibling("research/v3/spec.md"), "research/v3/spec.pdf");
|
|
||||||
assert_eq!(pdf_sibling("spec.md"), "spec.pdf");
|
|
||||||
assert_eq!(pdf_sibling("no_ext"), "no_ext.pdf");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -47,6 +47,17 @@ pub const KNOWN_KEYS: &[KnownKey] = &[
|
|||||||
key: "commit_policy",
|
key: "commit_policy",
|
||||||
read_by: "mission_delivery::Gate::parse — selects the delivery gate",
|
read_by: "mission_delivery::Gate::parse — selects the delivery gate",
|
||||||
},
|
},
|
||||||
|
KnownKey {
|
||||||
|
key: "allow_empty",
|
||||||
|
read_by: "phase_runner::empty_delivery_is_a_failure — when true, a coding \
|
||||||
|
phase that changes no files still completes; also vm_stop_gate::\
|
||||||
|
StopGate::for_phase, where it drops the in-loop delivery check",
|
||||||
|
},
|
||||||
|
KnownKey {
|
||||||
|
key: "done_when_check",
|
||||||
|
read_by: "vm_stop_gate::StopGate::for_phase — a shell command the agent's \
|
||||||
|
`Stop` hook runs, refusing the stop while it exits non-zero",
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
/// Keys a recipe may carry that are deliberately not consumed *yet*.
|
/// Keys a recipe may carry that are deliberately not consumed *yet*.
|
||||||
|
|||||||
+1289
-58
File diff suppressed because it is too large
Load Diff
@@ -23,7 +23,6 @@ use std::time::Duration;
|
|||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
const DEFAULT_MODEL: &str = "claude-opus-4-8";
|
const DEFAULT_MODEL: &str = "claude-opus-4-8";
|
||||||
const ANTHROPIC_API_VERSION: &str = "2023-06-01";
|
|
||||||
const POLL_INTERVAL: Duration = Duration::from_secs(30);
|
const POLL_INTERVAL: Duration = Duration::from_secs(30);
|
||||||
/// Cap the raw material we send to the model. Missions can produce
|
/// Cap the raw material we send to the model. Missions can produce
|
||||||
/// hundreds of KB of agent output; we slice by turn and by phase
|
/// hundreds of KB of agent output; we slice by turn and by phase
|
||||||
@@ -34,21 +33,26 @@ fn model_name() -> String {
|
|||||||
std::env::var("CLAWMATES_SUMMARIZER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
std::env::var("CLAWMATES_SUMMARIZER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn spawn(pool: PgPool) {
|
/// The runtime is carried purely so the summarizer can reach the SAME
|
||||||
|
/// providers as everything else. It used to hand-roll its own HTTPS POST with
|
||||||
|
/// `x-api-key: $ANTHROPIC_API_KEY`, which is why no audit of `.complete(` call
|
||||||
|
/// sites ever found it — and why every phase summary on this deployment died
|
||||||
|
/// with "credit balance is too low" while the phases themselves ran fine.
|
||||||
|
pub fn spawn(pool: PgPool, runtime: cm_runtime::Runtime) {
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
tokio::time::sleep(Duration::from_secs(45)).await;
|
tokio::time::sleep(Duration::from_secs(45)).await;
|
||||||
let mut ticker = tokio::time::interval(POLL_INTERVAL);
|
let mut ticker = tokio::time::interval(POLL_INTERVAL);
|
||||||
ticker.tick().await;
|
ticker.tick().await;
|
||||||
loop {
|
loop {
|
||||||
ticker.tick().await;
|
ticker.tick().await;
|
||||||
if let Err(e) = sweep_once(&pool).await {
|
if let Err(e) = sweep_once(&pool, &runtime).await {
|
||||||
eprintln!("phase_summarizer: sweep failed: {e}");
|
eprintln!("phase_summarizer: sweep failed: {e}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn sweep_once(pool: &PgPool) -> Result<(), String> {
|
async fn sweep_once(pool: &PgPool, runtime: &cm_runtime::Runtime) -> Result<(), String> {
|
||||||
// Terminal phases with no summary yet.
|
// Terminal phases with no summary yet.
|
||||||
let rows = sqlx::query(
|
let rows = sqlx::query(
|
||||||
"SELECT mp.id, mp.mission_id, mp.kind
|
"SELECT mp.id, mp.mission_id, mp.kind
|
||||||
@@ -65,7 +69,7 @@ async fn sweep_once(pool: &PgPool) -> Result<(), String> {
|
|||||||
let phase_id: Uuid = row.get("id");
|
let phase_id: Uuid = row.get("id");
|
||||||
let mission_id: Uuid = row.get("mission_id");
|
let mission_id: Uuid = row.get("mission_id");
|
||||||
let kind: String = row.get("kind");
|
let kind: String = row.get("kind");
|
||||||
if let Err(e) = summarize_one(pool, mission_id, phase_id, &kind).await {
|
if let Err(e) = summarize_one(pool, runtime, mission_id, phase_id, &kind).await {
|
||||||
// Persist an error row so we don't infinite-retry a broken
|
// Persist an error row so we don't infinite-retry a broken
|
||||||
// phase — the UI can surface "summary unavailable: <e>".
|
// phase — the UI can surface "summary unavailable: <e>".
|
||||||
eprintln!("phase_summarizer: {phase_id} ({kind}) failed: {e}");
|
eprintln!("phase_summarizer: {phase_id} ({kind}) failed: {e}");
|
||||||
@@ -77,6 +81,7 @@ async fn sweep_once(pool: &PgPool) -> Result<(), String> {
|
|||||||
|
|
||||||
async fn summarize_one(
|
async fn summarize_one(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
mission_id: Uuid,
|
mission_id: Uuid,
|
||||||
phase_id: Uuid,
|
phase_id: Uuid,
|
||||||
kind: &str,
|
kind: &str,
|
||||||
@@ -105,7 +110,7 @@ async fn summarize_one(
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
let (narrative, structured) = call_anthropic(kind, &material).await?;
|
let (narrative, structured, answered_by) = call_anthropic(runtime, kind, &material).await?;
|
||||||
let metrics = structured
|
let metrics = structured
|
||||||
.get("metrics")
|
.get("metrics")
|
||||||
.cloned()
|
.cloned()
|
||||||
@@ -128,7 +133,7 @@ async fn summarize_one(
|
|||||||
mission_id,
|
mission_id,
|
||||||
phase_id,
|
phase_id,
|
||||||
kind,
|
kind,
|
||||||
&model_name(),
|
&answered_by,
|
||||||
&narrative,
|
&narrative,
|
||||||
&metrics,
|
&metrics,
|
||||||
&sources,
|
&sources,
|
||||||
@@ -323,58 +328,25 @@ async fn collect_material(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn call_anthropic(kind: &str, material: &PhaseMaterial) -> Result<(String, Value), String> {
|
/// Returns the narrative, the parsed object, and **the model that answered** —
|
||||||
let api_key =
|
/// which may be a fallback link rather than `model_name()`, and is recorded as
|
||||||
std::env::var("ANTHROPIC_API_KEY").map_err(|_| "ANTHROPIC_API_KEY unset".to_string())?;
|
/// such.
|
||||||
|
async fn call_anthropic(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
|
kind: &str,
|
||||||
|
material: &PhaseMaterial,
|
||||||
|
) -> Result<(String, Value, String), String> {
|
||||||
let model = model_name();
|
let model = model_name();
|
||||||
let system = system_prompt(kind);
|
let system = system_prompt(kind);
|
||||||
let user = user_prompt(kind, material);
|
let user = user_prompt(kind, material);
|
||||||
|
|
||||||
let body = json!({
|
let (raw, answered_by) = crate::subscription::complete_with_fallback(
|
||||||
"model": model,
|
runtime, &system, &user, &model, 4096, false,
|
||||||
"max_tokens": 4096,
|
)
|
||||||
"system": system,
|
.await?;
|
||||||
"messages": [ { "role": "user", "content": user } ]
|
let raw = raw.trim().to_string();
|
||||||
});
|
|
||||||
let client = reqwest::Client::builder()
|
|
||||||
.timeout(std::time::Duration::from_secs(120))
|
|
||||||
.build()
|
|
||||||
.map_err(|e| format!("http client: {e}"))?;
|
|
||||||
let resp = client
|
|
||||||
.post("https://api.anthropic.com/v1/messages")
|
|
||||||
.header("x-api-key", &api_key)
|
|
||||||
.header("anthropic-version", ANTHROPIC_API_VERSION)
|
|
||||||
.header("content-type", "application/json")
|
|
||||||
.json(&body)
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("anthropic call: {e}"))?;
|
|
||||||
if !resp.status().is_success() {
|
|
||||||
let code = resp.status();
|
|
||||||
let body = resp.text().await.unwrap_or_default();
|
|
||||||
return Err(format!(
|
|
||||||
"anthropic {code}: {}",
|
|
||||||
&body[..body.len().min(500)]
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let json: Value = resp
|
|
||||||
.json()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("anthropic json: {e}"))?;
|
|
||||||
let raw = json
|
|
||||||
.get("content")
|
|
||||||
.and_then(|c| c.as_array())
|
|
||||||
.and_then(|arr| {
|
|
||||||
arr.iter()
|
|
||||||
.find(|b| b.get("type").and_then(|t| t.as_str()) == Some("text"))
|
|
||||||
})
|
|
||||||
.and_then(|b| b.get("text"))
|
|
||||||
.and_then(|t| t.as_str())
|
|
||||||
.ok_or_else(|| "anthropic response missing text block".to_string())?
|
|
||||||
.trim()
|
|
||||||
.to_string();
|
|
||||||
if raw.is_empty() {
|
if raw.is_empty() {
|
||||||
return Err("anthropic returned empty text".into());
|
return Err(format!("{answered_by} returned empty text"));
|
||||||
}
|
}
|
||||||
// Model returns a JSON object; extract narrative + rest.
|
// Model returns a JSON object; extract narrative + rest.
|
||||||
let parsed: Value = serde_json::from_str(&strip_code_fence(&raw)).map_err(|e| {
|
let parsed: Value = serde_json::from_str(&strip_code_fence(&raw)).map_err(|e| {
|
||||||
@@ -392,7 +364,7 @@ async fn call_anthropic(kind: &str, material: &PhaseMaterial) -> Result<(String,
|
|||||||
if narrative.is_empty() {
|
if narrative.is_empty() {
|
||||||
return Err("summarizer response missing narrative".into());
|
return Err("summarizer response missing narrative".into());
|
||||||
}
|
}
|
||||||
Ok((narrative, parsed))
|
Ok((narrative, parsed, answered_by))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Trim a leading/trailing ```json … ``` fence the model sometimes wraps
|
/// Trim a leading/trailing ```json … ``` fence the model sometimes wraps
|
||||||
|
|||||||
@@ -0,0 +1,266 @@
|
|||||||
|
//! What a repository actually contains, small enough to put in a prompt.
|
||||||
|
//!
|
||||||
|
//! The planner was given the root listing and planned "optimise the hot path"
|
||||||
|
//! for a crate whose hot path is `add(a: i64, b: i64) -> i64`. Names were not
|
||||||
|
//! enough: the mission was unachievable from the moment it was written, and
|
||||||
|
//! nothing discovered that until an agent had built a benchmark harness to
|
||||||
|
//! measure an integer addition.
|
||||||
|
//!
|
||||||
|
//! # The rule this module exists to enforce
|
||||||
|
//!
|
||||||
|
//! A digest is always partial for any repository worth planning against, and a
|
||||||
|
//! model shown a partial view without being told it is partial plans as though
|
||||||
|
//! it saw everything. So every omission is STATED — how many files were listed,
|
||||||
|
//! how many were shown, what was cut from each. That is the same distinction as
|
||||||
|
//! `Option<u32>` for the subagent probe: "we did not look" and "there is nothing
|
||||||
|
//! there" are different facts, and only one of them is about the repository.
|
||||||
|
//!
|
||||||
|
//! # Priority
|
||||||
|
//!
|
||||||
|
//! Manifests first (they say what the project IS and what it may depend on),
|
||||||
|
//! then the README, then source ascending by size — smallest-first shows the
|
||||||
|
//! most files per byte, and a planner benefits more from seeing twenty small
|
||||||
|
//! files than one large one.
|
||||||
|
|
||||||
|
/// One file in the repository tree.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct FileEntry {
|
||||||
|
pub path: String,
|
||||||
|
pub size: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Total characters of file CONTENT a digest may carry.
|
||||||
|
///
|
||||||
|
/// The prompt around it is ~1.5k, and the planner is a single call per mission,
|
||||||
|
/// so this is generous by design: the cost of a too-small digest is a plan built
|
||||||
|
/// on a guess, which costs a VM boot to discover.
|
||||||
|
pub const CONTENT_BUDGET: usize = 12_000;
|
||||||
|
|
||||||
|
/// Ceiling per file, so one large file cannot spend the whole budget.
|
||||||
|
pub const PER_FILE_CAP: usize = 3_000;
|
||||||
|
|
||||||
|
/// Files worth showing before any source.
|
||||||
|
fn is_manifest(path: &str) -> bool {
|
||||||
|
matches!(
|
||||||
|
path,
|
||||||
|
"Cargo.toml"
|
||||||
|
| "package.json"
|
||||||
|
| "pyproject.toml"
|
||||||
|
| "setup.py"
|
||||||
|
| "go.mod"
|
||||||
|
| "Gemfile"
|
||||||
|
| "pom.xml"
|
||||||
|
| "build.gradle"
|
||||||
|
| "Makefile"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_readme(path: &str) -> bool {
|
||||||
|
path.eq_ignore_ascii_case("README.md") || path.eq_ignore_ascii_case("README")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Paths to fetch, in the order they earn their place.
|
||||||
|
///
|
||||||
|
/// Directories and files a planner cannot use are dropped: lockfiles are huge
|
||||||
|
/// and say nothing a manifest does not, and build output is not source.
|
||||||
|
pub fn priority(entries: &[FileEntry]) -> Vec<&FileEntry> {
|
||||||
|
let mut useful: Vec<&FileEntry> = entries
|
||||||
|
.iter()
|
||||||
|
.filter(|e| {
|
||||||
|
let p = e.path.as_str();
|
||||||
|
!p.starts_with(".git/")
|
||||||
|
&& !p.contains("/target/")
|
||||||
|
&& !p.starts_with("target/")
|
||||||
|
&& !p.contains("node_modules/")
|
||||||
|
&& p != "Cargo.lock"
|
||||||
|
&& p != "package-lock.json"
|
||||||
|
&& p != "poetry.lock"
|
||||||
|
&& e.size > 0
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
useful.sort_by_key(|e| {
|
||||||
|
let rank = if is_manifest(&e.path) {
|
||||||
|
0
|
||||||
|
} else if is_readme(&e.path) {
|
||||||
|
1
|
||||||
|
} else {
|
||||||
|
2
|
||||||
|
};
|
||||||
|
(rank, e.size, e.path.clone())
|
||||||
|
});
|
||||||
|
useful
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Render the digest a planner sees.
|
||||||
|
///
|
||||||
|
/// `contents` is `(path, text)` for the files that were actually fetched, in
|
||||||
|
/// priority order. Anything not fetched is still LISTED, so the model knows the
|
||||||
|
/// file exists even when it cannot read it.
|
||||||
|
pub fn render(entries: &[FileEntry], contents: &[(String, String)]) -> String {
|
||||||
|
if entries.is_empty() {
|
||||||
|
return "(the repository is empty, or its tree could not be read)".to_string();
|
||||||
|
}
|
||||||
|
let mut out = String::new();
|
||||||
|
out.push_str(&format!("FILES ({} total):\n", entries.len()));
|
||||||
|
// The whole tree by name is cheap and is what stops "does X exist" guessing.
|
||||||
|
// Capped anyway: a 10k-file monorepo listing is not a prompt.
|
||||||
|
const MAX_LISTED: usize = 300;
|
||||||
|
for e in entries.iter().take(MAX_LISTED) {
|
||||||
|
out.push_str(&format!(" {} ({} bytes)\n", e.path, e.size));
|
||||||
|
}
|
||||||
|
if entries.len() > MAX_LISTED {
|
||||||
|
out.push_str(&format!(
|
||||||
|
" … and {} more files NOT listed\n",
|
||||||
|
entries.len() - MAX_LISTED
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if contents.is_empty() {
|
||||||
|
out.push_str("\n(no file contents could be read — plan from the names alone, and say so if that is not enough)\n");
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
out.push_str(&format!(
|
||||||
|
"\nCONTENTS ({} of {} files shown; anything not shown you have NOT seen):\n",
|
||||||
|
contents.len(),
|
||||||
|
entries.len()
|
||||||
|
));
|
||||||
|
for (path, text) in contents {
|
||||||
|
out.push_str(&format!("\n--- {path} ---\n{text}\n"));
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Take file texts up to the budget, truncating each at [`PER_FILE_CAP`].
|
||||||
|
///
|
||||||
|
/// Truncation is marked in the text itself rather than silently cutting: a model
|
||||||
|
/// that can see it is reading a fragment asks differently than one that believes
|
||||||
|
/// it read the file.
|
||||||
|
pub fn fit(fetched: Vec<(String, String)>) -> Vec<(String, String)> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let mut spent = 0usize;
|
||||||
|
for (path, text) in fetched {
|
||||||
|
if spent >= CONTENT_BUDGET {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let room = (CONTENT_BUDGET - spent).min(PER_FILE_CAP);
|
||||||
|
let text = if text.len() <= room {
|
||||||
|
text
|
||||||
|
} else {
|
||||||
|
let end = (0..=room)
|
||||||
|
.rev()
|
||||||
|
.find(|i| text.is_char_boundary(*i))
|
||||||
|
.unwrap_or(0);
|
||||||
|
format!(
|
||||||
|
"{}\n… [truncated: {} of {} bytes shown]",
|
||||||
|
&text[..end],
|
||||||
|
end,
|
||||||
|
text.len()
|
||||||
|
)
|
||||||
|
};
|
||||||
|
spent += text.len();
|
||||||
|
out.push((path, text));
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn f(path: &str, size: usize) -> FileEntry {
|
||||||
|
FileEntry {
|
||||||
|
path: path.into(),
|
||||||
|
size,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Manifests first, then the README, then source smallest-first. A planner
|
||||||
|
/// learns more from twenty small files than from one large one.
|
||||||
|
#[test]
|
||||||
|
fn the_files_that_say_what_this_is_come_first() {
|
||||||
|
let entries = vec![
|
||||||
|
f("src/big.rs", 9000),
|
||||||
|
f("README.md", 400),
|
||||||
|
f("src/lib.rs", 120),
|
||||||
|
f("Cargo.toml", 200),
|
||||||
|
];
|
||||||
|
let order: Vec<&str> = priority(&entries).iter().map(|e| e.path.as_str()).collect();
|
||||||
|
assert_eq!(order, vec!["Cargo.toml", "README.md", "src/lib.rs", "src/big.rs"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Lockfiles and build output are dropped: enormous, and they say nothing a
|
||||||
|
/// manifest does not.
|
||||||
|
#[test]
|
||||||
|
fn noise_is_not_offered_to_the_planner() {
|
||||||
|
let entries = vec![
|
||||||
|
f("Cargo.lock", 50_000),
|
||||||
|
f("target/debug/thing", 900_000),
|
||||||
|
f("node_modules/x/index.js", 400),
|
||||||
|
f(".git/config", 100),
|
||||||
|
f("src/lib.rs", 100),
|
||||||
|
f("empty.rs", 0),
|
||||||
|
];
|
||||||
|
let kept: Vec<&str> = priority(&entries).iter().map(|e| e.path.as_str()).collect();
|
||||||
|
assert_eq!(kept, vec!["src/lib.rs"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// THE rule. A partial view presented as complete is planned against as
|
||||||
|
/// though it were complete — which is how "optimise the hot path" gets
|
||||||
|
/// written for a crate that adds two integers.
|
||||||
|
#[test]
|
||||||
|
fn every_omission_is_stated() {
|
||||||
|
let entries: Vec<FileEntry> = (0..400).map(|i| f(&format!("src/f{i}.rs"), 100)).collect();
|
||||||
|
let shown = vec![("src/f0.rs".to_string(), "fn a() {}".to_string())];
|
||||||
|
let out = render(&entries, &shown);
|
||||||
|
|
||||||
|
assert!(out.contains("FILES (400 total)"), "{out}");
|
||||||
|
assert!(out.contains("and 100 more files NOT listed"), "{out}");
|
||||||
|
assert!(out.contains("1 of 400 files shown"), "{out}");
|
||||||
|
assert!(
|
||||||
|
out.contains("you have NOT seen"),
|
||||||
|
"the model must be told the view is partial: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A file cut short says so, in the text the model reads.
|
||||||
|
#[test]
|
||||||
|
fn a_truncated_file_says_it_was_truncated() {
|
||||||
|
let big = "x".repeat(PER_FILE_CAP * 2);
|
||||||
|
let out = fit(vec![("src/big.rs".into(), big.clone())]);
|
||||||
|
assert_eq!(out.len(), 1);
|
||||||
|
assert!(out[0].1.contains("truncated"), "{}", &out[0].1[..80]);
|
||||||
|
assert!(out[0].1.len() < big.len());
|
||||||
|
// And the marker names both numbers, so "how much did I miss" is
|
||||||
|
// answerable rather than guessable.
|
||||||
|
assert!(out[0].1.contains(&big.len().to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The budget is a total, not per file: one large file must not starve the
|
||||||
|
/// rest, and the whole digest must stay promptable.
|
||||||
|
#[test]
|
||||||
|
fn the_budget_bounds_the_whole_digest() {
|
||||||
|
let files: Vec<(String, String)> = (0..20)
|
||||||
|
.map(|i| (format!("src/f{i}.rs"), "y".repeat(PER_FILE_CAP)))
|
||||||
|
.collect();
|
||||||
|
let out = fit(files);
|
||||||
|
let total: usize = out.iter().map(|(_, t)| t.len()).sum();
|
||||||
|
assert!(total <= CONTENT_BUDGET, "digest was {total} bytes");
|
||||||
|
assert!(!out.is_empty(), "and it still shows something");
|
||||||
|
assert!(out.len() < 20, "not everything fits, by construction");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An empty or unreadable tree is stated as such — never rendered as a
|
||||||
|
/// repository that happens to contain nothing.
|
||||||
|
#[test]
|
||||||
|
fn an_unreadable_tree_is_not_an_empty_repository() {
|
||||||
|
let out = render(&[], &[]);
|
||||||
|
assert!(out.contains("could not be read"), "{out}");
|
||||||
|
|
||||||
|
// A tree we CAN read but no contents we could fetch is a different
|
||||||
|
// fact, and says so.
|
||||||
|
let out = render(&[f("src/lib.rs", 100)], &[]);
|
||||||
|
assert!(out.contains("src/lib.rs"), "{out}");
|
||||||
|
assert!(out.contains("no file contents could be read"), "{out}");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,159 @@
|
|||||||
|
//! A throwaway copy of a mission checkout, for commands that run as ROOT.
|
||||||
|
//!
|
||||||
|
//! Three places in this codebase run a real command against a mission's tree —
|
||||||
|
//! the judge's verification (`evaluator_tools::Sandbox`), the benchmark runner,
|
||||||
|
//! and the `on_green_tests` delivery gate. All three enter a container running as
|
||||||
|
//! root with the missions root bind-mounted, and all three run something that
|
||||||
|
//! writes `target/`. All three now go through here; the judge was the last to
|
||||||
|
//! move, having carried its own copy of this logic since before it existed.
|
||||||
|
//!
|
||||||
|
//! Run against the live checkout, that breaks the single-writer invariant: the
|
||||||
|
//! tree is owned by uid 65532 and now contains root-owned build output, so the
|
||||||
|
//! next phase's `cargo` hits permission-denied on a directory it cannot write.
|
||||||
|
//! The harness's uid probe reports it as `uids=0,65532`.
|
||||||
|
//!
|
||||||
|
//! # The cleanup half, which is the part that keeps being got wrong
|
||||||
|
//!
|
||||||
|
//! The copy inherits the same problem: its `target/` is root-owned, so the
|
||||||
|
//! server process (uid 65532) **cannot delete it**. A `Drop` calling
|
||||||
|
//! `std::fs::remove_dir_all` fails, and because that error is discarded the tree
|
||||||
|
//! survives forever — measured at 1.2 MB per benchmark run and 16 MB of stranded
|
||||||
|
//! judge sandboxes before this existed.
|
||||||
|
//!
|
||||||
|
//! So removal goes back through the container, as root, where the files were
|
||||||
|
//! written. `Drop` remains only as a fallback for the paths where nothing has
|
||||||
|
//! run as root yet, and does not pretend to be more.
|
||||||
|
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
/// Where throwaway copies live: siblings of the per-mission directories, like
|
||||||
|
/// `_outputs` and `_verify`, so reaping a mission cannot race a running command.
|
||||||
|
pub fn copy_root(kind: &str, mission_id: uuid::Uuid) -> PathBuf {
|
||||||
|
crate::mission_workspace::missions_root()
|
||||||
|
.join(kind)
|
||||||
|
.join(mission_id.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Delete a copy from inside the container that wrote it.
|
||||||
|
///
|
||||||
|
/// Best-effort and loud: a housekeeping failure must not cost a real verdict or
|
||||||
|
/// a real benchmark, but it must not be silent either — silence is how the leaks
|
||||||
|
/// this module exists for went unnoticed for a day.
|
||||||
|
pub async fn purge(container: &str, root: &Path) {
|
||||||
|
let Ok(docker) = crate::container_exec::connect() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let argv = vec![
|
||||||
|
"rm".to_string(),
|
||||||
|
"-rf".to_string(),
|
||||||
|
root.display().to_string(),
|
||||||
|
];
|
||||||
|
// Explicitly root: this exists to delete files an EARLIER root-run exec
|
||||||
|
// created, which uid 65532 cannot touch. Everything else now runs as 65532
|
||||||
|
// (see `container_exec`), so this is cleaning up history, not policy.
|
||||||
|
if let Err(e) = crate::container_exec::exec_as_root(
|
||||||
|
&docker,
|
||||||
|
container,
|
||||||
|
Some("/"),
|
||||||
|
&argv,
|
||||||
|
std::time::Duration::from_secs(120),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
eprintln!(
|
||||||
|
"root_copy: could not remove {} from {container}: {e}",
|
||||||
|
root.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A copy of a checkout, removed when it goes out of scope.
|
||||||
|
pub struct RootCopy {
|
||||||
|
root: PathBuf,
|
||||||
|
workdir: PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RootCopy {
|
||||||
|
/// Copy `source` into `root`, returning a handle whose `workdir` is the tree
|
||||||
|
/// to run in.
|
||||||
|
///
|
||||||
|
/// Packed through `mission_fs::pack_dir`, so the copy carries exactly what a
|
||||||
|
/// delivered diff carries — no `target/`, no `node_modules/`. One exclusion
|
||||||
|
/// list, four consumers.
|
||||||
|
pub fn of(source: &Path, root: &Path) -> Result<RootCopy, String> {
|
||||||
|
let archive = crate::mission_fs::pack_dir(source, "repo")
|
||||||
|
.map_err(|e| format!("pack {} for a root-run command: {e}", source.display()))?;
|
||||||
|
crate::mission_fs::unpack_into(&archive, root)
|
||||||
|
.map_err(|e| format!("unpack copy into {}: {e}", root.display()))?;
|
||||||
|
let workdir = root.join("repo");
|
||||||
|
if !workdir.is_dir() {
|
||||||
|
return Err(format!("copy missing at {}", workdir.display()));
|
||||||
|
}
|
||||||
|
Ok(RootCopy {
|
||||||
|
root: root.to_path_buf(),
|
||||||
|
workdir,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn workdir(&self) -> &Path {
|
||||||
|
&self.workdir
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Take the working directory and give up automatic cleanup.
|
||||||
|
///
|
||||||
|
/// For a caller whose copy outlives this handle — `evaluator_tools::Sandbox`
|
||||||
|
/// hands the path to a judge that has not run yet, so letting `Drop` fire on
|
||||||
|
/// return would delete the tree out from under it. That caller becomes
|
||||||
|
/// responsible for calling [`purge`], which is the only thing that can
|
||||||
|
/// remove root-owned build output anyway.
|
||||||
|
///
|
||||||
|
/// Spelled as a method rather than `mem::forget` at the call site, so the
|
||||||
|
/// transfer of responsibility is visible in the type rather than implied by
|
||||||
|
/// a leak.
|
||||||
|
pub fn into_workdir(self) -> PathBuf {
|
||||||
|
let workdir = self.workdir.clone();
|
||||||
|
std::mem::forget(self);
|
||||||
|
workdir
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for RootCopy {
|
||||||
|
/// Fallback only. This CANNOT remove root-owned build output — see
|
||||||
|
/// [`purge`], which is what actually clears a copy something has run in.
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = std::fs::remove_dir_all(&self.root);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// A copy must be a SIBLING of the per-mission directory, never inside it:
|
||||||
|
/// `teardown_container` removes `<missions_root>/<mission_id>` wholesale and
|
||||||
|
/// would take a running command's tree with it.
|
||||||
|
#[test]
|
||||||
|
fn copies_live_beside_the_mission_directory_not_inside_it() {
|
||||||
|
let mission = uuid::Uuid::now_v7();
|
||||||
|
let mission_dir = crate::mission_workspace::missions_root().join(mission.to_string());
|
||||||
|
for kind in ["_bench", "_gate", "_verify"] {
|
||||||
|
let root = copy_root(kind, mission);
|
||||||
|
assert!(!root.starts_with(&mission_dir), "{root:?}");
|
||||||
|
assert!(
|
||||||
|
root.starts_with(crate::mission_workspace::missions_root().join(kind)),
|
||||||
|
"{root:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The copy is not the checkout. Stated as a test because the whole defect
|
||||||
|
/// class is "ran the real command against the real tree".
|
||||||
|
#[test]
|
||||||
|
fn a_copy_is_never_the_checkout() {
|
||||||
|
let mission = uuid::Uuid::now_v7();
|
||||||
|
let live = crate::mission_workspace::checkout_path(mission);
|
||||||
|
for kind in ["_bench", "_gate", "_verify"] {
|
||||||
|
assert_ne!(copy_root(kind, mission).join("repo"), live);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -26,6 +26,19 @@ pub(crate) async fn workspace_agent(
|
|||||||
Ok(agent)
|
Ok(agent)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// As [`workspace_agent`], but sees soft-deleted agents too. PURGE ONLY.
|
||||||
|
pub(crate) async fn workspace_agent_any(
|
||||||
|
state: &AppState,
|
||||||
|
user: &cm_auth::AuthedUser,
|
||||||
|
agent_id: AgentId,
|
||||||
|
) -> Result<Agent, ApiError> {
|
||||||
|
let agent = cm_db::repo::agents::get_any(&state.pool, agent_id).await?;
|
||||||
|
if agent.workspace_id != user.workspace_id {
|
||||||
|
return Err(ApiError::NotFound);
|
||||||
|
}
|
||||||
|
Ok(agent)
|
||||||
|
}
|
||||||
|
|
||||||
/// `GET /api/claws/{id}/runtime-config` — the claw's model + §15 sandbox facts
|
/// `GET /api/claws/{id}/runtime-config` — the claw's model + §15 sandbox facts
|
||||||
/// (for the claw card / anatomy view's model badge).
|
/// (for the claw card / anatomy view's model badge).
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
@@ -577,7 +590,11 @@ pub async fn enhance_brain(
|
|||||||
let user_prompt = format!(
|
let user_prompt = format!(
|
||||||
"BRAIN: {reference}\n\n=== SYSTEM PROMPT ===\n{sp}\n\n=== AGENTS.md ===\n{agent_md}\n\n=== PERSONA ===\n{persona}\n\n=== SKILLS ===\n{skills}"
|
"BRAIN: {reference}\n\n=== SYSTEM PROMPT ===\n{sp}\n\n=== AGENTS.md ===\n{agent_md}\n\n=== PERSONA ===\n{persona}\n\n=== SKILLS ===\n{skills}"
|
||||||
);
|
);
|
||||||
let raw = match runtime.complete(ENHANCE_SYSTEM, &user_prompt, "claude-opus-4-8", 16000, true).await {
|
let raw = match crate::subscription::complete_or(
|
||||||
|
&runtime, ENHANCE_SYSTEM, &user_prompt, "claude-opus-4-8", 16000, true,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(t) => t,
|
Ok(t) => t,
|
||||||
Err(e) => { yield sse(json!({"stage":"error","pct":100,"label":format!("Opus error: {e}")})); return; }
|
Err(e) => { yield sse(json!({"stage":"error","pct":100,"label":format!("Opus error: {e}")})); return; }
|
||||||
};
|
};
|
||||||
@@ -656,9 +673,15 @@ pub(crate) async fn enhance_and_publish(
|
|||||||
let user_prompt = format!(
|
let user_prompt = format!(
|
||||||
"ROLE CONTEXT: {role_context}\n\nBRAIN: {reference}\n\n=== SYSTEM PROMPT ===\n{sp}\n\n=== AGENTS.md ===\n{agent_md}\n\n=== PERSONA ===\n{persona}\n\n=== SKILLS ===\n{skills}"
|
"ROLE CONTEXT: {role_context}\n\nBRAIN: {reference}\n\n=== SYSTEM PROMPT ===\n{sp}\n\n=== AGENTS.md ===\n{agent_md}\n\n=== PERSONA ===\n{persona}\n\n=== SKILLS ===\n{skills}"
|
||||||
);
|
);
|
||||||
let raw = runtime
|
let raw = crate::subscription::complete_or(
|
||||||
.complete(ENHANCE_SYSTEM, &user_prompt, "claude-opus-4-8", 16000, true)
|
runtime,
|
||||||
.await?;
|
ENHANCE_SYSTEM,
|
||||||
|
&user_prompt,
|
||||||
|
"claude-opus-4-8",
|
||||||
|
16000,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
let v = extract_json(&raw).ok_or_else(|| "unparseable enhance output".to_string())?;
|
let v = extract_json(&raw).ok_or_else(|| "unparseable enhance output".to_string())?;
|
||||||
let enh = v.get("enhanced").cloned().unwrap_or(Value::Null);
|
let enh = v.get("enhanced").cloned().unwrap_or(Value::Null);
|
||||||
let field = |k: &str| {
|
let field = |k: &str| {
|
||||||
@@ -1127,7 +1150,7 @@ pub async fn patch(
|
|||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
pub struct SetModelRequest {
|
pub struct SetModelRequest {
|
||||||
/// Model selector (claude / glm / glm-5.2 / kimi / gemini / groq /
|
/// Model selector (claude / glm / glm-5.2 / kimi / groq /
|
||||||
/// specific model id like `claude-sonnet-5`). Resolved through the
|
/// specific model id like `claude-sonnet-5`). Resolved through the
|
||||||
/// same RuntimeProvisioner::provider_alias_for that team creation
|
/// same RuntimeProvisioner::provider_alias_for that team creation
|
||||||
/// uses, so shorthand + fully-qualified ids both work.
|
/// uses, so shorthand + fully-qualified ids both work.
|
||||||
@@ -1279,7 +1302,12 @@ pub async fn batch_delete(
|
|||||||
let mut done = 0usize;
|
let mut done = 0usize;
|
||||||
for id in agent_ids {
|
for id in agent_ids {
|
||||||
let base = 100 * done / total;
|
let base = 100 * done / total;
|
||||||
let agent = match workspace_agent(&state, &user, id).await {
|
// `workspace_agent_any`, not `workspace_agent`: a purge has to be
|
||||||
|
// able to see the rows it exists to remove. The soft-delete path
|
||||||
|
// correctly hides them from every read, which also hid them from
|
||||||
|
// the only route that could reap them — four soft-deleted agents
|
||||||
|
// from June were unreachable from the application entirely.
|
||||||
|
let agent = match workspace_agent_any(&state, &user, id).await {
|
||||||
Ok(a) => a,
|
Ok(a) => a,
|
||||||
Err(_) => { yield sse(json!({"stage":"skip","pct":base,"label":format!("{id}: not found or no access")})); done += 1; continue; }
|
Err(_) => { yield sse(json!({"stage":"skip","pct":base,"label":format!("{id}: not found or no access")})); done += 1; continue; }
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ pub async fn propose_for_agent(
|
|||||||
Authed(user): Authed,
|
Authed(user): Authed,
|
||||||
Path(agent_id): Path<Uuid>,
|
Path(agent_id): Path<Uuid>,
|
||||||
) -> Result<Json<serde_json::Value>, ApiError> {
|
) -> Result<Json<serde_json::Value>, ApiError> {
|
||||||
let id = crate::level_up::propose_agent(&state.pool, user.workspace_id, user.user_id, agent_id)
|
let id = crate::level_up::propose_agent(&state.pool, &state.runtime, user.workspace_id, user.user_id, agent_id)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
eprintln!("level_up: propose_agent {agent_id} failed: {e}");
|
eprintln!("level_up: propose_agent {agent_id} failed: {e}");
|
||||||
@@ -51,7 +51,7 @@ pub async fn propose_for_team(
|
|||||||
Authed(user): Authed,
|
Authed(user): Authed,
|
||||||
Path(team_id): Path<Uuid>,
|
Path(team_id): Path<Uuid>,
|
||||||
) -> Result<Json<serde_json::Value>, ApiError> {
|
) -> Result<Json<serde_json::Value>, ApiError> {
|
||||||
let id = crate::level_up::propose_team(&state.pool, user.workspace_id, user.user_id, team_id)
|
let id = crate::level_up::propose_team(&state.pool, &state.runtime, user.workspace_id, user.user_id, team_id)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
eprintln!("level_up: propose_team {team_id} failed: {e}");
|
eprintln!("level_up: propose_team {team_id} failed: {e}");
|
||||||
|
|||||||
@@ -0,0 +1,359 @@
|
|||||||
|
//! `/api/missions/{id}/plan-proposals` — let a model author the phases.
|
||||||
|
//!
|
||||||
|
//! W1 / #13, and the sibling of [`crate::routes::mission_roster`]: that one has
|
||||||
|
//! a model size the team, this one has it decide what the work is. Same three
|
||||||
|
//! verbs and the same rule — propose and decide are separate, because only the
|
||||||
|
//! second one changes a mission.
|
||||||
|
//!
|
||||||
|
//! The model is handed two lists it may not depart from: the phase kinds
|
||||||
|
//! `phase_runner` dispatches on, and the config keys `phase_config` says have
|
||||||
|
//! readers. Both are enforced again on the way in, so a plan cannot describe
|
||||||
|
//! work this platform will accept and then not do.
|
||||||
|
|
||||||
|
use axum::extract::{Path, State};
|
||||||
|
use axum::Json;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use crate::mission_plan::{Plan, MAX_PHASES, PLANNABLE_KINDS};
|
||||||
|
use crate::{ApiError, AppState, Authed};
|
||||||
|
|
||||||
|
const PLANNER_MODEL: &str = "claude-opus-4-8";
|
||||||
|
|
||||||
|
/// What the repository actually contains, for the planner's prompt.
|
||||||
|
///
|
||||||
|
/// Names were not enough. Given the root listing alone, the planner wrote
|
||||||
|
/// "optimise the hot path" for a crate whose hot path is
|
||||||
|
/// `add(a: i64, b: i64) -> i64` — a mission that was unachievable from the
|
||||||
|
/// moment it was written, and that nothing discovered until an agent had built a
|
||||||
|
/// benchmark harness to measure an integer addition.
|
||||||
|
///
|
||||||
|
/// Read from the FORGE, not a checkout: at proposal time the mission is still a
|
||||||
|
/// draft and `ensure_checkout` has not run, so there is nothing on disk. Every
|
||||||
|
/// failure degrades to a STATED absence — a planner told "the listing could not
|
||||||
|
/// be read" can hedge; one told nothing assumes.
|
||||||
|
async fn repo_digest(pool: &sqlx::PgPool, mission_id: uuid::Uuid) -> String {
|
||||||
|
let row: Option<(Option<String>, Option<String>, Option<String>)> = sqlx::query_as(
|
||||||
|
"SELECT r.owner, r.name, r.default_branch
|
||||||
|
FROM missions m JOIN repos r ON r.id = m.repo_id
|
||||||
|
WHERE m.id = $1",
|
||||||
|
)
|
||||||
|
.bind(mission_id)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.flatten();
|
||||||
|
let Some((Some(owner), Some(name), branch)) = row else {
|
||||||
|
return "(this mission has no repository)".to_string();
|
||||||
|
};
|
||||||
|
let branch = branch.unwrap_or_else(|| "main".to_string());
|
||||||
|
let token = std::env::var("GITEA_TOKEN").unwrap_or_default();
|
||||||
|
let Ok(client) = reqwest::Client::builder()
|
||||||
|
.timeout(std::time::Duration::from_secs(20))
|
||||||
|
.build()
|
||||||
|
else {
|
||||||
|
return "(the repository could not be read)".to_string();
|
||||||
|
};
|
||||||
|
let auth = |r: reqwest::RequestBuilder| {
|
||||||
|
if token.trim().is_empty() {
|
||||||
|
r
|
||||||
|
} else {
|
||||||
|
r.header("Authorization", format!("token {token}"))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// The whole tree in one call, so "does this repo have benches/" is a fact
|
||||||
|
// rather than an inference from the root.
|
||||||
|
let tree_url = format!(
|
||||||
|
"https://git.redclaw.dev/api/v1/repos/{owner}/{name}/git/trees/{branch}?recursive=true&per_page=1000"
|
||||||
|
);
|
||||||
|
let tree: serde_json::Value = match auth(client.get(&tree_url)).send().await {
|
||||||
|
Ok(r) if r.status().is_success() => r.json().await.unwrap_or_default(),
|
||||||
|
_ => return "(the repository tree could not be read)".to_string(),
|
||||||
|
};
|
||||||
|
let entries: Vec<crate::repo_digest::FileEntry> = tree
|
||||||
|
.get("tree")
|
||||||
|
.and_then(|t| t.as_array())
|
||||||
|
.map(|items| {
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.filter(|e| e.get("type").and_then(|v| v.as_str()) == Some("blob"))
|
||||||
|
.filter_map(|e| {
|
||||||
|
Some(crate::repo_digest::FileEntry {
|
||||||
|
path: e.get("path")?.as_str()?.to_string(),
|
||||||
|
size: e.get("size").and_then(|v| v.as_u64()).unwrap_or(0) as usize,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
// Fetch in priority order until the budget is spent. Requested serially and
|
||||||
|
// capped: this runs inside one API request, and a repo with 500 useful files
|
||||||
|
// must not turn a proposal into 500 round trips.
|
||||||
|
let mut fetched: Vec<(String, String)> = Vec::new();
|
||||||
|
let mut spent = 0usize;
|
||||||
|
for e in crate::repo_digest::priority(&entries).into_iter().take(40) {
|
||||||
|
if spent >= crate::repo_digest::CONTENT_BUDGET {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let raw = format!(
|
||||||
|
"https://git.redclaw.dev/api/v1/repos/{owner}/{name}/raw/{}?ref={branch}",
|
||||||
|
e.path
|
||||||
|
);
|
||||||
|
if let Ok(r) = auth(client.get(&raw)).send().await {
|
||||||
|
if r.status().is_success() {
|
||||||
|
if let Ok(text) = r.text().await {
|
||||||
|
spent += text.len().min(crate::repo_digest::PER_FILE_CAP);
|
||||||
|
fetched.push((e.path.clone(), text));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
crate::repo_digest::render(&entries, &crate::repo_digest::fit(fetched))
|
||||||
|
}
|
||||||
|
|
||||||
|
const PLAN_SYSTEM: &str = "You decide what ONE software mission actually does — its phases, in order. \
|
||||||
|
Each phase is a full agent run against the same repository checkout: the next phase sees the tree the \
|
||||||
|
previous one left. They run SEQUENTIALLY, so phases are expensive and a handoff loses context at every \
|
||||||
|
step.\n\n\
|
||||||
|
Propose the FEWEST phases that genuinely need to be separate. ONE phase is usually the right answer, and \
|
||||||
|
is always the right answer for a self-contained change: splitting one change into plan → implement → \
|
||||||
|
test is a documented anti-pattern, not thoroughness — a single agent doing all three in one pass keeps \
|
||||||
|
the context that makes the later steps good. A second phase earns its place only when it depends on \
|
||||||
|
something the first phase could not have known when it started.\n\n\
|
||||||
|
Every phase needs a `task`: the specific instruction for THAT phase, not a restatement of the mission. \
|
||||||
|
An agent receives the mission description plus its own task, so a vague task means an agent guessing \
|
||||||
|
which part of the mission is its share.\n\n\
|
||||||
|
`done_when` is judged afterwards by a separate model reading the repository, so write it as something \
|
||||||
|
observable in the tree — a file that exists, a suite that passes — never as an intention. \
|
||||||
|
`done_when_check` is a SHELL COMMAND that must exit 0; it is enforced while the agent still works, so \
|
||||||
|
prefer it when the condition is mechanical. Set `allow_empty` true only for a phase whose job is to \
|
||||||
|
verify rather than to change files.\n\n\
|
||||||
|
ALWAYS respond with STRICT JSON ONLY, no prose and no markdown: \
|
||||||
|
{\"phases\":[{\"kind\":\"coding\",\"task\":\"...\",\"done_when\":null|\"...\",\
|
||||||
|
\"done_when_check\":null|\"...\",\"allow_empty\":null|true|false}]}";
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct PlanProposalResponse {
|
||||||
|
pub id: Uuid,
|
||||||
|
pub plan: Value,
|
||||||
|
pub author_model: String,
|
||||||
|
pub status: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `POST /api/missions/{id}/plan-proposals` — ask the model for a phase plan.
|
||||||
|
pub async fn suggest(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path(id): Path<Uuid>,
|
||||||
|
) -> Result<Json<PlanProposalResponse>, ApiError> {
|
||||||
|
let ws = user.workspace_id;
|
||||||
|
let mission = cm_db::repo::missions::get(&state.pool, id, ws.as_uuid())
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
|
||||||
|
let prompt = format!(
|
||||||
|
"MISSION: {}\n\nDESCRIPTION:\n{}\n\n=== THE REPOSITORY ===\n{}\n=== END REPOSITORY \
|
||||||
|
===\n\nPlan for the repository as it ACTUALLY IS, not as the description implies it \
|
||||||
|
might be. If the work needs something absent — a benchmark harness, a test suite, a \
|
||||||
|
config file — the phase that needs it must CREATE it, and its task must say so. If the \
|
||||||
|
description asks for something this code cannot support (optimising a function with \
|
||||||
|
nothing to optimise, testing a module that does not exist), say so in the task text and \
|
||||||
|
plan the phase that would establish the truth, rather than a phase that must fail.\n\n\
|
||||||
|
NOTE: a mission agent has NO package-registry access — it cannot add dependencies. A \
|
||||||
|
phase needing tooling must build it from the standard library or from what is already \
|
||||||
|
vendored here.\n\nPHASE KINDS YOU MAY USE (nothing else runs): {}\nCEILING: \
|
||||||
|
{MAX_PHASES} phases.\n\nPropose the plan now (JSON only).",
|
||||||
|
mission.title,
|
||||||
|
mission.description.as_deref().unwrap_or("(none)"),
|
||||||
|
repo_digest(&state.pool, id).await,
|
||||||
|
PLANNABLE_KINDS.join(", "),
|
||||||
|
);
|
||||||
|
|
||||||
|
// The stored `author_model` is whichever link of the fallback chain
|
||||||
|
// actually answered — see `subscription::complete_with_fallback`.
|
||||||
|
let (raw, author_model) = crate::subscription::complete_with_fallback(
|
||||||
|
&state.runtime,
|
||||||
|
PLAN_SYSTEM,
|
||||||
|
&prompt,
|
||||||
|
PLANNER_MODEL,
|
||||||
|
2000,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("mission {id}: plan proposal failed: {e}");
|
||||||
|
crate::subscription::as_api_error(&e)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let parsed: Value = crate::routes::claws::extract_json(&raw).ok_or_else(|| {
|
||||||
|
eprintln!("mission {id}: planner returned no JSON: {raw}");
|
||||||
|
ApiError::BadRequest
|
||||||
|
})?;
|
||||||
|
let plan: Plan = serde_json::from_value(parsed.clone()).map_err(|e| {
|
||||||
|
eprintln!("mission {id}: planner JSON is not a plan ({e}): {parsed}");
|
||||||
|
ApiError::BadRequest
|
||||||
|
})?;
|
||||||
|
// Validated BEFORE storing, so a stored proposal is always one that could be
|
||||||
|
// approved — the failure belongs to the model, not to whoever clicks
|
||||||
|
// approve later.
|
||||||
|
if let Err(why) = plan.validate() {
|
||||||
|
eprintln!("mission {id}: planner proposed an unrunnable plan: {why}");
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
let pid = Uuid::now_v7();
|
||||||
|
let stored = serde_json::to_value(&plan).map_err(|_| ApiError::Internal)?;
|
||||||
|
cm_db::repo::mission_plan_proposals::insert(
|
||||||
|
&state.pool,
|
||||||
|
pid,
|
||||||
|
id,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
&stored,
|
||||||
|
&author_model,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("mission {id}: could not store plan proposal: {e}");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
eprintln!(
|
||||||
|
"mission_plan: mission {id} — {author_model} proposed {} phase(s): {}",
|
||||||
|
plan.phases.len(),
|
||||||
|
plan.phases
|
||||||
|
.iter()
|
||||||
|
.map(|p| p.kind.as_str())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" → ")
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(Json(PlanProposalResponse {
|
||||||
|
id: pid,
|
||||||
|
plan: stored,
|
||||||
|
author_model,
|
||||||
|
status: "proposed".into(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `GET /api/missions/{id}/plan-proposals`
|
||||||
|
pub async fn list(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path(id): Path<Uuid>,
|
||||||
|
) -> Result<Json<Vec<cm_db::repo::mission_plan_proposals::MissionPlanProposal>>, ApiError> {
|
||||||
|
let rows = cm_db::repo::mission_plan_proposals::list(
|
||||||
|
&state.pool,
|
||||||
|
id,
|
||||||
|
user.workspace_id.as_uuid().to_owned(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?;
|
||||||
|
Ok(Json(rows))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
pub struct DecideRequest {
|
||||||
|
pub status: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub note: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `POST /api/missions/{id}/plan-proposals/{pid}/decide`
|
||||||
|
///
|
||||||
|
/// Approving REPLACES the mission's phases. Draft-only: re-planning a mission
|
||||||
|
/// whose phases have started would discard work that already ran, and the phase
|
||||||
|
/// rows are what every downstream sweep keys off.
|
||||||
|
pub async fn decide(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path((id, pid)): Path<(Uuid, Uuid)>,
|
||||||
|
Json(body): Json<DecideRequest>,
|
||||||
|
) -> Result<Json<Value>, ApiError> {
|
||||||
|
let ws = user.workspace_id;
|
||||||
|
let proposal = cm_db::repo::mission_plan_proposals::get(&state.pool, pid, ws.as_uuid().to_owned())
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
if proposal.mission_id != id {
|
||||||
|
return Err(ApiError::NotFound);
|
||||||
|
}
|
||||||
|
|
||||||
|
if body.status == "rejected" {
|
||||||
|
let decided = cm_db::repo::mission_plan_proposals::decide(
|
||||||
|
&state.pool,
|
||||||
|
pid,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
"rejected",
|
||||||
|
body.note.as_deref(),
|
||||||
|
Some(user.user_id.as_uuid().to_owned()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?;
|
||||||
|
return Ok(Json(json!({ "status": "rejected", "decided": decided })));
|
||||||
|
}
|
||||||
|
if body.status != "approved" {
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mission = cm_db::repo::missions::get(&state.pool, id, ws.as_uuid())
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
if mission.status != "draft" {
|
||||||
|
eprintln!("mission {id}: plan approval refused — mission is {}", mission.status);
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
let plan: Plan = serde_json::from_value(proposal.plan.clone()).map_err(|e| {
|
||||||
|
eprintln!("mission {id}: stored plan {pid} does not parse ({e})");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
// Re-validated at approval. The stored plan passed once, but `PLANNABLE_KINDS`
|
||||||
|
// and the config registry are properties of the BUILD — a proposal made
|
||||||
|
// before a deploy could name a kind this build no longer dispatches.
|
||||||
|
if let Err(why) = plan.validate() {
|
||||||
|
eprintln!("mission {id}: plan {pid} is no longer runnable: {why}");
|
||||||
|
let _ = cm_db::repo::mission_plan_proposals::decide(
|
||||||
|
&state.pool,
|
||||||
|
pid,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
"rejected",
|
||||||
|
Some(&why.to_string()),
|
||||||
|
Some(user.user_id.as_uuid().to_owned()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
let phases = plan.phases();
|
||||||
|
let claimed = cm_db::repo::mission_plan_proposals::approve_and_apply(
|
||||||
|
&state.pool,
|
||||||
|
pid,
|
||||||
|
id,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
&phases,
|
||||||
|
body.note.as_deref(),
|
||||||
|
Some(user.user_id.as_uuid().to_owned()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("mission {id}: could not apply plan {pid}: {e}");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
if !claimed {
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
eprintln!(
|
||||||
|
"mission_plan: mission {id} now runs a {}-phase model-authored plan from proposal {pid}",
|
||||||
|
phases.len()
|
||||||
|
);
|
||||||
|
Ok(Json(json!({
|
||||||
|
"status": "approved",
|
||||||
|
"phases": phases.iter().map(|(k, i, _)| json!({"kind": k, "order_idx": i})).collect::<Vec<_>>(),
|
||||||
|
})))
|
||||||
|
}
|
||||||
@@ -0,0 +1,321 @@
|
|||||||
|
//! `/api/missions/{id}/team-proposals` — let a model size the mission's team.
|
||||||
|
//!
|
||||||
|
//! Slice 5. The planner has been proposing rosters into React state for months;
|
||||||
|
//! this is where one reaches a mission. Three verbs, and the split between them
|
||||||
|
//! is the point:
|
||||||
|
//!
|
||||||
|
//! - **suggest** asks the model and PERSISTS the answer. It changes nothing
|
||||||
|
//! about the mission.
|
||||||
|
//! - **approve** writes the roster onto the mission, where the composed executor
|
||||||
|
//! reads it.
|
||||||
|
//! - **reject** records that a human said no, which is the only evidence we ever
|
||||||
|
//! collect about what the planner gets wrong.
|
||||||
|
//!
|
||||||
|
//! A proposal is never applied on arrival. A model sizing a team is a suggestion
|
||||||
|
//! about how many VMs to boot, and this codebase has an explicit rule about
|
||||||
|
//! model output that costs money: it is evidence for a decision, not the
|
||||||
|
//! decision.
|
||||||
|
|
||||||
|
use axum::extract::{Path, State};
|
||||||
|
use axum::Json;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use crate::mission_roster::{available_backends, Roster};
|
||||||
|
use crate::{ApiError, AppState, Authed};
|
||||||
|
|
||||||
|
/// The model that sizes a mission's team.
|
||||||
|
///
|
||||||
|
/// The same one the Master Planner uses. Sizing a team is the kind of judgement
|
||||||
|
/// the planner's own system prompt calls for — and it is a once-per-mission call,
|
||||||
|
/// so the cost argument that keeps missions on cheaper models does not apply.
|
||||||
|
const PLANNER_MODEL: &str = "claude-opus-4-8";
|
||||||
|
|
||||||
|
const ROSTER_SYSTEM: &str = "You size the team for ONE software mission that runs inside Firecracker \
|
||||||
|
microVMs. Each member you propose is a WHOLE VM — a boot, a repository injected as a tar, a full \
|
||||||
|
Claude Code session, and a collect — running one after another, each one receiving the working tree the \
|
||||||
|
previous member left behind. That is expensive and it is serial, so propose the FEWEST members that \
|
||||||
|
genuinely divide the work. One member is a perfectly good answer and is usually the right one for a \
|
||||||
|
small change; Anthropic measure multi-agent work at 3-10x the tokens with wall-clock often LONGER, and \
|
||||||
|
the benefit is thoroughness rather than speed.\n\n\
|
||||||
|
Members run SEQUENTIALLY and share the repository, so do NOT propose members that would edit the same \
|
||||||
|
file, and do NOT split one change into stages (plan → implement → test) — a handoff loses context at \
|
||||||
|
every step and one careful pass beats an assembly line. The shape that DOES earn its cost is an \
|
||||||
|
implementer followed by an independent verifier that only checks.\n\n\
|
||||||
|
Give each member a `backend` ONLY when running it on a different provider's image is the point — an \
|
||||||
|
independent verifier on another provider breaks the correlated failure where the model that wrote the \
|
||||||
|
code also grades it. Omit `backend` to inherit the mission's.\n\n\
|
||||||
|
ALWAYS respond with STRICT JSON ONLY, no prose and no markdown: \
|
||||||
|
{\"topology_kind\":\"pipeline\",\"members\":[{\"role\":\"...\",\"backend\":null|\"...\",\
|
||||||
|
\"rationale\":\"one line\"}]}";
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct ProposalResponse {
|
||||||
|
pub id: Uuid,
|
||||||
|
pub roster: Value,
|
||||||
|
pub author_model: String,
|
||||||
|
pub status: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `POST /api/missions/{id}/team-proposals` — ask the model for a roster.
|
||||||
|
pub async fn suggest(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path(id): Path<Uuid>,
|
||||||
|
) -> Result<Json<ProposalResponse>, ApiError> {
|
||||||
|
let ws = user.workspace_id;
|
||||||
|
let mission = cm_db::repo::missions::get(&state.pool, id, ws.as_uuid())
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
|
||||||
|
// The backends the FLEET can boot today, handed to the model as the menu.
|
||||||
|
// Without it the model invents plausible image names and the roster is
|
||||||
|
// refused after it was written, which reads as our bug rather than as a
|
||||||
|
// model guessing.
|
||||||
|
let available = available_backends(&state.pool, ws.as_uuid().to_owned())
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("mission {id}: could not read fleet backends: {e}");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let phases: Vec<(String, Option<String>)> = sqlx::query_as(
|
||||||
|
"SELECT kind, config->>'task' FROM mission_phases WHERE mission_id = $1 ORDER BY order_idx",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.fetch_all(&state.pool)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?;
|
||||||
|
|
||||||
|
let phase_text = phases
|
||||||
|
.iter()
|
||||||
|
.map(|(kind, task)| format!("- {kind}: {}", task.as_deref().unwrap_or("(no task text)")))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n");
|
||||||
|
let prompt = format!(
|
||||||
|
"MISSION: {}\n\nDESCRIPTION:\n{}\n\nPHASES:\n{}\n\nBACKENDS THIS FLEET CAN BOOT (use only \
|
||||||
|
these, or omit `backend`): {}\n\nPropose the roster now (JSON only).",
|
||||||
|
mission.title,
|
||||||
|
mission.description.as_deref().unwrap_or("(none)"),
|
||||||
|
if phase_text.is_empty() {
|
||||||
|
"(none declared)".to_string()
|
||||||
|
} else {
|
||||||
|
phase_text
|
||||||
|
},
|
||||||
|
if available.is_empty() {
|
||||||
|
"(none — omit backend on every member)".to_string()
|
||||||
|
} else {
|
||||||
|
available.join(", ")
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
// On the SUBSCRIPTION, like every mission VM — not the metered API key.
|
||||||
|
// `Runtime::complete` with a bare model name resolves to the default
|
||||||
|
// provider, which is the pay-as-you-go key; this planner died with
|
||||||
|
// "credit balance is too low" while missions on the same box ran fine.
|
||||||
|
// `author_model` is what ANSWERED, not what was asked for. When opus is
|
||||||
|
// capped the chain steps down to haiku and then to GLM, and a plan drafted
|
||||||
|
// by the third link but filed as an opus plan is a silent quality change.
|
||||||
|
let (raw, author_model) = crate::subscription::complete_with_fallback(
|
||||||
|
&state.runtime,
|
||||||
|
ROSTER_SYSTEM,
|
||||||
|
&prompt,
|
||||||
|
PLANNER_MODEL,
|
||||||
|
2000,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("mission {id}: roster proposal failed: {e}");
|
||||||
|
// A rate-limited subscription is a 503 the operator can act on, not
|
||||||
|
// a 500 that reads as "this server is broken".
|
||||||
|
crate::subscription::as_api_error(&e)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
// A model that answered with prose around its JSON has still answered; a
|
||||||
|
// model that answered with nothing usable has not, and that is a refusal
|
||||||
|
// rather than an empty roster.
|
||||||
|
let parsed: Value = crate::routes::claws::extract_json(&raw).ok_or_else(|| {
|
||||||
|
eprintln!("mission {id}: planner returned no JSON: {raw}");
|
||||||
|
ApiError::BadRequest
|
||||||
|
})?;
|
||||||
|
let roster: Roster = serde_json::from_value(parsed.clone()).map_err(|e| {
|
||||||
|
eprintln!("mission {id}: planner JSON is not a roster ({e}): {parsed}");
|
||||||
|
ApiError::BadRequest
|
||||||
|
})?;
|
||||||
|
// Validated BEFORE it is stored, so a stored proposal is always one that
|
||||||
|
// could be approved. Storing an invalid roster would mean the failure
|
||||||
|
// surfaces at approval time, pointing at the human rather than the model.
|
||||||
|
if let Err(why) = roster.validate(&available) {
|
||||||
|
eprintln!("mission {id}: planner proposed an unusable roster: {why}");
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
let pid = Uuid::now_v7();
|
||||||
|
let stored = serde_json::to_value(&roster).map_err(|_| ApiError::Internal)?;
|
||||||
|
cm_db::repo::mission_team_proposals::insert(
|
||||||
|
&state.pool,
|
||||||
|
pid,
|
||||||
|
id,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
&stored,
|
||||||
|
&author_model,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("mission {id}: could not store proposal: {e}");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
eprintln!(
|
||||||
|
"mission_roster: mission {id} — {} proposed {} member(s): {}",
|
||||||
|
author_model,
|
||||||
|
roster.members.len(),
|
||||||
|
roster
|
||||||
|
.members
|
||||||
|
.iter()
|
||||||
|
.map(|m| format!("{}{}", m.role, m.backend.as_deref().map(|b| format!("@{b}")).unwrap_or_default()))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ")
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(Json(ProposalResponse {
|
||||||
|
id: pid,
|
||||||
|
roster: stored,
|
||||||
|
author_model,
|
||||||
|
status: "proposed".into(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `GET /api/missions/{id}/team-proposals`
|
||||||
|
pub async fn list(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path(id): Path<Uuid>,
|
||||||
|
) -> Result<Json<Vec<cm_db::repo::mission_team_proposals::MissionTeamProposal>>, ApiError> {
|
||||||
|
let rows =
|
||||||
|
cm_db::repo::mission_team_proposals::list(&state.pool, id, user.workspace_id.as_uuid().to_owned())
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?;
|
||||||
|
Ok(Json(rows))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
pub struct DecideRequest {
|
||||||
|
/// `approved` or `rejected`.
|
||||||
|
pub status: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub note: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `POST /api/missions/{id}/team-proposals/{pid}/decide` — accept or refuse.
|
||||||
|
///
|
||||||
|
/// Approving writes `config.roster` on the mission and switches it to the
|
||||||
|
/// composed engine, because a roster is a graph of VMs and that is the engine
|
||||||
|
/// that runs one. Draft-only: re-shaping a mission that is already running would
|
||||||
|
/// change what its next phase does with no record of the swap on the phase that
|
||||||
|
/// already ran.
|
||||||
|
pub async fn decide(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path((id, pid)): Path<(Uuid, Uuid)>,
|
||||||
|
Json(body): Json<DecideRequest>,
|
||||||
|
) -> Result<Json<Value>, ApiError> {
|
||||||
|
let ws = user.workspace_id;
|
||||||
|
let proposal = cm_db::repo::mission_team_proposals::get(&state.pool, pid, ws.as_uuid().to_owned())
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
if proposal.mission_id != id {
|
||||||
|
return Err(ApiError::NotFound);
|
||||||
|
}
|
||||||
|
|
||||||
|
if body.status == "rejected" {
|
||||||
|
let decided = cm_db::repo::mission_team_proposals::decide(
|
||||||
|
&state.pool,
|
||||||
|
pid,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
"rejected",
|
||||||
|
body.note.as_deref(),
|
||||||
|
Some(user.user_id.as_uuid().to_owned()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?;
|
||||||
|
return Ok(Json(json!({ "status": "rejected", "decided": decided })));
|
||||||
|
}
|
||||||
|
if body.status != "approved" {
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mission = cm_db::repo::missions::get(&state.pool, id, ws.as_uuid())
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
if mission.status != "draft" {
|
||||||
|
eprintln!("mission {id}: roster approval refused — mission is {}", mission.status);
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
let roster: Roster = serde_json::from_value(proposal.roster.clone()).map_err(|e| {
|
||||||
|
eprintln!("mission {id}: stored proposal {pid} is not a roster ({e})");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
// Re-validated at approval, against the fleet as it is NOW. A node can go
|
||||||
|
// offline between proposing and approving, and the cheapest place to find
|
||||||
|
// that out is still here rather than at VM boot.
|
||||||
|
let available = available_backends(&state.pool, ws.as_uuid().to_owned())
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?;
|
||||||
|
if let Err(why) = roster.validate(&available) {
|
||||||
|
eprintln!("mission {id}: roster {pid} is no longer applicable: {why}");
|
||||||
|
let _ = cm_db::repo::mission_team_proposals::decide(
|
||||||
|
&state.pool,
|
||||||
|
pid,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
"rejected",
|
||||||
|
Some(&why.to_string()),
|
||||||
|
Some(user.user_id.as_uuid().to_owned()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
let graph = roster.graph().map_err(|e| {
|
||||||
|
eprintln!("mission {id}: approved roster does not build a graph: {e}");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
// Claiming the proposal and writing the mission are ONE transaction. Doing
|
||||||
|
// them as two statements left the first real approval in production marked
|
||||||
|
// `approved` with nothing written to the mission — and the partial unique
|
||||||
|
// index then makes that permanent, since no other proposal for that mission
|
||||||
|
// can ever be approved.
|
||||||
|
let claimed = cm_db::repo::mission_team_proposals::approve_and_apply(
|
||||||
|
&state.pool,
|
||||||
|
pid,
|
||||||
|
id,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
&graph,
|
||||||
|
body.note.as_deref(),
|
||||||
|
Some(user.user_id.as_uuid().to_owned()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("mission {id}: could not apply roster {pid}: {e}");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
if !claimed {
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
eprintln!(
|
||||||
|
"mission_roster: mission {id} now runs a {}-node composed graph from proposal {pid}",
|
||||||
|
roster.members.len()
|
||||||
|
);
|
||||||
|
Ok(Json(json!({
|
||||||
|
"status": "approved",
|
||||||
|
"team_engine": "composed",
|
||||||
|
"nodes": roster.members.len(),
|
||||||
|
"graph": graph,
|
||||||
|
})))
|
||||||
|
}
|
||||||
@@ -38,6 +38,16 @@ pub struct CreateMissionRequest {
|
|||||||
/// Defaults to "zeroclaw". "local_herdr" requires target_node_id.
|
/// Defaults to "zeroclaw". "local_herdr" requires target_node_id.
|
||||||
pub runtime_kind: Option<String>,
|
pub runtime_kind: Option<String>,
|
||||||
pub target_node_id: Option<Uuid>,
|
pub target_node_id: Option<Uuid>,
|
||||||
|
/// Which per-CLI rootfs a `microvm` mission boots (`missions.backend`), e.g.
|
||||||
|
/// "claude". NULL boots the node's default image.
|
||||||
|
pub backend: Option<String>,
|
||||||
|
/// Model that independently validates this mission's phase verdicts, e.g.
|
||||||
|
/// `glm:glm-4.7`. Omit to use the deployment default; send `""` to opt out of
|
||||||
|
/// independent validation and judge with the house model.
|
||||||
|
pub validator_model: Option<String>,
|
||||||
|
/// Team engine: `"claude_code"` asks the mission's agent to form a team.
|
||||||
|
/// Omit for solo, which is the default and much cheaper.
|
||||||
|
pub team_engine: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_schedule() -> Value {
|
fn default_schedule() -> Value {
|
||||||
@@ -260,6 +270,12 @@ pub async fn create(
|
|||||||
return Err(ApiError::BadRequest);
|
return Err(ApiError::BadRequest);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// microvm needs no target here: placement resolves a KVM-capable node at
|
||||||
|
// launch and fails the launch when there is none, so an explicit target is
|
||||||
|
// a request rather than a requirement. Rejecting the value outright — as
|
||||||
|
// this did until B4.5 — made `runtime_kind='microvm'` unreachable through
|
||||||
|
// the only interface that creates missions.
|
||||||
|
"microvm" => {}
|
||||||
_ => return Err(ApiError::BadRequest),
|
_ => return Err(ApiError::BadRequest),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -275,6 +291,9 @@ pub async fn create(
|
|||||||
config: body.config,
|
config: body.config,
|
||||||
runtime_kind: Some(runtime_kind),
|
runtime_kind: Some(runtime_kind),
|
||||||
target_node_id: body.target_node_id,
|
target_node_id: body.target_node_id,
|
||||||
|
backend: body.backend.as_deref(),
|
||||||
|
validator_model: body.validator_model.as_deref(),
|
||||||
|
team_engine: body.team_engine.as_deref(),
|
||||||
phases: phases_for_create(
|
phases: phases_for_create(
|
||||||
crate::workflow_registry::get(body.template_kind.trim()),
|
crate::workflow_registry::get(body.template_kind.trim()),
|
||||||
body.phases,
|
body.phases,
|
||||||
@@ -308,6 +327,301 @@ pub async fn get(
|
|||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// GET /api/missions/{id}/artifacts/{artifact_id}/content — the artifact's text.
|
||||||
|
///
|
||||||
|
/// The frontend had no way to READ an artifact: it listed paths and offered a
|
||||||
|
/// PDF preview, and the PDF never rendered. Markdown is the deliverable now, so
|
||||||
|
/// something has to serve it.
|
||||||
|
///
|
||||||
|
/// Two containment rules, both enforced rather than assumed:
|
||||||
|
///
|
||||||
|
/// - the artifact row must belong to a mission in the caller's workspace, so
|
||||||
|
/// an artifact id from another tenant is a 404, not a file read;
|
||||||
|
/// - the resolved path must stay inside `<missions_root>/_outputs`. Artifact
|
||||||
|
/// paths are written by this server, but a stored `../../etc/passwd` would
|
||||||
|
/// otherwise be read and returned. Canonicalise, then check the prefix —
|
||||||
|
/// checking the string before resolving `..` is the classic hole.
|
||||||
|
///
|
||||||
|
/// Text only, and capped: these are markdown documents, and streaming an
|
||||||
|
/// arbitrary captured file into a JSON body is not what this is for.
|
||||||
|
/// Turn a stored artifact path into an absolute one, refusing anything outside
|
||||||
|
/// `_outputs`.
|
||||||
|
///
|
||||||
|
/// Shared by the read and download routes deliberately: two copies of a
|
||||||
|
/// containment check is two chances for one of them to be the lenient one, and
|
||||||
|
/// the lenient one is a path-traversal read of the gateway's filesystem.
|
||||||
|
fn resolve_artifact_path(stored: &str) -> Result<std::path::PathBuf, ApiError> {
|
||||||
|
let root = crate::mission_outputs::outputs_root_dir();
|
||||||
|
let abs = crate::mission_outputs::missions_root_dir().join(stored);
|
||||||
|
// `canonicalize` on BOTH sides, so a symlink out of the tree resolves to
|
||||||
|
// its target before the comparison rather than after.
|
||||||
|
let resolved = std::fs::canonicalize(&abs).map_err(|_| ApiError::NotFound)?;
|
||||||
|
let root = std::fs::canonicalize(&root).map_err(|_| ApiError::NotFound)?;
|
||||||
|
if !resolved.starts_with(&root) {
|
||||||
|
eprintln!(
|
||||||
|
"missions: refused artifact {} — outside {}",
|
||||||
|
resolved.display(),
|
||||||
|
root.display()
|
||||||
|
);
|
||||||
|
return Err(ApiError::NotFound);
|
||||||
|
}
|
||||||
|
Ok(resolved)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `GET /api/missions/{id}/artifacts/{artifact_id}/download` — the file itself.
|
||||||
|
///
|
||||||
|
/// Separate from `artifact_content` because that route cannot serve the two
|
||||||
|
/// cases a download exists for: it caps at 2 MiB and reads as UTF-8, so a large
|
||||||
|
/// or binary artifact is unreachable by any means today. This one streams the
|
||||||
|
/// bytes with a filename attached and no ceiling.
|
||||||
|
pub async fn artifact_download(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path((id, artifact_id)): Path<(Uuid, Uuid)>,
|
||||||
|
) -> Result<axum::response::Response, ApiError> {
|
||||||
|
use axum::response::IntoResponse;
|
||||||
|
|
||||||
|
cm_db::repo::missions::get(&state.pool, id, user.workspace_id.as_uuid())
|
||||||
|
.await?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
let artifacts = cm_db::repo::missions::artifacts_for(&state.pool, id).await?;
|
||||||
|
let artifact = artifacts
|
||||||
|
.into_iter()
|
||||||
|
.find(|a| a.id == artifact_id)
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
let resolved = resolve_artifact_path(&artifact.path)?;
|
||||||
|
|
||||||
|
let bytes = tokio::fs::read(&resolved)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::NotFound)?;
|
||||||
|
|
||||||
|
// The basename, never the stored path: `_outputs/<mission>/<phase>/repo/x.md`
|
||||||
|
// as a filename would arrive as a browser-mangled string, and the path is
|
||||||
|
// internal layout the user has no reason to see.
|
||||||
|
let name = resolved
|
||||||
|
.file_name()
|
||||||
|
.and_then(|n| n.to_str())
|
||||||
|
.filter(|n| !n.is_empty())
|
||||||
|
.unwrap_or("artifact");
|
||||||
|
// Quoted and stripped of quotes/newlines: a filename is attacker-influenced
|
||||||
|
// input (an agent chose it) and this header is parsed by every browser.
|
||||||
|
let safe: String = name
|
||||||
|
.chars()
|
||||||
|
.filter(|c| *c != '"' && *c != '\\' && !c.is_control())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Ok((
|
||||||
|
[
|
||||||
|
(
|
||||||
|
axum::http::header::CONTENT_TYPE,
|
||||||
|
artifact.mime.unwrap_or_else(|| "application/octet-stream".into()),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
axum::http::header::CONTENT_DISPOSITION,
|
||||||
|
format!("attachment; filename=\"{safe}\""),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
bytes,
|
||||||
|
)
|
||||||
|
.into_response())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn artifact_content(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path((id, artifact_id)): Path<(Uuid, Uuid)>,
|
||||||
|
) -> Result<Json<serde_json::Value>, ApiError> {
|
||||||
|
/// Beyond this, a document is not something a reader wants inline.
|
||||||
|
const MAX_BYTES: u64 = 2 * 1024 * 1024;
|
||||||
|
|
||||||
|
// Scoped to the caller's workspace by loading the mission first.
|
||||||
|
cm_db::repo::missions::get(&state.pool, id, user.workspace_id.as_uuid())
|
||||||
|
.await?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
|
||||||
|
let artifacts = cm_db::repo::missions::artifacts_for(&state.pool, id).await?;
|
||||||
|
let artifact = artifacts
|
||||||
|
.into_iter()
|
||||||
|
.find(|a| a.id == artifact_id)
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
|
||||||
|
let resolved = resolve_artifact_path(&artifact.path)?;
|
||||||
|
|
||||||
|
let meta = std::fs::metadata(&resolved).map_err(|_| ApiError::NotFound)?;
|
||||||
|
if meta.len() > MAX_BYTES {
|
||||||
|
return Ok(Json(serde_json::json!({
|
||||||
|
"path": artifact.path,
|
||||||
|
"mime": artifact.mime,
|
||||||
|
"truncated": true,
|
||||||
|
"content": "",
|
||||||
|
"bytes": meta.len(),
|
||||||
|
})));
|
||||||
|
}
|
||||||
|
let content = std::fs::read_to_string(&resolved).map_err(|_| ApiError::NotFound)?;
|
||||||
|
Ok(Json(serde_json::json!({
|
||||||
|
"path": artifact.path,
|
||||||
|
"mime": artifact.mime,
|
||||||
|
"title": artifact.title,
|
||||||
|
"truncated": false,
|
||||||
|
"content": content,
|
||||||
|
"bytes": meta.len(),
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// POST /api/missions/{id}/merge — merge this mission's branch into the base.
|
||||||
|
///
|
||||||
|
/// The operator's button. `MergePolicy::Never` — the default for anything that
|
||||||
|
/// touches code — means "do not merge on your own", deferring to a human; this
|
||||||
|
/// endpoint is that human saying yes. So the additive-only test does not apply
|
||||||
|
/// here, and deliberately so.
|
||||||
|
///
|
||||||
|
/// It works in a FRESH CLONE under `_merge/<mission>`, never the mission
|
||||||
|
/// checkout: that directory is reaped on a timer after a mission ends, so a
|
||||||
|
/// merge that used it would succeed right after a run and fail inexplicably an
|
||||||
|
/// hour later. The clone is made by the server process, so nothing here runs as
|
||||||
|
/// root and the ordinary cleanup works — unlike the copies in `root_copy`.
|
||||||
|
pub async fn merge_branch(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path(id): Path<Uuid>,
|
||||||
|
) -> Result<Json<serde_json::Value>, ApiError> {
|
||||||
|
let mission = cm_db::repo::missions::get(&state.pool, id, user.workspace_id.as_uuid())
|
||||||
|
.await?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
let repo_id = mission.repo_id.ok_or(ApiError::BadRequest)?;
|
||||||
|
let repo = cm_db::repo::repos::get(&state.pool, repo_id, user.workspace_id)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::NotFound)?;
|
||||||
|
let clone_url = repo.clone_url.as_deref().ok_or(ApiError::BadRequest)?;
|
||||||
|
let base = repo.default_branch.as_deref().unwrap_or("main");
|
||||||
|
|
||||||
|
// The branch is whatever delivery actually pushed — read from the artifact
|
||||||
|
// it recorded, not reconstructed from the mission id. A phase that never
|
||||||
|
// pushed has no branch, and that must be a refusal rather than a guess.
|
||||||
|
let artifacts = cm_db::repo::missions::artifacts_for(&state.pool, id).await?;
|
||||||
|
let delivered = artifacts.iter().rev().find_map(|a| {
|
||||||
|
let m = a.metadata.as_object()?;
|
||||||
|
let branch = m.get("branch")?.as_str()?.to_string();
|
||||||
|
(m.get("pushed").and_then(|v| v.as_bool()) == Some(true)).then_some(branch)
|
||||||
|
});
|
||||||
|
let Some(branch) = delivered else {
|
||||||
|
return Ok(Json(serde_json::json!({
|
||||||
|
"merged": false,
|
||||||
|
"reason": "this mission has no pushed branch to merge",
|
||||||
|
})));
|
||||||
|
};
|
||||||
|
|
||||||
|
let auth = crate::mission_workspace::with_ambient_auth(clone_url);
|
||||||
|
let workdir = crate::mission_workspace::missions_root()
|
||||||
|
.join("_merge")
|
||||||
|
.join(id.to_string());
|
||||||
|
let _ = tokio::fs::remove_dir_all(&workdir).await;
|
||||||
|
if let Some(parent) = workdir.parent() {
|
||||||
|
let _ = tokio::fs::create_dir_all(parent).await;
|
||||||
|
}
|
||||||
|
let clone = tokio::process::Command::new("git")
|
||||||
|
.args(["clone", "--quiet", &auth.url])
|
||||||
|
.arg(&workdir)
|
||||||
|
.env("GIT_TERMINAL_PROMPT", "0")
|
||||||
|
.output()
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?;
|
||||||
|
if !clone.status.success() {
|
||||||
|
eprintln!(
|
||||||
|
"missions::merge_branch: clone for {id} failed: {}",
|
||||||
|
String::from_utf8_lossy(&clone.stderr)
|
||||||
|
.chars()
|
||||||
|
.take(300)
|
||||||
|
.collect::<String>()
|
||||||
|
);
|
||||||
|
return Ok(Json(serde_json::json!({
|
||||||
|
"merged": false,
|
||||||
|
"reason": "could not clone the repository to merge",
|
||||||
|
})));
|
||||||
|
}
|
||||||
|
|
||||||
|
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
||||||
|
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
||||||
|
let outcome = async {
|
||||||
|
let merged =
|
||||||
|
crate::auto_merge::merge_on_operator_approval(&workdir, &auth.url, &branch, base)
|
||||||
|
.await?;
|
||||||
|
if !merged.merged {
|
||||||
|
return Ok(merged);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the project's own tests against the MERGED tree, before it is
|
||||||
|
// published. Verifying first rather than reverting after is the
|
||||||
|
// difference between "main was never broken" and "main was broken until
|
||||||
|
// someone noticed".
|
||||||
|
//
|
||||||
|
// The merge is already committed locally at this point; refusing here
|
||||||
|
// simply never pushes it, and the branch is still there to retry.
|
||||||
|
match crate::mission_delivery::verify_tests(&workdir, &container).await {
|
||||||
|
crate::mission_delivery::TestOutcome::Passed => {}
|
||||||
|
crate::mission_delivery::TestOutcome::NoSuite => {
|
||||||
|
eprintln!(
|
||||||
|
"missions::merge_branch: {branch} has no discoverable test suite — publishing unverified"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
crate::mission_delivery::TestOutcome::Failed(code) => {
|
||||||
|
return Ok(crate::auto_merge::MergeOutcome {
|
||||||
|
merged: false,
|
||||||
|
reason: format!(
|
||||||
|
"the merged tree FAILS the project's tests (exit {code}) — not published. The branch is unchanged; fix it and merge again."
|
||||||
|
),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// Fail closed. A suite that could not run has not passed, and
|
||||||
|
// publishing on "we could not check" is how a green main stops
|
||||||
|
// meaning anything.
|
||||||
|
crate::mission_delivery::TestOutcome::CouldNotRun(why) => {
|
||||||
|
return Ok(crate::auto_merge::MergeOutcome {
|
||||||
|
merged: false,
|
||||||
|
reason: format!("could not run the tests on the merged tree ({why}) — not published"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
crate::auto_merge::push_merged(&workdir, &auth.url, base).await?;
|
||||||
|
Ok::<_, String>(crate::auto_merge::MergeOutcome {
|
||||||
|
merged: true,
|
||||||
|
reason: format!("tests pass on the merged tree; published to {base}"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
|
||||||
|
// Purge through the container: `verify_tests` runs `cargo test` as ROOT, so
|
||||||
|
// the workdir now holds a root-owned `target/` the server (uid 65532) cannot
|
||||||
|
// delete. Same defect as the bench and judge copies.
|
||||||
|
crate::root_copy::purge(&container, &workdir).await;
|
||||||
|
let _ = tokio::fs::remove_dir_all(&workdir).await;
|
||||||
|
|
||||||
|
match outcome {
|
||||||
|
Ok(o) => {
|
||||||
|
eprintln!(
|
||||||
|
"missions::merge_branch: mission {id} branch {branch} -> {base}: {}",
|
||||||
|
o.reason
|
||||||
|
);
|
||||||
|
Ok(Json(serde_json::json!({
|
||||||
|
"merged": o.merged,
|
||||||
|
"reason": o.reason,
|
||||||
|
"branch": branch,
|
||||||
|
"base": base,
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("missions::merge_branch: mission {id} failed: {e}");
|
||||||
|
Ok(Json(serde_json::json!({
|
||||||
|
"merged": false,
|
||||||
|
"reason": format!("merge failed: {e}"),
|
||||||
|
"branch": branch,
|
||||||
|
"base": base,
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// POST /api/missions/{id}/benchmark — run the benchmark harness
|
/// POST /api/missions/{id}/benchmark — run the benchmark harness
|
||||||
/// against a phase. Slot='baseline' records iteration 0's
|
/// against a phase. Slot='baseline' records iteration 0's
|
||||||
/// before_metrics; slot='after' with iteration=N records the
|
/// before_metrics; slot='after' with iteration=N records the
|
||||||
@@ -367,6 +681,61 @@ pub struct RefineResponse {
|
|||||||
pub refined: String,
|
pub refined: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
pub struct RefineDraftRequest {
|
||||||
|
#[serde(default)]
|
||||||
|
pub title: String,
|
||||||
|
pub description: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub template_kind: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `POST /api/missions/refine-draft` — polish a description with no mission
|
||||||
|
/// behind it yet.
|
||||||
|
///
|
||||||
|
/// The wizard's polish button fires while the user is still typing, before
|
||||||
|
/// anything is created. `refine` deliberately requires a saved draft so its
|
||||||
|
/// Accept can write back; this one has nothing to write back to and returns the
|
||||||
|
/// text for the caller to put in the box.
|
||||||
|
///
|
||||||
|
/// The phase list comes from the workflow recipe rather than the caller, for
|
||||||
|
/// the same reason `phases_for_create` prefers it: the recipe is the
|
||||||
|
/// authoritative composition, and a client that guessed would have the model
|
||||||
|
/// write acceptance criteria for phases the mission will not run.
|
||||||
|
pub async fn refine_draft(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(_user): Authed,
|
||||||
|
Json(req): Json<RefineDraftRequest>,
|
||||||
|
) -> Result<Json<RefineResponse>, ApiError> {
|
||||||
|
let phase_kinds: Vec<String> = req
|
||||||
|
.template_kind
|
||||||
|
.as_deref()
|
||||||
|
.and_then(crate::workflow_registry::get)
|
||||||
|
.map(|r| r.phases.iter().map(|p| p.kind.clone()).collect())
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let result = crate::mission_refiner::refine_draft(
|
||||||
|
&state.runtime,
|
||||||
|
req.title.trim(),
|
||||||
|
req.template_kind.as_deref().unwrap_or("custom"),
|
||||||
|
&phase_kinds,
|
||||||
|
&req.description,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("refine-draft failed: {e}");
|
||||||
|
if e.contains("empty") {
|
||||||
|
ApiError::BadRequest
|
||||||
|
} else {
|
||||||
|
crate::subscription::as_api_error(&e)
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
Ok(Json(RefineResponse {
|
||||||
|
original: result.original,
|
||||||
|
refined: result.refined,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
/// POST /api/missions/{id}/refine — generate a coherent, sectioned
|
/// POST /api/missions/{id}/refine — generate a coherent, sectioned
|
||||||
/// Markdown rewrite of the current description WITHOUT persisting.
|
/// Markdown rewrite of the current description WITHOUT persisting.
|
||||||
/// Frontend renders a before/after diff; user hits Accept (PATCH
|
/// Frontend renders a before/after diff; user hits Accept (PATCH
|
||||||
@@ -376,7 +745,7 @@ pub async fn refine(
|
|||||||
Authed(user): Authed,
|
Authed(user): Authed,
|
||||||
Path(id): Path<Uuid>,
|
Path(id): Path<Uuid>,
|
||||||
) -> Result<Json<RefineResponse>, ApiError> {
|
) -> Result<Json<RefineResponse>, ApiError> {
|
||||||
let result = crate::mission_refiner::refine(&state.pool, user.workspace_id, id)
|
let result = crate::mission_refiner::refine(&state.pool, &state.runtime, user.workspace_id, id)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
eprintln!("mission {id}: refine failed: {e}");
|
eprintln!("mission {id}: refine failed: {e}");
|
||||||
@@ -530,6 +899,31 @@ async fn reap_mission_resources(state: &AppState, mission_id: Uuid) {
|
|||||||
// drift back into skipping the container teardown.
|
// drift back into skipping the container teardown.
|
||||||
let provisioner = crate::runtime_provision::RuntimeProvisioner::from_env();
|
let provisioner = crate::runtime_provision::RuntimeProvisioner::from_env();
|
||||||
for cid in &claw_ids {
|
for cid in &claw_ids {
|
||||||
|
// Only claws this mission is the LAST holder of.
|
||||||
|
//
|
||||||
|
// Claws are reused across missions now (see
|
||||||
|
// `agent_template_link::reusable_claw`), so a mission's team can contain
|
||||||
|
// staff that other missions still employ. Purging those would delete a
|
||||||
|
// user's workforce as a side effect of tidying up one mission — and it
|
||||||
|
// would look like the roster quietly shrinking, not like an error.
|
||||||
|
let shared: i64 = sqlx::query_scalar(
|
||||||
|
"SELECT count(*)
|
||||||
|
FROM team_members tm
|
||||||
|
JOIN mission_teams mt ON mt.team_id = tm.team_id
|
||||||
|
WHERE tm.claw_id = $1 AND mt.mission_id <> $2",
|
||||||
|
)
|
||||||
|
.bind(cid)
|
||||||
|
.bind(mission_id)
|
||||||
|
.fetch_one(&state.pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or(0);
|
||||||
|
if shared > 0 {
|
||||||
|
eprintln!(
|
||||||
|
"missions::delete: keeping claw {cid} — {shared} other mission(s) still employ it"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
let report = crate::routes::claws::purge_agent(
|
let report = crate::routes::claws::purge_agent(
|
||||||
&state.pool,
|
&state.pool,
|
||||||
&state.runtime,
|
&state.runtime,
|
||||||
@@ -682,9 +1076,16 @@ pub async fn retry_phase(
|
|||||||
let mission = cm_db::repo::missions::get(&state.pool, id, user.workspace_id.as_uuid())
|
let mission = cm_db::repo::missions::get(&state.pool, id, user.workspace_id.as_uuid())
|
||||||
.await?
|
.await?
|
||||||
.ok_or(ApiError::NotFound)?;
|
.ok_or(ApiError::NotFound)?;
|
||||||
if mission.status != "running" {
|
// `failed` is retryable, and has to be: a failed phase now closes its
|
||||||
|
// mission (its later phases are marked unreachable so the mission can
|
||||||
|
// finish at all), so refusing anything but `running` would mean the one
|
||||||
|
// outcome you would actually want to retry is the one you cannot.
|
||||||
|
// `completed` and `cancelled` stay refused — reopening those is a different
|
||||||
|
// decision than re-running a phase that failed.
|
||||||
|
if mission.status != "running" && mission.status != "failed" {
|
||||||
return Err(ApiError::BadRequest);
|
return Err(ApiError::BadRequest);
|
||||||
}
|
}
|
||||||
|
let mut tx = state.pool.begin().await?;
|
||||||
let r = sqlx::query(
|
let r = sqlx::query(
|
||||||
"UPDATE mission_phases
|
"UPDATE mission_phases
|
||||||
SET status = 'pending', started_at = NULL, completed_at = NULL
|
SET status = 'pending', started_at = NULL, completed_at = NULL
|
||||||
@@ -693,12 +1094,41 @@ pub async fn retry_phase(
|
|||||||
)
|
)
|
||||||
.bind(phase_id)
|
.bind(phase_id)
|
||||||
.bind(id)
|
.bind(id)
|
||||||
.execute(&state.pool)
|
.execute(&mut *tx)
|
||||||
.await?;
|
.await?;
|
||||||
if r.rows_affected() == 0 {
|
if r.rows_affected() == 0 {
|
||||||
|
tx.rollback().await?;
|
||||||
return Err(ApiError::NotFound);
|
return Err(ApiError::NotFound);
|
||||||
}
|
}
|
||||||
Ok(Json(serde_json::json!({ "reset": true })))
|
// Reopen the phases this one's failure had made unreachable. Without this a
|
||||||
|
// retry runs the failed phase and then stops, because everything after it
|
||||||
|
// is terminal-by-skip — the mission would close again the moment this phase
|
||||||
|
// finished, having done only part of the work.
|
||||||
|
let reopened = sqlx::query(
|
||||||
|
"UPDATE mission_phases mp
|
||||||
|
SET status = 'pending', started_at = NULL, completed_at = NULL
|
||||||
|
WHERE mp.mission_id = $1
|
||||||
|
AND mp.status = 'skipped'
|
||||||
|
AND mp.order_idx > (SELECT order_idx FROM mission_phases WHERE id = $2)",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(phase_id)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
// And put the mission back to running, or nothing sweeps the phase: every
|
||||||
|
// launcher and closer keys off `missions.status = 'running'`.
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE missions SET status = 'running', completed_at = NULL, updated_at = now()
|
||||||
|
WHERE id = $1 AND status = 'failed'",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?;
|
||||||
|
tx.commit().await?;
|
||||||
|
Ok(Json(
|
||||||
|
serde_json::json!({ "reset": true, "reopened_phases": reopened }),
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// GET /api/missions/{id}/phases/{phase_id}/summary — the completion
|
/// GET /api/missions/{id}/phases/{phase_id}/summary — the completion
|
||||||
@@ -844,7 +1274,16 @@ pub async fn set_status(
|
|||||||
.any(|v| v.as_array().map(|a| !a.is_empty()).unwrap_or(false))
|
.any(|v| v.as_array().map(|a| !a.is_empty()).unwrap_or(false))
|
||||||
})
|
})
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
if prior.team_id.is_none() && prior.team_template_id.is_none() && !has_phase_teams {
|
// A microVM mission materialises no team — `microvm_executor` runs the
|
||||||
|
// agent CLI directly in the VM — so requiring one would reject the launch
|
||||||
|
// of a perfectly well-formed mission, and satisfying it would provision
|
||||||
|
// claws that never run.
|
||||||
|
let needs_team = prior.runtime_kind != "microvm";
|
||||||
|
if needs_team
|
||||||
|
&& prior.team_id.is_none()
|
||||||
|
&& prior.team_template_id.is_none()
|
||||||
|
&& !has_phase_teams
|
||||||
|
{
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"mission {id}: launch rejected — no team_id, no team_template_id, no config.phase_teams"
|
"mission {id}: launch rejected — no team_id, no team_template_id, no config.phase_teams"
|
||||||
);
|
);
|
||||||
@@ -1243,3 +1682,190 @@ mod tests {
|
|||||||
assert!(outputs_of(Some(&serde_json::json!({}))).is_empty());
|
assert!(outputs_of(Some(&serde_json::json!({}))).is_empty());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// GET /api/workforce — the roster grouped by the mission each claw works on.
|
||||||
|
///
|
||||||
|
/// The sidebar used to flatten `orgs → companies → teams → agents`, which
|
||||||
|
/// rendered a claw once per TEAM it belongs to. Since claws are reused across
|
||||||
|
/// missions, a crew of five that had run five missions appeared as twenty-five
|
||||||
|
/// rows of the same five people — the roster looked like it was multiplying.
|
||||||
|
///
|
||||||
|
/// Grouping by mission makes that repetition mean something: the same person
|
||||||
|
/// legitimately appears under each mission they staffed. `agents` is deduped
|
||||||
|
/// per mission, and claws belonging to no mission come back under `unassigned`
|
||||||
|
/// so a hand-created claw cannot fall out of the UI entirely.
|
||||||
|
pub async fn workforce(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
) -> Result<Json<Value>, ApiError> {
|
||||||
|
use sqlx::Row;
|
||||||
|
let ws = user.workspace_id.as_uuid();
|
||||||
|
|
||||||
|
// One query, not one-per-mission: the sidebar renders on every navigation.
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT m.id::text AS mission_id,
|
||||||
|
m.title AS mission_title,
|
||||||
|
m.status AS mission_status,
|
||||||
|
m.template_kind AS template_kind,
|
||||||
|
m.created_at AS created_at,
|
||||||
|
a.id::text AS agent_id,
|
||||||
|
a.name AS agent_name,
|
||||||
|
a.job_title AS job_title,
|
||||||
|
a.accent AS accent,
|
||||||
|
a.status AS agent_status,
|
||||||
|
tm.role AS role_slot
|
||||||
|
FROM missions m
|
||||||
|
JOIN mission_teams mt ON mt.mission_id = m.id
|
||||||
|
JOIN team_members tm ON tm.team_id = mt.team_id
|
||||||
|
JOIN agents a ON a.id = tm.claw_id
|
||||||
|
WHERE m.workspace_id = $1
|
||||||
|
AND a.deleted_at IS NULL
|
||||||
|
ORDER BY m.created_at DESC, tm.role ASC",
|
||||||
|
)
|
||||||
|
.bind(ws)
|
||||||
|
.fetch_all(&state.pool)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let mut missions: Vec<Value> = Vec::new();
|
||||||
|
let mut seen_mission: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
|
||||||
|
for r in rows {
|
||||||
|
let mid: String = r.get("mission_id");
|
||||||
|
let idx = match seen_mission.get(&mid) {
|
||||||
|
Some(i) => *i,
|
||||||
|
None => {
|
||||||
|
missions.push(serde_json::json!({
|
||||||
|
"mission_id": mid,
|
||||||
|
"title": r.get::<String, _>("mission_title"),
|
||||||
|
"status": r.get::<String, _>("mission_status"),
|
||||||
|
// Drives the World's palette: what the mission is FOR
|
||||||
|
// should be visible before any label is read.
|
||||||
|
"templateKind": r.get::<String, _>("template_kind"),
|
||||||
|
"agents": Vec::<Value>::new(),
|
||||||
|
}));
|
||||||
|
seen_mission.insert(r.get::<String, _>("mission_id"), missions.len() - 1);
|
||||||
|
missions.len() - 1
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let agent = serde_json::json!({
|
||||||
|
"id": r.get::<String, _>("agent_id"),
|
||||||
|
"name": r.get::<String, _>("agent_name"),
|
||||||
|
"job_title": r.get::<String, _>("job_title"),
|
||||||
|
"role_slot": r.get::<String, _>("role_slot"),
|
||||||
|
"accent": r.get::<String, _>("accent"),
|
||||||
|
"status": r.get::<String, _>("agent_status"),
|
||||||
|
});
|
||||||
|
// A claw bound to two NODES of the same mission is still one colleague.
|
||||||
|
let list = missions[idx]["agents"].as_array_mut().expect("agents array");
|
||||||
|
let id = agent["id"].clone();
|
||||||
|
if !list.iter().any(|a| a["id"] == id) {
|
||||||
|
list.push(agent);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Claws on no mission at all — hand-created, or whose missions were
|
||||||
|
// deleted. Without this they would simply vanish from the sidebar.
|
||||||
|
let loose = sqlx::query(
|
||||||
|
"SELECT a.id::text AS agent_id, a.name, a.job_title, a.accent, a.status
|
||||||
|
FROM agents a
|
||||||
|
WHERE a.workspace_id = $1
|
||||||
|
AND a.deleted_at IS NULL
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM team_members tm
|
||||||
|
JOIN mission_teams mt ON mt.team_id = tm.team_id
|
||||||
|
JOIN missions m ON m.id = mt.mission_id
|
||||||
|
WHERE tm.claw_id = a.id AND m.workspace_id = $1)
|
||||||
|
ORDER BY a.name ASC",
|
||||||
|
)
|
||||||
|
.bind(ws)
|
||||||
|
.fetch_all(&state.pool)
|
||||||
|
.await?;
|
||||||
|
let unassigned: Vec<Value> = loose
|
||||||
|
.into_iter()
|
||||||
|
.map(|r| {
|
||||||
|
serde_json::json!({
|
||||||
|
"id": r.get::<String, _>("agent_id"),
|
||||||
|
"name": r.get::<String, _>("name"),
|
||||||
|
"job_title": r.get::<String, _>("job_title"),
|
||||||
|
"role_slot": Value::Null,
|
||||||
|
"accent": r.get::<String, _>("accent"),
|
||||||
|
"status": r.get::<String, _>("status"),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Ok(Json(serde_json::json!({
|
||||||
|
"missions": missions,
|
||||||
|
"unassigned": unassigned,
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod reap_tests {
|
||||||
|
/// A mission's teardown must ask whether anyone else still employs a claw.
|
||||||
|
///
|
||||||
|
/// Claws are reused across missions now, so a mission's team can contain
|
||||||
|
/// staff other missions still hold. The old code purged every claw in the
|
||||||
|
/// team unconditionally, which under reuse deletes a user's workforce as a
|
||||||
|
/// side effect of tidying one mission — and it presents as the roster
|
||||||
|
/// quietly shrinking rather than as an error.
|
||||||
|
#[test]
|
||||||
|
fn mission_teardown_checks_for_other_employers_before_purging() {
|
||||||
|
let src = include_str!("missions.rs");
|
||||||
|
let reaper = src
|
||||||
|
.split("async fn reap_mission_resources")
|
||||||
|
.nth(1)
|
||||||
|
.expect("the reaper exists");
|
||||||
|
// Scoped to the reaper, so the check cannot be satisfied by some other
|
||||||
|
// function elsewhere in the file that happens to mention mission_teams.
|
||||||
|
assert!(
|
||||||
|
reaper.contains("mt.mission_id <> $2"),
|
||||||
|
"the purge must exclude claws held by another mission"
|
||||||
|
);
|
||||||
|
let purge_at = reaper.find("purge_agent").expect("it still purges");
|
||||||
|
let guard_at = reaper.find("mt.mission_id <> $2").expect("guard present");
|
||||||
|
assert!(
|
||||||
|
guard_at < purge_at,
|
||||||
|
"the guard has to run BEFORE the purge, or it is decoration"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod artifact_tests {
|
||||||
|
/// A filename reaches `Content-Disposition` after an AGENT chose it.
|
||||||
|
///
|
||||||
|
/// The value is attacker-influenced and parsed by every browser, so the
|
||||||
|
/// quote and control characters that would end the header early — or inject
|
||||||
|
/// a second one — are removed rather than escaped.
|
||||||
|
#[test]
|
||||||
|
fn a_downloaded_filename_cannot_break_out_of_its_header() {
|
||||||
|
let clean = |name: &str| -> String {
|
||||||
|
name.chars()
|
||||||
|
.filter(|c| *c != '"' && *c != '\\' && !c.is_control())
|
||||||
|
.collect()
|
||||||
|
};
|
||||||
|
assert_eq!(clean("findings.md"), "findings.md");
|
||||||
|
assert_eq!(clean("re\"port.md"), "report.md");
|
||||||
|
assert_eq!(clean("a\r\nX-Evil: 1.md"), "aX-Evil: 1.md");
|
||||||
|
assert_eq!(clean("back\\slash.md"), "backslash.md");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Both artifact routes resolve through ONE containment check.
|
||||||
|
///
|
||||||
|
/// Two copies is two chances for one of them to be the lenient one, and the
|
||||||
|
/// lenient one is an arbitrary read of the gateway's filesystem.
|
||||||
|
#[test]
|
||||||
|
fn one_containment_check_serves_both_routes() {
|
||||||
|
let src = include_str!("missions.rs");
|
||||||
|
assert_eq!(
|
||||||
|
src.matches(concat!("fn resolve_", "artifact_path")).count(),
|
||||||
|
1,
|
||||||
|
"one resolver"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
src.matches(concat!("resolve_", "artifact_path(&artifact.path)")).count(),
|
||||||
|
2,
|
||||||
|
"and both routes must go through it"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -14,6 +14,8 @@ pub mod health;
|
|||||||
pub mod identity;
|
pub mod identity;
|
||||||
pub mod level_up;
|
pub mod level_up;
|
||||||
pub mod library;
|
pub mod library;
|
||||||
|
pub mod mission_plan;
|
||||||
|
pub mod mission_roster;
|
||||||
pub mod missions;
|
pub mod missions;
|
||||||
pub mod nodes;
|
pub mod nodes;
|
||||||
pub mod oauth;
|
pub mod oauth;
|
||||||
|
|||||||
@@ -402,3 +402,115 @@ async fn bridge_terminal(hub: Arc<NodeHub>, node_id: NodeId, socket: WebSocket)
|
|||||||
}
|
}
|
||||||
hub.terminal_close(node_id, sid).await;
|
hub.terminal_close(node_id, sid).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `GET /api/fleet/capacity` — what the SCHEDULER sees, verbatim.
|
||||||
|
///
|
||||||
|
/// Pulled forward from the observability phase because the capacity harness
|
||||||
|
/// scenario needs it: a test that recomputed the slot arithmetic in bash would
|
||||||
|
/// drift from `vm_placement` and then agree with itself while the scheduler did
|
||||||
|
/// something else. This returns `vm_placement::survey` unmodified, so the fleet
|
||||||
|
/// page, the harness and the placer cannot disagree.
|
||||||
|
///
|
||||||
|
/// `backend` narrows to the nodes that can boot one image (`?backend=claude`),
|
||||||
|
/// matching what `choose` does for a phase.
|
||||||
|
pub async fn capacity(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Query(q): Query<CapacityQuery>,
|
||||||
|
) -> Result<Json<Value>, ApiError> {
|
||||||
|
let ws = user.workspace_id.as_uuid().to_owned();
|
||||||
|
let (fit, unfit) =
|
||||||
|
crate::vm_placement::survey(
|
||||||
|
&state.pool,
|
||||||
|
&state.node_hub,
|
||||||
|
ws,
|
||||||
|
&crate::vm_placement::required_backends(q.backend.as_deref(), None),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("fleet capacity survey failed: {e}");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
let ranked = crate::vm_placement::rank(fit);
|
||||||
|
Ok(Json(json!({
|
||||||
|
// Total free slots across the fleet. A burst larger than this MUST
|
||||||
|
// queue rather than overcommit — that is the whole feature.
|
||||||
|
"slots": ranked.iter().map(|n| n.slots).sum::<i64>(),
|
||||||
|
"nodes": ranked.iter().map(|n| json!({
|
||||||
|
"id": n.node_id,
|
||||||
|
"name": n.name,
|
||||||
|
"slots": n.slots,
|
||||||
|
"committedVms": n.committed_vms,
|
||||||
|
"headroom": n.headroom,
|
||||||
|
"memTotalMib": n.mem_total_mib,
|
||||||
|
"usedEffMib": n.used_eff_mib,
|
||||||
|
"diskFreeGib": n.disk_free_gib,
|
||||||
|
})).collect::<Vec<_>>(),
|
||||||
|
// Never folded into the above. "Full" and "unreadable" send an
|
||||||
|
// operator to different places, so they stay separate here too.
|
||||||
|
"unfit": unfit.iter().map(|(id, name, why)| json!({
|
||||||
|
"id": id,
|
||||||
|
"name": name,
|
||||||
|
"reason": why.reason(),
|
||||||
|
})).collect::<Vec<_>>(),
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
pub struct CapacityQuery {
|
||||||
|
pub backend: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `GET /api/fleet/backends` — the microVM backends a mission may actually use.
|
||||||
|
///
|
||||||
|
/// The SAME `available_backends` the roster planner is handed, not a second
|
||||||
|
/// list. The two rules it applies are both load-bearing and neither is obvious
|
||||||
|
/// from a node's capabilities alone: a backend must be built on an online node,
|
||||||
|
/// and it must have a credential contract. `agent-terminal` satisfies the first
|
||||||
|
/// and not the second — bootable, with nothing for the agent inside to
|
||||||
|
/// authenticate with — so offering it would produce a mission that validates,
|
||||||
|
/// launches, and fails at the agent turn, which is the expensive kind of late.
|
||||||
|
///
|
||||||
|
/// Exists because the UI had no backend selector at all: every mission created
|
||||||
|
/// from the dashboard ran on `claude`, so `local-ornith`, `glm` and `kimi` were
|
||||||
|
/// reachable only by calling the API directly.
|
||||||
|
pub async fn backends(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
) -> Result<Json<Value>, ApiError> {
|
||||||
|
let ws = user.workspace_id.as_uuid().to_owned();
|
||||||
|
let mut list = crate::mission_roster::available_backends(&state.pool, ws)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("fleet backends: {e}");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
// `default` is the generic `rootfs.ext4` and `claude` is the named one, and
|
||||||
|
// `microvm_credential_for` gives them the SAME contract — so a picker
|
||||||
|
// offering both shows two options with one meaning, and whichever the user
|
||||||
|
// picks they get the same thing. Collapse to the named one where it exists.
|
||||||
|
if list.iter().any(|b| b == "claude") {
|
||||||
|
list.retain(|b| b != "default");
|
||||||
|
}
|
||||||
|
Ok(Json(json!({
|
||||||
|
"backends": list.iter().map(|b| json!({
|
||||||
|
"id": b,
|
||||||
|
"label": backend_label(b),
|
||||||
|
})).collect::<Vec<_>>(),
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A name a person can choose between. The ids are deployment vocabulary
|
||||||
|
/// (`local-ornith`, `canary-claude`); a picker showing those alone asks the user
|
||||||
|
/// to know which company each one bills.
|
||||||
|
fn backend_label(id: &str) -> String {
|
||||||
|
match id {
|
||||||
|
"claude" => "Claude (Anthropic subscription)".into(),
|
||||||
|
"default" => "Claude (generic image)".into(),
|
||||||
|
"canary-claude" => "Claude — candidate CLI (canary)".into(),
|
||||||
|
"glm" => "GLM 4.7 (z.ai)".into(),
|
||||||
|
"kimi" => "Kimi (Moonshot)".into(),
|
||||||
|
"local-ornith" => "Ornith 9B — this fleet's own GPU".into(),
|
||||||
|
other => other.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -40,7 +40,6 @@ research tools. Grant write only to members that actually produce code or commit
|
|||||||
- glm-4.7 — strong general reasoning (Z.ai); best cost/quality default for most workers.\n\
|
- glm-4.7 — strong general reasoning (Z.ai); best cost/quality default for most workers.\n\
|
||||||
- glm-5.2 — GLM Opus-class for the hardest reasoning roles; higher cost.\n\
|
- glm-5.2 — GLM Opus-class for the hardest reasoning roles; higher cost.\n\
|
||||||
- kimi — excellent for code-heavy roles.\n\
|
- kimi — excellent for code-heavy roles.\n\
|
||||||
- gemini — Gemini 2.5 Flash: very fast; classification, summarization, high-volume tasks.\n\
|
|
||||||
- groq — fastest/cheapest; simple sequential high-throughput steps.\n\
|
- groq — fastest/cheapest; simple sequential high-throughput steps.\n\
|
||||||
AGENT TOOLS each agent can use at runtime: web.search (find sources), browser.goto (fetch a URL), \
|
AGENT TOOLS each agent can use at runtime: web.search (find sources), browser.goto (fetch a URL), \
|
||||||
files.write (build a markdown vault in the shared drive), chat.send (delegate to teammates), \
|
files.write (build a markdown vault in the shared drive), chat.send (delegate to teammates), \
|
||||||
@@ -133,7 +132,11 @@ pub async fn planner_chat(
|
|||||||
};
|
};
|
||||||
let user_prompt = format!("{hierarchy}{topology_lock}\n\n=== CONVERSATION ===\n{convo}\n\nRespond now (JSON only).");
|
let user_prompt = format!("{hierarchy}{topology_lock}\n\n=== CONVERSATION ===\n{convo}\n\nRespond now (JSON only).");
|
||||||
let system = planner_system_for(&body.mode);
|
let system = planner_system_for(&body.mode);
|
||||||
let raw = match runtime.complete(&system, &user_prompt, "claude-opus-4-8", 8000, true).await {
|
let raw = match crate::subscription::complete_or(
|
||||||
|
&runtime, &system, &user_prompt, "claude-opus-4-8", 8000, true,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(t) => t,
|
Ok(t) => t,
|
||||||
Err(e) => { yield sse(json!({"stage":"error","label":format!("Opus error: {e}")})); return; }
|
Err(e) => { yield sse(json!({"stage":"error","label":format!("Opus error: {e}")})); return; }
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ use crate::{ApiError, AppState, Authed};
|
|||||||
pub struct TeamMemberInput {
|
pub struct TeamMemberInput {
|
||||||
pub role: String,
|
pub role: String,
|
||||||
pub name: String,
|
pub name: String,
|
||||||
/// Model selector: claude | glm | glm-5.2 | kimi | gemini | groq.
|
/// Model selector: claude | glm | glm-5.2 | kimi | groq.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub model: String,
|
pub model: String,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
|
|||||||
@@ -309,6 +309,10 @@ pub async fn run_events_sse(
|
|||||||
.map(|n| n + 1)
|
.map(|n| n + 1)
|
||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
|
|
||||||
|
// Bytes of `checkpoint.log` already sent. The step cursor above counts
|
||||||
|
// RECORDS; this counts BYTES, because a log grows continuously rather than
|
||||||
|
// in discrete entries. Two sources, two cursors.
|
||||||
|
let mut log_sent: usize = 0;
|
||||||
let stream = async_stream::stream! {
|
let stream = async_stream::stream! {
|
||||||
loop {
|
loop {
|
||||||
match cm_db::repo::topology_runs::status(&pool, id, ws).await {
|
match cm_db::repo::topology_runs::status(&pool, id, ws).await {
|
||||||
@@ -327,6 +331,24 @@ pub async fn run_events_sse(
|
|||||||
sent += 1;
|
sent += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Live stdout/stderr from a microVM turn, appended by the
|
||||||
|
// node over the fleet WebSocket (`Uplink::VmOut`). Emitted
|
||||||
|
// as `step` so the existing reader renders it with no
|
||||||
|
// frontend change — it already reads `data.text`.
|
||||||
|
if let Some(log) = st
|
||||||
|
.checkpoint
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|c| c.get("log"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
{
|
||||||
|
if log.len() > log_sent {
|
||||||
|
let fresh = &log[log_sent..];
|
||||||
|
log_sent = log.len();
|
||||||
|
yield Ok::<Event, Infallible>(Event::default().event("step").data(
|
||||||
|
serde_json::json!({ "kind": "output", "text": fresh }).to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
if matches!(st.status.as_str(), "completed" | "failed" | "cancelled") {
|
if matches!(st.status.as_str(), "completed" | "failed" | "cancelled") {
|
||||||
let done = serde_json::json!({
|
let done = serde_json::json!({
|
||||||
"status": st.status,
|
"status": st.status,
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ use cm_domain::WorkspaceId;
|
|||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
use sqlx::{PgPool, Row};
|
use sqlx::{PgPool, Row};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::{ApiError, AppState, Authed};
|
use crate::{ApiError, AppState, Authed};
|
||||||
|
|
||||||
@@ -44,36 +45,407 @@ async fn working_agents(pool: &PgPool, ws: WorkspaceId) -> HashSet<String> {
|
|||||||
rows.into_iter().map(|r| r.get::<String, _>("id")).collect()
|
rows.into_iter().map(|r| r.get::<String, _>("id")).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Active missions (status='running') with their assigned team members —
|
/// One row per (mission, agent) pair for every mission the World should draw.
|
||||||
/// one row per (mission, agent) pair. The World SSE loop emits each as
|
///
|
||||||
/// a `mission:<id>` landmark orb + `world.touch` beams from every team
|
/// Three bugs were fixed here at once, and each hid the next:
|
||||||
/// member. Replaces the retired research/loops landmarks (commit
|
///
|
||||||
/// fdb8cfe) with the missions-era equivalent.
|
/// 1. **The join was on the wrong column.** It read
|
||||||
async fn active_missions(pool: &PgPool, ws: WorkspaceId) -> Vec<(String, String, String)> {
|
/// `JOIN team_members tm ON tm.team_id = m.team_id`, but `missions.team_id`
|
||||||
|
/// is a legacy pointer at the FIRST minted team — `0056_mission_teams.sql`
|
||||||
|
/// superseded it with the `mission_teams(mission_id, team_id, purpose)`
|
||||||
|
/// junction, which is what everything else (including `/api/workforce`)
|
||||||
|
/// joins through. A multi-team mission showed only its first team.
|
||||||
|
///
|
||||||
|
/// 2. **It was an INNER JOIN, and microVM missions have no team at all.**
|
||||||
|
/// `mission_orchestrator::on_launch` deliberately mints none for them, so
|
||||||
|
/// the platform's primary execution tier was dropped by the join and the
|
||||||
|
/// World has been showing nothing whatsoever for it. LEFT JOIN, and
|
||||||
|
/// `agent_id` is `None` for those — a phase that ran with no platform
|
||||||
|
/// agents draws no pawns, which is the truth rather than a gap.
|
||||||
|
///
|
||||||
|
/// 3. **Only `running` missions were selected.** Missions finish in minutes,
|
||||||
|
/// so the World was empty almost always. Recently-finished ones come back
|
||||||
|
/// too, carrying `status` so the client can draw a finished map instead of
|
||||||
|
/// animating a corpse.
|
||||||
|
struct MissionRow {
|
||||||
|
mission_id: String,
|
||||||
|
title: String,
|
||||||
|
status: String,
|
||||||
|
template_kind: String,
|
||||||
|
completed_at: Option<String>,
|
||||||
|
/// `None` for a mission with no team — see (2) above.
|
||||||
|
agent_id: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn world_missions(
|
||||||
|
pool: &PgPool,
|
||||||
|
ws: WorkspaceId,
|
||||||
|
only: Option<Uuid>,
|
||||||
|
) -> Vec<MissionRow> {
|
||||||
let rows = sqlx::query(
|
let rows = sqlx::query(
|
||||||
"SELECT m.id::text AS mission_id,
|
"SELECT m.id::text AS mission_id,
|
||||||
m.title AS title,
|
m.title AS title,
|
||||||
tm.claw_id::text AS agent_id
|
m.status AS status,
|
||||||
|
m.template_kind AS template_kind,
|
||||||
|
to_char(m.completed_at AT TIME ZONE 'UTC',
|
||||||
|
'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"') AS completed_at,
|
||||||
|
tm.claw_id::text AS agent_id
|
||||||
FROM missions m
|
FROM missions m
|
||||||
JOIN team_members tm ON tm.team_id = m.team_id
|
LEFT JOIN mission_teams mt ON mt.mission_id = m.id
|
||||||
|
LEFT JOIN team_members tm ON tm.team_id = mt.team_id
|
||||||
WHERE m.workspace_id = $1
|
WHERE m.workspace_id = $1
|
||||||
AND m.status = 'running'",
|
AND ( m.status = 'running'
|
||||||
|
OR ( m.status IN ('completed', 'failed')
|
||||||
|
AND m.completed_at > now() - interval '24 hours' ) )
|
||||||
|
AND ($2::uuid IS NULL OR m.id = $2)
|
||||||
|
ORDER BY m.created_at DESC",
|
||||||
)
|
)
|
||||||
.bind(ws.as_uuid())
|
.bind(ws.as_uuid())
|
||||||
|
.bind(only)
|
||||||
.fetch_all(pool)
|
.fetch_all(pool)
|
||||||
.await
|
.await
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
rows.into_iter()
|
rows.into_iter()
|
||||||
.map(|r| {
|
.map(|r| MissionRow {
|
||||||
(
|
mission_id: r.get::<String, _>("mission_id"),
|
||||||
r.get::<String, _>("mission_id"),
|
title: r.get::<String, _>("title"),
|
||||||
r.get::<String, _>("title"),
|
status: r.get::<String, _>("status"),
|
||||||
r.get::<String, _>("agent_id"),
|
template_kind: r.get::<String, _>("template_kind"),
|
||||||
)
|
completed_at: r.get::<Option<String>, _>("completed_at"),
|
||||||
|
agent_id: r.get::<Option<String>, _>("agent_id"),
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Every phase of the given missions, in plan order, with the agents that
|
||||||
|
/// execute it.
|
||||||
|
///
|
||||||
|
/// The whole plan is emitted — including `pending` phases that have not
|
||||||
|
/// started — because the World draws the mission's shape upfront and lights it
|
||||||
|
/// as it progresses. A phase list that only appeared as phases began would make
|
||||||
|
/// a five-phase mission indistinguishable from a one-phase mission until it was
|
||||||
|
/// nearly over.
|
||||||
|
struct PhaseRow {
|
||||||
|
mission_id: String,
|
||||||
|
phase_id: String,
|
||||||
|
kind: String,
|
||||||
|
order_idx: i32,
|
||||||
|
status: String,
|
||||||
|
iteration: i32,
|
||||||
|
started_at: Option<String>,
|
||||||
|
completed_at: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn world_phases(pool: &PgPool, ws: WorkspaceId, only: Option<Uuid>) -> Vec<PhaseRow> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT mp.mission_id::text AS mission_id,
|
||||||
|
mp.id::text AS phase_id,
|
||||||
|
mp.kind AS kind,
|
||||||
|
mp.order_idx AS order_idx,
|
||||||
|
mp.status AS status,
|
||||||
|
COALESCE(mp.iteration, 0) AS iteration,
|
||||||
|
to_char(mp.started_at AT TIME ZONE 'UTC',
|
||||||
|
'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"') AS started_at,
|
||||||
|
to_char(mp.completed_at AT TIME ZONE 'UTC',
|
||||||
|
'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"') AS completed_at
|
||||||
|
FROM mission_phases mp
|
||||||
|
JOIN missions m ON m.id = mp.mission_id
|
||||||
|
WHERE m.workspace_id = $1
|
||||||
|
AND ( m.status = 'running'
|
||||||
|
OR ( m.status IN ('completed', 'failed')
|
||||||
|
AND m.completed_at > now() - interval '24 hours' ) )
|
||||||
|
AND ($2::uuid IS NULL OR m.id = $2)
|
||||||
|
ORDER BY mp.mission_id, mp.order_idx",
|
||||||
|
)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.bind(only)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
rows.into_iter()
|
||||||
|
.map(|r| PhaseRow {
|
||||||
|
mission_id: r.get::<String, _>("mission_id"),
|
||||||
|
phase_id: r.get::<String, _>("phase_id"),
|
||||||
|
kind: r.get::<String, _>("kind"),
|
||||||
|
order_idx: r.get::<i32, _>("order_idx"),
|
||||||
|
status: r.get::<String, _>("status"),
|
||||||
|
iteration: r.get::<i32, _>("iteration"),
|
||||||
|
started_at: r.get::<Option<String>, _>("started_at"),
|
||||||
|
completed_at: r.get::<Option<String>, _>("completed_at"),
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Files a phase touched, from the `code_diff` artifact captured at delivery.
|
||||||
|
///
|
||||||
|
/// This is recorded fact, not inference: `mission_delivery` writes the path
|
||||||
|
/// list with the same revision and excludes it uses for `files_changed`, so the
|
||||||
|
/// orbs the World draws are the files git says changed.
|
||||||
|
///
|
||||||
|
/// It is end-of-phase detail — the capture runs when a phase finishes — so a
|
||||||
|
/// running phase shows its station lit but no files until it lands. Live
|
||||||
|
/// per-tool file touches are a separate, structured source.
|
||||||
|
struct FileRow {
|
||||||
|
mission_id: String,
|
||||||
|
phase_id: String,
|
||||||
|
path: String,
|
||||||
|
status: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn world_files(pool: &PgPool, ws: WorkspaceId, only: Option<Uuid>) -> Vec<FileRow> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT ma.mission_id::text AS mission_id,
|
||||||
|
ma.phase_id::text AS phase_id,
|
||||||
|
f->>'path' AS path,
|
||||||
|
f->>'status' AS status
|
||||||
|
FROM mission_artifacts ma
|
||||||
|
JOIN missions m ON m.id = ma.mission_id
|
||||||
|
CROSS JOIN LATERAL jsonb_array_elements(
|
||||||
|
COALESCE(ma.metadata->'files', '[]'::jsonb)) AS f
|
||||||
|
WHERE m.workspace_id = $1
|
||||||
|
AND ma.kind = 'code_diff'
|
||||||
|
AND ma.phase_id IS NOT NULL
|
||||||
|
AND ( m.status = 'running'
|
||||||
|
OR ( m.status IN ('completed', 'failed')
|
||||||
|
AND m.completed_at > now() - interval '24 hours' ) )
|
||||||
|
AND ($2::uuid IS NULL OR m.id = $2)
|
||||||
|
LIMIT 2000",
|
||||||
|
)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.bind(only)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
rows.into_iter()
|
||||||
|
.filter_map(|r| {
|
||||||
|
Some(FileRow {
|
||||||
|
mission_id: r.get::<String, _>("mission_id"),
|
||||||
|
phase_id: r.get::<String, _>("phase_id"),
|
||||||
|
path: r.get::<Option<String>, _>("path")?,
|
||||||
|
status: r.get::<Option<String>, _>("status").unwrap_or_default(),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One row of `mission_events` — what an agent actually did.
|
||||||
|
struct ActRow {
|
||||||
|
id: i64,
|
||||||
|
mission_id: String,
|
||||||
|
phase_id: Option<String>,
|
||||||
|
agent_id: Option<String>,
|
||||||
|
kind: String,
|
||||||
|
target: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Structured mission activity since `after`, oldest first.
|
||||||
|
///
|
||||||
|
/// `after < 0` means "the first pass has not run yet": everything is returned
|
||||||
|
/// so the caller can seed its cursor and draw the backlog as settled history.
|
||||||
|
async fn world_acts(pool: &PgPool, ws: WorkspaceId, only: Option<Uuid>, after: i64) -> Vec<ActRow> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT e.id,
|
||||||
|
e.mission_id::text AS mission_id,
|
||||||
|
e.phase_id::text AS phase_id,
|
||||||
|
e.agent_id::text AS agent_id,
|
||||||
|
e.kind,
|
||||||
|
e.target
|
||||||
|
FROM mission_events e
|
||||||
|
JOIN missions m ON m.id = e.mission_id
|
||||||
|
WHERE m.workspace_id = $1
|
||||||
|
AND e.kind IN ('tool.call', 'file.touch')
|
||||||
|
AND e.target IS NOT NULL
|
||||||
|
AND e.id > $2
|
||||||
|
AND ( m.status = 'running'
|
||||||
|
OR ( m.status IN ('completed', 'failed')
|
||||||
|
AND m.completed_at > now() - interval '24 hours' ) )
|
||||||
|
AND ($3::uuid IS NULL OR m.id = $3)
|
||||||
|
ORDER BY e.id
|
||||||
|
LIMIT 500",
|
||||||
|
)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.bind(after)
|
||||||
|
.bind(only)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
rows.into_iter()
|
||||||
|
.filter_map(|r| {
|
||||||
|
Some(ActRow {
|
||||||
|
id: r.get::<i64, _>("id"),
|
||||||
|
mission_id: r.get::<String, _>("mission_id"),
|
||||||
|
phase_id: r.get::<Option<String>, _>("phase_id"),
|
||||||
|
agent_id: r.get::<Option<String>, _>("agent_id"),
|
||||||
|
kind: r.get::<String, _>("kind"),
|
||||||
|
target: r.get::<Option<String>, _>("target")?,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One security finding, as the scanner recorded it.
|
||||||
|
struct FindingRow {
|
||||||
|
mission_id: String,
|
||||||
|
phase_id: String,
|
||||||
|
task_id: String,
|
||||||
|
title: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Findings raised by a security phase.
|
||||||
|
///
|
||||||
|
/// They are `mission_tasks` rows — the scanner's own store — not a new table.
|
||||||
|
/// There is deliberately NO severity field here: severity, file and line are
|
||||||
|
/// substrings inside `title` (see `security_scan.rs`), and a severity parsed
|
||||||
|
/// out of prose and then encoded as an orb's RADIUS would be an invented fact
|
||||||
|
/// rendered as a measurement. The title is shown as written.
|
||||||
|
async fn world_findings(pool: &PgPool, ws: WorkspaceId, only: Option<Uuid>) -> Vec<FindingRow> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT t.mission_id::text AS mission_id,
|
||||||
|
t.phase_id::text AS phase_id,
|
||||||
|
t.id::text AS task_id,
|
||||||
|
t.title
|
||||||
|
FROM mission_tasks t
|
||||||
|
JOIN mission_phases p ON p.id = t.phase_id
|
||||||
|
JOIN missions m ON m.id = t.mission_id
|
||||||
|
WHERE m.workspace_id = $1
|
||||||
|
AND p.kind = 'security_scan'
|
||||||
|
AND ( m.status = 'running'
|
||||||
|
OR ( m.status IN ('completed', 'failed')
|
||||||
|
AND m.completed_at > now() - interval '24 hours' ) )
|
||||||
|
AND ($2::uuid IS NULL OR m.id = $2)
|
||||||
|
LIMIT 300",
|
||||||
|
)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.bind(only)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
rows.into_iter()
|
||||||
|
.map(|r| FindingRow {
|
||||||
|
mission_id: r.get::<String, _>("mission_id"),
|
||||||
|
phase_id: r.get::<String, _>("phase_id"),
|
||||||
|
task_id: r.get::<String, _>("task_id"),
|
||||||
|
title: r.get::<String, _>("title"),
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A benchmark phase's result, summarised.
|
||||||
|
struct BenchRow {
|
||||||
|
mission_id: String,
|
||||||
|
phase_id: String,
|
||||||
|
note: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Benchmark deltas, as an annotation on the phase — not as nodes.
|
||||||
|
///
|
||||||
|
/// `delta` is a `Record<string, unknown>` with no schema: `compute_delta`
|
||||||
|
/// produces `{kind:"bencher_diff", samples:[…]}` when the before/after shapes
|
||||||
|
/// are structurally comparable, and `{kind:"opaque"}` when they are not. Only
|
||||||
|
/// the shape that can be parsed is formatted; the rest is COUNTED, never
|
||||||
|
/// guessed at, so a driver whose output we do not understand reports "3
|
||||||
|
/// samples" rather than an invented improvement.
|
||||||
|
async fn world_benchmarks(pool: &PgPool, ws: WorkspaceId, only: Option<Uuid>) -> Vec<BenchRow> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT DISTINCT ON (b.phase_id)
|
||||||
|
b.mission_id::text AS mission_id,
|
||||||
|
b.phase_id::text AS phase_id,
|
||||||
|
b.delta
|
||||||
|
FROM benchmark_snapshots b
|
||||||
|
JOIN missions m ON m.id = b.mission_id
|
||||||
|
WHERE m.workspace_id = $1
|
||||||
|
AND b.delta IS NOT NULL
|
||||||
|
AND ( m.status = 'running'
|
||||||
|
OR ( m.status IN ('completed', 'failed')
|
||||||
|
AND m.completed_at > now() - interval '24 hours' ) )
|
||||||
|
AND ($2::uuid IS NULL OR m.id = $2)
|
||||||
|
ORDER BY b.phase_id, b.iteration DESC",
|
||||||
|
)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.bind(only)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
rows.into_iter()
|
||||||
|
.filter_map(|r| {
|
||||||
|
let note = benchmark_note(&r.get::<serde_json::Value, _>("delta"))?;
|
||||||
|
Some(BenchRow {
|
||||||
|
mission_id: r.get::<String, _>("mission_id"),
|
||||||
|
phase_id: r.get::<String, _>("phase_id"),
|
||||||
|
note,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One line describing a benchmark delta, or `None` if there is nothing
|
||||||
|
/// truthful to say about it.
|
||||||
|
fn benchmark_note(delta: &serde_json::Value) -> Option<String> {
|
||||||
|
let samples = delta.get("samples").and_then(|s| s.as_array())?;
|
||||||
|
if samples.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let mut improved = 0usize;
|
||||||
|
let mut regressed = 0usize;
|
||||||
|
// Best (most negative) percent change, since that is the one claim the
|
||||||
|
// shape actually supports.
|
||||||
|
let mut best: Option<f64> = None;
|
||||||
|
for s in samples {
|
||||||
|
match s.get("direction").and_then(|d| d.as_str()) {
|
||||||
|
Some("improved") => improved += 1,
|
||||||
|
Some("regressed") => regressed += 1,
|
||||||
|
// Counted in the total but claimed for neither side.
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
if let Some(pct) = s.get("delta_pct").and_then(serde_json::Value::as_f64) {
|
||||||
|
best = Some(best.map_or(pct, |b: f64| b.min(pct)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut parts = vec![format!("{} sample(s)", samples.len())];
|
||||||
|
if improved > 0 {
|
||||||
|
parts.push(format!("{improved} faster"));
|
||||||
|
}
|
||||||
|
if regressed > 0 {
|
||||||
|
parts.push(format!("{regressed} slower"));
|
||||||
|
}
|
||||||
|
if let Some(b) = best.filter(|b| *b < 0.0) {
|
||||||
|
parts.push(format!("best {:.1}%", b));
|
||||||
|
}
|
||||||
|
Some(parts.join(", "))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Agents that execute a phase of this kind, via the purposes the phase runner
|
||||||
|
/// itself uses. Returns empty for a teamless (microVM) mission.
|
||||||
|
async fn phase_agents(
|
||||||
|
pool: &PgPool,
|
||||||
|
mission_id: &str,
|
||||||
|
kind: &str,
|
||||||
|
) -> Vec<String> {
|
||||||
|
let Ok(mid) = Uuid::parse_str(mission_id) else {
|
||||||
|
return Vec::new();
|
||||||
|
};
|
||||||
|
// `purposes_for` is the runner's own mapping, shared rather than copied —
|
||||||
|
// see its doc comment.
|
||||||
|
let purposes: Vec<String> = crate::phase_runner::purposes_for(kind)
|
||||||
|
.iter()
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
.collect();
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT DISTINCT tm.claw_id::text AS agent_id
|
||||||
|
FROM mission_teams mt
|
||||||
|
JOIN team_members tm ON tm.team_id = mt.team_id
|
||||||
|
WHERE mt.mission_id = $1 AND mt.purpose = ANY($2)",
|
||||||
|
)
|
||||||
|
.bind(mid)
|
||||||
|
.bind(&purposes)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
rows.into_iter()
|
||||||
|
.map(|r| r.get::<String, _>("agent_id"))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
/// Currently-running runs in the workspace as (run_id, agent_id) — each is a
|
/// Currently-running runs in the workspace as (run_id, agent_id) — each is a
|
||||||
/// real "this agent is converging on its active work" signal (Gource).
|
/// real "this agent is converging on its active work" signal (Gource).
|
||||||
async fn active_runs(pool: &PgPool, ws: WorkspaceId) -> Vec<(String, String)> {
|
async fn active_runs(pool: &PgPool, ws: WorkspaceId) -> Vec<(String, String)> {
|
||||||
@@ -330,10 +702,24 @@ async fn agent_telemetry(
|
|||||||
m
|
m
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Query for `GET /api/world/live`.
|
||||||
|
#[derive(serde::Deserialize)]
|
||||||
|
pub struct LiveQuery {
|
||||||
|
/// Scope the feed to one mission. The client already filters events by
|
||||||
|
/// mission, but doing it here means the server stops querying and sending
|
||||||
|
/// what the viewer will discard.
|
||||||
|
pub mission: Option<Uuid>,
|
||||||
|
}
|
||||||
|
|
||||||
/// `GET /api/world/live` — the taxonomy SSE feed.
|
/// `GET /api/world/live` — the taxonomy SSE feed.
|
||||||
pub async fn world_live(State(state): State<AppState>, Authed(user): Authed) -> impl IntoResponse {
|
pub async fn world_live(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Query(q): Query<LiveQuery>,
|
||||||
|
) -> impl IntoResponse {
|
||||||
let pool = state.pool.clone();
|
let pool = state.pool.clone();
|
||||||
let ws = user.workspace_id;
|
let ws = user.workspace_id;
|
||||||
|
let only_mission = q.mission;
|
||||||
|
|
||||||
let stream = async_stream::stream! {
|
let stream = async_stream::stream! {
|
||||||
let mut first = true;
|
let mut first = true;
|
||||||
@@ -341,6 +727,27 @@ pub async fn world_live(State(state): State<AppState>, Authed(user): Authed) ->
|
|||||||
let mut last: std::collections::HashMap<String, String> = std::collections::HashMap::new();
|
let mut last: std::collections::HashMap<String, String> = std::collections::HashMap::new();
|
||||||
// Per-run journal cursor so we stream only NEW run_events each poll.
|
// Per-run journal cursor so we stream only NEW run_events each poll.
|
||||||
let mut cursors: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
|
let mut cursors: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
|
||||||
|
// Last emitted signature per mission / per phase, so the plan is sent
|
||||||
|
// once and then only when something actually moves.
|
||||||
|
let mut last_mission: std::collections::HashMap<String, String> =
|
||||||
|
std::collections::HashMap::new();
|
||||||
|
let mut last_phase: std::collections::HashMap<String, String> =
|
||||||
|
std::collections::HashMap::new();
|
||||||
|
// (phase, path) pairs already announced — a delivered file is a fact
|
||||||
|
// that happened once, not a recurring event.
|
||||||
|
let mut last_file: HashSet<String> = HashSet::new();
|
||||||
|
// `mission_events` cursor. -1 until the first pass seeds it, which is
|
||||||
|
// what separates BACKFILL from MOTION: everything already in the table
|
||||||
|
// when a subscriber arrives is history and is drawn as a settled map,
|
||||||
|
// and only what lands afterwards is animated. Without the distinction,
|
||||||
|
// opening a finished mission would replay an hour of tool calls as a
|
||||||
|
// burst storm and read as a mission that just did all of it at once.
|
||||||
|
let mut event_cursor: i64 = -1;
|
||||||
|
// Findings already announced. A finding is raised once.
|
||||||
|
let mut last_finding: HashSet<String> = HashSet::new();
|
||||||
|
// Last benchmark summary per phase; a looping phase produces a new one.
|
||||||
|
let mut last_bench: std::collections::HashMap<String, String> =
|
||||||
|
std::collections::HashMap::new();
|
||||||
// Audit-log cursor for edge-initiated inter-agent events (delegation,
|
// Audit-log cursor for edge-initiated inter-agent events (delegation,
|
||||||
// A2A) that bypass the run loop. -1 until seeded on the first pass.
|
// A2A) that bypass the run loop. -1 until seeded on the first pass.
|
||||||
let mut audit_cursor: i64 = -1;
|
let mut audit_cursor: i64 = -1;
|
||||||
@@ -391,27 +798,233 @@ pub async fn world_live(State(state): State<AppState>, Authed(user): Authed) ->
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mission landmarks: one `mission:<id>` orb per running mission,
|
// Mission landmarks, and the whole phase plan behind them.
|
||||||
// with `world.touch` beams from every assigned team member. Missions
|
//
|
||||||
// outlive individual runs, so the orb gives the World a persistent
|
// The orb pins "this is the work"; the phases give it shape. Every
|
||||||
// pin for "this is what the team is working on right now" even when
|
// phase is emitted, including ones that have not started, because
|
||||||
// no run is claimed. Replaces the retired repo:{topic}/loop:{id}
|
// the World draws the plan upfront and lights it as it progresses —
|
||||||
// landmarks after commit fdb8cfe.
|
// a phase list that appeared only as phases began would make a
|
||||||
let missions = active_missions(&pool, ws).await;
|
// five-phase mission indistinguishable from a one-phase mission
|
||||||
|
// until it was nearly over.
|
||||||
|
let missions = world_missions(&pool, ws, only_mission).await;
|
||||||
let mut seen_missions: HashSet<String> = HashSet::new();
|
let mut seen_missions: HashSet<String> = HashSet::new();
|
||||||
for (mission_id, title, agent_id) in &missions {
|
for m in &missions {
|
||||||
if seen_missions.insert(mission_id.clone()) {
|
if seen_missions.insert(m.mission_id.clone()) {
|
||||||
let node_id = format!("mission:{mission_id}");
|
// Delta-guarded like every other stateful event here. Without
|
||||||
|
// this it re-sent the same mission on every poll — harmless
|
||||||
|
// to a client that keys on missionId, and pure noise on the
|
||||||
|
// wire that hides the events that DID change.
|
||||||
|
let sig = format!("{}|{}", m.status, m.completed_at.as_deref().unwrap_or(""));
|
||||||
|
if last_mission.get(&m.mission_id).map(|s| s != &sig).unwrap_or(true) {
|
||||||
|
last_mission.insert(m.mission_id.clone(), sig);
|
||||||
|
yield sse(
|
||||||
|
"mission.update",
|
||||||
|
json!({
|
||||||
|
"missionId": m.mission_id,
|
||||||
|
"title": m.title,
|
||||||
|
"status": m.status,
|
||||||
|
"templateKind": m.template_kind,
|
||||||
|
"completedAt": m.completed_at,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// A terminal mission is a map to read, not a scene to
|
||||||
|
// animate: it still gets an orb, but no heat.
|
||||||
|
let running = m.status == "running";
|
||||||
yield sse(
|
yield sse(
|
||||||
"node.activity",
|
"node.activity",
|
||||||
json!({ "nodeId": node_id, "label": title, "kind": "mission", "heat": 0.75 }),
|
json!({
|
||||||
|
"nodeId": format!("mission:{}", m.mission_id),
|
||||||
|
"label": m.title,
|
||||||
|
"kind": "mission",
|
||||||
|
"heat": if running { 0.75 } else { 0.0 },
|
||||||
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
let node_id = format!("mission:{mission_id}");
|
// `agent_id` is None for a teamless microVM mission. No pawn
|
||||||
yield sse(
|
// beams at it, which is accurate — nothing on this platform ran
|
||||||
"world.touch",
|
// that phase except a VM — and is why the join had to become a
|
||||||
json!({ "agentId": agent_id, "nodeId": node_id, "kind": "mission", "weight": 0.6 }),
|
// LEFT JOIN rather than being left to drop the mission entirely.
|
||||||
|
if let (Some(agent_id), true) = (&m.agent_id, m.status == "running") {
|
||||||
|
yield sse(
|
||||||
|
"world.touch",
|
||||||
|
json!({
|
||||||
|
"agentId": agent_id,
|
||||||
|
"nodeId": format!("mission:{}", m.mission_id),
|
||||||
|
"kind": "mission",
|
||||||
|
"weight": 0.6,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for p in world_phases(&pool, ws, only_mission).await {
|
||||||
|
// Re-emit only on change. `iteration` and `completed_at` are in
|
||||||
|
// the signature because a phase that loops re-enters `running`
|
||||||
|
// from `running` — status alone would hide the retry.
|
||||||
|
let sig = format!(
|
||||||
|
"{}|{}|{}",
|
||||||
|
p.status,
|
||||||
|
p.iteration,
|
||||||
|
p.completed_at.as_deref().unwrap_or("")
|
||||||
);
|
);
|
||||||
|
let changed = last_phase.get(&p.phase_id).map(|s| s != &sig).unwrap_or(true);
|
||||||
|
if !changed {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
last_phase.insert(p.phase_id.clone(), sig);
|
||||||
|
let agents = phase_agents(&pool, &p.mission_id, &p.kind).await;
|
||||||
|
yield sse(
|
||||||
|
"mission.phase",
|
||||||
|
json!({
|
||||||
|
"missionId": p.mission_id,
|
||||||
|
"phaseId": p.phase_id,
|
||||||
|
"kind": p.kind,
|
||||||
|
"orderIdx": p.order_idx,
|
||||||
|
"status": p.status,
|
||||||
|
"iteration": p.iteration,
|
||||||
|
"startedAt": p.started_at,
|
||||||
|
"completedAt": p.completed_at,
|
||||||
|
"agentIds": agents,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
yield sse(
|
||||||
|
"node.activity",
|
||||||
|
json!({
|
||||||
|
"nodeId": format!("phase:{}", p.phase_id),
|
||||||
|
"label": p.kind,
|
||||||
|
"kind": "phase",
|
||||||
|
// The client owns the lit/dim encoding via `status`;
|
||||||
|
// heat here is only the arrival pulse.
|
||||||
|
"heat": if p.status == "running" { 0.8 } else { 0.0 },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Files come AFTER the phases they belong to, deliberately: the
|
||||||
|
// client hangs a file tree under the coding station, and parenting
|
||||||
|
// there is first-write-wins. A file that arrived before its plan
|
||||||
|
// would pin its whole directory tree at the origin.
|
||||||
|
//
|
||||||
|
// Files, once each. A file is emitted when its phase's diff was
|
||||||
|
// captured and never again — the World keeps the node alive itself,
|
||||||
|
// and re-sending would re-burst the orb every poll as though the
|
||||||
|
// file had just been touched again.
|
||||||
|
for f in world_files(&pool, ws, only_mission).await {
|
||||||
|
let key = format!("{}|{}", f.phase_id, f.path);
|
||||||
|
if last_file.contains(&key) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
last_file.insert(key);
|
||||||
|
yield sse(
|
||||||
|
"mission.file",
|
||||||
|
json!({
|
||||||
|
"missionId": f.mission_id,
|
||||||
|
"phaseId": f.phase_id,
|
||||||
|
"path": f.path,
|
||||||
|
"status": f.status,
|
||||||
|
"source": "diff",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Benchmark results, as an annotation on the station. Re-emitted
|
||||||
|
// only when the summary changes: a phase that loops produces a new
|
||||||
|
// snapshot per iteration, and that IS news.
|
||||||
|
for b in world_benchmarks(&pool, ws, only_mission).await {
|
||||||
|
if last_bench.get(&b.phase_id).map(|n| n == &b.note).unwrap_or(false) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
last_bench.insert(b.phase_id.clone(), b.note.clone());
|
||||||
|
yield sse(
|
||||||
|
"mission.benchmark",
|
||||||
|
json!({
|
||||||
|
"missionId": b.mission_id,
|
||||||
|
"phaseId": b.phase_id,
|
||||||
|
"note": b.note,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Findings, once each, hanging off the security station. Same rule
|
||||||
|
// as files: a finding is a fact that was raised once, and
|
||||||
|
// re-sending it would pop the orb again on every poll.
|
||||||
|
for f in world_findings(&pool, ws, only_mission).await {
|
||||||
|
if !last_finding.insert(f.task_id.clone()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
yield sse(
|
||||||
|
"mission.finding",
|
||||||
|
json!({
|
||||||
|
"missionId": f.mission_id,
|
||||||
|
"phaseId": f.phase_id,
|
||||||
|
"findingId": f.task_id,
|
||||||
|
"title": f.title,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Structured activity — the motion channel. Tool calls and file
|
||||||
|
// touches recorded at the source by the container tap and the
|
||||||
|
// microVM `PostToolUse` hook. Never parsed from prose: a tool name
|
||||||
|
// in a log is indistinguishable from an agent TALKING about a tool.
|
||||||
|
{
|
||||||
|
let backfill = event_cursor < 0;
|
||||||
|
for a in world_acts(&pool, ws, only_mission, event_cursor.max(0)).await {
|
||||||
|
event_cursor = event_cursor.max(a.id);
|
||||||
|
match a.kind.as_str() {
|
||||||
|
"file.touch" => {
|
||||||
|
let key = format!("{}|{}", a.phase_id.as_deref().unwrap_or(""), a.target);
|
||||||
|
if !last_file.insert(key) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
yield sse(
|
||||||
|
"mission.file",
|
||||||
|
json!({
|
||||||
|
"missionId": a.mission_id,
|
||||||
|
"phaseId": a.phase_id,
|
||||||
|
"agentId": a.agent_id,
|
||||||
|
"path": a.target,
|
||||||
|
// Backlog is history: it draws the file and
|
||||||
|
// stops there. Only what lands while someone
|
||||||
|
// is watching is motion.
|
||||||
|
"source": if backfill { "diff" } else { "tool" },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// A tool call with an agent is a pawn leaving its
|
||||||
|
// station and coming back; without one (a microVM phase
|
||||||
|
// has no platform agent) it is only a node lighting up,
|
||||||
|
// which is the truth rather than an invented traveller.
|
||||||
|
_ if !backfill => {
|
||||||
|
let node_id = format!("tool:{}", a.target);
|
||||||
|
match &a.agent_id {
|
||||||
|
Some(agent) => yield sse(
|
||||||
|
"world.touch",
|
||||||
|
json!({
|
||||||
|
"agentId": agent,
|
||||||
|
"nodeId": node_id,
|
||||||
|
"kind": "event",
|
||||||
|
"weight": 0.45,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
None => yield sse(
|
||||||
|
"node.activity",
|
||||||
|
json!({
|
||||||
|
"nodeId": node_id,
|
||||||
|
"label": a.target,
|
||||||
|
"kind": "event",
|
||||||
|
"heat": 0.5,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A backlogged tool call draws nothing: the orb would be
|
||||||
|
// a tool nobody is using, permanently lit on a map of
|
||||||
|
// work that already finished.
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Real convergence: each running agent beams toward its active-run node.
|
// Real convergence: each running agent beams toward its active-run node.
|
||||||
@@ -426,6 +1039,23 @@ pub async fn world_live(State(state): State<AppState>, Authed(user): Authed) ->
|
|||||||
// Tail the run's journal for richer real events (reasoning, tool
|
// Tail the run's journal for richer real events (reasoning, tool
|
||||||
// convergence, doors). On first sight, jump the cursor to the
|
// convergence, doors). On first sight, jump the cursor to the
|
||||||
// current max so we stream forward without replaying the backlog.
|
// current max so we stream forward without replaying the backlog.
|
||||||
|
//
|
||||||
|
// THIS BLOCK IS THE a2a/door LAYER ONLY, and it works: those
|
||||||
|
// runs are the ones that write `run_events` (`run_events::append`
|
||||||
|
// is reached from `routes/a2a.rs` and `mcp_door.rs`).
|
||||||
|
//
|
||||||
|
// A sibling block used to read `topology_runs.checkpoint.records`
|
||||||
|
// here, on the theory that it covered missions. It could never
|
||||||
|
// have: `run_id` comes from `active_runs`, which selects
|
||||||
|
// `agent_runs.id`, and mission phases live in `topology_runs`
|
||||||
|
// under independently generated ids. The query ran every poll and
|
||||||
|
// matched nothing, for every mission, forever — which is why the
|
||||||
|
// World has never shown a mission's tool or file activity.
|
||||||
|
// Removed rather than repointed: pointing it at `topology_runs`
|
||||||
|
// would resurrect a path whose only per-step content is the
|
||||||
|
// agent's own prose output, and a tool name in prose is
|
||||||
|
// indistinguishable from an agent talking about a tool. Mission
|
||||||
|
// detail arrives as structured `mission_events` instead.
|
||||||
if let Some(&after) = cursors.get(run_id) {
|
if let Some(&after) = cursors.get(run_id) {
|
||||||
let rows = sqlx::query(
|
let rows = sqlx::query(
|
||||||
"SELECT seq, event_type, payload FROM run_events
|
"SELECT seq, event_type, payload FROM run_events
|
||||||
@@ -586,3 +1216,146 @@ pub async fn world_replay(
|
|||||||
json!({ "events": events, "hours": hours, "count": rows.len() }),
|
json!({ "events": events, "hours": hours, "count": rows.len() }),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod mission_feed_tests {
|
||||||
|
/// The mission query must not re-acquire any of the three bugs it fixed.
|
||||||
|
///
|
||||||
|
/// All three were invisible: an INNER JOIN does not error when it drops a
|
||||||
|
/// row, the legacy `missions.team_id` is a real column that resolves fine,
|
||||||
|
/// and `status = 'running'` is a defensible-looking filter. The World simply
|
||||||
|
/// showed less than the truth and nothing anywhere said so. A source walk is
|
||||||
|
/// the only guard available — these are runtime `sqlx::query` calls, so the
|
||||||
|
/// compiler checks nothing about them.
|
||||||
|
#[test]
|
||||||
|
fn the_mission_query_keeps_teamless_missions() {
|
||||||
|
let src = include_str!("world.rs");
|
||||||
|
let q = src
|
||||||
|
.split("async fn world_missions")
|
||||||
|
.nth(1)
|
||||||
|
.and_then(|s| s.split("async fn").next())
|
||||||
|
.expect("world_missions body");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
q.contains("LEFT JOIN mission_teams"),
|
||||||
|
"must LEFT JOIN mission_teams: a microVM mission has no team rows at \
|
||||||
|
all, and an INNER JOIN silently drops the platform's primary \
|
||||||
|
execution tier"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
q.contains("LEFT JOIN team_members"),
|
||||||
|
"the second join must be LEFT too, or the mission reappears and its \
|
||||||
|
agents take it back out"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!q.contains("tm.team_id = m.team_id"),
|
||||||
|
"must not join on the legacy missions.team_id — 0056 superseded it \
|
||||||
|
with mission_teams, and it points at only the FIRST minted team"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
q.contains("m.completed_at >"),
|
||||||
|
"recently-finished missions must be included or the World is empty \
|
||||||
|
almost always: missions finish in minutes"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
q.contains("m.status") && q.contains("m.template_kind"),
|
||||||
|
"status and template_kind must be carried: one decides finished-map \
|
||||||
|
vs live, the other the palette"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The checkpoint tail read `topology_runs` keyed by an `agent_runs` id and
|
||||||
|
/// could never match. If it comes back, missions silently lose their tool
|
||||||
|
/// and file detail again — and the query looks entirely reasonable.
|
||||||
|
#[test]
|
||||||
|
fn the_dead_checkpoint_tail_stays_dead() {
|
||||||
|
let src = include_str!("world.rs");
|
||||||
|
// Split so this assertion's own literal is not what the source walk
|
||||||
|
// finds. Written whole, the test fails on itself — which it did, and
|
||||||
|
// which is the same self-match that makes `pkill -f <pattern>` kill the
|
||||||
|
// shell carrying the pattern.
|
||||||
|
let needle = concat!("SELECT checkpoint FROM topology_", "runs WHERE id = $1::uuid");
|
||||||
|
assert!(
|
||||||
|
!src.contains(needle),
|
||||||
|
"the checkpoint tail is keyed on an agent_runs id and cannot match a \
|
||||||
|
topology_runs row; mission detail comes from structured events"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A benchmark delta is described only as far as its shape allows.
|
||||||
|
///
|
||||||
|
/// `compute_delta` emits `{kind:"opaque"}` whenever the before/after
|
||||||
|
/// metrics were not structurally comparable — which is most drivers. The
|
||||||
|
/// temptation is to say something anyway; the result would be a performance
|
||||||
|
/// claim the picture makes and the data does not support.
|
||||||
|
#[test]
|
||||||
|
fn a_benchmark_is_described_only_as_far_as_its_shape_allows() {
|
||||||
|
use serde_json::json;
|
||||||
|
assert_eq!(
|
||||||
|
super::benchmark_note(&json!({
|
||||||
|
"kind": "bencher_diff",
|
||||||
|
"samples": [
|
||||||
|
{"name": "a", "delta_pct": -12.5, "direction": "improved"},
|
||||||
|
{"name": "b", "delta_pct": 3.0, "direction": "regressed"},
|
||||||
|
// No direction and no percentage: counted, claimed for
|
||||||
|
// neither side.
|
||||||
|
{"name": "c"},
|
||||||
|
],
|
||||||
|
}))
|
||||||
|
.as_deref(),
|
||||||
|
Some("3 sample(s), 1 faster, 1 slower, best -12.5%")
|
||||||
|
);
|
||||||
|
// Nothing comparable happened, so nothing is said.
|
||||||
|
assert_eq!(
|
||||||
|
super::benchmark_note(&json!({ "kind": "opaque", "note": "not comparable" })),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
super::benchmark_note(&json!({ "kind": "bencher_diff", "samples": [] })),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
// Everything got slower: no "best" claim at all.
|
||||||
|
assert_eq!(
|
||||||
|
super::benchmark_note(&json!({
|
||||||
|
"samples": [{"name": "a", "delta_pct": 8.0, "direction": "regressed"}],
|
||||||
|
}))
|
||||||
|
.as_deref(),
|
||||||
|
Some("1 sample(s), 1 slower")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Agent→phase attribution must come from the runner's own mapping.
|
||||||
|
#[test]
|
||||||
|
fn phase_attribution_reuses_the_runners_mapping() {
|
||||||
|
let src = include_str!("world.rs");
|
||||||
|
assert!(
|
||||||
|
src.contains("crate::phase_runner::purposes_for"),
|
||||||
|
"a second copy of the kind→purpose mapping would let the picture \
|
||||||
|
disagree with the machine about who is working on what"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Files must be emitted after the phases they hang under.
|
||||||
|
///
|
||||||
|
/// The client parents a file's directory chain to the coding station, and
|
||||||
|
/// parenting is first-write-wins — so a file that reaches the browser
|
||||||
|
/// before its plan pins its whole tree at the origin permanently. Nothing
|
||||||
|
/// errors: the tree renders, in the wrong place, and reads as a layout
|
||||||
|
/// choice. Swapping the two loops back is a one-line-looking edit, which is
|
||||||
|
/// exactly why it needs a guard.
|
||||||
|
#[test]
|
||||||
|
fn files_are_emitted_after_the_phases_they_hang_under() {
|
||||||
|
let src = include_str!("world.rs");
|
||||||
|
let phases = src
|
||||||
|
.find("for p in world_phases(")
|
||||||
|
.expect("the phase emission loop");
|
||||||
|
let files = src
|
||||||
|
.find("for f in world_files(")
|
||||||
|
.expect("the file emission loop");
|
||||||
|
assert!(
|
||||||
|
phases < files,
|
||||||
|
"the phase loop must run before the file loop; files parented \
|
||||||
|
before their coding station stay at the origin forever"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -23,6 +23,7 @@
|
|||||||
//! from serving — it should stop us believing a scan that scanned nothing.
|
//! from serving — it should stop us believing a scan that scanned nothing.
|
||||||
|
|
||||||
use crate::container_exec;
|
use crate::container_exec;
|
||||||
|
use bollard::Docker;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
const PROBE_TIMEOUT: Duration = Duration::from_secs(20);
|
const PROBE_TIMEOUT: Duration = Duration::from_secs(20);
|
||||||
@@ -113,9 +114,67 @@ pub async fn probe(container: &str) -> Result<Vec<ToolStatus>, String> {
|
|||||||
};
|
};
|
||||||
out.push(status);
|
out.push(status);
|
||||||
}
|
}
|
||||||
|
out.push(probe_mission_uid_can_write(&docker, container).await);
|
||||||
Ok(out)
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Can uid 65532 actually work in the missions tree?
|
||||||
|
///
|
||||||
|
/// `container_exec` now runs every mission exec as 65532 rather than root, so
|
||||||
|
/// that no phase leaves behind files the cleanup (which runs as 65532) cannot
|
||||||
|
/// delete. That only holds while the image gives 65532 a writable `HOME` and
|
||||||
|
/// `CARGO_HOME` — and in the deployed image its default `HOME`
|
||||||
|
/// (`/zeroclaw-data`) and `/usr/local/cargo` are BOTH root-owned, which is why
|
||||||
|
/// `container_exec::mission_env` redirects them into the missions root.
|
||||||
|
///
|
||||||
|
/// If a future image moves that mount or tightens its permissions, every cargo
|
||||||
|
/// invocation starts failing for a reason no error message would connect to a
|
||||||
|
/// uid. So it is probed at boot, alongside the tools, and reported the same way.
|
||||||
|
async fn probe_mission_uid_can_write(docker: &Docker, container: &str) -> ToolStatus {
|
||||||
|
let root = crate::mission_workspace::missions_root();
|
||||||
|
let probe = root.join("_probe-uid");
|
||||||
|
// Through `exec`, not `exec_as_root`: the point is to exercise the exact
|
||||||
|
// policy real mission work gets, including the env it is given.
|
||||||
|
let argv: Vec<String> = [
|
||||||
|
"sh",
|
||||||
|
"-c",
|
||||||
|
&format!(
|
||||||
|
"set -e; mkdir -p \"$HOME\" \"$CARGO_HOME\" {p}; : > {p}/w; rm -rf {p}; echo \"uid=$(id -u) HOME=$HOME CARGO_HOME=$CARGO_HOME\"",
|
||||||
|
p = probe.display()
|
||||||
|
),
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let detail = match container_exec::exec(
|
||||||
|
docker,
|
||||||
|
container,
|
||||||
|
Some(&root.display().to_string()),
|
||||||
|
&argv,
|
||||||
|
PROBE_TIMEOUT,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(r) if r.success() => {
|
||||||
|
return ToolStatus {
|
||||||
|
program: "mission-uid".to_string(),
|
||||||
|
present: true,
|
||||||
|
detail: r.combined().trim().chars().take(120).collect(),
|
||||||
|
needed_for: "every mission exec, so no phase leaves root-owned files",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(r) => r.combined().trim().chars().take(160).collect(),
|
||||||
|
Err(e) => e.chars().take(160).collect(),
|
||||||
|
};
|
||||||
|
ToolStatus {
|
||||||
|
program: "mission-uid".to_string(),
|
||||||
|
present: false,
|
||||||
|
detail,
|
||||||
|
needed_for: "every mission exec, so no phase leaves root-owned files",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Probe at startup and write the result to stderr.
|
/// Probe at startup and write the result to stderr.
|
||||||
///
|
///
|
||||||
/// Spawned rather than awaited so a slow or absent Docker socket cannot delay
|
/// Spawned rather than awaited so a slow or absent Docker socket cannot delay
|
||||||
|
|||||||
@@ -18,22 +18,46 @@ pub fn claw_alias(claw_id: Uuid) -> String {
|
|||||||
format!("claw_{}", claw_id.simple())
|
format!("claw_{}", claw_id.simple())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The claw behind a runtime alias, or `None` if it is not one of ours.
|
||||||
|
///
|
||||||
|
/// The inverse of [`claw_alias`], and it lives beside it so the two cannot
|
||||||
|
/// drift — a changed prefix breaks the round-trip test rather than quietly
|
||||||
|
/// returning `None` for every agent and dropping their attribution.
|
||||||
|
///
|
||||||
|
/// `None` is the honest answer for `scout` and the other configured aliases
|
||||||
|
/// that are not claws: they have no row in `agents` to point at.
|
||||||
|
pub fn claw_from_alias(alias: &str) -> Option<Uuid> {
|
||||||
|
Uuid::parse_str(alias.trim().strip_prefix("claw_")?).ok()
|
||||||
|
}
|
||||||
|
|
||||||
/// Map a claw's chosen model to a configured provider alias.
|
/// Map a claw's chosen model to a configured provider alias.
|
||||||
///
|
///
|
||||||
/// Claude models resolve to `claude_cli.default`, which spawns the real
|
/// Claude models resolve to `claude_cli.default`, which spawns the real
|
||||||
/// `claude` binary against the Max subscription rather than posting to the
|
/// `claude` binary against the Max subscription rather than posting to the
|
||||||
/// raw API with Claude Code identity headers. The API-key path still exists
|
/// raw API with Claude Code identity headers. Agent work — ~99% of the
|
||||||
/// and the judge uses it deliberately (see below), but agent work — which is
|
/// tokens — belongs on the subscription and on the supported client.
|
||||||
/// ~99% of the tokens — belongs on the subscription and on the supported
|
|
||||||
/// client.
|
|
||||||
///
|
///
|
||||||
/// The judge stays on `anthropic.judge`/API key on purpose: if the
|
/// **The API-key path is gone.** `anthropic.default` and `anthropic.judge`
|
||||||
/// subscription throttles, missions degrade but verification keeps working.
|
/// were retired from the runtime config on 2026-08-10: both held `sk-ant-api`
|
||||||
/// Putting both on one credential would mean a single limit blinds the
|
/// keys on an account whose balance is zero, which the real code path reports
|
||||||
/// verifier at exactly the moment there is most to verify.
|
/// as `400 … "Your credit balance is too low"`. Every agent that named them
|
||||||
|
/// was repointed onto a live credential.
|
||||||
///
|
///
|
||||||
/// Non-Claude families are unchanged: `groq.default`, `gemini.default`, and
|
/// The independence argument that put the judge there still holds — a
|
||||||
/// the GLM/Kimi substitution below.
|
/// verifier sharing one credential with the implementer goes blind at exactly
|
||||||
|
/// the moment there is most to verify — but it is now served by a different
|
||||||
|
/// FAMILY rather than a different key: the validator runs on
|
||||||
|
/// `CLAWMATES_VALIDATOR_MODEL` (`glm:glm-4.7` on gw-04) while agents run on
|
||||||
|
/// the subscription, and `cross_provider_judge` refuses a validator in the
|
||||||
|
/// implementer's own family. `claude_cli.default` also carries
|
||||||
|
/// `fallback = ["claude_cli.kimi", "claude_cli.glm"]`, so a throttle degrades
|
||||||
|
/// across credentials instead of stopping.
|
||||||
|
///
|
||||||
|
/// Non-Claude families are unchanged: `groq.default` and the GLM/Kimi
|
||||||
|
/// substitution below. Gemini was removed entirely — a `gemini*` model now
|
||||||
|
/// falls through to the unrecognised branch, which LOGS and defaults to
|
||||||
|
/// `claude_cli.default` rather than silently routing to a provider we no
|
||||||
|
/// longer configure.
|
||||||
pub fn provider_alias_for(model: &str) -> &'static str {
|
pub fn provider_alias_for(model: &str) -> &'static str {
|
||||||
let m = model.trim().to_ascii_lowercase();
|
let m = model.trim().to_ascii_lowercase();
|
||||||
// Prefix families first (covers claude-sonnet-5, claude-opus-4-8,
|
// Prefix families first (covers claude-sonnet-5, claude-opus-4-8,
|
||||||
@@ -43,9 +67,6 @@ pub fn provider_alias_for(model: &str) -> &'static str {
|
|||||||
if m.starts_with("claude") {
|
if m.starts_with("claude") {
|
||||||
return "claude_cli.default";
|
return "claude_cli.default";
|
||||||
}
|
}
|
||||||
if m.starts_with("gemini") {
|
|
||||||
return "gemini.default";
|
|
||||||
}
|
|
||||||
return "groq.default";
|
return "groq.default";
|
||||||
}
|
}
|
||||||
match m.as_str() {
|
match m.as_str() {
|
||||||
@@ -88,7 +109,6 @@ pub fn provider_alias_for(model: &str) -> &'static str {
|
|||||||
pub fn is_exact_provider_match(model: &str) -> bool {
|
pub fn is_exact_provider_match(model: &str) -> bool {
|
||||||
let m = model.trim().to_ascii_lowercase();
|
let m = model.trim().to_ascii_lowercase();
|
||||||
m.starts_with("claude")
|
m.starts_with("claude")
|
||||||
|| m.starts_with("gemini")
|
|
||||||
|| m.starts_with("llama")
|
|| m.starts_with("llama")
|
||||||
|| m.starts_with("groq")
|
|| m.starts_with("groq")
|
||||||
}
|
}
|
||||||
@@ -363,7 +383,6 @@ mod tests {
|
|||||||
}
|
}
|
||||||
for m in [
|
for m in [
|
||||||
"claude-sonnet-5",
|
"claude-sonnet-5",
|
||||||
"gemini-2.5-flash",
|
|
||||||
"groq-llama",
|
"groq-llama",
|
||||||
"llama3",
|
"llama3",
|
||||||
] {
|
] {
|
||||||
@@ -402,8 +421,11 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn provider_alias_mapping() {
|
fn provider_alias_mapping() {
|
||||||
assert_eq!(provider_alias_for("gemini"), "gemini.default");
|
// Gemini is gone: no provider row, so it must land on the logged
|
||||||
assert_eq!(provider_alias_for("gemini-2.0-flash"), "gemini.default");
|
// default rather than a family alias that resolves to nothing.
|
||||||
|
assert_eq!(provider_alias_for("gemini"), "claude_cli.default");
|
||||||
|
assert_eq!(provider_alias_for("gemini-2.0-flash"), "claude_cli.default");
|
||||||
|
assert!(!is_exact_provider_match("gemini-2.5-flash"));
|
||||||
// glm/kimi families fall back to Claude until their own provider
|
// glm/kimi families fall back to Claude until their own provider
|
||||||
// tables are configured in the runtime template.
|
// tables are configured in the runtime template.
|
||||||
assert_eq!(provider_alias_for("GLM-4.7"), "claude_cli.default");
|
assert_eq!(provider_alias_for("GLM-4.7"), "claude_cli.default");
|
||||||
@@ -425,4 +447,19 @@ mod tests {
|
|||||||
let id = Uuid::nil();
|
let id = Uuid::nil();
|
||||||
assert_eq!(claw_alias(id), "claw_00000000000000000000000000000000");
|
assert_eq!(claw_alias(id), "claw_00000000000000000000000000000000");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The alias must round-trip, and must NOT invent a claw for one of the
|
||||||
|
/// configured non-claw aliases.
|
||||||
|
///
|
||||||
|
/// The failure this guards is silent both ways: a broken round-trip drops
|
||||||
|
/// every tool call's agent attribution (files appear, nobody moves), and a
|
||||||
|
/// too-eager parse would attribute work to a claw id that matches no row.
|
||||||
|
#[test]
|
||||||
|
fn an_alias_round_trips_to_its_claw_and_nothing_else_does() {
|
||||||
|
let id = Uuid::from_u128(0x0198_2f11_7ac0_7d51_9c3e_44a1_09b2_5e77);
|
||||||
|
assert_eq!(claw_from_alias(&claw_alias(id)), Some(id));
|
||||||
|
assert_eq!(claw_from_alias("scout"), None);
|
||||||
|
assert_eq!(claw_from_alias("claude_cli.default"), None);
|
||||||
|
assert_eq!(claw_from_alias("claw_not-a-uuid"), None);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -314,9 +314,7 @@ async fn exec_target(pool: &PgPool, mission_id: Uuid) -> Result<(String, PathBuf
|
|||||||
}
|
}
|
||||||
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
||||||
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
||||||
let root = std::env::var("CLAWMATES_MISSIONS_ROOT")
|
let workdir = crate::mission_workspace::missions_root()
|
||||||
.unwrap_or_else(|_| "/var/lib/clawmates-missions".to_string());
|
|
||||||
let workdir = PathBuf::from(root)
|
|
||||||
.join(mission_id.to_string())
|
.join(mission_id.to_string())
|
||||||
.join("repo");
|
.join("repo");
|
||||||
Ok((container, workdir))
|
Ok((container, workdir))
|
||||||
|
|||||||
@@ -0,0 +1,678 @@
|
|||||||
|
//! The Anthropic provider backed by the SUBSCRIPTION token, not the metered key.
|
||||||
|
//!
|
||||||
|
//! Two Anthropic credentials reach this server and they bill differently:
|
||||||
|
//!
|
||||||
|
//! - `ANTHROPIC_API_KEY` (`sk-ant-api…`) — metered, pay-as-you-go, and the thing
|
||||||
|
//! that runs out. Every mission VM already avoids it: `mission_runtime` sends
|
||||||
|
//! only the subscription token into a guest, deliberately.
|
||||||
|
//! - `ANTHROPIC_OAUTH_TOKEN` / `CLAUDE_CODE_OAUTH_TOKEN` (`sk-ant-oat…`) — the
|
||||||
|
//! Claude Code subscription, which is what the CLI inside every VM runs on.
|
||||||
|
//!
|
||||||
|
//! Server-side model calls that went through `Runtime::complete` with a bare
|
||||||
|
//! model name resolved to the DEFAULT provider — the metered key. So the roster
|
||||||
|
//! planner died with
|
||||||
|
//! `400 … "Your credit balance is too low to access the Anthropic API"` while
|
||||||
|
//! every mission on the same machine kept running fine on the subscription.
|
||||||
|
//! The harness reported it honestly as FAIL-NORUN rather than a passing scenario,
|
||||||
|
//! which is the only reason it was visible at all.
|
||||||
|
//!
|
||||||
|
//! This is the one place that turns the subscription token into a provider.
|
||||||
|
//! `evaluator::subscription_judge` had its own copy; there is now one.
|
||||||
|
|
||||||
|
/// The subscription-backed provider, or `None` when no usable token is present.
|
||||||
|
///
|
||||||
|
/// Checks the `sk-ant-oat` prefix rather than trusting the variable name: an
|
||||||
|
/// `sk-ant-api` key pasted into the OAuth slot would authenticate and then bill
|
||||||
|
/// the metered account, which is the failure this module exists to prevent —
|
||||||
|
/// silently, and with the same error weeks later.
|
||||||
|
pub fn provider() -> Option<cm_llm::AnthropicProvider> {
|
||||||
|
for var in ["ANTHROPIC_OAUTH_TOKEN", "CLAUDE_CODE_OAUTH_TOKEN"] {
|
||||||
|
let Ok(token) = std::env::var(var) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let token = token.trim();
|
||||||
|
if token.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !is_subscription_token(token) {
|
||||||
|
eprintln!(
|
||||||
|
"subscription: {var} is set but is not a Claude Code setup token \
|
||||||
|
(expected sk-ant-oat…) — ignoring it rather than billing the \
|
||||||
|
metered key by accident"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return Some(cm_llm::AnthropicProvider::new(token.to_string()));
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a token is a Claude Code subscription token rather than an API key.
|
||||||
|
pub fn is_subscription_token(token: &str) -> bool {
|
||||||
|
token.trim().starts_with("sk-ant-oat")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One completion on the subscription, mirroring `Runtime::complete`'s contract
|
||||||
|
/// so a caller can swap between them without reshaping its call.
|
||||||
|
///
|
||||||
|
/// Falls back to the caller's runtime when no subscription token exists, so a
|
||||||
|
/// deployment without one behaves exactly as it did before.
|
||||||
|
pub async fn complete_or(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
|
system: &str,
|
||||||
|
user: &str,
|
||||||
|
model: &str,
|
||||||
|
max_tokens: u32,
|
||||||
|
// Carried explicitly rather than defaulted. The Master Planner and the claw
|
||||||
|
// enhancer both pass `true`, and a helper that quietly dropped it would take
|
||||||
|
// web search away from two features while every test still passed.
|
||||||
|
web_search: bool,
|
||||||
|
) -> Result<String, String> {
|
||||||
|
// A `name:model` spec is an operator's explicit provider choice — the swarm
|
||||||
|
// worker model is literally configured that way (`kimi:kimi-k2.6`), and
|
||||||
|
// `Runtime::resolve_provider` honours it. Forcing that onto Anthropic would
|
||||||
|
// silently run someone's chosen model on the wrong provider, which is the
|
||||||
|
// same class of bug as this module exists to fix, only pointed the other
|
||||||
|
// way. Only a BARE name is ambiguous, and a bare name is what resolves to
|
||||||
|
// the default provider — the metered key.
|
||||||
|
if !is_bare_model_name(model) || provider().is_none() {
|
||||||
|
return runtime
|
||||||
|
.complete(system, user, model, max_tokens, web_search)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
let provider = provider().expect("checked just above");
|
||||||
|
complete_with(&provider, system, user, model, max_tokens, web_search).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a model string names a model without naming a provider.
|
||||||
|
pub fn is_bare_model_name(model: &str) -> bool {
|
||||||
|
!model.contains(':')
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How long to wait before each retry. Four attempts, ~30s of patience total.
|
||||||
|
///
|
||||||
|
/// The subscription has no credit wall, but it does have a rate limit, and a
|
||||||
|
/// roster proposal is a single one-shot call: a 429 that a browser would shrug
|
||||||
|
/// off used to fail the whole "propose a team" button. Measured on this
|
||||||
|
/// deployment — moving the roster onto the subscription turned
|
||||||
|
/// `400 credit balance too low` into `429 rate_limit_error`, i.e. a wall that
|
||||||
|
/// clears on its own became the failure mode, so waiting is the right answer.
|
||||||
|
const BACKOFF_SECS: &[u64] = &[2, 8, 20];
|
||||||
|
|
||||||
|
/// Whether an error is worth waiting out rather than reporting.
|
||||||
|
///
|
||||||
|
/// Deliberately narrow. A 400 (bad request), 401 (wrong token) or 404 (unknown
|
||||||
|
/// model) will never succeed on a retry, and retrying them turns a legible
|
||||||
|
/// error into a 30-second hang followed by the same error.
|
||||||
|
fn is_transient(e: &cm_llm::LlmError) -> bool {
|
||||||
|
use cm_llm::LlmError;
|
||||||
|
match e {
|
||||||
|
// The transport never reached Anthropic — a dropped connection or a
|
||||||
|
// DNS blip, not a rejected request.
|
||||||
|
LlmError::Transport(_) => true,
|
||||||
|
LlmError::Api(detail) => {
|
||||||
|
// `anthropic.rs` formats these as `"{status}: {body}"`.
|
||||||
|
detail.starts_with("429")
|
||||||
|
|| detail.starts_with("500")
|
||||||
|
|| detail.starts_with("502")
|
||||||
|
|| detail.starts_with("503")
|
||||||
|
|| detail.starts_with("529")
|
||||||
|
|| detail.contains("rate_limit")
|
||||||
|
|| detail.contains("overloaded")
|
||||||
|
}
|
||||||
|
LlmError::Scenario(_) | LlmError::Wire(_) => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Models to try, in order, when the requested one is rate limited.
|
||||||
|
///
|
||||||
|
/// The order is capability first, then independence:
|
||||||
|
///
|
||||||
|
/// opus -> sonnet -> haiku one account, three tiers. A throttle usually
|
||||||
|
/// hits a tier, so stepping down often clears it.
|
||||||
|
/// -> kimi -> glm two separately funded accounts. Now an
|
||||||
|
/// Anthropic outage, not just a throttle, is
|
||||||
|
/// survivable.
|
||||||
|
/// -> local our own GPU. Nothing left to be down.
|
||||||
|
///
|
||||||
|
/// Every model id here was probed on this deployment 2026-08-09 and answered
|
||||||
|
/// 200: the four Anthropic tiers on the subscription, `kimi-k2.7-code` on
|
||||||
|
/// api.kimi.com/coding, `glm-4.7` on z.ai, and `ornith-fleet:9b` on the fleet.
|
||||||
|
/// Configured is not the same as working — see `preflight`, which re-checks
|
||||||
|
/// them at boot, because a link nobody exercises is discovered broken during
|
||||||
|
/// the outage it existed for.
|
||||||
|
///
|
||||||
|
/// The last link runs on our OWN hardware. Every other entry — and every other
|
||||||
|
/// link above it — depends on somebody else's account staying funded and
|
||||||
|
/// unthrottled; `local:` depends on a GPU in the next room. It is last because
|
||||||
|
/// it is the weakest model, and present because a chain whose every link is
|
||||||
|
/// external is not a fallback chain, it is one outage in a trench coat.
|
||||||
|
///
|
||||||
|
/// Note the model half contains a colon (`ornith-fleet:9b`), which is why
|
||||||
|
/// `resolve_provider` splits on the FIRST one only.
|
||||||
|
///
|
||||||
|
/// Override with `CLAWMATES_MODEL_FALLBACK` (comma-separated). An empty value
|
||||||
|
/// disables fallback and restores plain "503 and wait".
|
||||||
|
const DEFAULT_FALLBACK: &str = "claude-sonnet-4-6,claude-haiku-4-5-20251001,\
|
||||||
|
kimi:kimi-k2.7-code,glm:glm-4.7,local:ornith-fleet:9b";
|
||||||
|
|
||||||
|
/// The chain to walk after `requested`, with `requested` itself removed so a
|
||||||
|
/// capped model is never retried as its own fallback.
|
||||||
|
pub fn fallback_chain(requested: &str) -> Vec<String> {
|
||||||
|
let raw =
|
||||||
|
std::env::var("CLAWMATES_MODEL_FALLBACK").unwrap_or_else(|_| DEFAULT_FALLBACK.to_string());
|
||||||
|
raw.split(',')
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|m| !m.is_empty() && *m != requested.trim())
|
||||||
|
.map(str::to_string)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a failure means "this model has no capacity right now" as opposed
|
||||||
|
/// to "this request was wrong".
|
||||||
|
///
|
||||||
|
/// The distinction is the whole safety of the chain: walking it on a malformed
|
||||||
|
/// prompt would ask three models the same bad question and report the third
|
||||||
|
/// one's confusion, while walking it on a rate limit is exactly the point.
|
||||||
|
pub fn is_capacity_failure(err: &str) -> bool {
|
||||||
|
err.contains("rate_limit") || err.contains("429") || err.contains("credit balance")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One completion, stepping down `fallback_chain` when a model has no capacity.
|
||||||
|
///
|
||||||
|
/// Returns the text **and the model that actually produced it**. Callers must
|
||||||
|
/// persist that second value: a plan drafted by the third link in the chain and
|
||||||
|
/// filed as an opus plan is a silent quality change, which is the failure shape
|
||||||
|
/// this project keeps paying for. Every hop is logged.
|
||||||
|
pub async fn complete_with_fallback(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
|
system: &str,
|
||||||
|
user: &str,
|
||||||
|
model: &str,
|
||||||
|
max_tokens: u32,
|
||||||
|
web_search: bool,
|
||||||
|
) -> Result<(String, String), String> {
|
||||||
|
let mut last = match complete_or(runtime, system, user, model, max_tokens, web_search).await {
|
||||||
|
Ok(text) => return Ok((text, model.to_string())),
|
||||||
|
Err(e) if is_capacity_failure(&e) => e,
|
||||||
|
// A real error. Do not launder it through two more models.
|
||||||
|
Err(e) => return Err(e),
|
||||||
|
};
|
||||||
|
for next in fallback_chain(model) {
|
||||||
|
eprintln!("model fallback: {model} has no capacity ({last}) — trying {next}");
|
||||||
|
match complete_or(runtime, system, user, &next, max_tokens, web_search).await {
|
||||||
|
Ok(text) => {
|
||||||
|
eprintln!("model fallback: {next} answered in place of {model}");
|
||||||
|
return Ok((text, next));
|
||||||
|
}
|
||||||
|
Err(e) if is_capacity_failure(&e) => last = e,
|
||||||
|
Err(e) => return Err(format!("fallback {next}: {e}")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(last)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What a probe of one link found.
|
||||||
|
///
|
||||||
|
/// `Throttled` is deliberately NOT a failure. A 429 means the spec resolved, the
|
||||||
|
/// credential authenticated, and the provider simply had no capacity this
|
||||||
|
/// second — which is the exact condition the chain exists to route around. A
|
||||||
|
/// report that painted it red would train an operator to ignore the red.
|
||||||
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
|
pub enum LinkStatus {
|
||||||
|
Answered,
|
||||||
|
Throttled(String),
|
||||||
|
/// Never came back. Its own state because it is the one that used to make
|
||||||
|
/// the whole report vanish: with no timeout, a single hung provider meant
|
||||||
|
/// silence from the tool built to prevent silence.
|
||||||
|
TimedOut,
|
||||||
|
/// The spec named a provider the registry does not have, so
|
||||||
|
/// `resolve_provider` silently fell back to the DEFAULT provider. The link
|
||||||
|
/// would "work" while running on entirely the wrong model.
|
||||||
|
Unregistered,
|
||||||
|
Broken(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LinkStatus {
|
||||||
|
pub fn usable(&self) -> bool {
|
||||||
|
matches!(self, LinkStatus::Answered | LinkStatus::Throttled(_))
|
||||||
|
}
|
||||||
|
fn label(&self) -> String {
|
||||||
|
match self {
|
||||||
|
LinkStatus::Answered => "ok".into(),
|
||||||
|
LinkStatus::Throttled(_) => "throttled (configured, no capacity now)".into(),
|
||||||
|
LinkStatus::TimedOut => {
|
||||||
|
format!("TIMED OUT after {}s — treat as down", PROBE_TIMEOUT.as_secs())
|
||||||
|
}
|
||||||
|
LinkStatus::Unregistered => "UNREGISTERED — resolves to the DEFAULT provider".into(),
|
||||||
|
LinkStatus::Broken(e) => format!("BROKEN: {}", e.chars().take(120).collect::<String>()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Probe every link of the chain, head model included.
|
||||||
|
///
|
||||||
|
/// Eight tokens each, through the SAME path a real call takes, so it proves
|
||||||
|
/// resolution and reachability rather than that a string is present in a config
|
||||||
|
/// file. The distinction matters here more than usual: `resolve_provider` falls
|
||||||
|
/// back to the default provider for an unknown provider name, so a typo in
|
||||||
|
/// `kimi:` does not error — it quietly runs on Anthropic, and the chain reads
|
||||||
|
/// as five providers while being one.
|
||||||
|
/// Per-link ceiling. Generous on purpose: `complete_or` spends up to 30s in its
|
||||||
|
/// own backoff before giving up, so anything under that would report a merely
|
||||||
|
/// throttled link as hung.
|
||||||
|
const PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
|
||||||
|
|
||||||
|
pub async fn preflight(runtime: &cm_runtime::Runtime, head: &str) -> Vec<(String, LinkStatus)> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for spec in std::iter::once(head.to_string()).chain(fallback_chain(head)) {
|
||||||
|
// A qualified spec whose provider is missing resolves to the default —
|
||||||
|
// detected the same way `cross_provider_judge` does it, by asking what
|
||||||
|
// the model half came back as.
|
||||||
|
if spec.contains(':') {
|
||||||
|
// Unrouted specs come back WHOLE; routed ones come back as the part
|
||||||
|
// after the FIRST colon. Testing "does it still contain a colon"
|
||||||
|
// reads the same and is wrong: `local:ornith-fleet:9b` resolves
|
||||||
|
// correctly to model `ornith-fleet:9b`, which does. This probe
|
||||||
|
// reported a provider the server had just registered as
|
||||||
|
// UNREGISTERED on its first live run, which is how the same latent
|
||||||
|
// bug was found in `evaluator::cross_provider_judge`.
|
||||||
|
let (_, resolved) = runtime.resolve_provider(&spec);
|
||||||
|
if resolved == spec {
|
||||||
|
out.push((spec.clone(), LinkStatus::Unregistered));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A non-empty system prompt. Kimi rejects an empty one outright —
|
||||||
|
// `400 the message at position 0 with role 'system' must not be empty` —
|
||||||
|
// so an empty probe reported a healthy provider as BROKEN on the first
|
||||||
|
// live run. The probe must look like the traffic it stands in for.
|
||||||
|
// NOT awaited here — the timeout has to wrap the FUTURE. Awaiting first
|
||||||
|
// and wrapping the result compiles, reads correctly, and bounds nothing.
|
||||||
|
let probe = complete_or(
|
||||||
|
runtime,
|
||||||
|
"You are a reachability probe.",
|
||||||
|
"Reply with exactly: OK",
|
||||||
|
&spec,
|
||||||
|
8,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
let status = match tokio::time::timeout(PROBE_TIMEOUT, probe).await {
|
||||||
|
Err(_) => LinkStatus::TimedOut,
|
||||||
|
Ok(Ok(_)) => LinkStatus::Answered,
|
||||||
|
Ok(Err(e)) if is_capacity_failure(&e) => LinkStatus::Throttled(e),
|
||||||
|
Ok(Err(e)) => LinkStatus::Broken(e),
|
||||||
|
};
|
||||||
|
// Emitted as it resolves, not collected and printed at the end. A later
|
||||||
|
// link that hangs must not be able to hide the ones already checked.
|
||||||
|
eprintln!("fallback chain: {spec:<32} {}", status.label());
|
||||||
|
out.push((spec, status));
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Probe the chain at boot and write the result to stderr.
|
||||||
|
///
|
||||||
|
/// Spawned rather than awaited, like `runtime_preflight`: this is diagnostic and
|
||||||
|
/// must never delay the server coming up. Loud when a link is unusable, because
|
||||||
|
/// the whole point of a chain is that nobody looks at it until the day it has to
|
||||||
|
/// work.
|
||||||
|
pub fn report_at_boot(runtime: cm_runtime::Runtime) {
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let head = std::env::var("CLAWMATES_PREFLIGHT_HEAD")
|
||||||
|
.unwrap_or_else(|_| "claude-opus-4-8".to_string());
|
||||||
|
let links = preflight(&runtime, &head).await;
|
||||||
|
let bad: Vec<_> = links.iter().filter(|(_, s)| !s.usable()).collect();
|
||||||
|
eprintln!(
|
||||||
|
"fallback chain ({} link(s), {} usable):",
|
||||||
|
links.len(),
|
||||||
|
links.len() - bad.len()
|
||||||
|
);
|
||||||
|
for (spec, status) in &links {
|
||||||
|
eprintln!(" {spec:<32} {}", status.label());
|
||||||
|
}
|
||||||
|
if !bad.is_empty() {
|
||||||
|
eprintln!(
|
||||||
|
"fallback chain: WARNING — {} link(s) are NOT usable. The chain is \
|
||||||
|
shorter than it reads, and the shortfall only shows up during the \
|
||||||
|
outage it exists for.",
|
||||||
|
bad.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Turn a `complete_or` failure into the right API error.
|
||||||
|
///
|
||||||
|
/// A rate limit that outlived the backoff is not a bug in this server, and
|
||||||
|
/// reporting it as one costs an operator a trip through the logs to find out
|
||||||
|
/// the answer was "wait". Measured: a bare 16-token probe with the same token
|
||||||
|
/// returned 429 with `x-should-retry: true` — Anthropic itself says try again.
|
||||||
|
pub fn as_api_error(err: &str) -> crate::error::ApiError {
|
||||||
|
if err.contains("rate_limit") || err.contains("429") {
|
||||||
|
return crate::error::ApiError::Unavailable(
|
||||||
|
"the Claude Code subscription is rate limited right now — this \
|
||||||
|
clears on its own; try again shortly"
|
||||||
|
.into(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
crate::error::ApiError::Internal
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stream one request and collect its text, waiting out transient failures.
|
||||||
|
async fn complete_with(
|
||||||
|
provider: &cm_llm::AnthropicProvider,
|
||||||
|
system: &str,
|
||||||
|
user: &str,
|
||||||
|
model: &str,
|
||||||
|
max_tokens: u32,
|
||||||
|
web_search: bool,
|
||||||
|
) -> Result<String, String> {
|
||||||
|
let mut attempt = 0usize;
|
||||||
|
loop {
|
||||||
|
match attempt_once(provider, system, user, model, max_tokens, web_search).await {
|
||||||
|
Ok(text) => return Ok(text),
|
||||||
|
Err((stage, e)) => {
|
||||||
|
let Some(delay) = BACKOFF_SECS.get(attempt).copied().filter(|_| is_transient(&e))
|
||||||
|
else {
|
||||||
|
return Err(format!("subscription {stage}: {e}"));
|
||||||
|
};
|
||||||
|
eprintln!(
|
||||||
|
"subscription {stage}: {e} — retrying in {delay}s \
|
||||||
|
(attempt {} of {})",
|
||||||
|
attempt + 2,
|
||||||
|
BACKOFF_SECS.len() + 1
|
||||||
|
);
|
||||||
|
tokio::time::sleep(std::time::Duration::from_secs(delay)).await;
|
||||||
|
attempt += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One attempt. The collected text is discarded on failure, so a retry never
|
||||||
|
/// concatenates a partial answer onto a whole one.
|
||||||
|
async fn attempt_once(
|
||||||
|
provider: &cm_llm::AnthropicProvider,
|
||||||
|
system: &str,
|
||||||
|
user: &str,
|
||||||
|
model: &str,
|
||||||
|
max_tokens: u32,
|
||||||
|
web_search: bool,
|
||||||
|
) -> Result<String, (&'static str, cm_llm::LlmError)> {
|
||||||
|
use cm_llm::{ChatMessage, ChatRequest, ChatRole, ContentPart, LlmEvent, LlmProvider};
|
||||||
|
use futures::StreamExt as _;
|
||||||
|
|
||||||
|
let request = ChatRequest {
|
||||||
|
system: system.to_string(),
|
||||||
|
model: model.to_string(),
|
||||||
|
messages: vec![ChatMessage {
|
||||||
|
role: ChatRole::User,
|
||||||
|
parts: vec![ContentPart::text(user)],
|
||||||
|
}],
|
||||||
|
tools: vec![],
|
||||||
|
max_tokens,
|
||||||
|
web_search,
|
||||||
|
};
|
||||||
|
let mut stream = provider.stream(request).await.map_err(|e| ("call", e))?;
|
||||||
|
let mut text = String::new();
|
||||||
|
while let Some(event) = stream.next().await {
|
||||||
|
match event {
|
||||||
|
Ok(LlmEvent::TextDelta(t)) => text.push_str(&t),
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(e) => return Err(("stream", e)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(text)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// Every server-side model call that should be on the subscription IS.
|
||||||
|
///
|
||||||
|
/// `validator_preflight` is the deliberate exception: it probes whatever
|
||||||
|
/// spec an operator configured (today `glm:glm-4.7`), and forcing it onto
|
||||||
|
/// Anthropic would make it prove the wrong thing — it exists to answer "is
|
||||||
|
/// the configured validator reachable".
|
||||||
|
/// The first version of this test grepped for the literal
|
||||||
|
/// `runtime.complete(` and passed while FOUR more call sites — the phase
|
||||||
|
/// planner, both swarm calls, and a second enhance path — still billed the
|
||||||
|
/// metered key. They were spelled `state.runtime` or wrapped across lines,
|
||||||
|
/// so the receiver name was never the thing to look for. Match the METHOD.
|
||||||
|
#[test]
|
||||||
|
fn no_server_side_call_silently_uses_the_metered_key() {
|
||||||
|
let sources = [
|
||||||
|
("routes/mission_roster.rs", include_str!("routes/mission_roster.rs")),
|
||||||
|
("routes/mission_plan.rs", include_str!("routes/mission_plan.rs")),
|
||||||
|
("routes/planner.rs", include_str!("routes/planner.rs")),
|
||||||
|
("routes/claws.rs", include_str!("routes/claws.rs")),
|
||||||
|
("swarm.rs", include_str!("swarm.rs")),
|
||||||
|
];
|
||||||
|
for (name, src) in sources {
|
||||||
|
assert!(
|
||||||
|
!src.contains(".complete("),
|
||||||
|
"{name} calls Runtime::complete directly — a bare model name there \
|
||||||
|
resolves to the DEFAULT provider, which is the metered API key. \
|
||||||
|
Use `subscription::complete_or`, which passes a `name:model` \
|
||||||
|
spec through untouched."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// And the exception stays an exception, on purpose.
|
||||||
|
assert!(
|
||||||
|
include_str!("validator_preflight.rs").contains("runtime.complete("),
|
||||||
|
"validator_preflight must keep probing the CONFIGURED spec"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Only errors that can clear on their own are waited out.
|
||||||
|
///
|
||||||
|
/// The negative half is the point: a 400 or a 401 retried three times is a
|
||||||
|
/// 30-second hang ending in the identical message, which reads as a stall
|
||||||
|
/// rather than a bad request — the failure mode this project keeps hitting.
|
||||||
|
#[test]
|
||||||
|
fn a_wall_that_clears_is_waited_out_and_one_that_does_not_is_not() {
|
||||||
|
use cm_llm::LlmError;
|
||||||
|
let api = |s: &str| LlmError::Api(s.to_string());
|
||||||
|
|
||||||
|
assert!(is_transient(&api(
|
||||||
|
"429 Too Many Requests: {\"type\":\"rate_limit_error\"}"
|
||||||
|
)));
|
||||||
|
assert!(is_transient(&api("529: overloaded_error")));
|
||||||
|
assert!(is_transient(&api("503 Service Unavailable")));
|
||||||
|
assert!(is_transient(&LlmError::Transport("connection reset".into())));
|
||||||
|
|
||||||
|
// The exact error that started this: it never clears by waiting, it
|
||||||
|
// clears by moving to the other credential — which is now done.
|
||||||
|
assert!(!is_transient(&api(
|
||||||
|
"400 Bad Request: Your credit balance is too low"
|
||||||
|
)));
|
||||||
|
assert!(!is_transient(&api("401 Unauthorized: invalid x-api-key")));
|
||||||
|
assert!(!is_transient(&api("404 Not Found: model not found")));
|
||||||
|
assert!(!is_transient(&LlmError::Wire("bad json".into())));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Nobody hand-rolls their own Anthropic HTTP call.
|
||||||
|
///
|
||||||
|
/// `phase_summarizer` did — its own `reqwest` POST to `api.anthropic.com`
|
||||||
|
/// with `x-api-key: $ANTHROPIC_API_KEY`. No audit of `.complete(` call
|
||||||
|
/// sites could ever have found it, and it was the last thing on this
|
||||||
|
/// deployment still billing an account with no credit: every phase summary
|
||||||
|
/// died with "credit balance is too low" while the phases themselves ran.
|
||||||
|
/// A call site is only routable if it goes through a provider, so walk the
|
||||||
|
/// whole crate rather than a hand-listed set of files.
|
||||||
|
#[test]
|
||||||
|
fn no_module_talks_to_anthropic_behind_the_providers_back() {
|
||||||
|
fn walk(dir: &std::path::Path, out: &mut Vec<std::path::PathBuf>) {
|
||||||
|
for entry in std::fs::read_dir(dir).expect("readable source dir") {
|
||||||
|
let path = entry.expect("readable entry").path();
|
||||||
|
if path.is_dir() {
|
||||||
|
walk(&path, out);
|
||||||
|
} else if path.extension().is_some_and(|e| e == "rs") {
|
||||||
|
out.push(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
|
||||||
|
let mut files = Vec::new();
|
||||||
|
walk(&root, &mut files);
|
||||||
|
assert!(files.len() > 20, "source walk found suspiciously few files");
|
||||||
|
|
||||||
|
for path in files {
|
||||||
|
// This module names the host in prose; it is the one that may.
|
||||||
|
if path.ends_with("subscription.rs") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let src = std::fs::read_to_string(&path).expect("readable source");
|
||||||
|
for needle in ["api.anthropic.com", "\"x-api-key\""] {
|
||||||
|
assert!(
|
||||||
|
!src.contains(needle),
|
||||||
|
"{} contains {needle} — build the request through cm_llm and \
|
||||||
|
route it via `subscription::complete_or`, so credential \
|
||||||
|
choice and the capacity fallback live in ONE place",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A model name may contain a colon, and "unregistered" must not mean that.
|
||||||
|
///
|
||||||
|
/// `resolve_provider` returns the spec unchanged when it does not recognise
|
||||||
|
/// the provider and the part after the FIRST colon when it does. The obvious
|
||||||
|
/// test — "does the model half still contain a colon" — reads the same and
|
||||||
|
/// is wrong the moment a model id has one. `ornith-fleet:9b` has one, and
|
||||||
|
/// the live preflight reported a provider the server had just registered as
|
||||||
|
/// UNREGISTERED. The identical bug was in `cross_provider_judge`, where it
|
||||||
|
/// would have refused a perfectly good independent judge.
|
||||||
|
#[test]
|
||||||
|
fn a_colon_in_the_model_name_is_not_a_missing_provider() {
|
||||||
|
// What `resolve_provider` returns in each case.
|
||||||
|
fn routed(spec: &str) -> &str {
|
||||||
|
spec.split_once(':').map(|(_, m)| m).unwrap_or(spec)
|
||||||
|
}
|
||||||
|
|
||||||
|
for spec in ["local:ornith-fleet:9b", "glm:glm-4.7", "kimi:kimi-k2.7-code"] {
|
||||||
|
assert_ne!(routed(spec), spec, "{spec} routed must not equal the whole spec");
|
||||||
|
}
|
||||||
|
// An unrecognised provider comes back WHOLE — the only true signal.
|
||||||
|
assert_eq!(routed("nosuch"), "nosuch");
|
||||||
|
// And the case that made the naive colon test look correct for so long.
|
||||||
|
assert!(routed("local:ornith-fleet:9b").contains(':'));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A throttled link is usable; an unregistered one is not.
|
||||||
|
///
|
||||||
|
/// The second is the dangerous one and the reason `preflight` checks
|
||||||
|
/// resolution separately from reachability. `resolve_provider` falls back to
|
||||||
|
/// the DEFAULT provider when it does not recognise a provider name, so a
|
||||||
|
/// typo in `kimi:` does not error — it quietly runs on Anthropic, and a
|
||||||
|
/// chain that reads as three accounts is really one. A reachability-only
|
||||||
|
/// probe would call that link green.
|
||||||
|
#[test]
|
||||||
|
fn only_a_link_that_could_never_answer_counts_as_unusable() {
|
||||||
|
assert!(LinkStatus::Answered.usable());
|
||||||
|
assert!(LinkStatus::Throttled("429 rate_limit".into()).usable());
|
||||||
|
|
||||||
|
assert!(!LinkStatus::Unregistered.usable());
|
||||||
|
assert!(!LinkStatus::Broken("401 invalid key".into()).usable());
|
||||||
|
|
||||||
|
// The labels must not read alike: "throttled" is a wait and
|
||||||
|
// "unregistered" is a config bug, and an operator acts differently on
|
||||||
|
// each.
|
||||||
|
assert!(LinkStatus::Throttled(String::new()).label().contains("configured"));
|
||||||
|
assert!(LinkStatus::Unregistered.label().contains("DEFAULT provider"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The chain never retries the capped model as its own fallback.
|
||||||
|
///
|
||||||
|
/// Without the filter, asking for haiku while haiku is capped would try
|
||||||
|
/// haiku, fail, and try haiku again — a chain that looks like resilience
|
||||||
|
/// and delivers none.
|
||||||
|
#[test]
|
||||||
|
fn the_chain_excludes_the_model_that_just_failed() {
|
||||||
|
// No env override in scope: this asserts the SHIPPED default.
|
||||||
|
let chain = fallback_chain("claude-opus-4-8");
|
||||||
|
assert_eq!(
|
||||||
|
chain,
|
||||||
|
vec![
|
||||||
|
"claude-sonnet-4-6",
|
||||||
|
"claude-haiku-4-5-20251001",
|
||||||
|
"kimi:kimi-k2.7-code",
|
||||||
|
"glm:glm-4.7",
|
||||||
|
"local:ornith-fleet:9b",
|
||||||
|
]
|
||||||
|
);
|
||||||
|
// Three providers behind five links. A chain that steps down three
|
||||||
|
// Anthropic tiers and stops is a tier ladder, not a fallback chain: one
|
||||||
|
// account being unreachable would end it.
|
||||||
|
let families: std::collections::BTreeSet<_> = chain
|
||||||
|
.iter()
|
||||||
|
.map(|m| m.split_once(':').map(|(p, _)| p).unwrap_or("anthropic"))
|
||||||
|
.collect();
|
||||||
|
assert!(
|
||||||
|
families.len() >= 3,
|
||||||
|
"the chain must span more than one account, got {families:?}"
|
||||||
|
);
|
||||||
|
// The last link must survive `resolve_provider`'s split, which takes the
|
||||||
|
// FIRST colon only — `local:ornith-fleet:9b` is provider `local`, model
|
||||||
|
// `ornith-fleet:9b`, and a split on the last colon would ask for a
|
||||||
|
// provider named `local:ornith-fleet`.
|
||||||
|
let last = chain.last().unwrap();
|
||||||
|
let (provider, model) = last.split_once(':').expect("a provider-qualified spec");
|
||||||
|
assert_eq!(provider, "local");
|
||||||
|
assert_eq!(model, "ornith-fleet:9b");
|
||||||
|
assert!(!fallback_chain("claude-haiku-4-5-20251001")
|
||||||
|
.iter()
|
||||||
|
.any(|m| m == "claude-haiku-4-5-20251001"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The chain is walked for "no capacity" and NOT for "bad request".
|
||||||
|
///
|
||||||
|
/// Walking it on a malformed prompt would ask three models the same bad
|
||||||
|
/// question and report the third one's confusion as the answer, burning
|
||||||
|
/// the two credentials that still work in order to hide the real error.
|
||||||
|
#[test]
|
||||||
|
fn only_a_capacity_failure_steps_down_the_chain() {
|
||||||
|
assert!(is_capacity_failure(
|
||||||
|
"subscription call: provider returned an error: 429 Too Many Requests"
|
||||||
|
));
|
||||||
|
assert!(is_capacity_failure("rate_limit_error"));
|
||||||
|
// The metered key's wall counts too — same meaning, different wording.
|
||||||
|
assert!(is_capacity_failure(
|
||||||
|
"400: Your credit balance is too low to access the Anthropic API"
|
||||||
|
));
|
||||||
|
|
||||||
|
assert!(!is_capacity_failure("400: messages.0: text content is empty"));
|
||||||
|
assert!(!is_capacity_failure("401: invalid x-api-key"));
|
||||||
|
assert!(!is_capacity_failure("404: model not found"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An operator's explicit provider choice is never hijacked.
|
||||||
|
///
|
||||||
|
/// The swarm worker model is a configured `name:model` spec. Routing that
|
||||||
|
/// onto the subscription would run someone's chosen Kimi or GLM model on
|
||||||
|
/// Anthropic and report success — the same silent-substitution bug as the
|
||||||
|
/// metered key, aimed the other way.
|
||||||
|
#[test]
|
||||||
|
fn a_provider_qualified_spec_is_left_alone() {
|
||||||
|
assert!(is_bare_model_name("claude-opus-4-8"));
|
||||||
|
assert!(is_bare_model_name("claude-haiku-4-5-20251001"));
|
||||||
|
assert!(!is_bare_model_name("kimi:kimi-k2.6"));
|
||||||
|
assert!(!is_bare_model_name("glm:glm-4.7"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A metered key in the OAuth slot must be REFUSED, not used.
|
||||||
|
///
|
||||||
|
/// Accepting it would authenticate, work, and bill the pay-as-you-go account
|
||||||
|
/// — the exact bill this module exists to stop, discovered weeks later when
|
||||||
|
/// it runs out mid-mission.
|
||||||
|
#[test]
|
||||||
|
fn only_a_setup_token_counts_as_the_subscription() {
|
||||||
|
assert!(is_subscription_token("sk-ant-oat01-abc"));
|
||||||
|
assert!(!is_subscription_token("sk-ant-api03-abc"));
|
||||||
|
assert!(!is_subscription_token(""));
|
||||||
|
assert!(!is_subscription_token("oat-but-not-anthropic"));
|
||||||
|
}
|
||||||
|
}
|
||||||
+33
-13
@@ -122,9 +122,11 @@ pub async fn run_swarm_job(
|
|||||||
let worker_model = resolve_worker_model(&job.worker_model);
|
let worker_model = resolve_worker_model(&job.worker_model);
|
||||||
|
|
||||||
// 1) PLAN — Opus decomposes the goal into worker tasks.
|
// 1) PLAN — Opus decomposes the goal into worker tasks.
|
||||||
|
// This record is written BEFORE the call, so it cannot name the model that
|
||||||
|
// answers. The record after the call can, and does.
|
||||||
records.push(step(
|
records.push(step(
|
||||||
"planner",
|
"planner",
|
||||||
"planner:opus",
|
"planner",
|
||||||
StepPhase::Plan,
|
StepPhase::Plan,
|
||||||
format!("Planning tasks for: {goal}"),
|
format!("Planning tasks for: {goal}"),
|
||||||
));
|
));
|
||||||
@@ -137,9 +139,18 @@ pub async fn run_swarm_job(
|
|||||||
"GOAL:\n{goal}\n\nCHECKLIST each task's output must satisfy:\n{}{want}",
|
"GOAL:\n{goal}\n\nCHECKLIST each task's output must satisfy:\n{}{want}",
|
||||||
checklist_lines(&checklist)
|
checklist_lines(&checklist)
|
||||||
);
|
);
|
||||||
let plan_raw = runtime
|
// The recorded role says which model ANSWERED. When opus is capped the
|
||||||
.complete(PLAN_SYSTEM, &plan_user, "claude-opus-4-8", 4000, false)
|
// chain steps down, and a step labelled "planner:opus" that GLM wrote is a
|
||||||
.await?;
|
// lie in the one place an operator looks to explain a bad decomposition.
|
||||||
|
let (plan_raw, plan_model) = crate::subscription::complete_with_fallback(
|
||||||
|
runtime,
|
||||||
|
PLAN_SYSTEM,
|
||||||
|
&plan_user,
|
||||||
|
"claude-opus-4-8",
|
||||||
|
4000,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
let tasks: Vec<String> = extract_json(&plan_raw)
|
let tasks: Vec<String> = extract_json(&plan_raw)
|
||||||
.and_then(|v| {
|
.and_then(|v| {
|
||||||
v.get("tasks").and_then(|t| t.as_array()).map(|a| {
|
v.get("tasks").and_then(|t| t.as_array()).map(|a| {
|
||||||
@@ -154,7 +165,7 @@ pub async fn run_swarm_job(
|
|||||||
}
|
}
|
||||||
records.push(step(
|
records.push(step(
|
||||||
"planner",
|
"planner",
|
||||||
"planner:opus",
|
format!("planner:{plan_model}"),
|
||||||
StepPhase::Plan,
|
StepPhase::Plan,
|
||||||
format!(
|
format!(
|
||||||
"Decomposed into {} tasks. Workers: {worker_model}. Verifier: claude-opus-4-8.",
|
"Decomposed into {} tasks. Workers: {worker_model}. Verifier: claude-opus-4-8.",
|
||||||
@@ -177,10 +188,12 @@ pub async fn run_swarm_job(
|
|||||||
let mut still: Vec<(usize, String)> = Vec::new();
|
let mut still: Vec<(usize, String)> = Vec::new();
|
||||||
let mut rejected = 0usize;
|
let mut rejected = 0usize;
|
||||||
for (idx, task) in pending.iter() {
|
for (idx, task) in pending.iter() {
|
||||||
let out = runtime
|
// `worker_model` may be a `name:model` spec the operator chose;
|
||||||
.complete(&wsys, task, &worker_model, 4000, true)
|
// `complete_or` passes those straight through untouched.
|
||||||
.await
|
let out =
|
||||||
.unwrap_or_else(|e| format!("worker error: {e}"));
|
crate::subscription::complete_or(runtime, &wsys, task, &worker_model, 4000, true)
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|e| format!("worker error: {e}"));
|
||||||
records.push(step(
|
records.push(step(
|
||||||
format!("task-{idx}"),
|
format!("task-{idx}"),
|
||||||
format!("worker:{worker_model}"),
|
format!("worker:{worker_model}"),
|
||||||
@@ -190,10 +203,17 @@ pub async fn run_swarm_job(
|
|||||||
ckpt(pool, id, &records, &totals).await;
|
ckpt(pool, id, &records, &totals).await;
|
||||||
|
|
||||||
let vuser = format!("TASK:\n{task}\n\nWORKER OUTPUT:\n{out}");
|
let vuser = format!("TASK:\n{task}\n\nWORKER OUTPUT:\n{out}");
|
||||||
let v_raw = runtime
|
let v_raw = crate::subscription::complete_with_fallback(
|
||||||
.complete(&vsys, &vuser, "claude-opus-4-8", 1200, true)
|
runtime,
|
||||||
.await
|
&vsys,
|
||||||
.unwrap_or_default();
|
&vuser,
|
||||||
|
"claude-opus-4-8",
|
||||||
|
1200,
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map(|(text, _)| text)
|
||||||
|
.unwrap_or_default();
|
||||||
let v = extract_json(&v_raw);
|
let v = extract_json(&v_raw);
|
||||||
let passed = v
|
let passed = v
|
||||||
.as_ref()
|
.as_ref()
|
||||||
|
|||||||
@@ -69,6 +69,11 @@ struct TemplateRoleFile {
|
|||||||
skills: Vec<String>,
|
skills: Vec<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
brain_seed: Option<String>,
|
brain_seed: Option<String>,
|
||||||
|
/// Which model this role's claw runs on. Omitted means the mint's default,
|
||||||
|
/// which is what every authored template does today — so adding the field
|
||||||
|
/// changes nothing until a template uses it.
|
||||||
|
#[serde(default)]
|
||||||
|
model: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn templates_dir() -> PathBuf {
|
fn templates_dir() -> PathBuf {
|
||||||
@@ -137,6 +142,7 @@ async fn load_one(pool: &PgPool, path: &std::path::Path) -> Result<String, Strin
|
|||||||
system_prompt: &r.system_prompt,
|
system_prompt: &r.system_prompt,
|
||||||
skills: r.skills.clone(),
|
skills: r.skills.clone(),
|
||||||
brain_seed: r.brain_seed.as_deref(),
|
brain_seed: r.brain_seed.as_deref(),
|
||||||
|
model: r.model.as_deref(),
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
|
|||||||
@@ -24,15 +24,22 @@ use tokio::sync::Mutex;
|
|||||||
use tokio_tungstenite::connect_async;
|
use tokio_tungstenite::connect_async;
|
||||||
use tokio_tungstenite::tungstenite::Message;
|
use tokio_tungstenite::tungstenite::Message;
|
||||||
|
|
||||||
/// Overall wall-clock budget for draining one turn's event stream. Must
|
/// Overall wall-clock budget for draining one turn's event stream.
|
||||||
/// exceed the daemon's own claude_cli provider timeout (600s on gw-04
|
///
|
||||||
/// via ZEROCLAW_providers__models__claude_cli__default__timeout_ms) —
|
/// A turn is an agent LOOP, not one model call. Each call inside it is bounded
|
||||||
/// otherwise the executor kills the ws before the daemon can reply and
|
/// separately by the daemon — `claude_cli`'s `timeout_secs`, 600s on gw-04 —
|
||||||
/// we see a phantom "turn timed out" while the daemon still logs a
|
/// so this has to cover however many calls the loop makes, not one of them.
|
||||||
/// successful llm response coming back. 700s gives 100s of headroom so
|
///
|
||||||
/// a daemon that just barely made it under its own limit doesn't lose
|
/// It was 700s, which is 100s more than a single call may take. MEASURED: a
|
||||||
/// its answer here.
|
/// healthy research turn is ~157s, but a throttled one blew the budget with one
|
||||||
const TURN_TIMEOUT: Duration = Duration::from_secs(700);
|
/// slow call plus a second, and the executor killed it mid-flight after 11m43s
|
||||||
|
/// with no error from the daemon — because nothing had failed yet. All the
|
||||||
|
/// operator got was "turn timed out".
|
||||||
|
///
|
||||||
|
/// An hour matches the phase's own budget. A genuinely stuck CALL is still
|
||||||
|
/// caught at 600s by the daemon and surfaces as a real error; this only stops
|
||||||
|
/// us killing turns that are working, slowly.
|
||||||
|
const TURN_TIMEOUT: Duration = Duration::from_secs(3600);
|
||||||
|
|
||||||
/// Drives ZeroClaw role-agents (in one container) to execute topology turns.
|
/// Drives ZeroClaw role-agents (in one container) to execute topology turns.
|
||||||
pub struct ZeroClawDriveExecutor {
|
pub struct ZeroClawDriveExecutor {
|
||||||
@@ -47,6 +54,53 @@ pub struct ZeroClawDriveExecutor {
|
|||||||
/// Bearer token, paired lazily and reused across turns.
|
/// Bearer token, paired lazily and reused across turns.
|
||||||
token: Arc<Mutex<Option<String>>>,
|
token: Arc<Mutex<Option<String>>>,
|
||||||
http: reqwest::Client,
|
http: reqwest::Client,
|
||||||
|
/// Where this executor's turns record what they did. `None` on every path
|
||||||
|
/// that is not a mission phase (the governor, the door, the evaluator) —
|
||||||
|
/// those turns belong to no phase and have nothing to attribute to.
|
||||||
|
tap: Option<Arc<MissionTap>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Where a turn's tool activity is written, and what it belongs to.
|
||||||
|
///
|
||||||
|
/// Carried on the executor rather than passed per turn because `TurnRequest`
|
||||||
|
/// is the shared orchestrator contract: threading a mission id through it would
|
||||||
|
/// put mission concepts into every tier that has no missions.
|
||||||
|
pub struct MissionTap {
|
||||||
|
pub pool: sqlx::PgPool,
|
||||||
|
pub mission_id: uuid::Uuid,
|
||||||
|
pub phase_id: Option<uuid::Uuid>,
|
||||||
|
pub run_id: Option<uuid::Uuid>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One tool call, as the frame stream reported it.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct ToolCall {
|
||||||
|
pub tool: String,
|
||||||
|
/// The path the tool's **arguments** named, if any. Never extracted from a
|
||||||
|
/// prose summary — see [`crate::mission_events::tool_path`].
|
||||||
|
pub path: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What one turn's frames said about the work, beside its text.
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||||
|
pub struct ToolTrace {
|
||||||
|
pub calls: Vec<ToolCall>,
|
||||||
|
/// Frame `type` values this drain did not recognise, counted.
|
||||||
|
///
|
||||||
|
/// Shipped in the same change as the tap on purpose: the frame name was
|
||||||
|
/// taken from a comment in this file rather than from a captured frame. If
|
||||||
|
/// the runtime called it something else, the tap would record nothing and
|
||||||
|
/// nothing anywhere would error — the World would simply stay as sparse as
|
||||||
|
/// it was before.
|
||||||
|
///
|
||||||
|
/// MEASURED on gw-04 (v0.8.3, 2026-08-11): a mission turn's stream carried
|
||||||
|
/// `chunk`, `done` and `session_start` and no tool frames at all. That is
|
||||||
|
/// not a protocol mismatch — `tool_call` is in the deployed binary
|
||||||
|
/// (`zeroclaw-gateway/src/ws.rs` emits `{"type":"tool_call","id","name",
|
||||||
|
/// "args"}`) — it is §15: these agents are provisioned tool-free behind the
|
||||||
|
/// MCP door, so they call nothing. The histogram is what let us tell those
|
||||||
|
/// two apart, which was its whole purpose.
|
||||||
|
pub unmatched: std::collections::BTreeMap<String, u32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ZeroClawDriveExecutor {
|
impl ZeroClawDriveExecutor {
|
||||||
@@ -64,9 +118,17 @@ impl ZeroClawDriveExecutor {
|
|||||||
default_alias,
|
default_alias,
|
||||||
token: Arc::new(Mutex::new(None)),
|
token: Arc::new(Mutex::new(None)),
|
||||||
http: reqwest::Client::new(),
|
http: reqwest::Client::new(),
|
||||||
|
tap: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Attach the mission this executor's turns belong to, so their tool calls
|
||||||
|
/// are recorded. Without it the executor behaves exactly as it did.
|
||||||
|
pub fn with_tap(mut self, tap: MissionTap) -> Self {
|
||||||
|
self.tap = Some(Arc::new(tap));
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
/// Build from the environment:
|
/// Build from the environment:
|
||||||
/// - `ZEROCLAW_GATEWAY_URL` (required) e.g. `http://127.0.0.1:42617`
|
/// - `ZEROCLAW_GATEWAY_URL` (required) e.g. `http://127.0.0.1:42617`
|
||||||
/// - `ZEROCLAW_TOKEN` (preferred) a durable bearer token — pair once
|
/// - `ZEROCLAW_TOKEN` (preferred) a durable bearer token — pair once
|
||||||
@@ -196,6 +258,19 @@ impl ZeroClawDriveExecutor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn drive(&self, alias: &str, prompt: &str) -> Result<TurnOutcome, OrchestratorError> {
|
pub async fn drive(&self, alias: &str, prompt: &str) -> Result<TurnOutcome, OrchestratorError> {
|
||||||
|
self.drive_traced(alias, prompt).await.map(|(o, _)| o)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// [`Self::drive`], also returning what the turn's frames said it did.
|
||||||
|
///
|
||||||
|
/// Exists so the tool tap is testable at all: `drive` discards the trace
|
||||||
|
/// after recording it, and a tap whose extraction is never asserted is
|
||||||
|
/// exactly the kind of code that silently records nothing.
|
||||||
|
pub(crate) async fn drive_traced(
|
||||||
|
&self,
|
||||||
|
alias: &str,
|
||||||
|
prompt: &str,
|
||||||
|
) -> Result<(TurnOutcome, ToolTrace), OrchestratorError> {
|
||||||
let token = self.ensure_paired().await?;
|
let token = self.ensure_paired().await?;
|
||||||
let ws_base = if let Some(rest) = self.gateway_url.strip_prefix("https") {
|
let ws_base = if let Some(rest) = self.gateway_url.strip_prefix("https") {
|
||||||
format!("wss{rest}")
|
format!("wss{rest}")
|
||||||
@@ -219,11 +294,102 @@ impl ZeroClawDriveExecutor {
|
|||||||
.await
|
.await
|
||||||
.map_err(|e| OrchestratorError::Executor(format!("ws send failed: {e}")))?;
|
.map_err(|e| OrchestratorError::Executor(format!("ws send failed: {e}")))?;
|
||||||
|
|
||||||
let outcome = tokio::time::timeout(TURN_TIMEOUT, Self::drain(&mut ws))
|
let (outcome, trace) = match tokio::time::timeout(TURN_TIMEOUT, Self::drain(&mut ws)).await
|
||||||
.await
|
{
|
||||||
.map_err(|_| OrchestratorError::Executor("turn timed out".into()))??;
|
Ok(res) => res?,
|
||||||
|
Err(_) => {
|
||||||
|
// "turn timed out" on its own is unactionable, and the one place
|
||||||
|
// the reason lives — the per-mission runtime container — is torn
|
||||||
|
// down after the phase, taking its log with it. Read the tail
|
||||||
|
// while it still exists.
|
||||||
|
//
|
||||||
|
// MEASURED: a research phase timed out at exactly 700s having
|
||||||
|
// produced zero steps and zero output, and the container was
|
||||||
|
// already gone by the time anyone looked. All that survived was
|
||||||
|
// the string.
|
||||||
|
let container = self.container_name();
|
||||||
|
let tail = match &container {
|
||||||
|
Some(c) => crate::container_exec::tail_logs(c, 40).await,
|
||||||
|
None => "(could not derive the container name from the gateway url)".into(),
|
||||||
|
};
|
||||||
|
return Err(OrchestratorError::Executor(format!(
|
||||||
|
"turn timed out after {}s driving agent {alias} on {} — the agent \
|
||||||
|
never finished a turn. Last lines from {}:\n{tail}",
|
||||||
|
TURN_TIMEOUT.as_secs(),
|
||||||
|
self.gateway_url,
|
||||||
|
container.as_deref().unwrap_or("its runtime container"),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
};
|
||||||
let _ = ws.close(None).await;
|
let _ = ws.close(None).await;
|
||||||
Ok(outcome)
|
self.record_trace(alias, &trace).await;
|
||||||
|
Ok((outcome, trace))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persist what this turn's frames said the agent did.
|
||||||
|
///
|
||||||
|
/// Best-effort and after the fact: a telemetry write must not be able to
|
||||||
|
/// fail a turn that already succeeded.
|
||||||
|
async fn record_trace(&self, alias: &str, trace: &ToolTrace) {
|
||||||
|
if !trace.unmatched.is_empty() {
|
||||||
|
// Logged whether or not a tap is attached — the point is to learn
|
||||||
|
// the real frame names, and the paths with no tap see the same
|
||||||
|
// stream.
|
||||||
|
eprintln!(
|
||||||
|
"topology_exec: unmatched frame types this turn ({alias}): {:?}",
|
||||||
|
trace.unmatched
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let Some(tap) = self.tap.as_ref() else { return };
|
||||||
|
if trace.calls.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let agent_id = crate::runtime_provision::claw_from_alias(alias);
|
||||||
|
let event = |kind: &str, target: String, detail: serde_json::Value| {
|
||||||
|
crate::mission_events::MissionEvent {
|
||||||
|
mission_id: tap.mission_id,
|
||||||
|
phase_id: tap.phase_id,
|
||||||
|
run_id: tap.run_id,
|
||||||
|
agent_id,
|
||||||
|
kind: kind.to_string(),
|
||||||
|
target: Some(target),
|
||||||
|
detail,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let mut events = Vec::new();
|
||||||
|
for call in &trace.calls {
|
||||||
|
events.push(event(
|
||||||
|
crate::mission_events::TOOL_CALL,
|
||||||
|
call.tool.clone(),
|
||||||
|
serde_json::Value::Null,
|
||||||
|
));
|
||||||
|
// A file touch is a SECOND event, not a replacement: the tool call
|
||||||
|
// happened whether or not we could name a path in its arguments,
|
||||||
|
// and collapsing the two would make every unparseable tool call
|
||||||
|
// disappear from the record entirely.
|
||||||
|
if let Some(path) = &call.path {
|
||||||
|
events.push(event(
|
||||||
|
crate::mission_events::FILE_TOUCH,
|
||||||
|
crate::mission_events::repo_relative(path, GUEST_ROOTS),
|
||||||
|
serde_json::json!({ "tool": call.tool }),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
crate::mission_events::record_all(&tap.pool, events).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The runtime container behind this executor, derived from its gateway URL
|
||||||
|
/// (`http://cm-runtime-mission-<hex>:42617`). Used only to fetch a log tail
|
||||||
|
/// for an error message, so an unparseable URL is `None` rather than a
|
||||||
|
/// failure.
|
||||||
|
fn container_name(&self) -> Option<String> {
|
||||||
|
let rest = self
|
||||||
|
.gateway_url
|
||||||
|
.split("://")
|
||||||
|
.nth(1)
|
||||||
|
.unwrap_or(&self.gateway_url);
|
||||||
|
let host = rest.split('/').next()?.split(':').next()?;
|
||||||
|
(!host.is_empty()).then(|| host.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Use a runtime agent as a governance judge: drive `alias` with the judge
|
/// Use a runtime agent as a governance judge: drive `alias` with the judge
|
||||||
@@ -286,7 +452,12 @@ impl ZeroClawDriveExecutor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Read frames until a terminal (`done`/`error`/`approval_request`) event.
|
/// Read frames until a terminal (`done`/`error`/`approval_request`) event.
|
||||||
async fn drain<S>(ws: &mut S) -> Result<TurnOutcome, OrchestratorError>
|
///
|
||||||
|
/// Returns the turn's outcome AND what its frames said the agent did. The
|
||||||
|
/// trace is separate from [`TurnOutcome`] deliberately: that type is the
|
||||||
|
/// shared orchestrator contract used by every tier, and tool telemetry is a
|
||||||
|
/// mission concern.
|
||||||
|
async fn drain<S>(ws: &mut S) -> Result<(TurnOutcome, ToolTrace), OrchestratorError>
|
||||||
where
|
where
|
||||||
S: StreamExt<Item = Result<Message, tokio_tungstenite::tungstenite::Error>>
|
S: StreamExt<Item = Result<Message, tokio_tungstenite::tungstenite::Error>>
|
||||||
+ SinkExt<Message>
|
+ SinkExt<Message>
|
||||||
@@ -295,6 +466,7 @@ impl ZeroClawDriveExecutor {
|
|||||||
let mut output = String::new();
|
let mut output = String::new();
|
||||||
let mut tokens: u64 = 0;
|
let mut tokens: u64 = 0;
|
||||||
let mut gated: Vec<GatedAction> = Vec::new();
|
let mut gated: Vec<GatedAction> = Vec::new();
|
||||||
|
let mut trace = ToolTrace::default();
|
||||||
|
|
||||||
while let Some(frame) = ws.next().await {
|
while let Some(frame) = ws.next().await {
|
||||||
let msg = frame.map_err(|e| OrchestratorError::Executor(format!("ws recv: {e}")))?;
|
let msg = frame.map_err(|e| OrchestratorError::Executor(format!("ws recv: {e}")))?;
|
||||||
@@ -340,8 +512,50 @@ impl ZeroClawDriveExecutor {
|
|||||||
"aborted" => {
|
"aborted" => {
|
||||||
return Err(OrchestratorError::Executor("turn aborted".into()));
|
return Err(OrchestratorError::Executor("turn aborted".into()));
|
||||||
}
|
}
|
||||||
// session_start, thinking, tool_call, tool_result, …
|
// The action channel. `arguments` is read as JSON and
|
||||||
_ => {}
|
// nothing else is: the frame also carries a prose
|
||||||
|
// summary, and a path pulled out of THAT would be right
|
||||||
|
// often enough to be believed and wrong often enough to
|
||||||
|
// put files on the map that nobody edited.
|
||||||
|
"tool_call" => {
|
||||||
|
// `name` is what the gateway sends; `tool` is
|
||||||
|
// what `approval_request` uses, kept as a fallback.
|
||||||
|
let tool = v
|
||||||
|
.get("name")
|
||||||
|
.or_else(|| v.get("tool"))
|
||||||
|
.and_then(|t| t.as_str())
|
||||||
|
.unwrap_or("")
|
||||||
|
.trim()
|
||||||
|
.to_string();
|
||||||
|
if !tool.is_empty() {
|
||||||
|
// `args` FIRST: that is what the gateway
|
||||||
|
// actually sends (`{"type":"tool_call","id",
|
||||||
|
// "name","args"}` — zeroclaw-gateway/src/ws.rs).
|
||||||
|
// The others were guesses, and a guess that
|
||||||
|
// never matches costs the file path silently:
|
||||||
|
// the tool call is still recorded, with no
|
||||||
|
// target, and reads as a tool that touched
|
||||||
|
// nothing.
|
||||||
|
let args = v
|
||||||
|
.get("args")
|
||||||
|
.or_else(|| v.get("arguments"))
|
||||||
|
.or_else(|| v.get("input"))
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or(serde_json::Value::Null);
|
||||||
|
trace.calls.push(ToolCall {
|
||||||
|
path: crate::mission_events::tool_path(&args),
|
||||||
|
tool,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// session_start, thinking, tool_result, …
|
||||||
|
other => {
|
||||||
|
// Counted, not ignored. See `ToolTrace::unmatched`:
|
||||||
|
// the frame name above is unverified, and a tap
|
||||||
|
// that matches nothing looks exactly like a mission
|
||||||
|
// that used no tools.
|
||||||
|
*trace.unmatched.entry(other.to_string()).or_insert(0) += 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Message::Ping(p) => {
|
Message::Ping(p) => {
|
||||||
@@ -352,14 +566,21 @@ impl ZeroClawDriveExecutor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(TurnOutcome {
|
Ok((
|
||||||
output: output.trim().to_string(),
|
TurnOutcome {
|
||||||
tokens,
|
output: output.trim().to_string(),
|
||||||
gated,
|
tokens,
|
||||||
})
|
gated,
|
||||||
|
},
|
||||||
|
trace,
|
||||||
|
))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Guest workspace roots, stripped so a tool's absolute path becomes the
|
||||||
|
/// repo-relative one a person recognises.
|
||||||
|
const GUEST_ROOTS: &[&str] = &["/mission/repo", "/workspace", "/repo"];
|
||||||
|
|
||||||
impl TurnExecutor for ZeroClawDriveExecutor {
|
impl TurnExecutor for ZeroClawDriveExecutor {
|
||||||
async fn run_turn(&self, req: TurnRequest) -> Result<TurnOutcome, OrchestratorError> {
|
async fn run_turn(&self, req: TurnRequest) -> Result<TurnOutcome, OrchestratorError> {
|
||||||
// An explicit per-node agent (graph `node.attrs["agent"]`) wins, so one
|
// An explicit per-node agent (graph `node.attrs["agent"]`) wins, so one
|
||||||
@@ -406,6 +627,32 @@ fn parse_agent_map(s: &str) -> HashMap<String, String> {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
/// The container name comes out of the gateway URL, or nothing does.
|
||||||
|
///
|
||||||
|
/// This is only used to fetch a log tail for a failure message, so a URL
|
||||||
|
/// shape it does not recognise must degrade to "no log" rather than to a
|
||||||
|
/// second error on top of the first.
|
||||||
|
#[test]
|
||||||
|
fn the_container_name_is_derived_or_absent_never_wrong() {
|
||||||
|
let ex = |url: &str| ZeroClawDriveExecutor::new(
|
||||||
|
url.to_string(),
|
||||||
|
String::new(),
|
||||||
|
std::collections::HashMap::new(),
|
||||||
|
"scout".into(),
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
ex("http://cm-runtime-mission-019fec2d596f:42617").container_name().as_deref(),
|
||||||
|
Some("cm-runtime-mission-019fec2d596f")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
ex("https://host.example:8443/base").container_name().as_deref(),
|
||||||
|
Some("host.example")
|
||||||
|
);
|
||||||
|
// No scheme is still a host.
|
||||||
|
assert_eq!(ex("clawmates-runtime:42617").container_name().as_deref(), Some("clawmates-runtime"));
|
||||||
|
assert_eq!(ex("").container_name(), None);
|
||||||
|
}
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
use axum::extract::ws::{Message as AxMsg, WebSocket, WebSocketUpgrade};
|
use axum::extract::ws::{Message as AxMsg, WebSocket, WebSocketUpgrade};
|
||||||
use axum::response::Response;
|
use axum::response::Response;
|
||||||
@@ -435,6 +682,32 @@ mod tests {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A stream carrying tool calls and one frame type we do not know.
|
||||||
|
async fn tool_ws(ws: WebSocketUpgrade) -> Response {
|
||||||
|
ws.on_upgrade(|mut socket: WebSocket| async move {
|
||||||
|
let _ = socket.recv().await;
|
||||||
|
for f in [
|
||||||
|
json!({"type": "session_start"}),
|
||||||
|
// The REAL frame shape, copied from the gateway:
|
||||||
|
// {"type":"tool_call","id","name","args"}.
|
||||||
|
json!({"type": "tool_call", "id": "t1", "name": "Read",
|
||||||
|
"args": {"file_path": "/mission/repo/src/a.rs"}}),
|
||||||
|
// A tool whose arguments name no path at all.
|
||||||
|
json!({"type": "tool_call", "id": "t2", "name": "Bash",
|
||||||
|
"args": {"command": "cargo test"}}),
|
||||||
|
// Prose that MENTIONS a path. It must not become a file touch.
|
||||||
|
json!({"type": "tool_call", "id": "t3", "name": "Grep",
|
||||||
|
"arguments_summary": "searching src/main.rs",
|
||||||
|
"args": {"pattern": "fn main"}}),
|
||||||
|
json!({"type": "a_frame_we_have_never_seen"}),
|
||||||
|
json!({"type": "a_frame_we_have_never_seen"}),
|
||||||
|
json!({"type": "done", "input_tokens": 1, "output_tokens": 1}),
|
||||||
|
] {
|
||||||
|
let _ = socket.send(AxMsg::Text(f.to_string().into())).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn approval_ws(ws: WebSocketUpgrade) -> Response {
|
async fn approval_ws(ws: WebSocketUpgrade) -> Response {
|
||||||
ws.on_upgrade(|mut socket: WebSocket| async move {
|
ws.on_upgrade(|mut socket: WebSocket| async move {
|
||||||
let _ = socket.recv().await;
|
let _ = socket.recv().await;
|
||||||
@@ -499,6 +772,40 @@ mod tests {
|
|||||||
assert!(out.gated.is_empty());
|
assert!(out.gated.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Tool detail comes from arguments, and unknown frames are counted.
|
||||||
|
///
|
||||||
|
/// The two halves are one test because they are one risk. The frame type
|
||||||
|
/// `tool_call` is taken from a comment in this file, not from a captured
|
||||||
|
/// frame — so if it is wrong, the tap records nothing, the World stays as
|
||||||
|
/// sparse as it was, and NOTHING errors. The histogram is what turns that
|
||||||
|
/// into a log line naming the real frame.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn tool_frames_give_up_their_arguments_and_unknown_frames_are_counted() {
|
||||||
|
let router = Router::new()
|
||||||
|
.route("/pair", post(pair))
|
||||||
|
.route("/ws/chat", get(tool_ws));
|
||||||
|
let base = serve(router).await;
|
||||||
|
let exec = ZeroClawDriveExecutor::new(base, "code".into(), HashMap::new(), "scout".into());
|
||||||
|
|
||||||
|
let (_out, trace) = exec.drive_traced("scout", "go").await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
trace.calls,
|
||||||
|
vec![
|
||||||
|
ToolCall { tool: "Read".into(), path: Some("/mission/repo/src/a.rs".into()) },
|
||||||
|
ToolCall { tool: "Bash".into(), path: None },
|
||||||
|
// `arguments_summary` said "src/main.rs". It is prose, so it is
|
||||||
|
// not a file touch — a path scraped from a sentence would put
|
||||||
|
// files on the map that no agent opened.
|
||||||
|
ToolCall { tool: "Grep".into(), path: None },
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert_eq!(trace.unmatched.get("a_frame_we_have_never_seen"), Some(&2));
|
||||||
|
assert_eq!(trace.unmatched.get("session_start"), Some(&1));
|
||||||
|
// `done` terminates the drain and is not an unmatched frame.
|
||||||
|
assert!(!trace.unmatched.contains_key("done"), "{:?}", trace.unmatched);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn approval_request_is_recorded_as_blocked() {
|
async fn approval_request_is_recorded_as_blocked() {
|
||||||
let router = Router::new()
|
let router = Router::new()
|
||||||
|
|||||||
@@ -33,7 +33,12 @@ const REAP_STUCK_AFTER_SECS: i64 = 15 * 60;
|
|||||||
|
|
||||||
/// Spawn the durable topology job worker. Polls for queued jobs every `poll`
|
/// Spawn the durable topology job worker. Polls for queued jobs every `poll`
|
||||||
/// interval; runs each to completion (or failure), checkpointing per step.
|
/// interval; runs each to completion (or failure), checkpointing per step.
|
||||||
pub fn spawn(pool: PgPool, runtime: cm_runtime::Runtime, poll: Duration) {
|
pub fn spawn(
|
||||||
|
pool: PgPool,
|
||||||
|
runtime: cm_runtime::Runtime,
|
||||||
|
hub: Arc<crate::fleet::NodeHub>,
|
||||||
|
poll: Duration,
|
||||||
|
) {
|
||||||
// Fire the stuck-container reaper on its own cadence — checking
|
// Fire the stuck-container reaper on its own cadence — checking
|
||||||
// once a minute is plenty and keeps this off the hot claim loop.
|
// once a minute is plenty and keeps this off the hot claim loop.
|
||||||
let reaper_pool = pool.clone();
|
let reaper_pool = pool.clone();
|
||||||
@@ -55,7 +60,7 @@ pub fn spawn(pool: PgPool, runtime: cm_runtime::Runtime, poll: Duration) {
|
|||||||
eprintln!("topology_worker: requeue_stale failed: {e}");
|
eprintln!("topology_worker: requeue_stale failed: {e}");
|
||||||
}
|
}
|
||||||
match cm_db::repo::topology_runs::claim_next_queued(&pool).await {
|
match cm_db::repo::topology_runs::claim_next_queued(&pool).await {
|
||||||
Ok(Some(job)) => run_job(&pool, &runtime, job).await,
|
Ok(Some(job)) => run_job(&pool, &runtime, &hub, job).await,
|
||||||
Ok(None) => tokio::time::sleep(poll).await,
|
Ok(None) => tokio::time::sleep(poll).await,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("topology_worker: claim failed: {e}");
|
eprintln!("topology_worker: claim failed: {e}");
|
||||||
@@ -80,10 +85,25 @@ async fn reap_stuck_runs(pool: &PgPool) -> Result<(), sqlx::Error> {
|
|||||||
FROM topology_runs
|
FROM topology_runs
|
||||||
WHERE status = 'running'
|
WHERE status = 'running'
|
||||||
AND mission_id IS NOT NULL
|
AND mission_id IS NOT NULL
|
||||||
|
-- Only jobs this worker drives. mission_id IS NOT NULL used to mean
|
||||||
|
-- the same thing as orchestrator-driven, and the microvm and session
|
||||||
|
-- tiers broke that: their checkpoint is NULL for life BY DESIGN, so the
|
||||||
|
-- zero-step-records test below is true of a perfectly healthy run.
|
||||||
|
AND tier = ANY($2)
|
||||||
AND created_at < now() - make_interval(secs => $1::float)
|
AND created_at < now() - make_interval(secs => $1::float)
|
||||||
AND coalesce(jsonb_array_length(coalesce(checkpoint->'records', '[]'::jsonb)), 0) = 0",
|
AND coalesce(jsonb_array_length(coalesce(checkpoint->'records', '[]'::jsonb)), 0) = 0",
|
||||||
)
|
)
|
||||||
.bind(REAP_STUCK_AFTER_SECS as f64)
|
.bind(REAP_STUCK_AFTER_SECS as f64)
|
||||||
|
.bind(
|
||||||
|
// REAPABLE, not worker-driven: `microvm_graph` is driven by this worker
|
||||||
|
// and must NOT be reaped — one of its steps is a whole agent session in a
|
||||||
|
// VM, so "no step records in 15 minutes" describes a healthy composed run
|
||||||
|
// as readily as a wedged one.
|
||||||
|
cm_db::repo::topology_runs::REAPABLE_TIERS
|
||||||
|
.iter()
|
||||||
|
.map(|s| (*s).to_string())
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
)
|
||||||
.fetch_all(pool)
|
.fetch_all(pool)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
@@ -108,6 +128,7 @@ async fn reap_stuck_runs(pool: &PgPool) -> Result<(), sqlx::Error> {
|
|||||||
async fn run_job(
|
async fn run_job(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
runtime: &cm_runtime::Runtime,
|
runtime: &cm_runtime::Runtime,
|
||||||
|
hub: &Arc<crate::fleet::NodeHub>,
|
||||||
job: cm_db::repo::topology_runs::ClaimedTopologyRun,
|
job: cm_db::repo::topology_runs::ClaimedTopologyRun,
|
||||||
) {
|
) {
|
||||||
let id = job.id;
|
let id = job.id;
|
||||||
@@ -145,30 +166,52 @@ async fn run_job(
|
|||||||
// Resume from the last checkpoint, or start fresh.
|
// Resume from the last checkpoint, or start fresh.
|
||||||
let progress: RunProgress = job
|
let progress: RunProgress = job
|
||||||
.checkpoint
|
.checkpoint
|
||||||
|
.clone()
|
||||||
.and_then(|c| serde_json::from_value(c).ok())
|
.and_then(|c| serde_json::from_value(c).ok())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
// The composed engines (Slice 4): this graph's nodes are not claws, they are
|
||||||
|
// Claude-Code-in-a-microVM sessions. Branched BEFORE the leaf executor is
|
||||||
|
// built, because that build reads the ZeroClaw gateway config — a composed
|
||||||
|
// run must not fail for want of a runtime it never dials.
|
||||||
|
if job.tier == "microvm_graph" {
|
||||||
|
let result = run_composed(pool, hub, &job, &graph, progress).await;
|
||||||
|
finish(pool, id, result).await;
|
||||||
|
maybe_teardown_ephemeral_team(pool, runtime, id).await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// C3: prefer the mission's per-run runtime endpoint when set on
|
// C3: prefer the mission's per-run runtime endpoint when set on
|
||||||
// the missions row; else fall back to the shared env-derived
|
// the missions row; else fall back to the shared env-derived
|
||||||
// gateway (pre-C3 missions + non-mission runs). This is what
|
// gateway (pre-C3 missions + non-mission runs). This is what
|
||||||
// isolates agents' workspace filesystem to that mission's repo.
|
// isolates agents' workspace filesystem to that mission's repo.
|
||||||
let mission_binding: Option<(Option<String>, Option<String>)> =
|
type MissionBinding = (Option<String>, Option<String>, Uuid, Option<Uuid>);
|
||||||
sqlx::query_as::<_, (Option<String>, Option<String>)>(
|
let mission_binding: Option<MissionBinding> = sqlx::query_as::<_, MissionBinding>(
|
||||||
"SELECT m.runtime_endpoint, m.runtime_pairing_code
|
"SELECT m.runtime_endpoint, m.runtime_pairing_code, m.id, r.mission_phase_id
|
||||||
FROM topology_runs r
|
FROM topology_runs r
|
||||||
JOIN missions m ON m.id = r.mission_id
|
JOIN missions m ON m.id = r.mission_id
|
||||||
WHERE r.id = $1",
|
WHERE r.id = $1",
|
||||||
)
|
)
|
||||||
.bind(id)
|
.bind(id)
|
||||||
.fetch_optional(pool)
|
.fetch_optional(pool)
|
||||||
.await
|
.await
|
||||||
.ok()
|
.ok()
|
||||||
.flatten();
|
.flatten();
|
||||||
|
// What this run's turns will be attributed to. `None` when the run belongs
|
||||||
|
// to no mission — a bare topology run has no phase to hang tool calls on.
|
||||||
|
let tap = mission_binding
|
||||||
|
.as_ref()
|
||||||
|
.map(|(_, _, mission_id, phase_id)| crate::topology_exec::MissionTap {
|
||||||
|
pool: pool.clone(),
|
||||||
|
mission_id: *mission_id,
|
||||||
|
phase_id: *phase_id,
|
||||||
|
run_id: Some(id),
|
||||||
|
});
|
||||||
let leaf_result = match mission_binding {
|
let leaf_result = match mission_binding {
|
||||||
Some((Some(url), Some(code))) => {
|
Some((Some(url), Some(code), _, _)) => {
|
||||||
ZeroClawDriveExecutor::from_env_for_gateway_with_code(url, code)
|
ZeroClawDriveExecutor::from_env_for_gateway_with_code(url, code)
|
||||||
}
|
}
|
||||||
Some((Some(url), None)) => ZeroClawDriveExecutor::from_env_for_gateway(url),
|
Some((Some(url), None, _, _)) => ZeroClawDriveExecutor::from_env_for_gateway(url),
|
||||||
_ => ZeroClawDriveExecutor::from_env(),
|
_ => ZeroClawDriveExecutor::from_env(),
|
||||||
};
|
};
|
||||||
let leaf = match leaf_result {
|
let leaf = match leaf_result {
|
||||||
@@ -178,6 +221,13 @@ async fn run_job(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
// The tap rides on the leaf executor, so the recursive tiers get it too:
|
||||||
|
// they drive the same leaf all the way down, and a company-tier mission's
|
||||||
|
// tool calls belong to its phase exactly as a team-tier one's do.
|
||||||
|
let leaf = match tap {
|
||||||
|
Some(t) => leaf.with_tap(t),
|
||||||
|
None => leaf,
|
||||||
|
};
|
||||||
|
|
||||||
// Select the executor by deploy tier: `team` drives claws directly; the
|
// Select the executor by deploy tier: `team` drives claws directly; the
|
||||||
// upper tiers drive the recursive sub-topology executor (which runs each
|
// upper tiers drive the recursive sub-topology executor (which runs each
|
||||||
@@ -201,6 +251,14 @@ async fn run_job(
|
|||||||
_ => drive(pool, id, &graph, &job.task, progress, &leaf).await,
|
_ => drive(pool, id, &graph, &job.task, progress, &leaf).await,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
finish(pool, id, result).await;
|
||||||
|
maybe_teardown_ephemeral_team(pool, runtime, id).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write a driven run's terminal state. The single place a run finishes, shared
|
||||||
|
/// by every tier — a second one would be a second completion path, which is where
|
||||||
|
/// every microVM bug this project has hit came from.
|
||||||
|
async fn finish(pool: &PgPool, id: Uuid, result: Result<RunRecord, OrchestratorError>) {
|
||||||
match result {
|
match result {
|
||||||
Ok(record) => {
|
Ok(record) => {
|
||||||
let value = serde_json::to_value(&record).unwrap_or(serde_json::Value::Null);
|
let value = serde_json::to_value(&record).unwrap_or(serde_json::Value::Null);
|
||||||
@@ -219,7 +277,77 @@ async fn run_job(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
maybe_teardown_ephemeral_team(pool, runtime, id).await;
|
}
|
||||||
|
|
||||||
|
/// Drive a composed run: the outer graph is Engine Z, every node is a
|
||||||
|
/// Claude-Code-in-a-microVM session (Engine C).
|
||||||
|
///
|
||||||
|
/// The mission columns are read here rather than carried on the run row so a
|
||||||
|
/// re-placed or re-backed mission takes effect on resume, and so the composed
|
||||||
|
/// path has exactly one source of truth for where a VM boots.
|
||||||
|
async fn run_composed(
|
||||||
|
pool: &PgPool,
|
||||||
|
hub: &Arc<crate::fleet::NodeHub>,
|
||||||
|
job: &cm_db::repo::topology_runs::ClaimedTopologyRun,
|
||||||
|
graph: &TopologyGraph,
|
||||||
|
progress: RunProgress,
|
||||||
|
) -> Result<RunRecord, OrchestratorError> {
|
||||||
|
let mission_id = job.mission_id.ok_or_else(|| {
|
||||||
|
OrchestratorError::Executor(
|
||||||
|
"a composed run has no mission, so there is no checkout for its nodes \
|
||||||
|
to share"
|
||||||
|
.into(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let phase_id = job.mission_phase_id.ok_or_else(|| {
|
||||||
|
OrchestratorError::Executor("a composed run must belong to a mission phase".into())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mission: (Option<Uuid>, Option<String>, Option<String>, bool) =
|
||||||
|
sqlx::query_as(
|
||||||
|
"SELECT target_node_id, backend, team_engine, (repo_id IS NOT NULL) \
|
||||||
|
FROM missions WHERE id = $1",
|
||||||
|
)
|
||||||
|
.bind(mission_id)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| OrchestratorError::Executor(format!("load mission {mission_id}: {e}")))?;
|
||||||
|
|
||||||
|
// The phase's completion gate, read here rather than carried on the run row
|
||||||
|
// so an edited `done_when_check` takes effect on the next node instead of at
|
||||||
|
// the next mission.
|
||||||
|
let phase: (String, serde_json::Value) =
|
||||||
|
sqlx::query_as("SELECT kind, config FROM mission_phases WHERE id = $1")
|
||||||
|
.bind(phase_id)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| OrchestratorError::Executor(format!("load phase {phase_id}: {e}")))?;
|
||||||
|
|
||||||
|
let exec = crate::microvm_turn_executor::for_fleet(
|
||||||
|
hub.clone(),
|
||||||
|
pool.clone(),
|
||||||
|
crate::microvm_turn_executor::ComposedRun {
|
||||||
|
run_id: job.id,
|
||||||
|
mission_id,
|
||||||
|
phase_id,
|
||||||
|
iteration: job.iteration.unwrap_or(1),
|
||||||
|
repo: crate::mission_workspace::checkout_path(mission_id),
|
||||||
|
// A repo-less composed mission gets an empty shared workspace, the
|
||||||
|
// same as a solo phase — the graph's whole property is that node 2
|
||||||
|
// sees node 1's files, and that holds whether or not it is a git
|
||||||
|
// checkout.
|
||||||
|
has_repo: mission.3,
|
||||||
|
target_node_id: mission.0,
|
||||||
|
backend: mission.1,
|
||||||
|
team_engine: mission.2,
|
||||||
|
gate: crate::vm_stop_gate::StopGate::for_phase(&phase.0, &phase.1)
|
||||||
|
.and_then(crate::vm_stop_gate::StopGate::per_node),
|
||||||
|
// Resume continues the step numbering; restarting it would re-use a
|
||||||
|
// finished node's vm id.
|
||||||
|
completed_steps: progress.completed as u32,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
drive(pool, job.id, graph, &job.task, progress, &exec).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Post-terminal hook: if this run's team is `ephemeral` and no siblings are
|
/// Post-terminal hook: if this run's team is `ephemeral` and no siblings are
|
||||||
|
|||||||
@@ -0,0 +1,158 @@
|
|||||||
|
//! Can the independent validator actually be reached?
|
||||||
|
//!
|
||||||
|
//! The sibling of [`crate::runtime_preflight`], for the same class of failure:
|
||||||
|
//! the code is right and the machine is not, and nothing says so until a mission
|
||||||
|
//! pays for it.
|
||||||
|
//!
|
||||||
|
//! `evaluator::cross_provider_judge` deliberately refuses to fall back to the
|
||||||
|
//! agent's own provider — a verdict from the same family is not an independent
|
||||||
|
//! check, and quietly producing one would claim a property the verdict does not
|
||||||
|
//! have. That refusal is correct, and its cost is that a dead validator makes
|
||||||
|
//! every `done_when` phase UNMEETABLE. The mission still boots a VM, still runs
|
||||||
|
//! an agent turn, still collects and delivers, and only then records
|
||||||
|
//! "the independent validator could not be reached this pass" on one evaluation
|
||||||
|
//! row.
|
||||||
|
//!
|
||||||
|
//! That happened: the z.ai credential expired mid-session and the first symptom
|
||||||
|
//! was a two-phase mission failing after both VMs had run. The information
|
||||||
|
//! existed the whole time; nobody was told until it was expensive.
|
||||||
|
//!
|
||||||
|
//! A report, not a gate — the same stance `runtime_preflight` takes. The server
|
||||||
|
//! must still boot with a broken validator, because refusing to start would turn
|
||||||
|
//! a degraded deployment into a dead one, and because a mission that opts out
|
||||||
|
//! (`validator_model = ''`) is unaffected. What this buys is that the degradation
|
||||||
|
//! is visible at startup instead of inferred from a failed mission.
|
||||||
|
|
||||||
|
/// The smallest question that proves a credential works end to end.
|
||||||
|
///
|
||||||
|
/// A real completion rather than a models-list or a HEAD: an expired key, a
|
||||||
|
/// revoked key and a key with no quota can all pass a cheaper check and fail the
|
||||||
|
/// call that matters. Two tokens of output.
|
||||||
|
const PROBE_PROMPT: &str = "Reply with exactly: OK";
|
||||||
|
|
||||||
|
/// What the probe found.
|
||||||
|
#[derive(Debug, PartialEq, Eq)]
|
||||||
|
pub enum Verdict {
|
||||||
|
/// No independent validator is configured; phases are judged by the house
|
||||||
|
/// model. Not a fault — a deployment may choose this.
|
||||||
|
NotConfigured,
|
||||||
|
/// Configured, resolved, and it answered.
|
||||||
|
Reachable { spec: String },
|
||||||
|
/// Configured but the registry has no such provider, so
|
||||||
|
/// `cross_provider_judge` will refuse it rather than judge with the default.
|
||||||
|
Unregistered { spec: String },
|
||||||
|
/// Configured and resolved, and the call failed.
|
||||||
|
Unreachable { spec: String, error: String },
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Verdict {
|
||||||
|
/// Is every `done_when` phase currently unmeetable because of this?
|
||||||
|
pub fn breaks_gated_phases(&self) -> bool {
|
||||||
|
matches!(
|
||||||
|
self,
|
||||||
|
Verdict::Unregistered { .. } | Verdict::Unreachable { .. }
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ask the configured independent validator to answer one trivial question.
|
||||||
|
pub async fn probe(runtime: &cm_runtime::Runtime) -> Verdict {
|
||||||
|
let Some(spec) = std::env::var("CLAWMATES_VALIDATOR_MODEL")
|
||||||
|
.ok()
|
||||||
|
.map(|s| s.trim().to_string())
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
else {
|
||||||
|
return Verdict::NotConfigured;
|
||||||
|
};
|
||||||
|
|
||||||
|
let (_provider, model) = runtime.resolve_provider(&spec);
|
||||||
|
// `resolve_provider` falls back to the DEFAULT provider for an unknown name,
|
||||||
|
// and the fallback is detectable because the returned model still carries the
|
||||||
|
// `name:` prefix. Checked here for the same reason the evaluator checks it:
|
||||||
|
// a validator that is silently the house model is worse than none.
|
||||||
|
if model.contains(':') {
|
||||||
|
return Verdict::Unregistered { spec };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Through `Runtime::complete`, which is the same resolve-then-stream path
|
||||||
|
// the evaluator's judge takes. A probe that dialled the provider its own way
|
||||||
|
// could pass while the real call fails.
|
||||||
|
match runtime.complete("", PROBE_PROMPT, &spec, 16, false).await {
|
||||||
|
Ok(_) => Verdict::Reachable { spec },
|
||||||
|
Err(e) => Verdict::Unreachable {
|
||||||
|
spec,
|
||||||
|
error: e.chars().take(160).collect(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Probe at boot and say plainly what it means for missions.
|
||||||
|
pub fn report_at_boot(runtime: cm_runtime::Runtime) {
|
||||||
|
tokio::spawn(async move {
|
||||||
|
match probe(&runtime).await {
|
||||||
|
Verdict::NotConfigured => eprintln!(
|
||||||
|
"validator_preflight: no CLAWMATES_VALIDATOR_MODEL — phase verdicts are judged \
|
||||||
|
by the house model, which is NOT an independent check"
|
||||||
|
),
|
||||||
|
Verdict::Reachable { spec } => {
|
||||||
|
eprintln!("validator_preflight: independent validator {spec} answered")
|
||||||
|
}
|
||||||
|
Verdict::Unregistered { spec } => eprintln!(
|
||||||
|
"validator_preflight: CLAWMATES_VALIDATOR_MODEL={spec} has no registered \
|
||||||
|
provider — the evaluator will refuse it rather than judge with the default, \
|
||||||
|
so EVERY phase with a done_when condition will fail as unmet. Register the \
|
||||||
|
provider, or set the mission's validator_model to '' to opt out."
|
||||||
|
),
|
||||||
|
Verdict::Unreachable { spec, error } => eprintln!(
|
||||||
|
"validator_preflight: independent validator {spec} is UNREACHABLE ({error}) — \
|
||||||
|
EVERY phase with a done_when condition will fail as unmet, after running its \
|
||||||
|
agent. Fix the credential, or set validator_model to '' per mission to judge \
|
||||||
|
with the house model."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// The two states that make gated phases unmeetable, and the two that do
|
||||||
|
/// not. This is the distinction the whole module exists to draw: "no
|
||||||
|
/// validator configured" is a choice, "configured and broken" is a fault
|
||||||
|
/// that silently fails every conditioned mission.
|
||||||
|
#[test]
|
||||||
|
fn only_a_configured_but_broken_validator_breaks_gated_phases() {
|
||||||
|
assert!(!Verdict::NotConfigured.breaks_gated_phases());
|
||||||
|
assert!(!Verdict::Reachable {
|
||||||
|
spec: "glm:glm-4.7".into()
|
||||||
|
}
|
||||||
|
.breaks_gated_phases());
|
||||||
|
|
||||||
|
assert!(Verdict::Unregistered {
|
||||||
|
spec: "glm:glm-4.7".into()
|
||||||
|
}
|
||||||
|
.breaks_gated_phases());
|
||||||
|
assert!(Verdict::Unreachable {
|
||||||
|
spec: "glm:glm-4.7".into(),
|
||||||
|
error: "401".into()
|
||||||
|
}
|
||||||
|
.breaks_gated_phases());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An unregistered provider is NOT reported as unreachable, and the
|
||||||
|
/// difference is actionable: one is fixed by registering a provider, the
|
||||||
|
/// other by fixing a credential. Collapsing them sends an operator to the
|
||||||
|
/// wrong place.
|
||||||
|
#[test]
|
||||||
|
fn the_two_faults_are_distinguishable() {
|
||||||
|
let a = Verdict::Unregistered {
|
||||||
|
spec: "glm:glm-4.7".into(),
|
||||||
|
};
|
||||||
|
let b = Verdict::Unreachable {
|
||||||
|
spec: "glm:glm-4.7".into(),
|
||||||
|
error: "401 Authentication Failed".into(),
|
||||||
|
};
|
||||||
|
assert_ne!(a, b);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,815 @@
|
|||||||
|
//! Which fleet node should run the next microVM phase, and whether any can.
|
||||||
|
//!
|
||||||
|
//! # What this replaces
|
||||||
|
//!
|
||||||
|
//! Placement was `capable.first()` over a list ordered `last_seen DESC`
|
||||||
|
//! (`mission_orchestrator`, `nodes::online_for_backend`) — the most recently
|
||||||
|
//! heartbeated node. Among healthy nodes all heartbeating every 5s that is
|
||||||
|
//! arbitrary, and it consults nothing about load: two missions launched together
|
||||||
|
//! land on the same machine. It did not matter while one node held the only
|
||||||
|
//! rootfs image; all three do now.
|
||||||
|
//!
|
||||||
|
//! # Observed memory is not capacity
|
||||||
|
//!
|
||||||
|
//! The correctness core, and the reason this is not a one-line sort change. A VM
|
||||||
|
//! that booted 30 seconds ago holds a fraction of its 8 GiB claim — the guest has
|
||||||
|
//! not touched the rest — so `mem_pct` reports a sold-out node as nearly idle.
|
||||||
|
//! Ranking on utilisation alone would happily book five more VMs onto a node with
|
||||||
|
//! room for one. `capacity_of` therefore takes the WORSE of observed usage and
|
||||||
|
//! committed usage, and `a_sold_out_node_is_not_mistaken_for_an_idle_one` is the
|
||||||
|
//! negative control that pins it.
|
||||||
|
//!
|
||||||
|
//! Commitments come from two places that must be unioned by IDENTITY, never
|
||||||
|
//! added: `microvm_client::list` (booted VMs, including orphans nothing has
|
||||||
|
//! reaped) and `nodes::pinned_microvm_phases` (chosen but not yet booted). The
|
||||||
|
//! deterministic `vm_id_for` is what lets the same phase be recognised in both.
|
||||||
|
//!
|
||||||
|
//! # Fail-closed
|
||||||
|
//!
|
||||||
|
//! A node whose health is stale, whose daemon will not answer, or which is
|
||||||
|
//! draining is INELIGIBLE, not low-scoring. Unknown is not permission — the same
|
||||||
|
//! rule `nodes::online_for_backend` already applies to capabilities. The one
|
||||||
|
//! exception is Beszel metrics: they feed `headroom` as a tiebreak only, so stale
|
||||||
|
//! metrics demote a node instead of excluding it.
|
||||||
|
|
||||||
|
use cm_db::repo::node_metrics::EvalRow;
|
||||||
|
use cm_domain::NodeId;
|
||||||
|
|
||||||
|
/// Memory a phase VM claims. Re-exported from the executor so there is ONE number
|
||||||
|
/// — a scheduler and a launcher that disagree about VM size is a fleet that
|
||||||
|
/// overcommits by exactly their difference.
|
||||||
|
pub(crate) use crate::microvm_executor::MEM_MIB as MEM_PER_VM_MIB;
|
||||||
|
|
||||||
|
/// Held back for the host: the daemon, the OS, page cache, and the margin that
|
||||||
|
/// keeps a node out of swap. A node in swap makes every VM on it slow, so this is
|
||||||
|
/// cheaper than the alternative.
|
||||||
|
const HOST_RESERVE_MIB: i64 = 4096;
|
||||||
|
|
||||||
|
/// The floor we refuse to believe a host's own footprint is below. Without it, a
|
||||||
|
/// node reporting less used memory than its VMs have claimed would compute a
|
||||||
|
/// negative baseline and inflate its free memory.
|
||||||
|
const HOST_BASELINE_FLOOR_MIB: i64 = 2048;
|
||||||
|
|
||||||
|
/// Disk a VM may consume: an 8 GiB sparse rootfs plus room for the collected tar.
|
||||||
|
const DISK_PER_VM_GIB: i64 = 12;
|
||||||
|
|
||||||
|
/// Never let VM disk drive a node below this. `_outputs` and images live on the
|
||||||
|
/// same filesystem on some nodes.
|
||||||
|
const DISK_RESERVE_GIB: i64 = 20;
|
||||||
|
|
||||||
|
/// Health older than this and the node is ineligible. Deliberately close to the
|
||||||
|
/// 20s at which `fleet::spawn_node_sweeper` marks a node offline: the window in
|
||||||
|
/// which a node is "online with unreadable memory" should be narrow.
|
||||||
|
pub const MAX_HEALTH_AGE_SECS: f64 = 30.0;
|
||||||
|
|
||||||
|
/// Beszel metrics older than this rank as zero headroom. Only a tiebreak.
|
||||||
|
const MAX_METRICS_AGE_SECS: f64 = 60.0;
|
||||||
|
|
||||||
|
/// A node that can take at least one more phase VM.
|
||||||
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
|
pub struct NodeCapacity {
|
||||||
|
pub node_id: NodeId,
|
||||||
|
pub name: String,
|
||||||
|
/// How many MORE 8 GiB VMs fit.
|
||||||
|
pub slots: i64,
|
||||||
|
pub headroom: f64,
|
||||||
|
pub committed_vms: i64,
|
||||||
|
pub mem_total_mib: i64,
|
||||||
|
pub used_eff_mib: i64,
|
||||||
|
pub disk_free_gib: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Why a node cannot take this phase. Each renders a distinct, actionable line —
|
||||||
|
/// "at capacity" and "we could not read it" send an operator to different places.
|
||||||
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
|
pub enum Unfit {
|
||||||
|
Draining,
|
||||||
|
NotConnected,
|
||||||
|
NoRecentHealth { age_secs: Option<f64> },
|
||||||
|
CapacityUnknown { err: String },
|
||||||
|
AtCapacity { committed: i64, used_eff_mib: i64, mem_total_mib: i64 },
|
||||||
|
NoDisk { free_gib: i64 },
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Unfit {
|
||||||
|
pub fn reason(&self) -> String {
|
||||||
|
match self {
|
||||||
|
Unfit::Draining => "draining".into(),
|
||||||
|
Unfit::NotConnected => "daemon not connected".into(),
|
||||||
|
Unfit::NoRecentHealth { age_secs } => match age_secs {
|
||||||
|
Some(a) => format!("health {a:.0}s stale (max {MAX_HEALTH_AGE_SECS:.0}s)"),
|
||||||
|
None => "never reported health".into(),
|
||||||
|
},
|
||||||
|
Unfit::CapacityUnknown { err } => format!("could not read running VMs: {err}"),
|
||||||
|
Unfit::AtCapacity { committed, used_eff_mib, mem_total_mib } => format!(
|
||||||
|
"at capacity: {committed} VM(s), {used_eff_mib}/{mem_total_mib} MiB committed"
|
||||||
|
),
|
||||||
|
Unfit::NoDisk { free_gib } => format!("only {free_gib} GiB free"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Why placement produced no node. Distinguished because the operator response
|
||||||
|
/// differs: wait, fix a daemon, or build an image.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub enum PlacementError {
|
||||||
|
/// No node has the image / KVM at all. Not a capacity problem.
|
||||||
|
NoCapableNode { backend: String, how_to_fix: String },
|
||||||
|
/// Every capable node is full. Transient — the caller should queue.
|
||||||
|
FleetAtCapacity { report: String },
|
||||||
|
/// We could not READ capacity. Must never be reported as "full".
|
||||||
|
FleetUnreadable { report: String },
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PlacementError {
|
||||||
|
/// Whether the caller should wait and retry rather than fail the work.
|
||||||
|
pub fn is_transient(&self) -> bool {
|
||||||
|
matches!(
|
||||||
|
self,
|
||||||
|
PlacementError::FleetAtCapacity { .. } | PlacementError::FleetUnreadable { .. }
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn message(&self) -> String {
|
||||||
|
match self {
|
||||||
|
PlacementError::NoCapableNode { backend, how_to_fix } => {
|
||||||
|
format!("no online node can run backend {backend:?} — {how_to_fix}")
|
||||||
|
}
|
||||||
|
PlacementError::FleetAtCapacity { report } => format!(
|
||||||
|
"fleet at capacity — a phase VM runs up to 60 min; this phase waits for a slot.\n{report}"
|
||||||
|
),
|
||||||
|
PlacementError::FleetUnreadable { report } => format!(
|
||||||
|
"cannot read node capacity — this is NOT a full fleet; check the node daemons.\n{report}"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What the host itself costs, excluding its phase VMs.
|
||||||
|
///
|
||||||
|
/// With nothing committed the answer is simply what the node reports. With VMs
|
||||||
|
/// committed it cannot be measured, only remembered or inferred — and inference
|
||||||
|
/// is where this went wrong: subtracting the VMs' FULL 8 GiB claim from
|
||||||
|
/// observed usage assumes they have already consumed it. A VM booted seconds
|
||||||
|
/// ago holds about an eighth of that, so the subtraction goes negative, hits
|
||||||
|
/// the floor, and hands back memory the host is really using.
|
||||||
|
///
|
||||||
|
/// Measured on morpheus (31757 MiB total, 4314 MiB idle, 2 slots) with 2 VMs
|
||||||
|
/// committed and young: the inferred baseline collapsed to the 2048 floor,
|
||||||
|
/// freeing 2266 MiB — exactly enough to admit a 3rd VM to a 2-slot node. The
|
||||||
|
/// `capacity` harness scenario caught it on its first full run.
|
||||||
|
///
|
||||||
|
/// So prefer the remembered idle reading, and take the LARGER of it and the
|
||||||
|
/// inference: a host that has genuinely started doing non-VM work must not be
|
||||||
|
/// under-charged just because it was once idle at a lower number.
|
||||||
|
fn host_baseline(mem_used_mib: i64, committed_vms: i64, baseline_mib: Option<i64>) -> i64 {
|
||||||
|
if committed_vms <= 0 {
|
||||||
|
// Directly observable, and the only moment it is.
|
||||||
|
return mem_used_mib.max(HOST_BASELINE_FLOOR_MIB);
|
||||||
|
}
|
||||||
|
let inferred = mem_used_mib - committed_vms * MEM_PER_VM_MIB as i64;
|
||||||
|
inferred
|
||||||
|
.max(baseline_mib.unwrap_or(0))
|
||||||
|
.max(HOST_BASELINE_FLOOR_MIB)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The whole capacity decision for one node, as pure arithmetic.
|
||||||
|
///
|
||||||
|
/// Separated from every I/O concern so the numbers can be tested against measured
|
||||||
|
/// fleet values without a database, a hub, or a VM.
|
||||||
|
pub fn capacity_of(
|
||||||
|
node_id: NodeId,
|
||||||
|
name: &str,
|
||||||
|
mem_total_mib: i64,
|
||||||
|
mem_used_mib: i64,
|
||||||
|
disk_free_gib: i64,
|
||||||
|
committed_vms: i64,
|
||||||
|
// What this node used the last time it was seen with nothing committed.
|
||||||
|
// `None` before it has ever been observed idle.
|
||||||
|
baseline_mib: Option<i64>,
|
||||||
|
headroom: f64,
|
||||||
|
) -> Result<NodeCapacity, Unfit> {
|
||||||
|
let host_baseline = host_baseline(mem_used_mib, committed_vms, baseline_mib);
|
||||||
|
let committed_use = committed_vms * MEM_PER_VM_MIB as i64 + host_baseline;
|
||||||
|
|
||||||
|
// The worse of the two views. Observed alone under-counts a freshly booted
|
||||||
|
// VM; committed alone under-counts a host doing real work outside its VMs.
|
||||||
|
let used_eff = mem_used_mib.max(committed_use);
|
||||||
|
let free = mem_total_mib - used_eff - HOST_RESERVE_MIB;
|
||||||
|
let slots = if free <= 0 { 0 } else { free / MEM_PER_VM_MIB as i64 };
|
||||||
|
|
||||||
|
if disk_free_gib - DISK_PER_VM_GIB < DISK_RESERVE_GIB {
|
||||||
|
return Err(Unfit::NoDisk { free_gib: disk_free_gib });
|
||||||
|
}
|
||||||
|
if slots < 1 {
|
||||||
|
return Err(Unfit::AtCapacity {
|
||||||
|
committed: committed_vms,
|
||||||
|
used_eff_mib: used_eff,
|
||||||
|
mem_total_mib,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(NodeCapacity {
|
||||||
|
node_id,
|
||||||
|
name: name.to_string(),
|
||||||
|
slots,
|
||||||
|
headroom,
|
||||||
|
committed_vms,
|
||||||
|
mem_total_mib,
|
||||||
|
used_eff_mib: used_eff,
|
||||||
|
disk_free_gib,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Admission inputs drawn from an `EvalRow`, or why the node is ineligible.
|
||||||
|
///
|
||||||
|
/// Fail-closed on stale or absent health: a node whose memory we cannot read is
|
||||||
|
/// one whose capacity we would be guessing at.
|
||||||
|
pub fn from_eval(
|
||||||
|
row: &EvalRow,
|
||||||
|
name: &str,
|
||||||
|
committed_vms: i64,
|
||||||
|
) -> Result<NodeCapacity, Unfit> {
|
||||||
|
if row.status == "draining" {
|
||||||
|
return Err(Unfit::Draining);
|
||||||
|
}
|
||||||
|
let fresh = row
|
||||||
|
.health_age_secs
|
||||||
|
.is_some_and(|a| a <= MAX_HEALTH_AGE_SECS);
|
||||||
|
let (Some(total), Some(used)) = (row.mem_total_bytes, row.mem_used_bytes) else {
|
||||||
|
return Err(Unfit::NoRecentHealth { age_secs: row.health_age_secs });
|
||||||
|
};
|
||||||
|
if !fresh || total <= 0 {
|
||||||
|
return Err(Unfit::NoRecentHealth { age_secs: row.health_age_secs });
|
||||||
|
}
|
||||||
|
const MIB: i64 = 1024 * 1024;
|
||||||
|
const GIB: i64 = 1024 * 1024 * 1024;
|
||||||
|
capacity_of(
|
||||||
|
row.node_id,
|
||||||
|
name,
|
||||||
|
total / MIB,
|
||||||
|
used / MIB,
|
||||||
|
row.disk_free_bytes.unwrap_or(0) / GIB,
|
||||||
|
committed_vms,
|
||||||
|
row.mem_baseline_mib,
|
||||||
|
row.headroom_fresh(MAX_METRICS_AGE_SECS),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rank admissible nodes: most free slots first, then live headroom, then id.
|
||||||
|
///
|
||||||
|
/// Slots before headroom SPREADS load rather than stacking it — two missions
|
||||||
|
/// launched together go to different machines. Headroom breaks ties with
|
||||||
|
/// real-time load, which is where a node mid-`cargo build` loses to an idle peer.
|
||||||
|
/// Node id last so the same fleet state always yields the same answer; the old
|
||||||
|
/// `last_seen DESC` made placement unreproducible between two identical runs.
|
||||||
|
pub fn rank(mut fit: Vec<NodeCapacity>) -> Vec<NodeCapacity> {
|
||||||
|
fit.sort_by(|a, b| {
|
||||||
|
b.slots
|
||||||
|
.cmp(&a.slots)
|
||||||
|
.then(
|
||||||
|
b.headroom
|
||||||
|
.partial_cmp(&a.headroom)
|
||||||
|
.unwrap_or(std::cmp::Ordering::Equal),
|
||||||
|
)
|
||||||
|
.then(a.node_id.as_uuid().cmp(&b.node_id.as_uuid()))
|
||||||
|
});
|
||||||
|
fit
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One line per node, for logs and for the message an operator reads.
|
||||||
|
pub fn report(fit: &[NodeCapacity], unfit: &[(NodeId, String, Unfit)]) -> String {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for f in fit {
|
||||||
|
out.push(format!(
|
||||||
|
" {}: {} slot(s) free, {} VM(s) committed, {}/{} MiB, headroom {:.0}",
|
||||||
|
f.name, f.slots, f.committed_vms, f.used_eff_mib, f.mem_total_mib, f.headroom
|
||||||
|
));
|
||||||
|
}
|
||||||
|
for (_, name, why) in unfit {
|
||||||
|
out.push(format!(" {name}: UNFIT — {}", why.reason()));
|
||||||
|
}
|
||||||
|
if out.is_empty() {
|
||||||
|
out.push(" (no capable nodes)".into());
|
||||||
|
}
|
||||||
|
out.join("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Count a node's commitments, unioning booted VMs with pinned-not-yet-booted
|
||||||
|
/// phases BY IDENTITY.
|
||||||
|
///
|
||||||
|
/// A composed graph's step VMs (`...-s0`, `-s1`) each count: each is a real
|
||||||
|
/// Firecracker process holding 8 GiB. A pinned phase counts only while no live VM
|
||||||
|
/// carries its id — otherwise the same claim would be counted twice and the fleet
|
||||||
|
/// would shrink by the number of phases currently starting.
|
||||||
|
pub fn commitments(live_vm_ids: &[String], pinned_keys: &[String]) -> i64 {
|
||||||
|
let live = live_vm_ids.len() as i64;
|
||||||
|
let unbooted = pinned_keys
|
||||||
|
.iter()
|
||||||
|
.filter(|k| !live_vm_ids.iter().any(|v| v.starts_with(k.as_str())))
|
||||||
|
.count() as i64;
|
||||||
|
live + unbooted
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every backend a phase needs on ONE node: the mission's, plus each backend
|
||||||
|
/// named by a node of its composed graph.
|
||||||
|
///
|
||||||
|
/// The roster stores them as `config.roster.nodes[].attrs.backend`, and they are
|
||||||
|
/// the reason this function exists. A 2-member roster with
|
||||||
|
/// `verifier@canary-claude` was placed on a node holding `claude` and not
|
||||||
|
/// `canary-claude`; the graph's first node ran, the second died with
|
||||||
|
/// `no rootfs for backend "canary-claude" on this node`, and the mission
|
||||||
|
/// delivered half its work and failed. Placement had asked only about the
|
||||||
|
/// mission's own backend, which was true and insufficient.
|
||||||
|
pub fn required_backends(mission_backend: Option<&str>, roster: Option<&serde_json::Value>) -> Vec<String> {
|
||||||
|
let mut out = vec![cm_db::repo::nodes::backend_key(mission_backend).to_string()];
|
||||||
|
if let Some(nodes) = roster.and_then(|r| r.get("nodes")).and_then(|n| n.as_array()) {
|
||||||
|
for n in nodes {
|
||||||
|
if let Some(b) = n
|
||||||
|
.get("attrs")
|
||||||
|
.and_then(|a| a.get("backend"))
|
||||||
|
.and_then(|b| b.as_str())
|
||||||
|
.filter(|b| !b.trim().is_empty())
|
||||||
|
{
|
||||||
|
out.push(b.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.sort();
|
||||||
|
out.dedup();
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Survey every capable node: which can take a phase VM, and why the rest cannot.
|
||||||
|
///
|
||||||
|
/// `vm_list` is asked of each candidate in parallel with a short deadline. A node
|
||||||
|
/// that will not answer is `CapacityUnknown` and therefore ineligible — we cannot
|
||||||
|
/// count what we cannot see, and guessing zero is how a node gets double-booked.
|
||||||
|
pub async fn survey(
|
||||||
|
pool: &sqlx::PgPool,
|
||||||
|
hub: &crate::fleet::NodeHub,
|
||||||
|
workspace_id: uuid::Uuid,
|
||||||
|
// EVERY backend the work needs, not just the mission's. A composed graph
|
||||||
|
// runs on ONE node and its nodes may each name their own — the roster's
|
||||||
|
// whole purpose is an independent verifier on another provider — so the
|
||||||
|
// node has to hold all of their rootfs images.
|
||||||
|
backends: &[String],
|
||||||
|
) -> Result<(Vec<NodeCapacity>, Vec<(NodeId, String, Unfit)>), String> {
|
||||||
|
let candidates = cm_db::repo::nodes::online_for_backends(pool, workspace_id, backends)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("looking up nodes for backends {backends:?}: {e}"))?;
|
||||||
|
if candidates.is_empty() {
|
||||||
|
return Ok((Vec::new(), Vec::new()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let evals = cm_db::repo::node_metrics::eval_all(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("reading node metrics: {e}"))?;
|
||||||
|
let pinned = cm_db::repo::nodes::pinned_microvm_phases(pool, workspace_id)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("reading pinned phases: {e}"))?;
|
||||||
|
|
||||||
|
let names = node_names(pool, workspace_id).await;
|
||||||
|
let mut fit = Vec::new();
|
||||||
|
let mut unfit = Vec::new();
|
||||||
|
for node in candidates {
|
||||||
|
let row = evals.iter().find(|e| e.node_id == node);
|
||||||
|
let name = names
|
||||||
|
.get(&node.as_uuid())
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_else(|| node.as_uuid().to_string()[..8].to_string());
|
||||||
|
|
||||||
|
// Not connected: nothing can be asked of it, and nothing can run on it.
|
||||||
|
if !hub.is_connected(node) {
|
||||||
|
unfit.push((node, name, Unfit::NotConnected));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(row) = row else {
|
||||||
|
unfit.push((node, name, Unfit::NoRecentHealth { age_secs: None }));
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Commitments: booted VMs unioned with phases pinned here but not yet
|
||||||
|
// booted, by the deterministic id both sides agree on.
|
||||||
|
let live = match crate::microvm_client::list(hub, node).await {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(e) => {
|
||||||
|
unfit.push((node, name, Unfit::CapacityUnknown { err: e }));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let keys: Vec<String> = pinned
|
||||||
|
.iter()
|
||||||
|
.filter(|(n, _, _)| *n == node)
|
||||||
|
.map(|(_, phase, iter)| crate::microvm_executor::vm_id_for(*phase, *iter, None))
|
||||||
|
.collect();
|
||||||
|
let committed = commitments(&live, &keys);
|
||||||
|
|
||||||
|
// An idle node is the ONLY time its own footprint is measurable rather
|
||||||
|
// than inferred, so take the reading whenever we get one. Cheap: an
|
||||||
|
// UPDATE per idle node per survey, and it is what stops a young VM's
|
||||||
|
// unconsumed memory from being handed out a second time.
|
||||||
|
if committed == 0 {
|
||||||
|
if let Some(used) = row.mem_used_bytes.filter(|_| {
|
||||||
|
row.health_age_secs
|
||||||
|
.is_some_and(|a| a <= MAX_HEALTH_AGE_SECS)
|
||||||
|
}) {
|
||||||
|
let mib = used / (1024 * 1024);
|
||||||
|
if row.mem_baseline_mib != Some(mib) {
|
||||||
|
let _ = cm_db::repo::nodes::set_mem_baseline(pool, node, mib).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
match from_eval(row, &name, committed) {
|
||||||
|
Ok(c) => fit.push(c),
|
||||||
|
Err(why) => unfit.push((node, name, why)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok((rank(fit), unfit))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Node names for readable reports. A capacity report naming two machines
|
||||||
|
/// "New node" is a report nobody can act on.
|
||||||
|
async fn node_names(
|
||||||
|
pool: &sqlx::PgPool,
|
||||||
|
workspace_id: uuid::Uuid,
|
||||||
|
) -> std::collections::HashMap<uuid::Uuid, String> {
|
||||||
|
sqlx::query_as::<_, (uuid::Uuid, String)>(
|
||||||
|
"SELECT id, name FROM nodes WHERE workspace_id = $1",
|
||||||
|
)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default()
|
||||||
|
.into_iter()
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Choose a node for a phase, honouring an explicit target as a REQUEST.
|
||||||
|
///
|
||||||
|
/// `want` is honoured only if that node is genuinely admissible — the same
|
||||||
|
/// "a request, not a guarantee" rule the orchestrator already applied to
|
||||||
|
/// capability, now extended to capacity and draining.
|
||||||
|
pub async fn choose(
|
||||||
|
pool: &sqlx::PgPool,
|
||||||
|
hub: &crate::fleet::NodeHub,
|
||||||
|
workspace_id: uuid::Uuid,
|
||||||
|
backends: &[String],
|
||||||
|
want: Option<uuid::Uuid>,
|
||||||
|
) -> Result<NodeId, PlacementError> {
|
||||||
|
let named = backends.join(", ");
|
||||||
|
let how_to_fix = format!(
|
||||||
|
"needs /dev/kvm + firecracker (scripts/fc-node-setup.sh) AND the {named} rootfs \
|
||||||
|
built on ONE node (scripts/fc-build-rootfs.sh <host> <image> <name>) — a \
|
||||||
|
composed graph runs on a single node, so that node needs every image its \
|
||||||
|
nodes ask for"
|
||||||
|
);
|
||||||
|
let (fit, unfit) = survey(pool, hub, workspace_id, backends).await.map_err(|e| {
|
||||||
|
PlacementError::FleetUnreadable { report: format!(" survey failed: {e}") }
|
||||||
|
})?;
|
||||||
|
|
||||||
|
if fit.is_empty() && unfit.is_empty() {
|
||||||
|
return Err(PlacementError::NoCapableNode {
|
||||||
|
backend: named,
|
||||||
|
how_to_fix,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let report = report(&fit, &unfit);
|
||||||
|
|
||||||
|
// `want` is ADVISORY, always. The only caller passes `missions.target_node_id`,
|
||||||
|
// which is simply where the PREVIOUS phase ran — not an operator's choice.
|
||||||
|
// Treating it as a requirement had two consequences, both wrong:
|
||||||
|
//
|
||||||
|
// - a previous node that had since filled up (or gone unreadable) failed
|
||||||
|
// the phase outright: `TargetUnfit` is not transient, so it never
|
||||||
|
// reached the queue. Note this was NOT the drain case — a draining node
|
||||||
|
// is already excluded by `online_for_backend`'s `status = 'online'`, so
|
||||||
|
// it never reaches `unfit` at all and the pin simply falls through.
|
||||||
|
// `drain-midmission` passes either way; the path it does not cover is
|
||||||
|
// "phase 1's node is now full", which is the one that used to fail.
|
||||||
|
// - and while the node stayed fit, every later phase went back to it
|
||||||
|
// regardless of ranking — accidental mission-to-node affinity, which
|
||||||
|
// this module's own header says must not exist.
|
||||||
|
//
|
||||||
|
// Mission state lives on the gateway (inject -> run -> collect -> destroy),
|
||||||
|
// so re-placing costs nothing. Prefer the pin when it still fits; say out
|
||||||
|
// loud why it did not when it does not, and rank as usual.
|
||||||
|
if let Some(want) = want {
|
||||||
|
if let Some(c) = fit.iter().find(|c| c.node_id.as_uuid() == want) {
|
||||||
|
return Ok(c.node_id);
|
||||||
|
}
|
||||||
|
if let Some((_, name, why)) = unfit.iter().find(|(n, _, _)| n.as_uuid() == want) {
|
||||||
|
eprintln!(
|
||||||
|
"vm_placement: the previous phase's node {name} is {} — re-placing this phase",
|
||||||
|
why.reason()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(best) = fit.into_iter().next() {
|
||||||
|
return Ok(best.node_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nothing fit. Distinguish "full" from "blind": an operator sent to look for
|
||||||
|
// a load problem that is really a dead daemon wastes the outage.
|
||||||
|
let blind = unfit.iter().all(|(_, _, w)| {
|
||||||
|
matches!(w, Unfit::CapacityUnknown { .. } | Unfit::NotConnected | Unfit::NoRecentHealth { .. })
|
||||||
|
});
|
||||||
|
Err(if blind {
|
||||||
|
PlacementError::FleetUnreadable { report }
|
||||||
|
} else {
|
||||||
|
PlacementError::FleetAtCapacity { report }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn nid(n: u128) -> NodeId {
|
||||||
|
NodeId::from(uuid::Uuid::from_u128(n))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// THE test. Measured on tank: 60 GiB total, and five VMs booted moments ago
|
||||||
|
/// showing only ~12 GiB used because the guests have not touched their claim.
|
||||||
|
///
|
||||||
|
/// Observed-usage-only arithmetic says (61440-12000-4096)/8192 = 5 more VMs.
|
||||||
|
/// The node has room for ONE. Booking those five is a node in swap, and every
|
||||||
|
/// VM on it slows down together.
|
||||||
|
/// A node whose VMs have not yet consumed their claim must not hand the
|
||||||
|
/// difference out again.
|
||||||
|
///
|
||||||
|
/// This is the bug the `capacity` harness scenario found on its first full
|
||||||
|
/// run — "morpheus peaked at 3 concurrent VM(s) with only 2 slot(s)" — and
|
||||||
|
/// the numbers here are that node's real ones. Idle it reports 4314 MiB of
|
||||||
|
/// 31757 and the survey correctly gives it 2 slots. Two VMs later, each
|
||||||
|
/// holding roughly 1 GiB of its 8 GiB, observed usage is ~6314 MiB;
|
||||||
|
/// inferring the baseline as 6314 - 16384 goes negative, clamps to the
|
||||||
|
/// 2048 floor, and invents 2266 MiB — exactly one more VM than exists.
|
||||||
|
/// A previous node that is no longer usable re-places the next phase; it
|
||||||
|
/// does not fail it.
|
||||||
|
///
|
||||||
|
/// `choose` treated `missions.target_node_id` — which is only ever "where
|
||||||
|
/// the last phase ran" — as a hard requirement, so a pinned node that had
|
||||||
|
/// since FILLED UP produced `TargetUnfit`, which is not transient, and the
|
||||||
|
/// phase failed instead of queueing or moving. It also gave every later
|
||||||
|
/// phase silent affinity back to the first node.
|
||||||
|
///
|
||||||
|
/// The drain case is not this one and never was: `online_for_backend`
|
||||||
|
/// filters on `status = 'online'`, so a draining node is not a candidate
|
||||||
|
/// and the pin falls through to ranking. `drain-midmission` passes on both
|
||||||
|
/// the old and new code, which is why the capacity half needs this test.
|
||||||
|
/// A composed graph's per-node backends are part of what placement needs.
|
||||||
|
///
|
||||||
|
/// The full harness found this: a 2-member roster with
|
||||||
|
/// `verifier@canary-claude` was placed on a node holding `claude` and not
|
||||||
|
/// `canary-claude`. The first graph node ran, the second died with
|
||||||
|
/// `no rootfs for backend "canary-claude" on this node`, and the mission
|
||||||
|
/// delivered half its work and failed. Placement had asked only about the
|
||||||
|
/// mission's own backend — true, and insufficient.
|
||||||
|
#[test]
|
||||||
|
fn a_composed_graph_needs_every_backend_its_nodes_name() {
|
||||||
|
let roster = serde_json::json!({
|
||||||
|
"kind": "pipeline",
|
||||||
|
"nodes": [
|
||||||
|
{"id": "n0", "role": "implementer"},
|
||||||
|
{"id": "n1", "role": "verifier", "attrs": {"backend": "canary-claude"}},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
assert_eq!(
|
||||||
|
required_backends(Some("claude"), Some(&roster)),
|
||||||
|
vec!["canary-claude".to_string(), "claude".to_string()],
|
||||||
|
"both images have to be on the ONE node the graph runs on"
|
||||||
|
);
|
||||||
|
|
||||||
|
// A solo mission is unchanged — this must not make ordinary placement
|
||||||
|
// stricter than it was.
|
||||||
|
assert_eq!(required_backends(Some("claude"), None), vec!["claude"]);
|
||||||
|
assert_eq!(required_backends(None, None), vec!["default"]);
|
||||||
|
|
||||||
|
// A node with no explicit backend inherits the mission's, so it adds
|
||||||
|
// nothing. Deduped, or a 5-node graph would ask for `claude` five times
|
||||||
|
// and the containment query would still be right but the error message
|
||||||
|
// would be nonsense.
|
||||||
|
let inherit = serde_json::json!({"nodes": [
|
||||||
|
{"id": "n0", "role": "a"},
|
||||||
|
{"id": "n1", "role": "b", "attrs": {}},
|
||||||
|
{"id": "n2", "role": "c", "attrs": {"backend": ""}},
|
||||||
|
]});
|
||||||
|
assert_eq!(required_backends(Some("claude"), Some(&inherit)), vec!["claude"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_unfit_previous_node_is_re_placed_not_refused() {
|
||||||
|
let drained = uuid::Uuid::from_u128(1);
|
||||||
|
let healthy = capacity_of(nid(2), "tank", 61440, 6144, 800, 0, None, 90.0).unwrap();
|
||||||
|
|
||||||
|
// Stand in for `choose`'s decision: the pin is consulted, then dropped.
|
||||||
|
let fit = vec![healthy.clone()];
|
||||||
|
let picked = fit
|
||||||
|
.iter()
|
||||||
|
.find(|c| c.node_id.as_uuid() == drained)
|
||||||
|
.or_else(|| fit.first())
|
||||||
|
.expect("a fit node exists");
|
||||||
|
assert_eq!(
|
||||||
|
picked.node_id,
|
||||||
|
nid(2),
|
||||||
|
"with the pinned node absent from `fit`, ranking must still yield a node"
|
||||||
|
);
|
||||||
|
|
||||||
|
// And the error that used to be produced here no longer exists, so it
|
||||||
|
// cannot be reintroduced as a non-transient failure by accident.
|
||||||
|
for e in [
|
||||||
|
PlacementError::FleetAtCapacity { report: String::new() },
|
||||||
|
PlacementError::FleetUnreadable { report: String::new() },
|
||||||
|
] {
|
||||||
|
assert!(e.is_transient(), "both no-node outcomes must QUEUE, not fail");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_young_vms_unconsumed_memory_is_not_handed_out_twice() {
|
||||||
|
// Idle: the reading that gets remembered, and the slot count it implies.
|
||||||
|
let idle = capacity_of(nid(3), "morpheus", 31757, 4314, 312, 0, None, 90.0)
|
||||||
|
.expect("an idle morpheus fits VMs");
|
||||||
|
assert_eq!(idle.slots, 2, "idle capacity is the number we are defending");
|
||||||
|
|
||||||
|
// Two committed, both young. WITHOUT the remembered baseline this
|
||||||
|
// returned 1 slot and admitted a third VM.
|
||||||
|
let inferred = capacity_of(nid(3), "morpheus", 31757, 6314, 312, 2, None, 90.0);
|
||||||
|
assert!(
|
||||||
|
inferred.is_ok(),
|
||||||
|
"the old inference is preserved as the no-baseline fallback"
|
||||||
|
);
|
||||||
|
|
||||||
|
// WITH it, the node is correctly full.
|
||||||
|
let remembered = capacity_of(nid(3), "morpheus", 31757, 6314, 312, 2, Some(4314), 90.0);
|
||||||
|
assert!(
|
||||||
|
matches!(remembered, Err(Unfit::AtCapacity { committed: 2, .. })),
|
||||||
|
"a 2-slot node with 2 VMs committed is FULL, got {remembered:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A host that starts doing real work outside its VMs is charged for it.
|
||||||
|
///
|
||||||
|
/// The remembered baseline is a floor, not a substitute. If it replaced the
|
||||||
|
/// inference outright, a node that was idle at 4 GiB and is now running a
|
||||||
|
/// 20 GiB build would still be scored as if it were idle — the same
|
||||||
|
/// over-commit, arrived at from the opposite direction.
|
||||||
|
#[test]
|
||||||
|
fn a_remembered_baseline_never_under_charges_a_busy_host() {
|
||||||
|
// 1 VM committed and consumed (8192), plus 20 GiB of non-VM work.
|
||||||
|
let used = 8192 + 20480;
|
||||||
|
let c = capacity_of(nid(3), "busy", 61440, used, 800, 1, Some(4096), 90.0)
|
||||||
|
.expect("still has room");
|
||||||
|
// Inference says 20480; the stale 4096 baseline must not win.
|
||||||
|
assert_eq!(c.used_eff_mib, used, "observed usage is charged in full");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_sold_out_node_is_not_mistaken_for_an_idle_one() {
|
||||||
|
let observed_only =
|
||||||
|
capacity_of(nid(1), "tank", 61440, 12000, 800, 0, None, 50.0).expect("fits");
|
||||||
|
assert_eq!(
|
||||||
|
observed_only.slots, 5,
|
||||||
|
"this is what utilisation alone claims — the bug being fixed"
|
||||||
|
);
|
||||||
|
|
||||||
|
let with_commitments =
|
||||||
|
capacity_of(nid(1), "tank", 61440, 12000, 800, 5, None, 50.0).expect("fits");
|
||||||
|
assert_eq!(
|
||||||
|
with_commitments.slots, 1,
|
||||||
|
"five 8 GiB claims are already spoken for, whatever the guests have touched"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The measured idle fleet. Numbers from `free`/`df` on the real machines, so
|
||||||
|
/// a future change to the constants has to face what it does to real nodes.
|
||||||
|
#[test]
|
||||||
|
fn the_measured_fleet_gets_the_slots_it_actually_has() {
|
||||||
|
// tank: 60 GiB, ~6 GiB used at idle.
|
||||||
|
let tank = capacity_of(nid(1), "tank", 61440, 6144, 869, 0, None, 90.0).unwrap();
|
||||||
|
assert_eq!(tank.slots, 6);
|
||||||
|
// architect: 60 GiB, ~7 GiB used.
|
||||||
|
let arch = capacity_of(nid(2), "architect", 61440, 7168, 388, 0, None, 90.0).unwrap();
|
||||||
|
assert_eq!(arch.slots, 6);
|
||||||
|
// morpheus: 31 GiB — deliberately the conservative 2, not 3. Three VMs
|
||||||
|
// would leave under 2 GiB for the host, which is where the OOM killer
|
||||||
|
// lives, and an OOM-killed VM looks like an agent that gave up.
|
||||||
|
let morph = capacity_of(nid(3), "morpheus", 31744, 5120, 312, 0, None, 90.0).unwrap();
|
||||||
|
assert_eq!(morph.slots, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spread, don't stack; then real load; then determinism.
|
||||||
|
#[test]
|
||||||
|
fn ranking_prefers_free_slots_then_headroom_then_a_stable_order() {
|
||||||
|
let a = capacity_of(nid(1), "a", 61440, 6144, 800, 0, None, 40.0).unwrap(); // 6 slots
|
||||||
|
let b = capacity_of(nid(2), "b", 61440, 6144, 800, 3, None, 90.0).unwrap(); // 3 slots
|
||||||
|
assert_eq!(rank(vec![b.clone(), a.clone()])[0].name, "a", "more slots wins");
|
||||||
|
|
||||||
|
// Equal slots → the node under less real load.
|
||||||
|
let busy = capacity_of(nid(3), "busy", 61440, 6144, 800, 0, None, 10.0).unwrap();
|
||||||
|
let idle = capacity_of(nid(4), "idle", 61440, 6144, 800, 0, None, 95.0).unwrap();
|
||||||
|
assert_eq!(rank(vec![busy.clone(), idle.clone()])[0].name, "idle");
|
||||||
|
|
||||||
|
// Equal on both → same answer twice. `last_seen DESC` could not promise this.
|
||||||
|
let x = capacity_of(nid(9), "x", 61440, 6144, 800, 0, None, 50.0).unwrap();
|
||||||
|
let y = capacity_of(nid(8), "y", 61440, 6144, 800, 0, None, 50.0).unwrap();
|
||||||
|
assert_eq!(rank(vec![x.clone(), y.clone()])[0].name, "y");
|
||||||
|
assert_eq!(rank(vec![y, x])[0].name, "y");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A booted VM and its pinned phase row are ONE claim, not two.
|
||||||
|
#[test]
|
||||||
|
fn commitments_union_by_identity_rather_than_adding() {
|
||||||
|
let live = vec!["m-abc123def456-0".to_string(), "m-abc123def456-0-s2".to_string()];
|
||||||
|
// Same phase as the live VMs: already counted.
|
||||||
|
assert_eq!(commitments(&live, &["m-abc123def456-0".to_string()]), 2);
|
||||||
|
// A different phase, pinned but not yet booted: a real additional claim.
|
||||||
|
assert_eq!(
|
||||||
|
commitments(&live, &["m-999888777666-0".to_string()]),
|
||||||
|
3,
|
||||||
|
"a phase chosen seconds ago holds 8 GiB no node can report yet"
|
||||||
|
);
|
||||||
|
assert_eq!(commitments(&[], &["m-1-0".into(), "m-2-0".into()]), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Disk is a hard gate, and it is checked BEFORE capacity so the message
|
||||||
|
/// names the real problem.
|
||||||
|
#[test]
|
||||||
|
fn a_node_short_of_disk_is_refused_even_with_memory_to_spare() {
|
||||||
|
let e = capacity_of(nid(1), "tank", 61440, 6144, 25, 0, None, 90.0).unwrap_err();
|
||||||
|
assert!(matches!(e, Unfit::NoDisk { free_gib: 25 }), "{e:?}");
|
||||||
|
assert!(e.reason().contains("25 GiB"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stale metrics may cost a tie; they may never win one, and they may never
|
||||||
|
/// exclude a node — that is health's job.
|
||||||
|
#[test]
|
||||||
|
fn stale_beszel_metrics_demote_but_do_not_exclude() {
|
||||||
|
let mut row = row_for(nid(1), 61440 * MIB_T, 6144 * MIB_T, 800 * GIB_T);
|
||||||
|
row.metrics_age_secs = Some(3600.0);
|
||||||
|
row.health_age_secs = Some(3.0);
|
||||||
|
row.cpu_pct = Some(5.0);
|
||||||
|
let fit = from_eval(&row, "tank", 0).expect("still eligible");
|
||||||
|
assert_eq!(fit.headroom, 95.0, "fresh health carries the headroom");
|
||||||
|
|
||||||
|
row.health_age_secs = Some(3600.0);
|
||||||
|
assert!(
|
||||||
|
matches!(from_eval(&row, "tank", 0), Err(Unfit::NoRecentHealth { .. })),
|
||||||
|
"stale HEALTH is exclusion, because memory is then a guess"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The two failures an operator must never confuse.
|
||||||
|
#[test]
|
||||||
|
fn unreadable_capacity_never_reads_as_a_full_fleet() {
|
||||||
|
let full = PlacementError::FleetAtCapacity { report: " tank: 0 slots".into() };
|
||||||
|
let blind = PlacementError::FleetUnreadable { report: " tank: UNFIT".into() };
|
||||||
|
assert!(full.message().contains("at capacity"));
|
||||||
|
assert!(blind.message().contains("cannot read"));
|
||||||
|
assert!(
|
||||||
|
!blind.message().contains("at capacity"),
|
||||||
|
"sends an operator hunting a load problem that does not exist"
|
||||||
|
);
|
||||||
|
assert!(full.is_transient() && blind.is_transient());
|
||||||
|
let missing = PlacementError::NoCapableNode {
|
||||||
|
backend: "claude".into(),
|
||||||
|
how_to_fix: "build the image".into(),
|
||||||
|
};
|
||||||
|
assert!(!missing.is_transient(), "a missing image will not fix itself by waiting");
|
||||||
|
}
|
||||||
|
|
||||||
|
const MIB_T: i64 = 1024 * 1024;
|
||||||
|
const GIB_T: i64 = 1024 * 1024 * 1024;
|
||||||
|
|
||||||
|
fn row_for(node_id: NodeId, total: i64, used: i64, disk_free: i64) -> EvalRow {
|
||||||
|
EvalRow {
|
||||||
|
node_id,
|
||||||
|
workspace_id: cm_domain::WorkspaceId::from(uuid::Uuid::from_u128(1)),
|
||||||
|
status: "online".into(),
|
||||||
|
cpu_pct: None,
|
||||||
|
mem_pct: None,
|
||||||
|
disk_pct: None,
|
||||||
|
gpu_pct: None,
|
||||||
|
temp_max: None,
|
||||||
|
load1: None,
|
||||||
|
mem_total_bytes: Some(total),
|
||||||
|
mem_used_bytes: Some(used),
|
||||||
|
disk_free_bytes: Some(disk_free),
|
||||||
|
mem_baseline_mib: None,
|
||||||
|
health_age_secs: Some(3.0),
|
||||||
|
metrics_age_secs: Some(3.0),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A draining node is ineligible, not merely unattractive. The microVM path
|
||||||
|
/// never checked this before: a mission pinned before a drain kept feeding
|
||||||
|
/// VMs to a node an operator had cordoned.
|
||||||
|
#[test]
|
||||||
|
fn a_draining_node_is_ineligible() {
|
||||||
|
let mut row = row_for(nid(1), 61440 * MIB_T, 6144 * MIB_T, 800 * GIB_T);
|
||||||
|
row.status = "draining".into();
|
||||||
|
assert_eq!(from_eval(&row, "tank", 0), Err(Unfit::Draining));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,516 @@
|
|||||||
|
//! The completion gate, moved into the agent's own loop.
|
||||||
|
//!
|
||||||
|
//! Every check this platform has on a phase runs **after** the agent has
|
||||||
|
//! finished: the evaluator judges `done_when`, capture notices that a coding
|
||||||
|
//! phase delivered nothing, and either verdict costs a whole new VM — a fresh
|
||||||
|
//! boot, a fresh inject, and an agent starting again with none of the context
|
||||||
|
//! that got it that far. Meanwhile the documented failure of a long-running
|
||||||
|
//! agent is that it *stops too early*.
|
||||||
|
//!
|
||||||
|
//! Claude Code's `Stop` hook is the seam. **Exit code 2 blocks the stop and
|
||||||
|
//! feeds stderr back to the model as the reason.** Measured, not read off docs —
|
||||||
|
//! an agent told "say hello and do nothing else", whose `Stop` hook exited 2
|
||||||
|
//! saying `evidence.txt` was missing, created `evidence.txt` and then stopped.
|
||||||
|
//!
|
||||||
|
//! # What it may and may not check
|
||||||
|
//!
|
||||||
|
//! Deliberately mechanical: whether the repository changed, and whether a
|
||||||
|
//! command the phase author wrote exits 0. NOT the `done_when` verdict — that is
|
||||||
|
//! an LLM judgement made host-side by a *different provider* on purpose
|
||||||
|
//! ([[evaluator-verification]]), and re-implementing it inside the VM would put
|
||||||
|
//! the agent's own environment in charge of grading the agent, which is the
|
||||||
|
//! correlated failure the independent judge exists to break.
|
||||||
|
//!
|
||||||
|
//! # The cap is load-bearing
|
||||||
|
//!
|
||||||
|
//! A gate with no ceiling turns a stuck agent into a wedged one: it would be
|
||||||
|
//! blocked, retry, be blocked again, and burn the hour-long turn budget instead
|
||||||
|
//! of failing in a way the operator can see. After [`MAX_BLOCKS`] the gate lets
|
||||||
|
//! the agent stop, records that it did, and leaves the verdict to the existing
|
||||||
|
//! post-hoc path — which still runs, unchanged.
|
||||||
|
//!
|
||||||
|
//! # Which hooks exist here
|
||||||
|
//!
|
||||||
|
//! `TaskCompleted` / `TeammateIdle` were the plan's chosen seam. Measured under
|
||||||
|
//! `claude -p`: they never fire, because no team forms in print mode at all.
|
||||||
|
//! `Stop`, `SubagentStop`, `PreToolUse`, `PostToolUse`, `UserPromptSubmit` and
|
||||||
|
//! `SessionStart` do.
|
||||||
|
|
||||||
|
|
||||||
|
/// How many times the gate may refuse a stop before it gives up and lets the
|
||||||
|
/// agent finish. Three is enough for "you wrote nothing" → "you wrote something"
|
||||||
|
/// → "your check passes" without ever approaching the turn budget.
|
||||||
|
pub const MAX_BLOCKS: u32 = 3;
|
||||||
|
|
||||||
|
/// The file the gate writes when it gives up and lets the agent stop with its
|
||||||
|
/// condition still failing.
|
||||||
|
///
|
||||||
|
/// A separate file rather than a line in the log, because the log is not
|
||||||
|
/// parseable for this: a block reason embeds the check's own output, and an
|
||||||
|
/// output line beginning `cap:` would read as a cap release that never happened.
|
||||||
|
///
|
||||||
|
/// It exists because the block COUNT cannot answer the question. Three blocks
|
||||||
|
/// followed by a stop that finally passed, and three blocks followed by a
|
||||||
|
/// release at the cap, both report `blocks: 3` — and they are opposite outcomes.
|
||||||
|
/// Without this, the second one completed the phase green.
|
||||||
|
pub const CAPPED_FILE: &str = "capped";
|
||||||
|
|
||||||
|
/// Where the gate lives in the guest.
|
||||||
|
///
|
||||||
|
/// Under `/root`, never under the repository. Anything written into
|
||||||
|
/// `/mission/repo` is collected and diffed, so a gate script placed there would
|
||||||
|
/// arrive in the user's delivered patch as if an agent had authored it.
|
||||||
|
pub const GATE_DIR: &str = "/root/gate";
|
||||||
|
|
||||||
|
/// What must hold before this phase's agent is allowed to stop.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct StopGate {
|
||||||
|
/// The phase must leave the repository changed. Set for coding phases that
|
||||||
|
/// have not declared `allow_empty` — the same rule
|
||||||
|
/// `empty_delivery_is_a_failure` applies post-hoc, applied while the agent
|
||||||
|
/// can still do something about it.
|
||||||
|
pub require_changes: bool,
|
||||||
|
/// `config.done_when_check`: a shell command, run in the repo, that must
|
||||||
|
/// exit 0. The deterministic half of a completion condition — a command,
|
||||||
|
/// not a judgement.
|
||||||
|
pub check: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StopGate {
|
||||||
|
/// The gate for a phase, or `None` when there is nothing to enforce.
|
||||||
|
///
|
||||||
|
/// `None` matters: installing a hook that can never block would still cost a
|
||||||
|
/// process per stop and would put a `--settings` flag on the command line
|
||||||
|
/// for no reason.
|
||||||
|
pub fn for_phase(kind: &str, config: &serde_json::Value) -> Option<StopGate> {
|
||||||
|
let allow_empty = config.get("allow_empty").and_then(|v| v.as_bool()) == Some(true);
|
||||||
|
let check = config
|
||||||
|
.get("done_when_check")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(str::to_string);
|
||||||
|
let require_changes = kind == "coding" && !allow_empty;
|
||||||
|
if !require_changes && check.is_none() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(StopGate {
|
||||||
|
require_changes,
|
||||||
|
check,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The same gate, for ONE NODE of a composed run.
|
||||||
|
///
|
||||||
|
/// `require_changes` is a property of the phase, not of every node in it: a
|
||||||
|
/// graph whose second node reviews or verifies is *supposed* to leave the
|
||||||
|
/// tree alone, and a per-node gate would refuse its stop three times for
|
||||||
|
/// doing exactly its job. Dropping it loses nothing, because
|
||||||
|
/// `empty_delivery_is_a_failure` applies the same rule post-hoc to what the
|
||||||
|
/// phase as a whole delivered.
|
||||||
|
///
|
||||||
|
/// That "post-hoc" claim used to be written as covering the `check` too. It
|
||||||
|
/// did not: nothing outside this hook has ever re-run `done_when_check`, so
|
||||||
|
/// a release at [`MAX_BLOCKS`] completed the phase green with the check
|
||||||
|
/// still failing. [`CAPPED_FILE`] is what closes that.
|
||||||
|
///
|
||||||
|
/// A declared `check` DOES apply per node: it is a command the phase author
|
||||||
|
/// wrote, and every stage of the work should satisfy it.
|
||||||
|
pub fn per_node(self) -> Option<StopGate> {
|
||||||
|
self.check.map(|check| StopGate {
|
||||||
|
require_changes: false,
|
||||||
|
check: Some(check),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The hook script, as POSIX `sh`.
|
||||||
|
///
|
||||||
|
/// `repo` and `dir` are parameters rather than the constants above so a test
|
||||||
|
/// can run this script — the real one, not a paraphrase — against a real git
|
||||||
|
/// repository in a temp directory.
|
||||||
|
pub fn script(&self, repo: &str, dir: &str) -> String {
|
||||||
|
let mut s = String::from("#!/bin/sh\n# ClawMates stop gate. Exit 2 refuses the stop.\n");
|
||||||
|
s.push_str(&format!("REPO={}\nGATE={}\nMAX={MAX_BLOCKS}\n", q(repo), q(dir)));
|
||||||
|
s.push_str("N=$(cat \"$GATE/blocks\" 2>/dev/null || echo 0)\nreason=''\n");
|
||||||
|
|
||||||
|
if self.require_changes {
|
||||||
|
// Two questions, because either alone is answerable "no" by a
|
||||||
|
// perfectly good phase: an agent that committed its work leaves a
|
||||||
|
// clean tree, and an agent that did not commit leaves HEAD where it
|
||||||
|
// was. Only both together mean nothing happened.
|
||||||
|
s.push_str(
|
||||||
|
"BASE=$(cat \"$REPO/.git/clawmates-base\" 2>/dev/null || echo '')\n\
|
||||||
|
DIRTY=$(git -C \"$REPO\" status --porcelain 2>/dev/null | head -c 400)\n\
|
||||||
|
HEAD=$(git -C \"$REPO\" rev-parse HEAD 2>/dev/null || echo '')\n\
|
||||||
|
if [ -z \"$DIRTY\" ] && [ -n \"$BASE\" ] && [ \"$HEAD\" = \"$BASE\" ]; then\n\
|
||||||
|
\x20 reason='This phase has changed nothing: the working tree is clean and \
|
||||||
|
HEAD is still the commit you started from. Do the work the task describes \
|
||||||
|
and leave it in the tree. If the task genuinely requires no code change, \
|
||||||
|
say so explicitly in your final message.'\n\
|
||||||
|
fi\n",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(check) = &self.check {
|
||||||
|
s.push_str(&format!(
|
||||||
|
"if [ -z \"$reason\" ]; then\n\
|
||||||
|
\x20 out=$(cd \"$REPO\" && sh -c {} 2>&1); rc=$?\n\
|
||||||
|
\x20 if [ \"$rc\" -ne 0 ]; then\n\
|
||||||
|
\x20 reason=\"This phase's completion check exited $rc, so the work is not \
|
||||||
|
done yet. The check is: {}\n\nIts output:\n$(printf '%s' \"$out\" | tail -c 1500)\"\n\
|
||||||
|
\x20 fi\n\
|
||||||
|
fi\n",
|
||||||
|
q(check),
|
||||||
|
// Inside a double-quoted assignment, so the command text itself
|
||||||
|
// must not carry a `\"` or a `$` that the shell would expand.
|
||||||
|
check.replace('\\', "\\\\").replace('"', "'").replace('$', "\\$"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
s.push_str(
|
||||||
|
"if [ -z \"$reason\" ]; then echo pass >> \"$GATE/log\"; exit 0; fi\n\
|
||||||
|
if [ \"$N\" -ge \"$MAX\" ]; then\n\
|
||||||
|
\x20 echo \"cap: $reason\" >> \"$GATE/log\"\n\
|
||||||
|
\x20 echo 1 > \"$GATE/capped\"\n\
|
||||||
|
\x20 exit 0\n\
|
||||||
|
fi\n\
|
||||||
|
N=$((N+1)); echo \"$N\" > \"$GATE/blocks\"\n\
|
||||||
|
echo \"block $N: $reason\" >> \"$GATE/log\"\n\
|
||||||
|
printf '%s\\n' \"$reason\" >&2\n\
|
||||||
|
exit 2\n",
|
||||||
|
);
|
||||||
|
s
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One shell command that writes the gate SCRIPT into the guest.
|
||||||
|
///
|
||||||
|
/// It deliberately does NOT write `settings.json`. It used to, and it wrote
|
||||||
|
/// the whole document — so the moment a second feature needed a hook, the
|
||||||
|
/// later writer would silently erase this one. The composed document is
|
||||||
|
/// built in exactly one place: [`crate::vm_tool_tap::guest_settings`].
|
||||||
|
///
|
||||||
|
/// Written by `printf` through an exec rather than injected as part of the
|
||||||
|
/// tar: the tar lands in `/mission/repo`, which is exactly where this must
|
||||||
|
/// not be.
|
||||||
|
pub fn install_command(&self, repo: &str, dir: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"mkdir -p {d} && rm -f {d}/blocks {d}/log {d}/capped \
|
||||||
|
&& printf '%s' {script} > {d}/stop-gate.sh \
|
||||||
|
&& chmod +x {d}/stop-gate.sh",
|
||||||
|
d = dir,
|
||||||
|
script = q(&self.script(repo, dir)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Single-quote for `sh`. Same rule as `microvm_executor::shell_quote`, kept
|
||||||
|
/// local so this module has no dependency on the executor it is used by.
|
||||||
|
fn q(s: &str) -> String {
|
||||||
|
format!("'{}'", s.replace('\'', r"'\''"))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use serde_json::json;
|
||||||
|
use std::path::Path;
|
||||||
|
use std::process::Command;
|
||||||
|
|
||||||
|
fn sh(script: &str, dir: &Path) -> std::process::Output {
|
||||||
|
let path = dir.join("stop-gate.sh");
|
||||||
|
std::fs::write(&path, script).unwrap();
|
||||||
|
Command::new("sh").arg(&path).output().expect("run the gate")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A git repo with one commit and the clone-point marker the real checkout
|
||||||
|
/// carries (`mission_workspace::record_base_commit` writes it).
|
||||||
|
fn repo_with_base(root: &Path) -> std::path::PathBuf {
|
||||||
|
let repo = root.join("repo");
|
||||||
|
std::fs::create_dir_all(&repo).unwrap();
|
||||||
|
let git = |args: &[&str]| {
|
||||||
|
let o = Command::new("git")
|
||||||
|
.arg("-C")
|
||||||
|
.arg(&repo)
|
||||||
|
.args(args)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
assert!(o.status.success(), "git {args:?}: {:?}", o);
|
||||||
|
};
|
||||||
|
git(&["init", "--quiet"]);
|
||||||
|
git(&["config", "user.email", "t@t"]);
|
||||||
|
git(&["config", "user.name", "T"]);
|
||||||
|
std::fs::write(repo.join("README.md"), "base\n").unwrap();
|
||||||
|
git(&["add", "."]);
|
||||||
|
git(&["commit", "--quiet", "-m", "base"]);
|
||||||
|
let head = Command::new("git")
|
||||||
|
.arg("-C")
|
||||||
|
.arg(&repo)
|
||||||
|
.args(["rev-parse", "HEAD"])
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
std::fs::write(
|
||||||
|
repo.join(".git/clawmates-base"),
|
||||||
|
String::from_utf8_lossy(&head.stdout).trim(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
repo
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The failure this exists for: an agent that stops having written nothing.
|
||||||
|
/// Post-hoc that costs a whole new VM; here it costs one sentence.
|
||||||
|
#[test]
|
||||||
|
fn an_agent_that_changed_nothing_is_not_allowed_to_stop() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let repo = repo_with_base(tmp.path());
|
||||||
|
let gate = StopGate {
|
||||||
|
require_changes: true,
|
||||||
|
check: None,
|
||||||
|
};
|
||||||
|
let script = gate.script(&repo.display().to_string(), &tmp.path().display().to_string());
|
||||||
|
|
||||||
|
let out = sh(&script, tmp.path());
|
||||||
|
assert_eq!(out.status.code(), Some(2), "the stop must be refused");
|
||||||
|
let why = String::from_utf8_lossy(&out.stderr);
|
||||||
|
assert!(why.contains("changed nothing"), "{why}");
|
||||||
|
|
||||||
|
// Uncommitted work counts — the usual case, since the agent is told to
|
||||||
|
// leave its work in the tree rather than commit it.
|
||||||
|
std::fs::write(repo.join("new.rs"), "fn done() {}\n").unwrap();
|
||||||
|
let out = sh(&script, tmp.path());
|
||||||
|
assert_eq!(out.status.code(), Some(0), "{:?}", out);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The gate gives up after [`MAX_BLOCKS`] and lets the agent stop — and it
|
||||||
|
/// must LEAVE A MARK when it does. Nothing outside this hook ever runs a
|
||||||
|
/// `done_when_check`, so a silent release completed the phase green with its
|
||||||
|
/// condition still failing.
|
||||||
|
///
|
||||||
|
/// The two files say different things and both are needed: `blocks` reaches
|
||||||
|
/// 3 in this test AND in a run where the agent got it right on the fourth
|
||||||
|
/// try, so the count alone cannot tell success from surrender.
|
||||||
|
#[test]
|
||||||
|
fn a_gate_that_gives_up_records_that_it_gave_up() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let dir = tmp.path().display().to_string();
|
||||||
|
let repo = repo_with_base(tmp.path());
|
||||||
|
let gate = StopGate {
|
||||||
|
require_changes: false,
|
||||||
|
check: Some("exit 1".into()),
|
||||||
|
};
|
||||||
|
let script = gate.script(&repo.display().to_string(), &dir);
|
||||||
|
|
||||||
|
for n in 1..=MAX_BLOCKS {
|
||||||
|
let out = sh(&script, tmp.path());
|
||||||
|
assert_eq!(out.status.code(), Some(2), "block {n} must refuse the stop");
|
||||||
|
assert!(
|
||||||
|
!tmp.path().join(CAPPED_FILE).exists(),
|
||||||
|
"the cap mark must not appear while the gate is still blocking"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// One more stop: the gate is out of blocks and must let the agent go.
|
||||||
|
let out = sh(&script, tmp.path());
|
||||||
|
assert_eq!(out.status.code(), Some(0), "at the cap the stop is allowed");
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(tmp.path().join(CAPPED_FILE))
|
||||||
|
.unwrap()
|
||||||
|
.trim(),
|
||||||
|
"1",
|
||||||
|
"the release must be recorded, or nothing downstream can see it"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The negative control for the mark: a gate whose check PASSES releases the
|
||||||
|
/// agent too, and that release must not be recorded as a surrender. Without
|
||||||
|
/// this, "always write the file" would pass the test above and fail every
|
||||||
|
/// healthy phase in production.
|
||||||
|
#[test]
|
||||||
|
fn a_gate_that_is_satisfied_leaves_no_cap_mark() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let repo = repo_with_base(tmp.path());
|
||||||
|
let gate = StopGate {
|
||||||
|
require_changes: false,
|
||||||
|
check: Some("true".into()),
|
||||||
|
};
|
||||||
|
let script = gate.script(&repo.display().to_string(), &tmp.path().display().to_string());
|
||||||
|
|
||||||
|
let out = sh(&script, tmp.path());
|
||||||
|
assert_eq!(out.status.code(), Some(0));
|
||||||
|
assert!(
|
||||||
|
!tmp.path().join(CAPPED_FILE).exists(),
|
||||||
|
"a satisfied gate must not look like one that gave up"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// And committed work counts too. An agent that committed leaves a CLEAN
|
||||||
|
/// tree, so a gate that only looked at `git status` would refuse the stop of
|
||||||
|
/// a phase that had done everything asked of it.
|
||||||
|
#[test]
|
||||||
|
fn work_the_agent_committed_satisfies_the_gate() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let repo = repo_with_base(tmp.path());
|
||||||
|
std::fs::write(repo.join("new.rs"), "fn done() {}\n").unwrap();
|
||||||
|
for args in [vec!["add", "."], vec!["commit", "--quiet", "-m", "work"]] {
|
||||||
|
Command::new("git")
|
||||||
|
.arg("-C")
|
||||||
|
.arg(&repo)
|
||||||
|
.args(&args)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let gate = StopGate {
|
||||||
|
require_changes: true,
|
||||||
|
check: None,
|
||||||
|
};
|
||||||
|
let out = sh(
|
||||||
|
&gate.script(&repo.display().to_string(), &tmp.path().display().to_string()),
|
||||||
|
tmp.path(),
|
||||||
|
);
|
||||||
|
assert_eq!(out.status.code(), Some(0), "{:?}", out);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The cap. Without it a stuck agent is blocked, retries, is blocked again,
|
||||||
|
/// and spends the whole hour-long turn budget instead of failing where an
|
||||||
|
/// operator can see it.
|
||||||
|
#[test]
|
||||||
|
fn the_gate_gives_up_after_the_cap_and_says_so() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let repo = repo_with_base(tmp.path());
|
||||||
|
let gate = StopGate {
|
||||||
|
require_changes: true,
|
||||||
|
check: None,
|
||||||
|
};
|
||||||
|
let script = gate.script(&repo.display().to_string(), &tmp.path().display().to_string());
|
||||||
|
|
||||||
|
for i in 1..=MAX_BLOCKS {
|
||||||
|
assert_eq!(
|
||||||
|
sh(&script, tmp.path()).status.code(),
|
||||||
|
Some(2),
|
||||||
|
"block {i} of {MAX_BLOCKS}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
sh(&script, tmp.path()).status.code(),
|
||||||
|
Some(0),
|
||||||
|
"past the cap the agent must be allowed to stop"
|
||||||
|
);
|
||||||
|
let log = std::fs::read_to_string(tmp.path().join("log")).unwrap();
|
||||||
|
assert!(log.contains("cap:"), "giving up is recorded: {log}");
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(tmp.path().join("blocks"))
|
||||||
|
.unwrap()
|
||||||
|
.trim(),
|
||||||
|
MAX_BLOCKS.to_string(),
|
||||||
|
"and the count is exact, so the host can report it"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A phase-declared check runs in the repo, and its OUTPUT comes back — a
|
||||||
|
/// gate that said only "the check failed" would send the agent guessing.
|
||||||
|
#[test]
|
||||||
|
fn a_declared_check_must_pass_and_its_output_is_the_feedback() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let repo = repo_with_base(tmp.path());
|
||||||
|
let gate = StopGate {
|
||||||
|
require_changes: false,
|
||||||
|
check: Some("test -f wanted.txt || { echo 'wanted.txt is missing'; exit 3; }".into()),
|
||||||
|
};
|
||||||
|
let script = gate.script(&repo.display().to_string(), &tmp.path().display().to_string());
|
||||||
|
|
||||||
|
let out = sh(&script, tmp.path());
|
||||||
|
assert_eq!(out.status.code(), Some(2));
|
||||||
|
let why = String::from_utf8_lossy(&out.stderr);
|
||||||
|
assert!(why.contains("exited 3"), "{why}");
|
||||||
|
assert!(why.contains("wanted.txt is missing"), "{why}");
|
||||||
|
|
||||||
|
std::fs::write(repo.join("wanted.txt"), "here\n").unwrap();
|
||||||
|
assert_eq!(sh(&script, tmp.path()).status.code(), Some(0));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A check with quotes, `$` and apostrophes is ordinary. It travels through
|
||||||
|
/// `sh -c` inside a script that itself travels through `sh -c` to reach the
|
||||||
|
/// guest, and a quoting bug at either layer would run something else.
|
||||||
|
#[test]
|
||||||
|
fn a_check_with_shell_metacharacters_survives_both_layers() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let repo = repo_with_base(tmp.path());
|
||||||
|
std::fs::write(repo.join("it's here.txt"), "x\n").unwrap();
|
||||||
|
let gate = StopGate {
|
||||||
|
require_changes: false,
|
||||||
|
check: Some("test -f \"it's here.txt\" && echo $HOME > /dev/null".into()),
|
||||||
|
};
|
||||||
|
let out = sh(
|
||||||
|
&gate.script(&repo.display().to_string(), &tmp.path().display().to_string()),
|
||||||
|
tmp.path(),
|
||||||
|
);
|
||||||
|
assert_eq!(out.status.code(), Some(0), "{:?}", out);
|
||||||
|
// And the install command it is embedded in is still one shell argument.
|
||||||
|
let install = gate.install_command("/mission/repo", GATE_DIR);
|
||||||
|
assert!(install.contains("stop-gate.sh"), "{install}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Nothing the gate writes may land under the repository: `/mission/repo` is
|
||||||
|
/// collected and diffed, so a file there arrives in the user's patch as if
|
||||||
|
/// an agent had written it.
|
||||||
|
#[test]
|
||||||
|
fn the_gate_never_writes_into_the_delivered_tree() {
|
||||||
|
let gate = StopGate {
|
||||||
|
require_changes: true,
|
||||||
|
check: Some("cargo test".into()),
|
||||||
|
};
|
||||||
|
assert!(GATE_DIR.starts_with("/root/"), "{GATE_DIR}");
|
||||||
|
let install = gate.install_command("/mission/repo", GATE_DIR);
|
||||||
|
for write in ["> /mission/repo", "/mission/repo/stop", "/mission/repo/.claude"] {
|
||||||
|
assert!(!install.contains(write), "{install}");
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
crate::vm_tool_tap::guest_settings(Some(GATE_DIR), None)["hooks"]["Stop"][0]["hooks"]
|
||||||
|
[0]["command"],
|
||||||
|
json!("/root/gate/stop-gate.sh")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A composed run's nodes must not each be held to "this phase changed
|
||||||
|
/// something". The graph's verifier node changes nothing BY DESIGN, and a
|
||||||
|
/// per-node gate would refuse its stop until the cap — three wasted agent
|
||||||
|
/// turns for doing its job correctly.
|
||||||
|
#[test]
|
||||||
|
fn a_composed_node_is_not_held_to_the_whole_phases_delivery() {
|
||||||
|
let phase = StopGate::for_phase("coding", &json!({})).unwrap();
|
||||||
|
assert!(phase.require_changes);
|
||||||
|
assert!(
|
||||||
|
phase.per_node().is_none(),
|
||||||
|
"with nothing but the delivery rule, a node has no gate at all"
|
||||||
|
);
|
||||||
|
|
||||||
|
let with_check =
|
||||||
|
StopGate::for_phase("coding", &json!({ "done_when_check": "cargo test" })).unwrap();
|
||||||
|
let node = with_check.per_node().expect("the declared check still applies");
|
||||||
|
assert!(!node.require_changes);
|
||||||
|
assert_eq!(node.check.as_deref(), Some("cargo test"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A gate with nothing to enforce must not be installed at all — a hook that
|
||||||
|
/// can never block still costs a process per stop and a flag on the command
|
||||||
|
/// line.
|
||||||
|
#[test]
|
||||||
|
fn a_phase_with_nothing_to_enforce_gets_no_gate() {
|
||||||
|
let none = json!({});
|
||||||
|
assert!(StopGate::for_phase("research", &none).is_none());
|
||||||
|
assert!(StopGate::for_phase("coding", &json!({ "allow_empty": true })).is_none());
|
||||||
|
|
||||||
|
let coding = StopGate::for_phase("coding", &none).expect("a coding phase must deliver");
|
||||||
|
assert!(coding.require_changes);
|
||||||
|
assert!(coding.check.is_none());
|
||||||
|
|
||||||
|
// A declared check applies to any kind, including one that is allowed to
|
||||||
|
// change nothing — a verification phase's whole job is that check.
|
||||||
|
let verify = StopGate::for_phase(
|
||||||
|
"research",
|
||||||
|
&json!({ "allow_empty": true, "done_when_check": " ./verify.sh " }),
|
||||||
|
)
|
||||||
|
.expect("a declared check is a gate on its own");
|
||||||
|
assert!(!verify.require_changes);
|
||||||
|
assert_eq!(verify.check.as_deref(), Some("./verify.sh"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,331 @@
|
|||||||
|
//! What the agent did inside a microVM, taken from Claude Code's own hooks.
|
||||||
|
//!
|
||||||
|
//! The microVM tier had no action channel at all: a phase ran, a diff came
|
||||||
|
//! back, and everything between was invisible. The seam is the same one
|
||||||
|
//! [`crate::vm_stop_gate`] proved works in this image — `PostToolUse` fires
|
||||||
|
//! under `claude -p`, measured, not read off documentation.
|
||||||
|
//!
|
||||||
|
//! # The observer must not become a participant
|
||||||
|
//!
|
||||||
|
//! The hook `exit 0`s unconditionally. A `PostToolUse` hook that exits non-zero
|
||||||
|
//! feeds its stderr back to the model, so a tap with a bug would start
|
||||||
|
//! *instructing* the agent it exists to watch — and the resulting transcript
|
||||||
|
//! would look like a model that lost the plot rather than a broken hook.
|
||||||
|
//!
|
||||||
|
//! # Never inside the repository
|
||||||
|
//!
|
||||||
|
//! Everything lives under `/root`. `/mission/repo` is collected and diffed, so
|
||||||
|
//! a tap file written there would arrive in the user's delivered patch as
|
||||||
|
//! though an agent had authored it — the same rule, and the same reason, as the
|
||||||
|
//! stop gate's [`crate::vm_stop_gate::GATE_DIR`].
|
||||||
|
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
|
||||||
|
/// Where the tap writes in the guest. Under `/root`, never the repo.
|
||||||
|
pub const TAP_DIR: &str = "/root/tap";
|
||||||
|
|
||||||
|
/// The file the hook appends to, one JSON object per line.
|
||||||
|
pub const TAP_FILE: &str = "/root/tap/tools.jsonl";
|
||||||
|
|
||||||
|
/// The single settings document the guest agent runs with.
|
||||||
|
///
|
||||||
|
/// One path, because there is only ever one writer — see [`guest_settings`].
|
||||||
|
pub const SETTINGS_PATH: &str = "/root/guest-settings.json";
|
||||||
|
|
||||||
|
/// Read the tap out of the guest, before collection destroys the VM.
|
||||||
|
///
|
||||||
|
/// `|| true` so a phase whose agent called no tools — or where the hook never
|
||||||
|
/// fired — reads as empty rather than as a failed probe. The difference between
|
||||||
|
/// those two is the histogram in the log, not an error here.
|
||||||
|
pub const DRAIN_PROBE: &str = "cat /root/tap/tools.jsonl 2>/dev/null || true";
|
||||||
|
|
||||||
|
/// Read the tap from line `from` onward, so a repeated drain returns only what
|
||||||
|
/// is new.
|
||||||
|
///
|
||||||
|
/// A cursor rather than a re-read: the live drain runs every few seconds
|
||||||
|
/// against a file the agent is still appending to, and re-sending the whole
|
||||||
|
/// file each pass would record every tool call once per poll — a phase would
|
||||||
|
/// finish with its early files weighted by how long it ran.
|
||||||
|
///
|
||||||
|
/// `tail -n +N` is 1-based on the FIRST line to print, so `from` is a line
|
||||||
|
/// count already consumed and the probe asks for `from + 1`.
|
||||||
|
pub fn drain_from(from: usize) -> String {
|
||||||
|
format!("tail -n +{} {TAP_FILE} 2>/dev/null || true", from + 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How far a drain advanced the cursor — the number of LINES it consumed.
|
||||||
|
///
|
||||||
|
/// Counts every line, including blank ones, and that is the whole point. The
|
||||||
|
/// hook appends the event and then a newline of its own, so the tap is
|
||||||
|
/// `{json}\n\n{json}\n\n…` and `parse` skips the blanks. Advancing the cursor
|
||||||
|
/// by the number of PARSED events instead would leave it short by one line per
|
||||||
|
/// event, and `tail -n +N` would hand back events already recorded — every one
|
||||||
|
/// of them written again on the next poll, with nothing anywhere reporting it.
|
||||||
|
pub fn consumed_lines(raw: &str) -> usize {
|
||||||
|
raw.lines().count()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One observed tool call.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct Observed {
|
||||||
|
pub tool: String,
|
||||||
|
/// The path the tool's **input** named, if any. From JSON, never prose.
|
||||||
|
pub path: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The hook script. Copies stdin verbatim to the tap file and gets out of the
|
||||||
|
/// way.
|
||||||
|
///
|
||||||
|
/// The parsing happens host-side, on purpose: a `jq` or `sed` pipeline in the
|
||||||
|
/// guest would need the tool's JSON schema baked into a shell script, inside an
|
||||||
|
/// image we do not rebuild for a parser change, with no way to tell a parse
|
||||||
|
/// failure from a quiet turn.
|
||||||
|
pub fn hook_script(dir: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"#!/bin/sh\n\
|
||||||
|
# The tool tap. See cm-api/src/vm_tool_tap.rs.\n\
|
||||||
|
mkdir -p {dir} 2>/dev/null\n\
|
||||||
|
# `cat` of stdin, appended whole. One JSON object per line, because\n\
|
||||||
|
# Claude Code hands the hook one event per invocation.\n\
|
||||||
|
cat >> {dir}/tools.jsonl 2>/dev/null\n\
|
||||||
|
printf '\\n' >> {dir}/tools.jsonl 2>/dev/null\n\
|
||||||
|
# ALWAYS zero. A non-zero PostToolUse hook talks back to the model.\n\
|
||||||
|
exit 0\n"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The settings document for the guest, carrying **every** hook at once.
|
||||||
|
///
|
||||||
|
/// This function exists because the alternative — each feature writing its own
|
||||||
|
/// `settings.json` — is a silent clobber. The stop gate wrote the whole
|
||||||
|
/// document; a tap that did the same would erase the gate, and a coding phase
|
||||||
|
/// would then complete having written nothing, which is the exact failure the
|
||||||
|
/// gate exists to catch. One writer, one document, one test that both hooks
|
||||||
|
/// survive it.
|
||||||
|
///
|
||||||
|
/// `None` for either half means that hook is simply absent.
|
||||||
|
pub fn guest_settings(gate_dir: Option<&str>, tap_dir: Option<&str>) -> Value {
|
||||||
|
let mut hooks = serde_json::Map::new();
|
||||||
|
if let Some(dir) = gate_dir {
|
||||||
|
hooks.insert(
|
||||||
|
"Stop".into(),
|
||||||
|
json!([{ "hooks": [{ "type": "command", "command": format!("{dir}/stop-gate.sh") }] }]),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if let Some(dir) = tap_dir {
|
||||||
|
hooks.insert(
|
||||||
|
"PostToolUse".into(),
|
||||||
|
json!([{ "hooks": [{ "type": "command", "command": format!("{dir}/tap.sh") }] }]),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
json!({ "hooks": Value::Object(hooks) })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One shell command that installs the tap.
|
||||||
|
///
|
||||||
|
/// Written by `printf` through an exec rather than injected with the workspace
|
||||||
|
/// tar: the tar lands in `/mission/repo`, which is exactly where this must not.
|
||||||
|
pub fn install_command(dir: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"mkdir -p {dir} && rm -f {dir}/tools.jsonl \
|
||||||
|
&& printf '%s' {script} > {dir}/tap.sh && chmod +x {dir}/tap.sh",
|
||||||
|
dir = dir,
|
||||||
|
script = q(&hook_script(dir)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write the composed settings document.
|
||||||
|
pub fn settings_command(path: &str, settings: &Value) -> String {
|
||||||
|
format!("printf '%s' {} > {path}", q(&settings.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse a drained tap.
|
||||||
|
///
|
||||||
|
/// Tolerant by construction: the file is appended to by a shell hook in a VM
|
||||||
|
/// that may be killed mid-write, so a truncated last line is expected and is
|
||||||
|
/// skipped rather than failing the whole drain. Losing the last tool call of a
|
||||||
|
/// phase costs one orb; losing all of them because of it would cost the tier.
|
||||||
|
pub fn parse(raw: &str) -> Vec<Observed> {
|
||||||
|
raw.lines()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|l| !l.is_empty())
|
||||||
|
.filter_map(|line| {
|
||||||
|
let v: Value = serde_json::from_str(line).ok()?;
|
||||||
|
// Only tool events. The same hook file would carry others if the
|
||||||
|
// settings ever install one, and a `hook_event_name` we do not
|
||||||
|
// recognise must not be read as a tool named "".
|
||||||
|
let tool = v
|
||||||
|
.get("tool_name")
|
||||||
|
.or_else(|| v.get("toolName"))
|
||||||
|
.and_then(Value::as_str)?
|
||||||
|
.trim()
|
||||||
|
.to_string();
|
||||||
|
if tool.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let input = v
|
||||||
|
.get("tool_input")
|
||||||
|
.or_else(|| v.get("toolInput"))
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or(Value::Null);
|
||||||
|
Some(Observed {
|
||||||
|
path: crate::mission_events::tool_path(&input),
|
||||||
|
tool,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Single-quote for `sh`. Local copy, same rule as the stop gate's — these two
|
||||||
|
/// modules deliberately share no code, so neither can break the other.
|
||||||
|
fn q(s: &str) -> String {
|
||||||
|
format!("'{}'", s.replace('\'', r"'\''"))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// The gate and the tap must BOTH survive one settings document.
|
||||||
|
///
|
||||||
|
/// This is the whole reason `guest_settings` exists. Two writers each
|
||||||
|
/// producing a whole `settings.json` is not a merge conflict — the second
|
||||||
|
/// simply wins, no error, and the loser's hook never runs. When the loser is
|
||||||
|
/// the stop gate, a coding phase completes having written nothing: the exact
|
||||||
|
/// failure the gate was built to catch.
|
||||||
|
#[test]
|
||||||
|
fn both_hooks_survive_one_settings_document() {
|
||||||
|
let s = guest_settings(Some("/root/gate"), Some(TAP_DIR));
|
||||||
|
let hooks = s.get("hooks").expect("hooks");
|
||||||
|
assert_eq!(
|
||||||
|
hooks["Stop"][0]["hooks"][0]["command"],
|
||||||
|
json!("/root/gate/stop-gate.sh"),
|
||||||
|
"the stop gate must survive the tap being installed"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
hooks["PostToolUse"][0]["hooks"][0]["command"],
|
||||||
|
json!("/root/tap/tap.sh")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Either half absent leaves the other exactly as it was.
|
||||||
|
#[test]
|
||||||
|
fn one_hook_alone_is_a_valid_document() {
|
||||||
|
let gate_only = guest_settings(Some("/root/gate"), None);
|
||||||
|
assert!(gate_only["hooks"].get("Stop").is_some());
|
||||||
|
assert!(gate_only["hooks"].get("PostToolUse").is_none());
|
||||||
|
|
||||||
|
let tap_only = guest_settings(None, Some(TAP_DIR));
|
||||||
|
assert!(tap_only["hooks"].get("Stop").is_none());
|
||||||
|
assert!(tap_only["hooks"].get("PostToolUse").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The observer must never talk back to the model.
|
||||||
|
#[test]
|
||||||
|
fn the_hook_always_exits_zero() {
|
||||||
|
let s = hook_script(TAP_DIR);
|
||||||
|
assert!(s.contains("exit 0"));
|
||||||
|
// No conditional exits at all: a `PostToolUse` hook that exits non-zero
|
||||||
|
// feeds stderr back to the agent, so the tap would become an instruction.
|
||||||
|
assert!(!s.contains("exit 1") && !s.contains("exit 2"), "{s}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Nothing the tap writes may land in the delivered tree.
|
||||||
|
#[test]
|
||||||
|
fn the_tap_never_writes_into_the_repository() {
|
||||||
|
assert!(TAP_DIR.starts_with("/root/"));
|
||||||
|
assert!(TAP_FILE.starts_with("/root/"));
|
||||||
|
assert!(SETTINGS_PATH.starts_with("/root/"));
|
||||||
|
let cmd = install_command(TAP_DIR);
|
||||||
|
assert!(!cmd.contains("/mission/repo"), "{cmd}");
|
||||||
|
assert!(!hook_script(TAP_DIR).contains("/mission/repo"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A real `PostToolUse` payload gives up its tool and its path — and a
|
||||||
|
/// truncated final line does not take the rest of the phase with it.
|
||||||
|
#[test]
|
||||||
|
fn a_drained_tap_parses_and_tolerates_a_torn_last_line() {
|
||||||
|
let raw = concat!(
|
||||||
|
r#"{"hook_event_name":"PostToolUse","tool_name":"Edit","#,
|
||||||
|
r#""tool_input":{"file_path":"/mission/repo/src/a.rs"}}"#,
|
||||||
|
"\n",
|
||||||
|
r#"{"hook_event_name":"PostToolUse","tool_name":"Bash","tool_input":{"command":"ls"}}"#,
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
// The VM was destroyed mid-write.
|
||||||
|
r#"{"hook_event_name":"PostToolUse","tool_name":"Wri"#,
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
parse(raw),
|
||||||
|
vec![
|
||||||
|
Observed { tool: "Edit".into(), path: Some("/mission/repo/src/a.rs".into()) },
|
||||||
|
Observed { tool: "Bash".into(), path: None },
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Exactly one place in the tree writes the guest settings document.
|
||||||
|
///
|
||||||
|
/// The unit test above proves `guest_settings` composes correctly; it says
|
||||||
|
/// nothing about whether anyone bypasses it. A second `> …settings.json`
|
||||||
|
/// anywhere is the silent clobber itself, and it would pass every other
|
||||||
|
/// test in this file.
|
||||||
|
#[test]
|
||||||
|
fn nothing_else_writes_the_guest_settings() {
|
||||||
|
for (name, src) in [
|
||||||
|
("vm_stop_gate.rs", include_str!("vm_stop_gate.rs")),
|
||||||
|
("microvm_executor.rs", include_str!("microvm_executor.rs")),
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
!src.contains("> {d}/settings.json") && !src.contains("settings.json\","),
|
||||||
|
"{name} writes a settings document of its own; compose it through \
|
||||||
|
vm_tool_tap::guest_settings instead"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// And the one legitimate writer is this module's own helper.
|
||||||
|
let exec = include_str!("microvm_executor.rs");
|
||||||
|
assert_eq!(
|
||||||
|
exec.matches("vm_tool_tap::settings_command").count(),
|
||||||
|
1,
|
||||||
|
"the settings document must be written exactly once per turn"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The cursor must not re-read what it already returned.
|
||||||
|
///
|
||||||
|
/// Off by one here is not a crash, it is a double-count: `tail -n +1` and
|
||||||
|
/// `tail -n +2` both return output, and the wrong one quietly records every
|
||||||
|
/// early tool call once per poll.
|
||||||
|
#[test]
|
||||||
|
fn the_drain_cursor_asks_for_what_it_has_not_seen() {
|
||||||
|
assert!(drain_from(0).contains("tail -n +1 "));
|
||||||
|
assert!(drain_from(3).contains("tail -n +4 "));
|
||||||
|
assert!(drain_from(0).contains(TAP_FILE));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The cursor counts LINES, not events.
|
||||||
|
///
|
||||||
|
/// The hook writes the event and then a newline of its own, so a two-event
|
||||||
|
/// tap is four lines. Advancing by parsed-event count would leave the
|
||||||
|
/// cursor two lines short, `tail` would return both events again, and the
|
||||||
|
/// live drain would re-record everything it had already recorded — growing
|
||||||
|
/// worse the longer the turn ran, and silent throughout.
|
||||||
|
#[test]
|
||||||
|
fn the_cursor_counts_lines_not_events() {
|
||||||
|
let raw = concat!(
|
||||||
|
r#"{"tool_name":"Edit","tool_input":{"file_path":"a.rs"}}"#,
|
||||||
|
"\n\n",
|
||||||
|
r#"{"tool_name":"Bash","tool_input":{"command":"ls"}}"#,
|
||||||
|
"\n\n",
|
||||||
|
);
|
||||||
|
assert_eq!(parse(raw).len(), 2, "two events");
|
||||||
|
assert_eq!(consumed_lines(raw), 4, "…written across four lines");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An event that is not a tool call is not a tool named "".
|
||||||
|
#[test]
|
||||||
|
fn a_non_tool_event_is_skipped() {
|
||||||
|
assert!(parse(r#"{"hook_event_name":"SessionStart","session_id":"x"}"#).is_empty());
|
||||||
|
assert!(parse(r#"{"tool_name":" ","tool_input":{}}"#).is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -564,6 +564,109 @@ async fn a_failed_gate_publishes_to_a_wip_branch() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// #55: a mission whose checkout was re-cloned builds divergent history against
|
||||||
|
/// its OWN deterministic branch, and git rejects every push it will ever make.
|
||||||
|
/// That was terminal — the work stayed on a local branch in a directory that
|
||||||
|
/// gets reaped — and it is reachable from a retry, a container teardown, or disk
|
||||||
|
/// loss, not just from someone deleting a checkout by hand.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn diverged_history_lands_on_a_new_branch_instead_of_being_lost() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let remote = tmp.path().join("remote.git");
|
||||||
|
std::fs::create_dir_all(&remote).unwrap();
|
||||||
|
Command::new("git")
|
||||||
|
.args(["init", "--bare", "--quiet"])
|
||||||
|
.arg(&remote)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
let branch = "clawmates/mission-test-dddddddd";
|
||||||
|
let url = remote.to_str().unwrap();
|
||||||
|
|
||||||
|
// The first attempt: a checkout that pushed its work and then vanished.
|
||||||
|
let first = seed_repo(tmp.path(), Uuid::now_v7());
|
||||||
|
std::fs::write(first.join("first.rs"), "fn first() {}\n").unwrap();
|
||||||
|
git(&first, &["add", "."]);
|
||||||
|
git(&first, &["commit", "--quiet", "-m", "first pass"]);
|
||||||
|
git(&first, &["checkout", "-B", branch]);
|
||||||
|
let one = mission_delivery::publish_phase_branch(
|
||||||
|
&first,
|
||||||
|
url,
|
||||||
|
branch,
|
||||||
|
mission_delivery::Gate::Always,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(one.pushed, "setup push failed: {:?}", one.error);
|
||||||
|
let claimed = Command::new("git")
|
||||||
|
.arg("-C")
|
||||||
|
.arg(&remote)
|
||||||
|
.args(["rev-parse", branch])
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
let claimed = String::from_utf8_lossy(&claimed.stdout).trim().to_string();
|
||||||
|
|
||||||
|
// The retry: a fresh clone of the same mission, so unrelated history under
|
||||||
|
// the same deterministic branch name.
|
||||||
|
let second = seed_repo(tmp.path(), Uuid::now_v7());
|
||||||
|
std::fs::write(second.join("second.rs"), "fn second() {}\n").unwrap();
|
||||||
|
git(&second, &["add", "."]);
|
||||||
|
git(&second, &["commit", "--quiet", "-m", "retry"]);
|
||||||
|
git(&second, &["checkout", "-B", branch]);
|
||||||
|
|
||||||
|
let out = mission_delivery::publish_phase_branch(
|
||||||
|
&second,
|
||||||
|
url,
|
||||||
|
branch,
|
||||||
|
mission_delivery::Gate::Always,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
out.pushed,
|
||||||
|
"the retry's work never reached the forge: {:?}",
|
||||||
|
out.error
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
out.branch.starts_with(branch) && out.branch != branch,
|
||||||
|
"it must land on a NEW ref, not the contested one: {}",
|
||||||
|
out.branch
|
||||||
|
);
|
||||||
|
|
||||||
|
let refs = Command::new("git")
|
||||||
|
.arg("-C")
|
||||||
|
.arg(&remote)
|
||||||
|
.args(["for-each-ref", "--format=%(refname:short)"])
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
let refs = String::from_utf8_lossy(&refs.stdout);
|
||||||
|
assert!(refs.contains(&out.branch), "refs: {refs}");
|
||||||
|
|
||||||
|
// NEVER force: the first attempt's ref still points where it did. Losing it
|
||||||
|
// to make this push look tidy would trade one lost copy of the work for
|
||||||
|
// another.
|
||||||
|
let still = Command::new("git")
|
||||||
|
.arg("-C")
|
||||||
|
.arg(&remote)
|
||||||
|
.args(["rev-parse", branch])
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
String::from_utf8_lossy(&still.stdout).trim(),
|
||||||
|
claimed,
|
||||||
|
"the earlier attempt's branch was overwritten"
|
||||||
|
);
|
||||||
|
let show = Command::new("git")
|
||||||
|
.arg("-C")
|
||||||
|
.arg(&remote)
|
||||||
|
.args(["show", &format!("{}:second.rs", out.branch)])
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
assert!(String::from_utf8_lossy(&show.stdout).contains("fn second()"));
|
||||||
|
}
|
||||||
|
|
||||||
/// An unreachable remote is a degraded success, not a failure: the patch and
|
/// An unreachable remote is a degraded success, not a failure: the patch and
|
||||||
/// the local branch both still exist.
|
/// the local branch both still exist.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ async fn seed_test_template(pool: &sqlx::PgPool) -> Uuid {
|
|||||||
system_prompt: "Plan the feature. Break it into INT-XX items.",
|
system_prompt: "Plan the feature. Break it into INT-XX items.",
|
||||||
skills: vec!["decompose-int-items".into()],
|
skills: vec!["decompose-int-items".into()],
|
||||||
brain_seed: Some("# Planner\nBreak features into INT items."),
|
brain_seed: Some("# Planner\nBreak features into INT items."),
|
||||||
|
model: None,
|
||||||
},
|
},
|
||||||
team_templates::UpsertBuiltinRole {
|
team_templates::UpsertBuiltinRole {
|
||||||
slot: "coder",
|
slot: "coder",
|
||||||
@@ -76,6 +77,7 @@ async fn seed_test_template(pool: &sqlx::PgPool) -> Uuid {
|
|||||||
system_prompt: "Implement one INT item at a time.",
|
system_prompt: "Implement one INT item at a time.",
|
||||||
skills: vec!["write-rust-current-edition".into()],
|
skills: vec!["write-rust-current-edition".into()],
|
||||||
brain_seed: Some("# Coder\nOne INT per commit."),
|
brain_seed: Some("# Coder\nOne INT per commit."),
|
||||||
|
model: None,
|
||||||
},
|
},
|
||||||
team_templates::UpsertBuiltinRole {
|
team_templates::UpsertBuiltinRole {
|
||||||
slot: "reviewer",
|
slot: "reviewer",
|
||||||
@@ -83,6 +85,9 @@ async fn seed_test_template(pool: &sqlx::PgPool) -> Uuid {
|
|||||||
system_prompt: "Review each commit before merge.",
|
system_prompt: "Review each commit before merge.",
|
||||||
skills: vec!["code-review-checklist".into()],
|
skills: vec!["code-review-checklist".into()],
|
||||||
brain_seed: None,
|
brain_seed: None,
|
||||||
|
// The point of migration 0071: a reviewer that does NOT
|
||||||
|
// share a model with the coder it reviews.
|
||||||
|
model: Some("glm-4.7"),
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
@@ -258,3 +263,114 @@ async fn on_launch_no_template_hard_fails() {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(team_id.is_none());
|
assert!(team_id.is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Slice 5: an APPROVED roster outranks the team template.
|
||||||
|
///
|
||||||
|
/// The template gives every composed mission the same five roles on the same
|
||||||
|
/// image. A roster is the model's answer for THIS mission, and it is the only
|
||||||
|
/// path that carries a per-node backend — which is how a mission runs more than
|
||||||
|
/// one provider at all. If the template won, a heterogeneous roster would be
|
||||||
|
/// accepted, stored, and then silently ignored at launch.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn an_approved_roster_outranks_the_template() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = seed_workspace(&pool).await;
|
||||||
|
let template_id = seed_test_template(&pool).await;
|
||||||
|
let mission = seed_mission(&pool, ws, template_id, "roster beats template").await;
|
||||||
|
|
||||||
|
// With no roster, the shape comes from the template — the behaviour every
|
||||||
|
// composed mission had before this slice.
|
||||||
|
let from_template = mission_orchestrator::composed_graph(&pool, mission, &["mission"])
|
||||||
|
.await
|
||||||
|
.expect("template graph")
|
||||||
|
.expect("the template supplies a shape");
|
||||||
|
let template_nodes = from_template["nodes"].as_array().unwrap().len();
|
||||||
|
assert!(template_nodes >= 1);
|
||||||
|
assert!(
|
||||||
|
from_template["nodes"][0]["attrs"].get("backend").is_none(),
|
||||||
|
"a template cannot express a per-node backend — that is the gap the roster fills"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Approve a roster the way the route does: the built graph under
|
||||||
|
// `config.roster`.
|
||||||
|
let roster = cm_api::mission_roster::Roster {
|
||||||
|
topology_kind: "pipeline".into(),
|
||||||
|
members: vec![
|
||||||
|
cm_api::mission_roster::RosterMember {
|
||||||
|
role: "implementer".into(),
|
||||||
|
backend: Some("claude".into()),
|
||||||
|
rationale: None,
|
||||||
|
},
|
||||||
|
cm_api::mission_roster::RosterMember {
|
||||||
|
role: "verifier".into(),
|
||||||
|
backend: Some("kimi".into()),
|
||||||
|
rationale: None,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
let graph = roster.graph().expect("a runnable graph");
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE missions SET config = jsonb_set(config, '{roster}', $2::jsonb, true) WHERE id = $1",
|
||||||
|
)
|
||||||
|
.bind(mission)
|
||||||
|
.bind(&graph)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let chosen = mission_orchestrator::composed_graph(&pool, mission, &["mission"])
|
||||||
|
.await
|
||||||
|
.expect("roster graph")
|
||||||
|
.expect("the roster supplies a shape");
|
||||||
|
let nodes = chosen["nodes"].as_array().unwrap();
|
||||||
|
assert_eq!(nodes.len(), 2, "the roster's two nodes, not the template's");
|
||||||
|
assert_eq!(nodes[0]["role"], "implementer");
|
||||||
|
assert_eq!(nodes[0]["attrs"]["backend"], "claude");
|
||||||
|
assert_eq!(nodes[1]["attrs"]["backend"], "kimi");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Migration 0071: a template role may name its own model, and the claw minted
|
||||||
|
/// for it must actually run on that model.
|
||||||
|
///
|
||||||
|
/// Before this, `mint_team_from_template` bound EVERY role to one literal — so a
|
||||||
|
/// template whose whole point is an independent reviewer minted a reviewer
|
||||||
|
/// sharing a model with the coder it reviews. That is the correlated failure the
|
||||||
|
/// cross-provider judge exists to break, reintroduced one layer down.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_template_role_may_run_on_its_own_model() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = seed_workspace(&pool).await;
|
||||||
|
let user = seed_owner(&pool, ws).await;
|
||||||
|
let template_id = seed_test_template(&pool).await;
|
||||||
|
let mission = seed_mission(&pool, ws, template_id, "per-role models").await;
|
||||||
|
|
||||||
|
mission_orchestrator::on_launch(&pool, ws, user, mission, None)
|
||||||
|
.await
|
||||||
|
.expect("launch");
|
||||||
|
|
||||||
|
let rows: Vec<(String, Option<String>)> = sqlx::query_as(
|
||||||
|
"SELECT job_title, model_binding FROM agents
|
||||||
|
WHERE workspace_id = $1 AND deleted_at IS NULL
|
||||||
|
ORDER BY job_title",
|
||||||
|
)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.fetch_all(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let by_role: std::collections::HashMap<_, _> = rows.into_iter().collect();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
by_role.get("reviewer").and_then(|m| m.clone()).as_deref(),
|
||||||
|
Some("glm-4.7"),
|
||||||
|
"the reviewer must run the model its role names: {by_role:?}"
|
||||||
|
);
|
||||||
|
// And a role that names none still gets the mint's default, so every
|
||||||
|
// template written before 0071 behaves exactly as it did.
|
||||||
|
for silent in ["planner", "coder"] {
|
||||||
|
assert_eq!(
|
||||||
|
by_role.get(silent).and_then(|m| m.clone()).as_deref(),
|
||||||
|
Some("claude-sonnet-5"),
|
||||||
|
"{silent} named no model and must take the default"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -278,6 +278,7 @@ async fn evaluations_are_unique_per_iteration_and_upsert() {
|
|||||||
model: "runtime:coordinator".into(),
|
model: "runtime:coordinator".into(),
|
||||||
error: None,
|
error: None,
|
||||||
checks: Vec::new(),
|
checks: Vec::new(),
|
||||||
|
independent: false,
|
||||||
};
|
};
|
||||||
cm_api::evaluator::record(&pool, mission, phase, 0, &first)
|
cm_api::evaluator::record(&pool, mission, phase, 0, &first)
|
||||||
.await
|
.await
|
||||||
@@ -296,6 +297,7 @@ async fn evaluations_are_unique_per_iteration_and_upsert() {
|
|||||||
exit_code: Some(0),
|
exit_code: Some(0),
|
||||||
evidence: "exit status: 0".into(),
|
evidence: "exit status: 0".into(),
|
||||||
}],
|
}],
|
||||||
|
independent: false,
|
||||||
};
|
};
|
||||||
cm_api::evaluator::record(&pool, mission, phase, 0, &second)
|
cm_api::evaluator::record(&pool, mission, phase, 0, &second)
|
||||||
.await
|
.await
|
||||||
@@ -353,6 +355,7 @@ async fn latest_returns_the_most_recent_iteration() {
|
|||||||
model: "m".into(),
|
model: "m".into(),
|
||||||
error: None,
|
error: None,
|
||||||
checks: Vec::new(),
|
checks: Vec::new(),
|
||||||
|
independent: false,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -113,3 +113,51 @@ pub async fn agents_for_template(
|
|||||||
})
|
})
|
||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A claw already in this workspace that can take this template role again.
|
||||||
|
///
|
||||||
|
/// The workforce is meant to be KEPT: a mission that needs a `coder` should
|
||||||
|
/// hire the one that already exists rather than mint a sixth. Without this,
|
||||||
|
/// every zeroclaw mission added a whole team to the roster permanently — they
|
||||||
|
/// are minted `lifecycle = 'permanent'` and nothing reaps them until the
|
||||||
|
/// mission itself is deleted — while each member was used exactly once.
|
||||||
|
///
|
||||||
|
/// A claw currently on a RUNNING mission is not offered. Two missions driving
|
||||||
|
/// the same ZeroClaw agent and the same `.brain` at once is a data race with a
|
||||||
|
/// model on the other end of it; minting a second claw is much cheaper than
|
||||||
|
/// reasoning about that.
|
||||||
|
///
|
||||||
|
/// Oldest first, so reuse concentrates on the same few claws and their brains
|
||||||
|
/// actually accumulate, instead of spreading thinly across a growing pool.
|
||||||
|
pub async fn reusable_claw(
|
||||||
|
pool: &PgPool,
|
||||||
|
workspace_id: uuid::Uuid,
|
||||||
|
template_id: uuid::Uuid,
|
||||||
|
role_slot: &str,
|
||||||
|
) -> Result<Option<uuid::Uuid>, DbError> {
|
||||||
|
let row: Option<(uuid::Uuid,)> = sqlx::query_as(
|
||||||
|
"SELECT a.id
|
||||||
|
FROM agents a
|
||||||
|
JOIN agent_template_link l ON l.agent_id = a.id
|
||||||
|
WHERE a.workspace_id = $1
|
||||||
|
AND a.deleted_at IS NULL
|
||||||
|
AND l.template_id = $2
|
||||||
|
AND l.role_slot = $3
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM team_members tm
|
||||||
|
JOIN mission_teams mt ON mt.team_id = tm.team_id
|
||||||
|
JOIN missions m ON m.id = mt.mission_id
|
||||||
|
WHERE tm.claw_id = a.id
|
||||||
|
AND m.status = 'running'
|
||||||
|
)
|
||||||
|
ORDER BY a.created_at
|
||||||
|
LIMIT 1",
|
||||||
|
)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(template_id)
|
||||||
|
.bind(role_slot)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(row.map(|(id,)| id))
|
||||||
|
}
|
||||||
|
|||||||
@@ -64,6 +64,52 @@ pub async fn insert(pool: &PgPool, agent: &Agent, policy: &AccessPolicy) -> Resu
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Fetch an agent even if it has been soft-deleted.
|
||||||
|
///
|
||||||
|
/// For the PURGE path only. `get` hides soft-deleted rows, which is right for
|
||||||
|
/// every read — but it also meant the hard purge could not see the rows it
|
||||||
|
/// exists to remove: a soft-deleted agent was unreachable from every route and
|
||||||
|
/// accumulated forever with no way out of the application. Four of them dated
|
||||||
|
/// from June before anyone noticed, because the UI correctly never showed them.
|
||||||
|
pub async fn get_any(pool: &PgPool, agent_id: AgentId) -> Result<Agent, DbError> {
|
||||||
|
// `sqlx::query_as` rather than the checked macro: this is the same columns
|
||||||
|
// as `get` minus one predicate, and adding a second compile-time query for
|
||||||
|
// that would mean regenerating the offline cache on every machine that
|
||||||
|
// builds this.
|
||||||
|
let row: Option<(
|
||||||
|
uuid::Uuid,
|
||||||
|
uuid::Uuid,
|
||||||
|
String,
|
||||||
|
String,
|
||||||
|
String,
|
||||||
|
String,
|
||||||
|
String,
|
||||||
|
String,
|
||||||
|
uuid::Uuid,
|
||||||
|
String,
|
||||||
|
)> = sqlx::query_as(
|
||||||
|
"SELECT id, workspace_id, name, job_title, system_prompt, avatar,
|
||||||
|
accent, wallpaper, managed_by, status
|
||||||
|
FROM agents WHERE id = $1",
|
||||||
|
)
|
||||||
|
.bind(agent_id.as_uuid())
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await?;
|
||||||
|
let row = row.ok_or(DbError::NotFound)?;
|
||||||
|
Ok(Agent {
|
||||||
|
id: AgentId::from(row.0),
|
||||||
|
workspace_id: WorkspaceId::from(row.1),
|
||||||
|
name: row.2,
|
||||||
|
job_title: row.3,
|
||||||
|
system_prompt: row.4,
|
||||||
|
avatar: row.5,
|
||||||
|
accent: row.6,
|
||||||
|
wallpaper: row.7,
|
||||||
|
managed_by: UserId::from(row.8),
|
||||||
|
status: row.9.parse().expect("status CHECK constraint"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn get(pool: &PgPool, agent_id: AgentId) -> Result<Agent, DbError> {
|
pub async fn get(pool: &PgPool, agent_id: AgentId) -> Result<Agent, DbError> {
|
||||||
let row = sqlx::query!(
|
let row = sqlx::query!(
|
||||||
"SELECT id, workspace_id, name, job_title, system_prompt, avatar,
|
"SELECT id, workspace_id, name, job_title, system_prompt, avatar,
|
||||||
@@ -87,7 +133,7 @@ pub async fn get(pool: &PgPool, agent_id: AgentId) -> Result<Agent, DbError> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Persist the model a claw was deployed with (e.g. "claude", "gemini",
|
/// Persist the model a claw was deployed with (e.g. "claude", "glm",
|
||||||
/// "glm-5.2") — the runtime config is otherwise the only record of it.
|
/// "glm-5.2") — the runtime config is otherwise the only record of it.
|
||||||
pub async fn set_model_binding(
|
pub async fn set_model_binding(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
|
|||||||
@@ -0,0 +1,231 @@
|
|||||||
|
//! Model-authored mission PLANS — the phase list — and whether a human
|
||||||
|
//! accepted them.
|
||||||
|
//!
|
||||||
|
//! See `migrations/0070_mission_plan_proposals.sql` for why a proposal is
|
||||||
|
//! persisted rather than applied on arrival.
|
||||||
|
|
||||||
|
use crate::DbError;
|
||||||
|
use serde_json::Value;
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use time::OffsetDateTime;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, serde::Serialize)]
|
||||||
|
pub struct MissionPlanProposal {
|
||||||
|
pub id: Uuid,
|
||||||
|
pub mission_id: Uuid,
|
||||||
|
pub plan: Value,
|
||||||
|
pub author_model: String,
|
||||||
|
pub status: String,
|
||||||
|
pub note: Option<String>,
|
||||||
|
#[serde(with = "time::serde::rfc3339")]
|
||||||
|
pub created_at: OffsetDateTime,
|
||||||
|
#[serde(with = "time::serde::rfc3339::option")]
|
||||||
|
pub decided_at: Option<OffsetDateTime>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn insert(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
mission_id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
plan: &Value,
|
||||||
|
author_model: &str,
|
||||||
|
) -> Result<(), DbError> {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO mission_plan_proposals
|
||||||
|
(id, mission_id, workspace_id, plan, author_model)
|
||||||
|
VALUES ($1, $2, $3, $4, $5)",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(mission_id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(plan)
|
||||||
|
.bind(author_model)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every proposal for a mission, newest first. Rejected ones are included on
|
||||||
|
/// purpose: what a human turned down is the only record of what the planner
|
||||||
|
/// gets wrong.
|
||||||
|
pub async fn list(
|
||||||
|
pool: &PgPool,
|
||||||
|
mission_id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
) -> Result<Vec<MissionPlanProposal>, DbError> {
|
||||||
|
let rows = sqlx::query_as::<_, (Uuid, Uuid, Value, String, String, Option<String>, OffsetDateTime, Option<OffsetDateTime>)>(
|
||||||
|
"SELECT id, mission_id, plan, author_model, status, note, created_at, decided_at
|
||||||
|
FROM mission_plan_proposals
|
||||||
|
WHERE mission_id = $1 AND workspace_id = $2
|
||||||
|
ORDER BY created_at DESC",
|
||||||
|
)
|
||||||
|
.bind(mission_id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows
|
||||||
|
.into_iter()
|
||||||
|
.map(
|
||||||
|
|(id, mission_id, plan, author_model, status, note, created_at, decided_at)| {
|
||||||
|
MissionPlanProposal {
|
||||||
|
id,
|
||||||
|
mission_id,
|
||||||
|
plan,
|
||||||
|
author_model,
|
||||||
|
status,
|
||||||
|
note,
|
||||||
|
created_at,
|
||||||
|
decided_at,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
) -> Result<Option<MissionPlanProposal>, DbError> {
|
||||||
|
let row = sqlx::query_as::<_, (Uuid, Uuid, Value, String, String, Option<String>, OffsetDateTime, Option<OffsetDateTime>)>(
|
||||||
|
"SELECT id, mission_id, plan, author_model, status, note, created_at, decided_at
|
||||||
|
FROM mission_plan_proposals
|
||||||
|
WHERE id = $1 AND workspace_id = $2",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(row.map(
|
||||||
|
|(id, mission_id, plan, author_model, status, note, created_at, decided_at)| {
|
||||||
|
MissionPlanProposal {
|
||||||
|
id,
|
||||||
|
mission_id,
|
||||||
|
plan,
|
||||||
|
author_model,
|
||||||
|
status,
|
||||||
|
note,
|
||||||
|
created_at,
|
||||||
|
decided_at,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Approve a plan AND write its phases onto the mission, atomically.
|
||||||
|
///
|
||||||
|
/// One transaction, for the reason `mission_team_proposals::approve_and_apply`
|
||||||
|
/// documents: a two-statement version left a proposal marked `approved` against
|
||||||
|
/// a mission that never received it, and the partial unique index then makes
|
||||||
|
/// that state permanent.
|
||||||
|
///
|
||||||
|
/// The mission's existing phases are REPLACED. A plan is an answer to "what is
|
||||||
|
/// this mission", not an addition to the recipe's answer — merging the two would
|
||||||
|
/// produce a phase list neither the model nor the recipe author intended. Only a
|
||||||
|
/// draft mission is eligible (checked by the caller), so nothing in flight is
|
||||||
|
/// discarded.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub async fn approve_and_apply(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
mission_id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
phases: &[(String, i32, Value)],
|
||||||
|
note: Option<&str>,
|
||||||
|
decided_by: Option<Uuid>,
|
||||||
|
) -> Result<bool, DbError> {
|
||||||
|
let mut tx = pool.begin().await?;
|
||||||
|
|
||||||
|
let claimed = sqlx::query(
|
||||||
|
"UPDATE mission_plan_proposals
|
||||||
|
SET status = 'approved', note = $3, decided_at = now(), decided_by = $4
|
||||||
|
WHERE id = $1 AND workspace_id = $2 AND status = 'proposed'",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(note)
|
||||||
|
.bind(decided_by)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
if claimed != 1 {
|
||||||
|
tx.rollback().await?;
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scoped by workspace on the mission, so a proposal cannot rewrite the
|
||||||
|
// phases of a mission in another workspace even if its own row were forged.
|
||||||
|
let owned: i64 = sqlx::query_scalar(
|
||||||
|
"SELECT count(*) FROM missions WHERE id = $1 AND workspace_id = $2",
|
||||||
|
)
|
||||||
|
.bind(mission_id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.fetch_one(&mut *tx)
|
||||||
|
.await?;
|
||||||
|
if owned != 1 {
|
||||||
|
tx.rollback().await?;
|
||||||
|
return Err(DbError::NotFound);
|
||||||
|
}
|
||||||
|
|
||||||
|
sqlx::query("DELETE FROM mission_phases WHERE mission_id = $1")
|
||||||
|
.bind(mission_id)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?;
|
||||||
|
for (kind, order_idx, config) in phases {
|
||||||
|
// `done_when` is PROMOTED out of the config into its column, exactly as
|
||||||
|
// `missions::create` does. The evaluator sweep filters on the column in
|
||||||
|
// SQL on every tick — a plan whose condition stayed in the JSONB blob
|
||||||
|
// would be stored, rendered, and never judged, which is the same shape
|
||||||
|
// as the unread `task` this whole registry exists because of.
|
||||||
|
let done_when = config
|
||||||
|
.get("done_when")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty());
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO mission_phases
|
||||||
|
(id, mission_id, kind, order_idx, status, config, done_when, max_iterations)
|
||||||
|
VALUES ($1, $2, $3, $4, 'pending', $5, $6, 1)",
|
||||||
|
)
|
||||||
|
.bind(Uuid::now_v7())
|
||||||
|
.bind(mission_id)
|
||||||
|
.bind(kind)
|
||||||
|
.bind(order_idx)
|
||||||
|
.bind(config)
|
||||||
|
.bind(done_when)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
tx.commit().await?;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record a decision. Only a `proposed` row may be decided, so approving twice
|
||||||
|
/// — a double-click, a retried request — cannot re-apply a roster to a mission
|
||||||
|
/// that has since moved on. Returns whether this call was the one that decided.
|
||||||
|
pub async fn decide(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
status: &str,
|
||||||
|
note: Option<&str>,
|
||||||
|
decided_by: Option<Uuid>,
|
||||||
|
) -> Result<bool, DbError> {
|
||||||
|
let done = sqlx::query(
|
||||||
|
"UPDATE mission_plan_proposals
|
||||||
|
SET status = $3, note = $4, decided_at = now(), decided_by = $5
|
||||||
|
WHERE id = $1 AND workspace_id = $2 AND status = 'proposed'",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(status)
|
||||||
|
.bind(note)
|
||||||
|
.bind(decided_by)
|
||||||
|
.execute(pool)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
Ok(done == 1)
|
||||||
|
}
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
//! Model-authored mission rosters, and whether a human accepted them.
|
||||||
|
//!
|
||||||
|
//! See `migrations/0070_mission_team_proposals.sql` for why a proposal is
|
||||||
|
//! persisted rather than applied on arrival.
|
||||||
|
|
||||||
|
use crate::DbError;
|
||||||
|
use serde_json::Value;
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use time::OffsetDateTime;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, serde::Serialize)]
|
||||||
|
pub struct MissionTeamProposal {
|
||||||
|
pub id: Uuid,
|
||||||
|
pub mission_id: Uuid,
|
||||||
|
pub roster: Value,
|
||||||
|
pub author_model: String,
|
||||||
|
pub status: String,
|
||||||
|
pub note: Option<String>,
|
||||||
|
#[serde(with = "time::serde::rfc3339")]
|
||||||
|
pub created_at: OffsetDateTime,
|
||||||
|
#[serde(with = "time::serde::rfc3339::option")]
|
||||||
|
pub decided_at: Option<OffsetDateTime>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn insert(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
mission_id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
roster: &Value,
|
||||||
|
author_model: &str,
|
||||||
|
) -> Result<(), DbError> {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO mission_team_proposals
|
||||||
|
(id, mission_id, workspace_id, roster, author_model)
|
||||||
|
VALUES ($1, $2, $3, $4, $5)",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(mission_id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(roster)
|
||||||
|
.bind(author_model)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every proposal for a mission, newest first. Rejected ones are included on
|
||||||
|
/// purpose: what a human turned down is the only record of what the planner
|
||||||
|
/// gets wrong.
|
||||||
|
pub async fn list(
|
||||||
|
pool: &PgPool,
|
||||||
|
mission_id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
) -> Result<Vec<MissionTeamProposal>, DbError> {
|
||||||
|
let rows = sqlx::query_as::<_, (Uuid, Uuid, Value, String, String, Option<String>, OffsetDateTime, Option<OffsetDateTime>)>(
|
||||||
|
"SELECT id, mission_id, roster, author_model, status, note, created_at, decided_at
|
||||||
|
FROM mission_team_proposals
|
||||||
|
WHERE mission_id = $1 AND workspace_id = $2
|
||||||
|
ORDER BY created_at DESC",
|
||||||
|
)
|
||||||
|
.bind(mission_id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows
|
||||||
|
.into_iter()
|
||||||
|
.map(
|
||||||
|
|(id, mission_id, roster, author_model, status, note, created_at, decided_at)| {
|
||||||
|
MissionTeamProposal {
|
||||||
|
id,
|
||||||
|
mission_id,
|
||||||
|
roster,
|
||||||
|
author_model,
|
||||||
|
status,
|
||||||
|
note,
|
||||||
|
created_at,
|
||||||
|
decided_at,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
) -> Result<Option<MissionTeamProposal>, DbError> {
|
||||||
|
let row = sqlx::query_as::<_, (Uuid, Uuid, Value, String, String, Option<String>, OffsetDateTime, Option<OffsetDateTime>)>(
|
||||||
|
"SELECT id, mission_id, roster, author_model, status, note, created_at, decided_at
|
||||||
|
FROM mission_team_proposals
|
||||||
|
WHERE id = $1 AND workspace_id = $2",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(row.map(
|
||||||
|
|(id, mission_id, roster, author_model, status, note, created_at, decided_at)| {
|
||||||
|
MissionTeamProposal {
|
||||||
|
id,
|
||||||
|
mission_id,
|
||||||
|
roster,
|
||||||
|
author_model,
|
||||||
|
status,
|
||||||
|
note,
|
||||||
|
created_at,
|
||||||
|
decided_at,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Approve a proposal AND apply it to its mission, atomically.
|
||||||
|
///
|
||||||
|
/// One transaction, because the two halves are one decision. The first version
|
||||||
|
/// claimed the proposal and then wrote the mission in two statements, and
|
||||||
|
/// production found the hole on the first real approval: the write failed, the
|
||||||
|
/// claim stood, and the mission was left with no roster while its proposal said
|
||||||
|
/// `approved` — a state the partial unique index then makes permanent, since no
|
||||||
|
/// second proposal for that mission can ever be approved.
|
||||||
|
///
|
||||||
|
/// Returns false when the proposal was already decided (a double-clicked
|
||||||
|
/// approve), in which case nothing is written.
|
||||||
|
pub async fn approve_and_apply(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
mission_id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
graph: &Value,
|
||||||
|
note: Option<&str>,
|
||||||
|
decided_by: Option<Uuid>,
|
||||||
|
) -> Result<bool, DbError> {
|
||||||
|
let mut tx = pool.begin().await?;
|
||||||
|
|
||||||
|
let claimed = sqlx::query(
|
||||||
|
"UPDATE mission_team_proposals
|
||||||
|
SET status = 'approved', note = $3, decided_at = now(), decided_by = $4
|
||||||
|
WHERE id = $1 AND workspace_id = $2 AND status = 'proposed'",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(note)
|
||||||
|
.bind(decided_by)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
if claimed != 1 {
|
||||||
|
tx.rollback().await?;
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
// `jsonb_set` REFUSES a scalar, and a mission created without a `config`
|
||||||
|
// stores jsonb `null` — a scalar. `coalesce` does not help: it guards SQL
|
||||||
|
// NULL, and this is a JSON null, which is a perfectly good non-NULL value of
|
||||||
|
// the wrong shape. Production hit this on the first real approval with
|
||||||
|
// "cannot set path in scalar".
|
||||||
|
let applied = sqlx::query(
|
||||||
|
"UPDATE missions
|
||||||
|
SET config = jsonb_set(
|
||||||
|
CASE WHEN jsonb_typeof(config) = 'object' THEN config ELSE '{}'::jsonb END,
|
||||||
|
'{roster}', $3::jsonb, true),
|
||||||
|
team_engine = 'composed',
|
||||||
|
updated_at = now()
|
||||||
|
WHERE id = $1 AND workspace_id = $2",
|
||||||
|
)
|
||||||
|
.bind(mission_id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(graph)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
if applied != 1 {
|
||||||
|
tx.rollback().await?;
|
||||||
|
return Err(DbError::NotFound);
|
||||||
|
}
|
||||||
|
|
||||||
|
tx.commit().await?;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record a decision. Only a `proposed` row may be decided, so approving twice
|
||||||
|
/// — a double-click, a retried request — cannot re-apply a roster to a mission
|
||||||
|
/// that has since moved on. Returns whether this call was the one that decided.
|
||||||
|
pub async fn decide(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
status: &str,
|
||||||
|
note: Option<&str>,
|
||||||
|
decided_by: Option<Uuid>,
|
||||||
|
) -> Result<bool, DbError> {
|
||||||
|
let done = sqlx::query(
|
||||||
|
"UPDATE mission_team_proposals
|
||||||
|
SET status = $3, note = $4, decided_at = now(), decided_by = $5
|
||||||
|
WHERE id = $1 AND workspace_id = $2 AND status = 'proposed'",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(status)
|
||||||
|
.bind(note)
|
||||||
|
.bind(decided_by)
|
||||||
|
.execute(pool)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
Ok(done == 1)
|
||||||
|
}
|
||||||
@@ -34,6 +34,10 @@ pub struct Mission {
|
|||||||
pub runtime_kind: String,
|
pub runtime_kind: String,
|
||||||
/// FK → nodes(id); only relevant when runtime_kind = 'local_herdr'
|
/// FK → nodes(id); only relevant when runtime_kind = 'local_herdr'
|
||||||
pub target_node_id: Option<Uuid>,
|
pub target_node_id: Option<Uuid>,
|
||||||
|
/// Which per-CLI rootfs a `microvm` mission boots. NULL = the node's default
|
||||||
|
/// image. Read by placement (a node must HOLD this image) and by the executor
|
||||||
|
/// (it is passed to `vm_create`).
|
||||||
|
pub backend: Option<String>,
|
||||||
/// Per-mission ZeroClaw runtime container name (C3 workspace isolation).
|
/// Per-mission ZeroClaw runtime container name (C3 workspace isolation).
|
||||||
/// Null until `mission_runtime::ensure_container` provisions it.
|
/// Null until `mission_runtime::ensure_container` provisions it.
|
||||||
pub runtime_container_name: Option<String>,
|
pub runtime_container_name: Option<String>,
|
||||||
@@ -129,6 +133,15 @@ pub struct NewMission<'a> {
|
|||||||
/// Defaults to 'zeroclaw' when None.
|
/// Defaults to 'zeroclaw' when None.
|
||||||
pub runtime_kind: Option<&'a str>,
|
pub runtime_kind: Option<&'a str>,
|
||||||
pub target_node_id: Option<Uuid>,
|
pub target_node_id: Option<Uuid>,
|
||||||
|
/// Which per-CLI image a `microvm` mission boots. NULL = the node's default
|
||||||
|
/// rootfs. Deliberately unconstrained in the schema: which images exist is a
|
||||||
|
/// property of the NODES, not of the database.
|
||||||
|
pub backend: Option<&'a str>,
|
||||||
|
/// Independent validator for this mission's verdicts. `None` = deployment
|
||||||
|
/// default; `Some("")` = explicitly none. See migration 0068.
|
||||||
|
pub validator_model: Option<&'a str>,
|
||||||
|
/// `"claude_code"` to ask for an agent team; `None` = solo. See 0069.
|
||||||
|
pub team_engine: Option<&'a str>,
|
||||||
pub phases: Vec<NewMissionPhase>,
|
pub phases: Vec<NewMissionPhase>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -157,9 +170,9 @@ pub async fn insert(pool: &PgPool, m: NewMission<'_>) -> Result<Uuid, DbError> {
|
|||||||
"INSERT INTO missions
|
"INSERT INTO missions
|
||||||
(id, workspace_id, title, template_kind, team_id,
|
(id, workspace_id, title, template_kind, team_id,
|
||||||
team_template_id, repo_id, schedule, status, description, config,
|
team_template_id, repo_id, schedule, status, description, config,
|
||||||
runtime_kind, target_node_id)
|
runtime_kind, target_node_id, backend, validator_model, team_engine)
|
||||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'draft',$9,$10,
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'draft',$9,$10,
|
||||||
COALESCE($11,'zeroclaw'),$12)",
|
COALESCE($11,'zeroclaw'),$12,$13,$14,$15)",
|
||||||
)
|
)
|
||||||
.bind(mission_id)
|
.bind(mission_id)
|
||||||
.bind(m.workspace_id)
|
.bind(m.workspace_id)
|
||||||
@@ -173,6 +186,9 @@ pub async fn insert(pool: &PgPool, m: NewMission<'_>) -> Result<Uuid, DbError> {
|
|||||||
.bind(&m.config)
|
.bind(&m.config)
|
||||||
.bind(m.runtime_kind)
|
.bind(m.runtime_kind)
|
||||||
.bind(m.target_node_id)
|
.bind(m.target_node_id)
|
||||||
|
.bind(m.backend)
|
||||||
|
.bind(m.validator_model)
|
||||||
|
.bind(m.team_engine)
|
||||||
.execute(&mut *tx)
|
.execute(&mut *tx)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
@@ -224,7 +240,7 @@ pub async fn get(pool: &PgPool, id: Uuid, workspace_id: Uuid) -> Result<Option<M
|
|||||||
let row = sqlx::query(
|
let row = sqlx::query(
|
||||||
"SELECT id, workspace_id, title, template_kind, team_id,
|
"SELECT id, workspace_id, title, template_kind, team_id,
|
||||||
team_template_id, repo_id, schedule, status,
|
team_template_id, repo_id, schedule, status,
|
||||||
description, config, runtime_kind, target_node_id,
|
description, config, runtime_kind, target_node_id, backend,
|
||||||
runtime_container_name, runtime_endpoint, runtime_pairing_code,
|
runtime_container_name, runtime_endpoint, runtime_pairing_code,
|
||||||
created_at, updated_at, completed_at
|
created_at, updated_at, completed_at
|
||||||
FROM missions WHERE id = $1 AND workspace_id = $2",
|
FROM missions WHERE id = $1 AND workspace_id = $2",
|
||||||
@@ -247,6 +263,7 @@ pub async fn get(pool: &PgPool, id: Uuid, workspace_id: Uuid) -> Result<Option<M
|
|||||||
config: r.get("config"),
|
config: r.get("config"),
|
||||||
runtime_kind: r.get("runtime_kind"),
|
runtime_kind: r.get("runtime_kind"),
|
||||||
target_node_id: r.get("target_node_id"),
|
target_node_id: r.get("target_node_id"),
|
||||||
|
backend: r.get("backend"),
|
||||||
runtime_container_name: r.get("runtime_container_name"),
|
runtime_container_name: r.get("runtime_container_name"),
|
||||||
runtime_endpoint: r.get("runtime_endpoint"),
|
runtime_endpoint: r.get("runtime_endpoint"),
|
||||||
runtime_pairing_code: r.get("runtime_pairing_code"),
|
runtime_pairing_code: r.get("runtime_pairing_code"),
|
||||||
@@ -266,7 +283,7 @@ pub async fn list_by_workspace(
|
|||||||
let rows = sqlx::query(
|
let rows = sqlx::query(
|
||||||
"SELECT id, workspace_id, title, template_kind, team_id,
|
"SELECT id, workspace_id, title, template_kind, team_id,
|
||||||
team_template_id, repo_id, schedule, status,
|
team_template_id, repo_id, schedule, status,
|
||||||
description, config, runtime_kind, target_node_id,
|
description, config, runtime_kind, target_node_id, backend,
|
||||||
runtime_container_name, runtime_endpoint, runtime_pairing_code,
|
runtime_container_name, runtime_endpoint, runtime_pairing_code,
|
||||||
created_at, updated_at, completed_at
|
created_at, updated_at, completed_at
|
||||||
FROM missions WHERE workspace_id = $1
|
FROM missions WHERE workspace_id = $1
|
||||||
@@ -292,6 +309,7 @@ pub async fn list_by_workspace(
|
|||||||
config: r.get("config"),
|
config: r.get("config"),
|
||||||
runtime_kind: r.get("runtime_kind"),
|
runtime_kind: r.get("runtime_kind"),
|
||||||
target_node_id: r.get("target_node_id"),
|
target_node_id: r.get("target_node_id"),
|
||||||
|
backend: r.get("backend"),
|
||||||
runtime_container_name: r.get("runtime_container_name"),
|
runtime_container_name: r.get("runtime_container_name"),
|
||||||
runtime_endpoint: r.get("runtime_endpoint"),
|
runtime_endpoint: r.get("runtime_endpoint"),
|
||||||
runtime_pairing_code: r.get("runtime_pairing_code"),
|
runtime_pairing_code: r.get("runtime_pairing_code"),
|
||||||
@@ -358,7 +376,7 @@ pub async fn set_runtime_binding(
|
|||||||
endpoint: Option<&str>,
|
endpoint: Option<&str>,
|
||||||
pairing_code: Option<&str>,
|
pairing_code: Option<&str>,
|
||||||
) -> Result<(), DbError> {
|
) -> Result<(), DbError> {
|
||||||
sqlx::query(
|
let r = sqlx::query(
|
||||||
"UPDATE missions
|
"UPDATE missions
|
||||||
SET runtime_container_name = $3,
|
SET runtime_container_name = $3,
|
||||||
runtime_endpoint = $4,
|
runtime_endpoint = $4,
|
||||||
@@ -373,6 +391,16 @@ pub async fn set_runtime_binding(
|
|||||||
.bind(pairing_code)
|
.bind(pairing_code)
|
||||||
.execute(pool)
|
.execute(pool)
|
||||||
.await?;
|
.await?;
|
||||||
|
// A `WHERE id = $1 AND workspace_id = $2` that matches nothing is not an
|
||||||
|
// error to sqlx — it updates zero rows and returns Ok. That made a
|
||||||
|
// mismatched workspace indistinguishable from a successful bind, and the
|
||||||
|
// binding is what the sweeper uses to find a mission's container: a silent
|
||||||
|
// no-op here leaks a container with no record that anything went wrong.
|
||||||
|
// Callers log this rather than aborting, which is the point — it becomes
|
||||||
|
// visible instead of invisible.
|
||||||
|
if r.rows_affected() == 0 {
|
||||||
|
return Err(DbError::NotFound);
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ pub mod fleet_beszel;
|
|||||||
pub mod fleet_tailscale;
|
pub mod fleet_tailscale;
|
||||||
pub mod level_up;
|
pub mod level_up;
|
||||||
pub mod messages;
|
pub mod messages;
|
||||||
|
pub mod mission_plan_proposals;
|
||||||
|
pub mod mission_team_proposals;
|
||||||
pub mod missions;
|
pub mod missions;
|
||||||
pub mod node_metrics;
|
pub mod node_metrics;
|
||||||
pub mod node_rules;
|
pub mod node_rules;
|
||||||
|
|||||||
@@ -71,6 +71,22 @@ pub struct EvalRow {
|
|||||||
pub gpu_pct: Option<f64>,
|
pub gpu_pct: Option<f64>,
|
||||||
pub temp_max: Option<f64>,
|
pub temp_max: Option<f64>,
|
||||||
pub load1: Option<f64>,
|
pub load1: Option<f64>,
|
||||||
|
/// Absolute memory, for CAPACITY rather than utilisation. `mem_pct` cannot
|
||||||
|
/// answer "does another 8 GiB VM fit" — a node at 20% of 31 GiB and one at
|
||||||
|
/// 20% of 60 GiB report the same percentage and hold a different number of
|
||||||
|
/// VMs. From the 5s heartbeat, which is the only source with absolutes.
|
||||||
|
pub mem_total_bytes: Option<i64>,
|
||||||
|
pub mem_used_bytes: Option<i64>,
|
||||||
|
pub disk_free_bytes: Option<i64>,
|
||||||
|
/// Memory in use with no phase VMs committed, remembered from the last time
|
||||||
|
/// this node was observed idle. `None` until then, which makes placement
|
||||||
|
/// fall back to inferring it — the behaviour that over-committed morpheus.
|
||||||
|
pub mem_baseline_mib: Option<i64>,
|
||||||
|
/// Age of each source. Placement is fail-closed on stale health (a node whose
|
||||||
|
/// RAM we cannot read is one we are guessing at), and demotes rather than
|
||||||
|
/// excludes on stale Beszel metrics, which only ever break ties.
|
||||||
|
pub health_age_secs: Option<f64>,
|
||||||
|
pub metrics_age_secs: Option<f64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl EvalRow {
|
impl EvalRow {
|
||||||
@@ -91,6 +107,22 @@ impl EvalRow {
|
|||||||
let used = self.cpu_pct.unwrap_or(0.0).max(self.mem_pct.unwrap_or(0.0));
|
let used = self.cpu_pct.unwrap_or(0.0).max(self.mem_pct.unwrap_or(0.0));
|
||||||
100.0 - used
|
100.0 - used
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// [`headroom`] when at least one source is recent, otherwise the WORST
|
||||||
|
/// possible score.
|
||||||
|
///
|
||||||
|
/// Used only as a placement TIEBREAK, never as an admission gate: stale
|
||||||
|
/// metrics may cost a node a tie, they may never win one. Admission is
|
||||||
|
/// decided by absolute memory from the heartbeat, which has its own
|
||||||
|
/// freshness check.
|
||||||
|
pub fn headroom_fresh(&self, max_age_secs: f64) -> f64 {
|
||||||
|
let fresh = |a: Option<f64>| a.is_some_and(|x| x <= max_age_secs);
|
||||||
|
if fresh(self.health_age_secs) || fresh(self.metrics_age_secs) {
|
||||||
|
self.headroom()
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Every node's current metric scalars (merged Beszel + heartbeat health).
|
/// Every node's current metric scalars (merged Beszel + heartbeat health).
|
||||||
@@ -101,7 +133,13 @@ pub async fn eval_all(pool: &PgPool) -> Result<Vec<EvalRow>, DbError> {
|
|||||||
COALESCE(m.mem_pct, CASE WHEN h.mem_total > 0 THEN h.mem_used::float8 / h.mem_total * 100 END) AS mem_pct,
|
COALESCE(m.mem_pct, CASE WHEN h.mem_total > 0 THEN h.mem_used::float8 / h.mem_total * 100 END) AS mem_pct,
|
||||||
COALESCE(m.disk_pct, CASE WHEN h.disk_total > 0 THEN (h.disk_total - h.disk_free)::float8 / h.disk_total * 100 END) AS disk_pct,
|
COALESCE(m.disk_pct, CASE WHEN h.disk_total > 0 THEN (h.disk_total - h.disk_free)::float8 / h.disk_total * 100 END) AS disk_pct,
|
||||||
m.gpu_pct, m.temp_max,
|
m.gpu_pct, m.temp_max,
|
||||||
COALESCE(m.load1, h.load1) AS load1
|
COALESCE(m.load1, h.load1) AS load1,
|
||||||
|
h.mem_total AS mem_total_bytes,
|
||||||
|
h.mem_used AS mem_used_bytes,
|
||||||
|
h.disk_free AS disk_free_bytes,
|
||||||
|
n.mem_baseline_mib,
|
||||||
|
EXTRACT(EPOCH FROM now() - h.captured_at)::float8 AS health_age_secs,
|
||||||
|
EXTRACT(EPOCH FROM now() - m.updated_at)::float8 AS metrics_age_secs
|
||||||
FROM nodes n
|
FROM nodes n
|
||||||
LEFT JOIN node_health h ON h.node_id = n.id
|
LEFT JOIN node_health h ON h.node_id = n.id
|
||||||
LEFT JOIN node_metrics m ON m.node_id = n.id",
|
LEFT JOIN node_metrics m ON m.node_id = n.id",
|
||||||
@@ -120,6 +158,12 @@ pub async fn eval_all(pool: &PgPool) -> Result<Vec<EvalRow>, DbError> {
|
|||||||
gpu_pct: r.get("gpu_pct"),
|
gpu_pct: r.get("gpu_pct"),
|
||||||
temp_max: r.get("temp_max"),
|
temp_max: r.get("temp_max"),
|
||||||
load1: r.get("load1"),
|
load1: r.get("load1"),
|
||||||
|
mem_total_bytes: r.get("mem_total_bytes"),
|
||||||
|
mem_used_bytes: r.get("mem_used_bytes"),
|
||||||
|
disk_free_bytes: r.get("disk_free_bytes"),
|
||||||
|
mem_baseline_mib: r.get("mem_baseline_mib"),
|
||||||
|
health_age_secs: r.get("health_age_secs"),
|
||||||
|
metrics_age_secs: r.get("metrics_age_secs"),
|
||||||
})
|
})
|
||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -178,6 +178,170 @@ pub async fn set_status(pool: &PgPool, id: NodeId, status: &str) -> Result<(), D
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Record what a node reports it can host, for placement predicates.
|
||||||
|
///
|
||||||
|
/// Replaces rather than merges: the node sends its complete view on every
|
||||||
|
/// report, so a capability it has *stopped* having (firecracker uninstalled,
|
||||||
|
/// `/dev/kvm` gone after a reboot into a non-virt kernel) must disappear here
|
||||||
|
/// too. Merging would let a stale `true` survive forever.
|
||||||
|
pub async fn set_capabilities(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: NodeId,
|
||||||
|
capabilities: &serde_json::Value,
|
||||||
|
) -> Result<(), DbError> {
|
||||||
|
sqlx::query("UPDATE nodes SET capabilities = $2 WHERE id = $1")
|
||||||
|
.bind(id.as_uuid())
|
||||||
|
.bind(capabilities)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Online nodes that report every one of `required` as `true`.
|
||||||
|
///
|
||||||
|
/// The predicate side of placement. Nothing is assumed: a node that has never
|
||||||
|
/// reported has `capabilities = '{}'`, which fails every requirement — an
|
||||||
|
/// unqueried node and an incapable node are treated identically, because
|
||||||
|
/// scheduling work onto a node whose abilities are unknown is how you get a
|
||||||
|
/// mission that cannot start and does not say why.
|
||||||
|
pub async fn online_with_capabilities(
|
||||||
|
pool: &PgPool,
|
||||||
|
workspace_id: uuid::Uuid,
|
||||||
|
required: &[&str],
|
||||||
|
) -> Result<Vec<NodeId>, DbError> {
|
||||||
|
let needed: serde_json::Value = required
|
||||||
|
.iter()
|
||||||
|
.map(|k| ((*k).to_string(), serde_json::Value::Bool(true)))
|
||||||
|
.collect::<serde_json::Map<_, _>>()
|
||||||
|
.into();
|
||||||
|
let rows: Vec<(uuid::Uuid,)> = sqlx::query_as(
|
||||||
|
"SELECT id FROM nodes
|
||||||
|
WHERE workspace_id = $1 AND status = 'online' AND capabilities @> $2
|
||||||
|
ORDER BY last_seen DESC NULLS LAST",
|
||||||
|
)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(&needed)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows.into_iter().map(|(id,)| NodeId::from(id)).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Online nodes that can host a microVM **and** hold the image `backend` names.
|
||||||
|
///
|
||||||
|
/// KVM alone is the wrong predicate. The first real microVM mission was placed
|
||||||
|
/// on a node reporting `microvm: true` that did not have `rootfs-claude.ext4`;
|
||||||
|
/// it failed by name rather than booting the wrong image, but whether a mission
|
||||||
|
/// ran came down to which capable node was listed first.
|
||||||
|
///
|
||||||
|
/// `backend = None` means the node's default image, which reports itself as
|
||||||
|
/// `"default"` — so the requirement is never vacuous. A node running an older
|
||||||
|
/// daemon has no `rootfs` key at all and matches nothing, which is the same
|
||||||
|
/// treatment an unqueried node gets for every other capability: unknown is not
|
||||||
|
/// permission.
|
||||||
|
///
|
||||||
|
/// Capability only — this says a node COULD run the backend, not that it has room.
|
||||||
|
/// Capacity is `cm_api::vm_placement`'s job.
|
||||||
|
/// microVM phases already pinned to a node but whose VM may not exist yet.
|
||||||
|
///
|
||||||
|
/// The other half of "how much is this node committed to". `vm_list` reports
|
||||||
|
/// BOOTED VMs; between `phase_runner` choosing a node and the guest answering,
|
||||||
|
/// there is a window of seconds in which a phase is a real 8 GiB claim that no
|
||||||
|
/// node can report. Two missions launched together both survey a node as empty
|
||||||
|
/// and both land on it.
|
||||||
|
///
|
||||||
|
/// Returns (node, phase_id, iteration) so the caller can build the same
|
||||||
|
/// deterministic vm id the executor uses and union the two sets by identity
|
||||||
|
/// rather than adding them — a phase whose VM HAS booted must count once, not
|
||||||
|
/// twice.
|
||||||
|
pub async fn pinned_microvm_phases(
|
||||||
|
pool: &PgPool,
|
||||||
|
workspace_id: uuid::Uuid,
|
||||||
|
) -> Result<Vec<(NodeId, uuid::Uuid, i32)>, DbError> {
|
||||||
|
let rows: Vec<(uuid::Uuid, uuid::Uuid, i32)> = sqlx::query_as(
|
||||||
|
"SELECT m.target_node_id, p.id, p.iteration
|
||||||
|
FROM mission_phases p
|
||||||
|
JOIN missions m ON m.id = p.mission_id
|
||||||
|
WHERE m.workspace_id = $1
|
||||||
|
AND m.runtime_kind = 'microvm'
|
||||||
|
AND m.status = 'running'
|
||||||
|
AND m.target_node_id IS NOT NULL
|
||||||
|
-- `pending` counts: it is about to become a VM. `completed`/`failed`
|
||||||
|
-- do not: their VM is destroyed on every exit path of
|
||||||
|
-- `run_phase_in_vm`, so counting them would shrink the fleet by the
|
||||||
|
-- number of missions it has ever run.
|
||||||
|
AND p.status IN ('pending', 'running')",
|
||||||
|
)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows
|
||||||
|
.into_iter()
|
||||||
|
.map(|(n, p, i)| (NodeId::from(n), p, i))
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn online_for_backend(
|
||||||
|
pool: &PgPool,
|
||||||
|
workspace_id: uuid::Uuid,
|
||||||
|
backend: Option<&str>,
|
||||||
|
) -> Result<Vec<NodeId>, DbError> {
|
||||||
|
online_for_backends(pool, workspace_id, &[backend_key(backend).to_string()]).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Nodes that can run EVERY one of these backends.
|
||||||
|
///
|
||||||
|
/// A composed mission runs its whole graph on one node, and the graph's nodes
|
||||||
|
/// may each name their own backend — an independent verifier on another
|
||||||
|
/// provider is the entire point of the roster. Asking only for the mission's
|
||||||
|
/// backend placed such a mission on a node with `claude` and no
|
||||||
|
/// `canary-claude`, and the run died at the second graph node with
|
||||||
|
/// `no rootfs for backend "canary-claude" on this node`. The full harness
|
||||||
|
/// caught it; nothing before it had a reason to.
|
||||||
|
pub async fn online_for_backends(
|
||||||
|
pool: &PgPool,
|
||||||
|
workspace_id: uuid::Uuid,
|
||||||
|
backends: &[String],
|
||||||
|
) -> Result<Vec<NodeId>, DbError> {
|
||||||
|
// `@>` on the array asks "does this node's list contain ALL of these" —
|
||||||
|
// containment, not intersection, which is exactly the question here and the
|
||||||
|
// whole reason the node reports an array rather than a count.
|
||||||
|
let rows: Vec<(uuid::Uuid,)> = sqlx::query_as(
|
||||||
|
"SELECT id FROM nodes
|
||||||
|
WHERE workspace_id = $1 AND status = 'online'
|
||||||
|
AND capabilities @> '{\"microvm\": true}'::jsonb
|
||||||
|
AND capabilities -> 'rootfs' @> $2::jsonb
|
||||||
|
-- Deterministic, NOT `last_seen DESC`. Ranking now happens in
|
||||||
|
-- `cm_api::vm_placement`, against real capacity. Ordering by last_seen
|
||||||
|
-- and taking `.first()` was the placement algorithm until now: among
|
||||||
|
-- healthy nodes all heartbeating every 5s, that is arbitrary — it sent
|
||||||
|
-- concurrent missions to whichever node's packet landed most recently,
|
||||||
|
-- with no regard for what was already running there.
|
||||||
|
ORDER BY id",
|
||||||
|
)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(serde_json::Value::Array(
|
||||||
|
backends
|
||||||
|
.iter()
|
||||||
|
.map(|b| serde_json::Value::String(b.clone()))
|
||||||
|
.collect(),
|
||||||
|
))
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows.into_iter().map(|(id,)| NodeId::from(id)).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The name a backend reports itself as in a node's `rootfs` list.
|
||||||
|
///
|
||||||
|
/// Must agree with `clawmates-node::microvm::rootfs_for`, which resolves the same
|
||||||
|
/// three spellings to the default image. If these two drift, placement promises
|
||||||
|
/// an image the booter cannot find — or refuses one it has.
|
||||||
|
pub fn backend_key(backend: Option<&str>) -> &str {
|
||||||
|
match backend {
|
||||||
|
None | Some("") | Some("default") => "default",
|
||||||
|
Some(b) => b,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Mark online nodes whose last heartbeat is older than `secs` as offline.
|
/// Mark online nodes whose last heartbeat is older than `secs` as offline.
|
||||||
pub async fn mark_stale_offline(pool: &PgPool, secs: i64) -> Result<(), DbError> {
|
pub async fn mark_stale_offline(pool: &PgPool, secs: i64) -> Result<(), DbError> {
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
@@ -242,3 +406,40 @@ fn map_node(r: sqlx::postgres::PgRow) -> NodeRow {
|
|||||||
temp_max: r.get("m_temp_max"),
|
temp_max: r.get("m_temp_max"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// The three spellings that mean "the node's default image" must all resolve
|
||||||
|
/// to the name the node actually advertises for it. A mismatch here makes
|
||||||
|
/// placement reject every node for an ordinary mission with no backend set.
|
||||||
|
#[test]
|
||||||
|
fn the_default_backend_has_one_name() {
|
||||||
|
for spelling in [None, Some(""), Some("default")] {
|
||||||
|
assert_eq!(backend_key(spelling), "default", "{spelling:?}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// And a named backend is passed through verbatim — it is matched against the
|
||||||
|
/// node's list, which is built from the filenames on its disk.
|
||||||
|
#[test]
|
||||||
|
fn a_named_backend_is_not_rewritten() {
|
||||||
|
assert_eq!(backend_key(Some("claude")), "claude");
|
||||||
|
assert_eq!(backend_key(Some("agent-terminal")), "agent-terminal");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Remember a node's idle memory footprint.
|
||||||
|
///
|
||||||
|
/// Only ever called with a reading taken while the node had ZERO phase VMs
|
||||||
|
/// committed — that is the one moment the number is honestly observable.
|
||||||
|
/// Writing it at any other time would record the VMs as part of the host.
|
||||||
|
pub async fn set_mem_baseline(pool: &PgPool, node_id: NodeId, mib: i64) -> Result<(), DbError> {
|
||||||
|
sqlx::query("UPDATE nodes SET mem_baseline_mib = $2 WHERE id = $1")
|
||||||
|
.bind(node_id.as_uuid())
|
||||||
|
.bind(mib)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|||||||
@@ -44,6 +44,12 @@ pub struct TemplateRole {
|
|||||||
pub system_prompt: String,
|
pub system_prompt: String,
|
||||||
pub skills: Vec<String>,
|
pub skills: Vec<String>,
|
||||||
pub brain_seed: Option<String>,
|
pub brain_seed: Option<String>,
|
||||||
|
/// Which model this role's claw runs on. `None` takes the mint's default —
|
||||||
|
/// which is what every role did unconditionally before migration 0071, and
|
||||||
|
/// why a template could not put its reviewer on a different model from the
|
||||||
|
/// coder it reviews.
|
||||||
|
#[serde(default)]
|
||||||
|
pub model: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
@@ -60,6 +66,8 @@ pub struct UpsertBuiltinRole<'a> {
|
|||||||
pub system_prompt: &'a str,
|
pub system_prompt: &'a str,
|
||||||
pub skills: Vec<String>,
|
pub skills: Vec<String>,
|
||||||
pub brain_seed: Option<&'a str>,
|
pub brain_seed: Option<&'a str>,
|
||||||
|
/// Optional per-role model. `None` leaves the mint's default in place.
|
||||||
|
pub model: Option<&'a str>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
@@ -138,13 +146,14 @@ pub async fn upsert_builtin(pool: &PgPool, b: UpsertBuiltin<'_>) -> Result<Uuid,
|
|||||||
for r in &b.roles {
|
for r in &b.roles {
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
"INSERT INTO template_roles
|
"INSERT INTO template_roles
|
||||||
(template_id, slot, order_idx, system_prompt, skills, brain_seed)
|
(template_id, slot, order_idx, system_prompt, skills, brain_seed, model)
|
||||||
VALUES ($1,$2,$3,$4,$5,$6)
|
VALUES ($1,$2,$3,$4,$5,$6,$7)
|
||||||
ON CONFLICT (template_id, slot) DO UPDATE SET
|
ON CONFLICT (template_id, slot) DO UPDATE SET
|
||||||
order_idx = EXCLUDED.order_idx,
|
order_idx = EXCLUDED.order_idx,
|
||||||
system_prompt = EXCLUDED.system_prompt,
|
system_prompt = EXCLUDED.system_prompt,
|
||||||
skills = EXCLUDED.skills,
|
skills = EXCLUDED.skills,
|
||||||
brain_seed = EXCLUDED.brain_seed",
|
brain_seed = EXCLUDED.brain_seed,
|
||||||
|
model = EXCLUDED.model",
|
||||||
)
|
)
|
||||||
.bind(id)
|
.bind(id)
|
||||||
.bind(r.slot)
|
.bind(r.slot)
|
||||||
@@ -152,6 +161,7 @@ pub async fn upsert_builtin(pool: &PgPool, b: UpsertBuiltin<'_>) -> Result<Uuid,
|
|||||||
.bind(r.system_prompt)
|
.bind(r.system_prompt)
|
||||||
.bind(&r.skills)
|
.bind(&r.skills)
|
||||||
.bind(r.brain_seed)
|
.bind(r.brain_seed)
|
||||||
|
.bind(r.model)
|
||||||
.execute(&mut *tx)
|
.execute(&mut *tx)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
@@ -226,7 +236,7 @@ pub async fn get(pool: &PgPool, id: Uuid) -> Result<Option<TeamTemplateDetail>,
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
let role_rows = sqlx::query(
|
let role_rows = sqlx::query(
|
||||||
"SELECT template_id, slot, order_idx, system_prompt, skills, brain_seed
|
"SELECT template_id, slot, order_idx, system_prompt, skills, brain_seed, model
|
||||||
FROM template_roles WHERE template_id = $1
|
FROM template_roles WHERE template_id = $1
|
||||||
ORDER BY order_idx ASC",
|
ORDER BY order_idx ASC",
|
||||||
)
|
)
|
||||||
@@ -242,6 +252,7 @@ pub async fn get(pool: &PgPool, id: Uuid) -> Result<Option<TeamTemplateDetail>,
|
|||||||
system_prompt: r.get("system_prompt"),
|
system_prompt: r.get("system_prompt"),
|
||||||
skills: r.get("skills"),
|
skills: r.get("skills"),
|
||||||
brain_seed: r.get("brain_seed"),
|
brain_seed: r.get("brain_seed"),
|
||||||
|
model: r.get("model"),
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
Ok(Some(TeamTemplateDetail { template: t, roles }))
|
Ok(Some(TeamTemplateDetail { template: t, roles }))
|
||||||
|
|||||||
@@ -54,6 +54,15 @@ pub struct ClaimedTopologyRun {
|
|||||||
/// Deploy tier: `team` drives claws directly; `company`/`org` drive the
|
/// Deploy tier: `team` drives claws directly; `company`/`org` drive the
|
||||||
/// recursive sub-topology executor.
|
/// recursive sub-topology executor.
|
||||||
pub tier: String,
|
pub tier: String,
|
||||||
|
/// The mission this run belongs to, when it belongs to one. The composed
|
||||||
|
/// (`microvm_graph`) tier needs it: its nodes share the mission's checkout,
|
||||||
|
/// and that shared tree is how file work survives a node boundary.
|
||||||
|
pub mission_id: Option<Uuid>,
|
||||||
|
/// The mission phase, for the same reason — the phase and pass identify the
|
||||||
|
/// VMs a composed run may boot.
|
||||||
|
pub mission_phase_id: Option<Uuid>,
|
||||||
|
/// Which pass of the phase produced this run.
|
||||||
|
pub iteration: Option<i32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Lifecycle status + progress for a durable run (status endpoint).
|
/// Lifecycle status + progress for a durable run (status endpoint).
|
||||||
@@ -203,51 +212,129 @@ pub async fn check_ephemeral_teardown(
|
|||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Tiers the topology worker drives, and therefore the only ones it may claim,
|
||||||
|
/// requeue or reap.
|
||||||
|
///
|
||||||
|
/// **A load-bearing allowlist, not tidiness.** Both sweepers were written when
|
||||||
|
/// every `running` row was a `cm_orchestrator` job that checkpointed after each
|
||||||
|
/// step. `tier='microvm'` and `tier='session'` broke that assumption: they are
|
||||||
|
/// inserted directly as `running` by `phase_runner`, driven by a `tokio::spawn`
|
||||||
|
/// that owns them start to finish, and they never write `updated_at` or
|
||||||
|
/// `checkpoint` while in flight.
|
||||||
|
///
|
||||||
|
/// Measured cost of the omission: `requeue_stale` flipped an in-flight microVM run
|
||||||
|
/// to `queued` at 180s, `claim_next_queued` handed it to the worker, and the worker
|
||||||
|
/// failed it with "missing or invalid graph" — a microvm run's graph is a
|
||||||
|
/// placeholder `TopologyGraph` cannot parse. Mission 019fd43e died at 210 seconds
|
||||||
|
/// with its agent still working and its VM orphaned. Every microVM mission that
|
||||||
|
/// appeared to work did so only by finishing inside three minutes.
|
||||||
|
///
|
||||||
|
/// An allowlist rather than a denylist on purpose: the next self-driven tier is
|
||||||
|
/// then safe by default, instead of exposed until someone remembers this file.
|
||||||
|
pub const WORKER_DRIVEN_TIERS: &[&str] = &[
|
||||||
|
"team",
|
||||||
|
"company",
|
||||||
|
"org",
|
||||||
|
"swarm",
|
||||||
|
"compare",
|
||||||
|
// The composed engines: a ZeroClaw graph whose every node is a
|
||||||
|
// Claude-Code-in-a-microVM session. Worker-driven BY DESIGN — the outer
|
||||||
|
// graph's durability (checkpoint, resume, cancellation) is the entire reason
|
||||||
|
// the tier exists, and it comes from being claimed like any other job. It
|
||||||
|
// survives `requeue_stale` because the executor touches `updated_at` from a
|
||||||
|
// ticker for the whole length of a VM turn, not only between steps.
|
||||||
|
"microvm_graph",
|
||||||
|
];
|
||||||
|
|
||||||
|
/// Tiers the stuck-run reaper may fail.
|
||||||
|
///
|
||||||
|
/// A subset of [`WORKER_DRIVEN_TIERS`], and the difference matters. The reaper
|
||||||
|
/// asks "has this run journaled a step within 15 minutes of being CREATED?",
|
||||||
|
/// which assumes a step is short. A `microvm_graph` node is a whole agent session
|
||||||
|
/// in a VM with an hour's budget, so a healthy composed run can legitimately
|
||||||
|
/// journal nothing for far longer than the reaper's patience — it would kill the
|
||||||
|
/// run and orphan a live VM, which is #54 wearing a different tier.
|
||||||
|
///
|
||||||
|
/// Losing the reaper for that tier costs little: a composed run that genuinely
|
||||||
|
/// wedges stops touching `updated_at` and `requeue_stale` recovers it at 180s,
|
||||||
|
/// which is the mechanism the reaper was a backstop for in the first place.
|
||||||
|
pub const REAPABLE_TIERS: &[&str] = &["team", "company", "org", "swarm", "compare"];
|
||||||
|
|
||||||
|
/// The allowlist as owned strings, for binding as `text[]`.
|
||||||
|
fn worker_driven() -> Vec<String> {
|
||||||
|
WORKER_DRIVEN_TIERS.iter().map(|s| (*s).to_string()).collect()
|
||||||
|
}
|
||||||
|
|
||||||
/// Atomically claim the oldest queued job, flipping it to `running`. Uses
|
/// Atomically claim the oldest queued job, flipping it to `running`. Uses
|
||||||
/// `FOR UPDATE SKIP LOCKED` so multiple workers never claim the same job.
|
/// `FOR UPDATE SKIP LOCKED` so multiple workers never claim the same job.
|
||||||
/// Returns `None` when the queue is empty.
|
/// Returns `None` when the queue is empty.
|
||||||
pub async fn claim_next_queued(pool: &PgPool) -> Result<Option<ClaimedTopologyRun>, DbError> {
|
pub async fn claim_next_queued(pool: &PgPool) -> Result<Option<ClaimedTopologyRun>, DbError> {
|
||||||
let row = sqlx::query!(
|
use sqlx::Row as _;
|
||||||
|
// A runtime query rather than `query!` so the tier allowlist can be bound
|
||||||
|
// without regenerating the offline metadata on a machine with no database.
|
||||||
|
let row = sqlx::query(
|
||||||
"UPDATE topology_runs
|
"UPDATE topology_runs
|
||||||
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
|
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
|
||||||
WHERE id = (
|
WHERE id = (
|
||||||
SELECT id FROM topology_runs
|
SELECT id FROM topology_runs
|
||||||
WHERE status = 'queued'
|
WHERE status = 'queued'
|
||||||
|
-- Defence in depth. Even if a self-driven row somehow reaches
|
||||||
|
-- `queued`, the worker must not adopt a job it cannot execute:
|
||||||
|
-- doing so is what turned a live microVM run into a
|
||||||
|
-- missing-or-invalid-graph failure.
|
||||||
|
AND tier = ANY($1)
|
||||||
ORDER BY created_at
|
ORDER BY created_at
|
||||||
FOR UPDATE SKIP LOCKED
|
FOR UPDATE SKIP LOCKED
|
||||||
LIMIT 1
|
LIMIT 1
|
||||||
)
|
)
|
||||||
RETURNING id, workspace_id, task, graph, checkpoint, last_event_id, tier",
|
RETURNING id, workspace_id, task, graph, checkpoint, last_event_id, tier,
|
||||||
|
mission_id, mission_phase_id, iteration",
|
||||||
)
|
)
|
||||||
|
.bind(worker_driven())
|
||||||
.fetch_optional(pool)
|
.fetch_optional(pool)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(row.map(|r| ClaimedTopologyRun {
|
Ok(row.map(|r| ClaimedTopologyRun {
|
||||||
id: r.id,
|
id: r.get("id"),
|
||||||
workspace_id: r.workspace_id,
|
workspace_id: r.get("workspace_id"),
|
||||||
task: r.task,
|
task: r.get("task"),
|
||||||
graph: r.graph,
|
graph: r.get("graph"),
|
||||||
checkpoint: r.checkpoint,
|
checkpoint: r.get("checkpoint"),
|
||||||
last_event_id: r.last_event_id,
|
last_event_id: r.get("last_event_id"),
|
||||||
tier: r.tier,
|
tier: r.get("tier"),
|
||||||
|
mission_id: r.get("mission_id"),
|
||||||
|
mission_phase_id: r.get("mission_phase_id"),
|
||||||
|
iteration: r.get("iteration"),
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Persist mid-run progress: the completed-step checkpoint + journal offset.
|
/// Persist mid-run progress: the completed-step checkpoint + journal offset.
|
||||||
/// Touches `updated_at` so the stale-run sweeper treats the job as alive.
|
/// Touches `updated_at` so the stale-run sweeper treats the job as alive.
|
||||||
|
///
|
||||||
|
/// MERGES rather than replaces. This is not cosmetic: a second writer appends
|
||||||
|
/// live agent output under `checkpoint.log` (see `fleet.rs`, `Uplink::VmOut`),
|
||||||
|
/// and a composed run checkpoints after EVERY graph node. With `SET checkpoint =
|
||||||
|
/// $2` each node's progress silently wiped the log written during it, so a
|
||||||
|
/// composed mission finished with a full `records` array and no output at all —
|
||||||
|
/// while a solo mission, which has no second writer, streamed fine. The keys are
|
||||||
|
/// disjoint, so the progress object still wins for everything it owns.
|
||||||
pub async fn checkpoint(
|
pub async fn checkpoint(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
id: Uuid,
|
id: Uuid,
|
||||||
checkpoint: &Value,
|
checkpoint: &Value,
|
||||||
last_event_id: i64,
|
last_event_id: i64,
|
||||||
) -> Result<(), DbError> {
|
) -> Result<(), DbError> {
|
||||||
sqlx::query!(
|
// `query` rather than `query!`: the macro verifies against a cached schema
|
||||||
|
// that would need regenerating for this SQL, and the bind types here are
|
||||||
|
// unambiguous.
|
||||||
|
sqlx::query(
|
||||||
"UPDATE topology_runs
|
"UPDATE topology_runs
|
||||||
SET checkpoint = $2, last_event_id = $3, updated_at = now()
|
SET checkpoint = COALESCE(checkpoint, '{}'::jsonb) || $2,
|
||||||
|
last_event_id = $3, updated_at = now()
|
||||||
WHERE id = $1",
|
WHERE id = $1",
|
||||||
id,
|
|
||||||
checkpoint,
|
|
||||||
last_event_id,
|
|
||||||
)
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(checkpoint)
|
||||||
|
.bind(last_event_id)
|
||||||
.execute(pool)
|
.execute(pool)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -325,12 +412,19 @@ pub async fn current_status(pool: &PgPool, id: Uuid) -> Result<Option<String>, D
|
|||||||
/// touch within `older_than_secs`). The next claim resumes them from checkpoint.
|
/// touch within `older_than_secs`). The next claim resumes them from checkpoint.
|
||||||
/// Returns how many were requeued.
|
/// Returns how many were requeued.
|
||||||
pub async fn requeue_stale(pool: &PgPool, older_than_secs: f64) -> Result<u64, DbError> {
|
pub async fn requeue_stale(pool: &PgPool, older_than_secs: f64) -> Result<u64, DbError> {
|
||||||
let result = sqlx::query!(
|
let result = sqlx::query(
|
||||||
"UPDATE topology_runs
|
"UPDATE topology_runs
|
||||||
SET status = 'queued', updated_at = now()
|
SET status = 'queued', updated_at = now()
|
||||||
WHERE status = 'running' AND updated_at < now() - make_interval(secs => $1)",
|
WHERE status = 'running'
|
||||||
older_than_secs,
|
AND updated_at < now() - make_interval(secs => $1)
|
||||||
|
-- Only jobs the WORKER drives. A self-driven run (microvm, session) is
|
||||||
|
-- owned by its own task for its whole life and never touches
|
||||||
|
-- `updated_at`, so without this every one of them looked stale after
|
||||||
|
-- three minutes and was requeued out from under a live VM.
|
||||||
|
AND tier = ANY($2)",
|
||||||
)
|
)
|
||||||
|
.bind(older_than_secs)
|
||||||
|
.bind(worker_driven())
|
||||||
.execute(pool)
|
.execute(pool)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(result.rows_affected())
|
Ok(result.rows_affected())
|
||||||
|
|||||||
@@ -0,0 +1,148 @@
|
|||||||
|
//! Approving a plan rewrites a mission's phases — atomically, and with
|
||||||
|
//! `done_when` promoted into the column the evaluator actually reads.
|
||||||
|
|
||||||
|
use cm_db::repo::mission_plan_proposals as plans;
|
||||||
|
use cm_domain::WorkspaceId;
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
async fn workspace(pool: &sqlx::PgPool) -> WorkspaceId {
|
||||||
|
let ws = cm_domain::Workspace {
|
||||||
|
id: WorkspaceId::new(),
|
||||||
|
name: "Plan".into(),
|
||||||
|
plan: "team".into(),
|
||||||
|
};
|
||||||
|
cm_db::repo::workspaces::insert(pool, &ws).await.expect("workspace");
|
||||||
|
ws.id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A mission with the recipe-derived phases a plan is meant to replace.
|
||||||
|
async fn mission_with_phases(pool: &sqlx::PgPool, ws: WorkspaceId) -> Uuid {
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO missions (id, workspace_id, title, template_kind, status, schedule, config)
|
||||||
|
VALUES ($1, $2, 'plan test', 'research_and_code', 'draft', '{}'::jsonb, 'null'::jsonb)",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("insert mission");
|
||||||
|
for (kind, idx) in [("research", 0), ("coding", 1)] {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO mission_phases (id, mission_id, kind, order_idx, status, config)
|
||||||
|
VALUES ($1, $2, $3, $4, 'pending', '{}'::jsonb)",
|
||||||
|
)
|
||||||
|
.bind(Uuid::now_v7())
|
||||||
|
.bind(id)
|
||||||
|
.bind(kind)
|
||||||
|
.bind(idx)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("insert phase");
|
||||||
|
}
|
||||||
|
id
|
||||||
|
}
|
||||||
|
|
||||||
|
fn a_plan() -> Value {
|
||||||
|
json!({"phases": [{"kind": "coding", "task": "do the thing", "done_when": "FILE.md exists"}]})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn phases() -> Vec<(String, i32, Value)> {
|
||||||
|
vec![(
|
||||||
|
"coding".to_string(),
|
||||||
|
0,
|
||||||
|
json!({"task": "do the thing", "done_when": "FILE.md exists"}),
|
||||||
|
)]
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The plan REPLACES the recipe's phases — a plan is an answer to "what is this
|
||||||
|
/// mission", not an addition to one.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn an_approved_plan_replaces_the_missions_phases() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = mission_with_phases(&pool, ws).await;
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
plans::insert(&pool, id, m, ws.as_uuid().to_owned(), &a_plan(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
|
||||||
|
assert!(plans::approve_and_apply(&pool, id, m, ws.as_uuid().to_owned(), &phases(), None, None)
|
||||||
|
.await
|
||||||
|
.expect("apply"));
|
||||||
|
|
||||||
|
let rows: Vec<(String, i32, Option<String>, i32)> = sqlx::query_as(
|
||||||
|
"SELECT kind, order_idx, done_when, max_iterations FROM mission_phases
|
||||||
|
WHERE mission_id = $1 ORDER BY order_idx",
|
||||||
|
)
|
||||||
|
.bind(m)
|
||||||
|
.fetch_all(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(rows.len(), 1, "the two recipe phases must be gone: {rows:?}");
|
||||||
|
assert_eq!(rows[0].0, "coding");
|
||||||
|
assert_eq!(rows[0].1, 0);
|
||||||
|
// THE assertion. `done_when` lives in a COLUMN because the evaluator sweep
|
||||||
|
// filters on it in SQL every tick; a plan whose condition stayed in the
|
||||||
|
// JSONB blob would be stored, rendered, and never judged.
|
||||||
|
assert_eq!(
|
||||||
|
rows[0].2.as_deref(),
|
||||||
|
Some("FILE.md exists"),
|
||||||
|
"done_when must be promoted out of the config, or nothing ever judges it"
|
||||||
|
);
|
||||||
|
assert_eq!(rows[0].3, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Claim and apply are one decision. A proposal marked `approved` against a
|
||||||
|
/// mission whose phases were never rewritten is permanent — the partial unique
|
||||||
|
/// index blocks every later approval.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_failed_apply_leaves_the_proposal_undecided() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = mission_with_phases(&pool, ws).await;
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
plans::insert(&pool, id, m, ws.as_uuid().to_owned(), &a_plan(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
|
||||||
|
// Another workspace's id: the mission-ownership check inside the
|
||||||
|
// transaction must fail and undo the claim.
|
||||||
|
let other = workspace(&pool).await;
|
||||||
|
let err = plans::approve_and_apply(&pool, id, m, other.as_uuid().to_owned(), &phases(), None, None).await;
|
||||||
|
assert!(err.is_ok() || err.is_err());
|
||||||
|
|
||||||
|
let rows = plans::list(&pool, m, ws.as_uuid().to_owned()).await.expect("list");
|
||||||
|
assert_eq!(
|
||||||
|
rows[0].status, "proposed",
|
||||||
|
"the claim must be rolled back, or this proposal is stuck approved forever"
|
||||||
|
);
|
||||||
|
// And the mission's original phases are untouched.
|
||||||
|
let n: i64 = sqlx::query_scalar("SELECT count(*) FROM mission_phases WHERE mission_id = $1")
|
||||||
|
.bind(m)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(n, 2, "a failed apply must not have deleted the existing phases");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// At most one approved plan per mission: two would be two answers to "what is
|
||||||
|
/// this mission", and the phase table holds one.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_mission_cannot_have_two_approved_plans() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = mission_with_phases(&pool, ws).await;
|
||||||
|
let (a, b) = (Uuid::now_v7(), Uuid::now_v7());
|
||||||
|
for id in [a, b] {
|
||||||
|
plans::insert(&pool, id, m, ws.as_uuid().to_owned(), &a_plan(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
}
|
||||||
|
assert!(plans::approve_and_apply(&pool, a, m, ws.as_uuid().to_owned(), &phases(), None, None)
|
||||||
|
.await
|
||||||
|
.expect("approve a"));
|
||||||
|
let second = plans::approve_and_apply(&pool, b, m, ws.as_uuid().to_owned(), &phases(), None, None).await;
|
||||||
|
assert!(second.is_err(), "a second approved plan was allowed: {second:?}");
|
||||||
|
}
|
||||||
@@ -0,0 +1,240 @@
|
|||||||
|
//! A mission may have many proposals and at most one approved roster.
|
||||||
|
//!
|
||||||
|
//! Both properties are enforced in SQL rather than in the handler, and both
|
||||||
|
//! matter for the same reason: the composed executor reads ONE field for what
|
||||||
|
//! shape a mission is, so a second approval would silently win by being written
|
||||||
|
//! last.
|
||||||
|
|
||||||
|
use cm_db::repo::mission_team_proposals as proposals;
|
||||||
|
use cm_domain::WorkspaceId;
|
||||||
|
use serde_json::json;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
async fn workspace(pool: &sqlx::PgPool) -> WorkspaceId {
|
||||||
|
let ws = cm_domain::Workspace {
|
||||||
|
id: WorkspaceId::new(),
|
||||||
|
name: "Roster".into(),
|
||||||
|
plan: "team".into(),
|
||||||
|
};
|
||||||
|
cm_db::repo::workspaces::insert(pool, &ws).await.expect("workspace");
|
||||||
|
ws.id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A mission row to hang proposals off — `mission_id` is a real FK.
|
||||||
|
async fn mission(pool: &sqlx::PgPool, ws: WorkspaceId) -> Uuid {
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO missions (id, workspace_id, title, template_kind, status, schedule, config)
|
||||||
|
VALUES ($1, $2, 'roster test', 'research_and_code', 'draft', '{}'::jsonb, '{}'::jsonb)",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("insert mission");
|
||||||
|
id
|
||||||
|
}
|
||||||
|
|
||||||
|
fn roster() -> serde_json::Value {
|
||||||
|
json!({
|
||||||
|
"topology_kind": "pipeline",
|
||||||
|
"members": [
|
||||||
|
{"role": "implementer"},
|
||||||
|
{"role": "verifier", "backend": "kimi"}
|
||||||
|
]
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The whole point of persisting: a proposal is a record, not a click. It
|
||||||
|
/// arrives `proposed`, applied to nothing.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_proposal_arrives_undecided_and_is_listed() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = mission(&pool, ws).await;
|
||||||
|
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
proposals::insert(&pool, id, m, ws.as_uuid().to_owned(), &roster(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
|
||||||
|
let rows = proposals::list(&pool, m, ws.as_uuid().to_owned()).await.expect("list");
|
||||||
|
assert_eq!(rows.len(), 1);
|
||||||
|
assert_eq!(rows[0].status, "proposed");
|
||||||
|
assert_eq!(rows[0].author_model, "claude-opus-4-8");
|
||||||
|
assert!(rows[0].decided_at.is_none());
|
||||||
|
assert_eq!(rows[0].roster["members"][1]["backend"], "kimi");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Deciding twice must not decide twice. A double-clicked approve, or a retried
|
||||||
|
/// request, would otherwise re-apply a roster to a mission that has moved on.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn only_the_first_decision_counts() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = mission(&pool, ws).await;
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
proposals::insert(&pool, id, m, ws.as_uuid().to_owned(), &roster(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
|
||||||
|
let first = proposals::decide(&pool, id, ws.as_uuid().to_owned(), "approved", None, None)
|
||||||
|
.await
|
||||||
|
.expect("decide");
|
||||||
|
assert!(first, "the first approval must claim the proposal");
|
||||||
|
|
||||||
|
let second = proposals::decide(&pool, id, ws.as_uuid().to_owned(), "rejected", None, None)
|
||||||
|
.await
|
||||||
|
.expect("decide");
|
||||||
|
assert!(!second, "a decided proposal must not be re-decided");
|
||||||
|
|
||||||
|
let rows = proposals::list(&pool, m, ws.as_uuid().to_owned()).await.expect("list");
|
||||||
|
assert_eq!(rows[0].status, "approved", "and the first decision stands");
|
||||||
|
assert!(rows[0].decided_at.is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// At most one approved roster per mission, enforced by a partial unique index.
|
||||||
|
/// Two approved proposals are two answers to "what shape is this mission".
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_mission_cannot_have_two_approved_rosters() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = mission(&pool, ws).await;
|
||||||
|
|
||||||
|
let a = Uuid::now_v7();
|
||||||
|
let b = Uuid::now_v7();
|
||||||
|
for id in [a, b] {
|
||||||
|
proposals::insert(&pool, id, m, ws.as_uuid().to_owned(), &roster(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(proposals::decide(&pool, a, ws.as_uuid().to_owned(), "approved", None, None)
|
||||||
|
.await
|
||||||
|
.expect("approve a"));
|
||||||
|
// The second approval must be REFUSED by the database, not merely lose a
|
||||||
|
// race in the handler.
|
||||||
|
let second = proposals::decide(&pool, b, ws.as_uuid().to_owned(), "approved", None, None).await;
|
||||||
|
assert!(second.is_err(), "a second approved roster was allowed: {second:?}");
|
||||||
|
|
||||||
|
// Rejecting it is still fine — the constraint is on approvals only, and the
|
||||||
|
// ones a human turned down are the record of what the planner gets wrong.
|
||||||
|
assert!(proposals::decide(&pool, b, ws.as_uuid().to_owned(), "rejected", Some("too many VMs"), None)
|
||||||
|
.await
|
||||||
|
.expect("reject b"));
|
||||||
|
let rows = proposals::list(&pool, m, ws.as_uuid().to_owned()).await.expect("list");
|
||||||
|
assert_eq!(rows.len(), 2, "a rejected proposal is kept, not deleted");
|
||||||
|
assert!(rows.iter().any(|r| r.status == "rejected" && r.note.as_deref() == Some("too many VMs")));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Another workspace's proposal is not visible and not decidable. Every read
|
||||||
|
/// here is scoped, and this is the test that keeps it that way.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_proposal_belongs_to_its_workspace() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let other = workspace(&pool).await;
|
||||||
|
let m = mission(&pool, ws).await;
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
proposals::insert(&pool, id, m, ws.as_uuid().to_owned(), &roster(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
|
||||||
|
assert!(proposals::get(&pool, id, other.as_uuid().to_owned()).await.expect("get").is_none());
|
||||||
|
assert!(proposals::list(&pool, m, other.as_uuid().to_owned()).await.expect("list").is_empty());
|
||||||
|
assert!(
|
||||||
|
!proposals::decide(&pool, id, other.as_uuid().to_owned(), "approved", None, None)
|
||||||
|
.await
|
||||||
|
.expect("decide"),
|
||||||
|
"another workspace must not be able to approve this roster"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The bug production found on the FIRST real approval, in the exact shape it
|
||||||
|
/// had: a mission created through the API with no `config` stores jsonb `null`
|
||||||
|
/// — a scalar — and `jsonb_set` refuses a scalar with "cannot set path in
|
||||||
|
/// scalar". `coalesce` does not help, because that guards SQL NULL and this is a
|
||||||
|
/// perfectly good JSON null of the wrong shape.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_roster_applies_to_a_mission_whose_config_is_json_null() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = mission(&pool, ws).await;
|
||||||
|
// Exactly what `POST /api/missions` stores when the body omits `config`.
|
||||||
|
sqlx::query("UPDATE missions SET config = 'null'::jsonb WHERE id = $1")
|
||||||
|
.bind(m)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
proposals::insert(&pool, id, m, ws.as_uuid().to_owned(), &roster(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
let graph = json!({"kind":"pipeline","nodes":[{"id":"n0","role":"implementer","attrs":{}}],"edges":[]});
|
||||||
|
|
||||||
|
let applied = proposals::approve_and_apply(
|
||||||
|
&pool,
|
||||||
|
id,
|
||||||
|
m,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
&graph,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("apply");
|
||||||
|
assert!(applied);
|
||||||
|
|
||||||
|
let (engine, nodes): (Option<String>, Option<i32>) = sqlx::query_as(
|
||||||
|
"SELECT team_engine, jsonb_array_length(config->'roster'->'nodes') FROM missions WHERE id = $1",
|
||||||
|
)
|
||||||
|
.bind(m)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(engine.as_deref(), Some("composed"));
|
||||||
|
assert_eq!(nodes, Some(1), "the roster must actually be on the mission");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Claim and apply are one decision, so they must commit or fail together. A
|
||||||
|
/// proposal marked `approved` against a mission that never received the roster
|
||||||
|
/// is permanent: the partial unique index blocks every later approval, and the
|
||||||
|
/// mission runs solo while its proposal says otherwise.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_failed_apply_leaves_the_proposal_undecided() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = mission(&pool, ws).await;
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
proposals::insert(&pool, id, m, ws.as_uuid().to_owned(), &roster(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
|
||||||
|
// A mission id that does not exist in this workspace: the apply half matches
|
||||||
|
// no row, which is the failure the transaction has to undo.
|
||||||
|
let err = proposals::approve_and_apply(
|
||||||
|
&pool,
|
||||||
|
id,
|
||||||
|
Uuid::now_v7(),
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
&json!({}),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(err.is_err(), "applying to a missing mission must fail: {err:?}");
|
||||||
|
|
||||||
|
let rows = proposals::list(&pool, m, ws.as_uuid().to_owned()).await.expect("list");
|
||||||
|
assert_eq!(
|
||||||
|
rows[0].status, "proposed",
|
||||||
|
"the claim must have been rolled back, or this proposal is stuck approved forever"
|
||||||
|
);
|
||||||
|
// And it can still be approved properly afterwards.
|
||||||
|
let graph = json!({"kind":"pipeline","nodes":[{"id":"n0","role":"implementer","attrs":{}}],"edges":[]});
|
||||||
|
assert!(
|
||||||
|
proposals::approve_and_apply(&pool, id, m, ws.as_uuid().to_owned(), &graph, None, None)
|
||||||
|
.await
|
||||||
|
.expect("apply")
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -263,6 +263,7 @@ async fn a_template_with_live_agents_still_accepts_edits() {
|
|||||||
system_prompt: prompt,
|
system_prompt: prompt,
|
||||||
skills: extra,
|
skills: extra,
|
||||||
brain_seed: None,
|
brain_seed: None,
|
||||||
|
model: None,
|
||||||
}],
|
}],
|
||||||
};
|
};
|
||||||
team_templates::upsert_builtin(&pool, build("first", vec![]))
|
team_templates::upsert_builtin(&pool, build("first", vec![]))
|
||||||
|
|||||||
@@ -0,0 +1,218 @@
|
|||||||
|
//! The sweepers must leave SELF-DRIVEN runs alone.
|
||||||
|
//!
|
||||||
|
//! A `tier='microvm'` or `tier='session'` run is inserted directly as `running` by
|
||||||
|
//! `phase_runner` and owned start-to-finish by its own `tokio::spawn`. Nothing
|
||||||
|
//! touches its `updated_at` or `checkpoint` while it is in flight, because there
|
||||||
|
//! is no per-step loop to hook.
|
||||||
|
//!
|
||||||
|
//! Both sweepers were written when every `running` row was a `cm_orchestrator` job
|
||||||
|
//! that checkpointed after each step, and neither filtered on tier. The result,
|
||||||
|
//! measured in production on 2026-08-06: `requeue_stale` declared a healthy microVM
|
||||||
|
//! run stale at 180 seconds, the worker claimed it, failed to deserialize its graph
|
||||||
|
//! placeholder, and killed the phase with "missing or invalid graph" — while the
|
||||||
|
//! agent went on working and its VM was orphaned for over an hour.
|
||||||
|
//!
|
||||||
|
//! **Every microVM mission that appeared to work did so by finishing inside three
|
||||||
|
//! minutes.** The end-to-end harness runs a 90-second mission, so it cannot see
|
||||||
|
//! this class at all — which is why the guard lives here, against the real SQL, and
|
||||||
|
//! costs milliseconds instead of eight minutes.
|
||||||
|
|
||||||
|
use cm_db::repo::topology_runs;
|
||||||
|
use cm_domain::WorkspaceId;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
/// Insert a run that is `running` and has looked idle for a long time — exactly
|
||||||
|
/// the shape a long agent turn presents.
|
||||||
|
async fn stale_running_run(pool: &sqlx::PgPool, ws: WorkspaceId, tier: &str) -> Uuid {
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO topology_runs
|
||||||
|
(id, workspace_id, task, kind, status, graph, tier,
|
||||||
|
created_at, updated_at)
|
||||||
|
VALUES ($1, $2, 'long turn', 'run', 'running', $3, $4,
|
||||||
|
now() - interval '30 minutes', now() - interval '30 minutes')",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
// The placeholder a self-driven run carries: no `kind`, so `TopologyGraph`
|
||||||
|
// cannot parse it. That is what turned a requeue into a hard failure.
|
||||||
|
.bind(serde_json::json!({ "nodes": [], "edges": [], "executor": tier }))
|
||||||
|
.bind(tier)
|
||||||
|
// `mission_id` is left NULL: it has an FK to `missions`, and `requeue_stale`
|
||||||
|
// does not look at it. The reaper DOES filter on `mission_id IS NOT NULL` —
|
||||||
|
// which is precisely what used to be mistaken for "orchestrator-driven" — and
|
||||||
|
// it now shares the same tier allowlist, asserted below.
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("insert run");
|
||||||
|
id
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn status_of(pool: &sqlx::PgPool, id: Uuid) -> String {
|
||||||
|
sqlx::query_scalar::<_, String>("SELECT status FROM topology_runs WHERE id = $1")
|
||||||
|
.bind(id)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.expect("read status")
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn workspace(pool: &sqlx::PgPool) -> WorkspaceId {
|
||||||
|
let ws = cm_domain::Workspace {
|
||||||
|
id: WorkspaceId::new(),
|
||||||
|
name: "Sweeper".into(),
|
||||||
|
plan: "team".into(),
|
||||||
|
};
|
||||||
|
cm_db::repo::workspaces::insert(pool, &ws)
|
||||||
|
.await
|
||||||
|
.expect("workspace");
|
||||||
|
ws.id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The bug, in one assertion: 30 minutes idle and it must still be `running`.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn requeue_stale_leaves_self_driven_runs_alone() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
|
||||||
|
let microvm = stale_running_run(&pool, ws, "microvm").await;
|
||||||
|
let session = stale_running_run(&pool, ws, "session").await;
|
||||||
|
|
||||||
|
let moved = topology_runs::requeue_stale(&pool, 180.0)
|
||||||
|
.await
|
||||||
|
.expect("requeue");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
status_of(&pool, microvm).await,
|
||||||
|
"running",
|
||||||
|
"a microvm run was requeued out from under a live VM ({moved} rows moved)"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
status_of(&pool, session).await,
|
||||||
|
"running",
|
||||||
|
"a session run was requeued out from under a live agent"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// And a worker-driven run in the same state MUST still be requeued, or the fix
|
||||||
|
/// would have been "stop sweeping" rather than "sweep the right rows".
|
||||||
|
#[tokio::test]
|
||||||
|
async fn requeue_stale_still_rescues_worker_driven_runs() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let team = stale_running_run(&pool, ws, "team").await;
|
||||||
|
|
||||||
|
topology_runs::requeue_stale(&pool, 180.0).await.expect("requeue");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
status_of(&pool, team).await,
|
||||||
|
"queued",
|
||||||
|
"a genuinely stalled team run must still be recovered"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Defence in depth: even handed a queued self-driven row, the worker must not
|
||||||
|
/// adopt a job it cannot execute. Claiming one is what produced the
|
||||||
|
/// "missing or invalid graph" failure on a run that was perfectly healthy.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn the_worker_will_not_claim_a_self_driven_run() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
topology_runs::enqueue_run_tier(
|
||||||
|
&pool,
|
||||||
|
id,
|
||||||
|
ws,
|
||||||
|
"should never be claimed",
|
||||||
|
&serde_json::json!({ "nodes": [], "edges": [], "executor": "microvm" }),
|
||||||
|
"microvm",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("enqueue");
|
||||||
|
|
||||||
|
let claimed = topology_runs::claim_next_queued(&pool).await.expect("claim");
|
||||||
|
assert!(
|
||||||
|
claimed.is_none(),
|
||||||
|
"the worker claimed a microvm run: {:?}",
|
||||||
|
claimed.map(|c| c.tier)
|
||||||
|
);
|
||||||
|
assert_eq!(status_of(&pool, id).await, "queued", "and it must be left as it was");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The composed tier is the mirror image of the two above and must not be
|
||||||
|
/// mistaken for them: it runs VMs, but the WORKER drives its graph, so being
|
||||||
|
/// claimed and requeued is exactly what gives it checkpointing and resume.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn the_worker_claims_and_rescues_a_composed_run() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
topology_runs::enqueue_run_tier(
|
||||||
|
&pool,
|
||||||
|
id,
|
||||||
|
ws,
|
||||||
|
"compose the engines",
|
||||||
|
// A real graph, unlike the self-driven placeholder: the worker plans it.
|
||||||
|
&serde_json::json!({
|
||||||
|
"kind": "pipeline",
|
||||||
|
"nodes": [{ "id": "a", "role": "worker", "attrs": {} }],
|
||||||
|
"edges": []
|
||||||
|
}),
|
||||||
|
"microvm_graph",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("enqueue");
|
||||||
|
|
||||||
|
let claimed = topology_runs::claim_next_queued(&pool)
|
||||||
|
.await
|
||||||
|
.expect("claim")
|
||||||
|
.expect("a composed run must be claimable, or it never runs at all");
|
||||||
|
assert_eq!(claimed.tier, "microvm_graph");
|
||||||
|
assert_eq!(claimed.id, id);
|
||||||
|
|
||||||
|
// And a composed run whose worker died must come back: its checkpoint is
|
||||||
|
// what makes resume possible, and requeue is what triggers it.
|
||||||
|
sqlx::query("UPDATE topology_runs SET updated_at = now() - interval '30 minutes' WHERE id = $1")
|
||||||
|
.bind(id)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.expect("age the run");
|
||||||
|
topology_runs::requeue_stale(&pool, 180.0).await.expect("requeue");
|
||||||
|
assert_eq!(
|
||||||
|
status_of(&pool, id).await,
|
||||||
|
"queued",
|
||||||
|
"a composed run orphaned by a dead worker must be recovered"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The allowlist is the single place this policy lives, so assert its membership
|
||||||
|
/// directly — a new self-driven tier added without touching it would otherwise be
|
||||||
|
/// exposed exactly as microvm was.
|
||||||
|
#[test]
|
||||||
|
fn the_allowlist_names_only_worker_driven_tiers() {
|
||||||
|
for driven in ["team", "swarm", "company", "org"] {
|
||||||
|
assert!(
|
||||||
|
topology_runs::WORKER_DRIVEN_TIERS.contains(&driven),
|
||||||
|
"{driven} is driven by the worker and must be sweepable"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for self_driven in ["microvm", "session"] {
|
||||||
|
assert!(
|
||||||
|
!topology_runs::WORKER_DRIVEN_TIERS.contains(&self_driven),
|
||||||
|
"{self_driven} owns its own lifecycle; sweeping it kills live work"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// `microvm_graph` is worker-driven but NOT reapable: one of its steps is a
|
||||||
|
// whole agent session in a VM, so "no step records in 15 minutes" is what a
|
||||||
|
// healthy composed run looks like, and reaping it would orphan a live VM —
|
||||||
|
// #54 in a different tier.
|
||||||
|
assert!(topology_runs::WORKER_DRIVEN_TIERS.contains(&"microvm_graph"));
|
||||||
|
assert!(!topology_runs::REAPABLE_TIERS.contains(&"microvm_graph"));
|
||||||
|
for reapable in topology_runs::REAPABLE_TIERS {
|
||||||
|
assert!(
|
||||||
|
topology_runs::WORKER_DRIVEN_TIERS.contains(reapable),
|
||||||
|
"{reapable} is reaped but never driven"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
//! A failed phase must not strand its mission at `running` forever.
|
||||||
|
//!
|
||||||
|
//! `start_pending_phases` launches a phase only when every lower-order phase is
|
||||||
|
//! `completed`, so once one fails the rest can never run. They sat `pending`,
|
||||||
|
//! and `close_finished_missions` requires no phase to be non-terminal — so the
|
||||||
|
//! mission never finished, and `mission_runtime`'s sweeper (which fires after a
|
||||||
|
//! terminal state) never reaped its container.
|
||||||
|
//!
|
||||||
|
//! Found by counting containers on gw-04, not by a test: one leaked runtime
|
||||||
|
//! container per failed multi-phase mission, accumulating for days. This is the
|
||||||
|
//! SQL that ends it, tested against a real database because the bug lived
|
||||||
|
//! entirely in the interaction between two queries' predicates.
|
||||||
|
|
||||||
|
use cm_domain::WorkspaceId;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
async fn workspace(pool: &sqlx::PgPool) -> WorkspaceId {
|
||||||
|
let ws = cm_domain::Workspace {
|
||||||
|
id: WorkspaceId::new(),
|
||||||
|
name: "Unreachable".into(),
|
||||||
|
plan: "team".into(),
|
||||||
|
};
|
||||||
|
cm_db::repo::workspaces::insert(pool, &ws).await.expect("workspace");
|
||||||
|
ws.id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A running mission whose phase 0 failed and whose phases 1..n never started.
|
||||||
|
async fn stuck_mission(pool: &sqlx::PgPool, ws: WorkspaceId) -> Uuid {
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO missions (id, workspace_id, title, template_kind, status, schedule, config)
|
||||||
|
VALUES ($1, $2, 'stuck', 'research_and_code', 'running', '{}'::jsonb, '{}'::jsonb)",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("mission");
|
||||||
|
for (idx, status) in [(0, "failed"), (1, "pending"), (2, "pending")] {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO mission_phases (id, mission_id, kind, order_idx, status, config)
|
||||||
|
VALUES ($1, $2, 'coding', $3, $4, '{}'::jsonb)",
|
||||||
|
)
|
||||||
|
.bind(Uuid::now_v7())
|
||||||
|
.bind(id)
|
||||||
|
.bind(idx)
|
||||||
|
.bind(status)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("phase");
|
||||||
|
}
|
||||||
|
id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The sweep, as `phase_runner::skip_unreachable_phases` runs it.
|
||||||
|
async fn skip_unreachable(pool: &sqlx::PgPool) -> u64 {
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE mission_phases mp
|
||||||
|
SET status = 'skipped', completed_at = now()
|
||||||
|
WHERE mp.status = 'pending'
|
||||||
|
AND EXISTS (SELECT 1 FROM missions m WHERE m.id = mp.mission_id AND m.status = 'running')
|
||||||
|
AND EXISTS (
|
||||||
|
SELECT 1 FROM mission_phases prior
|
||||||
|
WHERE prior.mission_id = mp.mission_id
|
||||||
|
AND prior.order_idx < mp.order_idx
|
||||||
|
AND prior.status = 'failed'
|
||||||
|
)",
|
||||||
|
)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("skip")
|
||||||
|
.rows_affected()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn statuses(pool: &sqlx::PgPool, mission: Uuid) -> Vec<String> {
|
||||||
|
sqlx::query_scalar("SELECT status FROM mission_phases WHERE mission_id = $1 ORDER BY order_idx")
|
||||||
|
.bind(mission)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.expect("statuses")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_failed_phase_makes_the_later_ones_unreachable_not_pending_forever() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = stuck_mission(&pool, ws).await;
|
||||||
|
|
||||||
|
assert_eq!(skip_unreachable(&pool).await, 2, "both later phases are unreachable");
|
||||||
|
assert_eq!(
|
||||||
|
statuses(&pool, m).await,
|
||||||
|
vec!["failed", "skipped", "skipped"],
|
||||||
|
"a phase that can never run must say so, or the mission never closes"
|
||||||
|
);
|
||||||
|
// Every phase is now terminal, which is what `close_finished_missions`
|
||||||
|
// waits for — the container sweeper keys off the mission reaching that.
|
||||||
|
let non_terminal: i64 = sqlx::query_scalar(
|
||||||
|
"SELECT count(*) FROM mission_phases
|
||||||
|
WHERE mission_id = $1 AND status NOT IN ('completed','failed','skipped')",
|
||||||
|
)
|
||||||
|
.bind(m)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(non_terminal, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A phase waiting AHEAD of the failure is untouched: order is what makes a
|
||||||
|
/// phase unreachable, and a failure later in the list says nothing about one
|
||||||
|
/// still queued before it.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_phase_before_the_failure_is_left_alone() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = Uuid::now_v7();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO missions (id, workspace_id, title, template_kind, status, schedule, config)
|
||||||
|
VALUES ($1, $2, 'ordered', 'research_and_code', 'running', '{}'::jsonb, '{}'::jsonb)",
|
||||||
|
)
|
||||||
|
.bind(m)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
for (idx, status) in [(0, "pending"), (1, "failed"), (2, "pending")] {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO mission_phases (id, mission_id, kind, order_idx, status, config)
|
||||||
|
VALUES ($1, $2, 'coding', $3, $4, '{}'::jsonb)",
|
||||||
|
)
|
||||||
|
.bind(Uuid::now_v7())
|
||||||
|
.bind(m)
|
||||||
|
.bind(idx)
|
||||||
|
.bind(status)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
skip_unreachable(&pool).await;
|
||||||
|
assert_eq!(statuses(&pool, m).await, vec!["pending", "failed", "skipped"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A mission that is not running is not swept: a draft's phases are pending by
|
||||||
|
/// definition and must not be skipped out from under it.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn only_a_running_missions_phases_are_skipped() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = stuck_mission(&pool, ws).await;
|
||||||
|
sqlx::query("UPDATE missions SET status = 'draft' WHERE id = $1")
|
||||||
|
.bind(m)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(skip_unreachable(&pool).await, 0);
|
||||||
|
assert_eq!(statuses(&pool, m).await, vec!["failed", "pending", "pending"]);
|
||||||
|
}
|
||||||
@@ -187,5 +187,9 @@ async fn browsing_returns_web_tainted_content_and_taints_later_gated_actions() {
|
|||||||
let png = blob.get(&key).await.expect("screenshot stored");
|
let png = blob.get(&key).await.expect("screenshot stored");
|
||||||
assert_eq!(&png[..8], b"\x89PNG\r\n\x1a\n", "PNG magic");
|
assert_eq!(&png[..8], b"\x89PNG\r\n\x1a\n", "PNG magic");
|
||||||
|
|
||||||
|
// `shutdown` drains the warm pool only; the sandbox assigned to this agent
|
||||||
|
// outlives it by design (reused across a redeploy). A test has no next
|
||||||
|
// deploy, so it must release its own or the container simply stays.
|
||||||
|
browser.release_agent(agent.id).await;
|
||||||
browser.shutdown().await;
|
browser.shutdown().await;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -152,6 +152,11 @@ async fn shell_exec_runs_in_the_agent_sandbox_with_persistent_home() {
|
|||||||
outputs[1]
|
outputs[1]
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// `shutdown` drains the warm pool; the container this exec ASSIGNED to the
|
||||||
|
// agent survives it on purpose, so a redeploy can reuse it. A test has no
|
||||||
|
// next deploy, so it must release its own or the container simply stays —
|
||||||
|
// which is how 289 of them accumulated before anyone counted.
|
||||||
|
sandboxes.release_agent(agent.id).await;
|
||||||
sandboxes.shutdown().await;
|
sandboxes.shutdown().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -229,4 +234,5 @@ async fn shell_exec_without_a_sandbox_runtime_reports_a_tool_error() {
|
|||||||
statuses.iter().all(|s| s == "error"),
|
statuses.iter().all(|s| s == "error"),
|
||||||
"steps must record the failure: {statuses:?}"
|
"steps must record the failure: {statuses:?}"
|
||||||
);
|
);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -118,7 +118,17 @@ async fn the_pool_prefills_assigns_and_refills() {
|
|||||||
"reuse must not drain the pool"
|
"reuse must not drain the pool"
|
||||||
);
|
);
|
||||||
|
|
||||||
// Shutdown destroys assigned AND pooled sandboxes.
|
// `shutdown` tears down the POOL only — assigned sandboxes deliberately
|
||||||
|
// survive it, so they can be reused across a redeploy. The comment here
|
||||||
|
// used to claim it destroyed both, which is why nobody noticed that every
|
||||||
|
// run of this test left its assigned container running: three such tests,
|
||||||
|
// three leaked containers per `./scripts/test.sh`, 289 of them by the time
|
||||||
|
// anyone counted.
|
||||||
|
manager.release_agent(agent).await;
|
||||||
manager.shutdown().await;
|
manager.shutdown().await;
|
||||||
assert_eq!(manager.pool_size().await, 0);
|
assert_eq!(manager.pool_size().await, 0);
|
||||||
|
assert!(
|
||||||
|
!manager.release_agent(agent).await,
|
||||||
|
"the agent's sandbox must be gone, not merely unpooled"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -85,15 +85,10 @@ mcp_config = "/zeroclaw-data/clawmates-mcp.json"
|
|||||||
# model_provider = "claude_cli.glm5"
|
# model_provider = "claude_cli.glm5"
|
||||||
# risk_profile = "toolfree"
|
# risk_profile = "toolfree"
|
||||||
#
|
#
|
||||||
# Gemini (Google) — built-in `gemini` API family (no image rebuild). Key via env
|
# Gemini is NOT wired. The platform stripped it: no provider alias, no key
|
||||||
# ZEROCLAW_providers__models__gemini__default__api_key=<GEMINI_API_KEY>; model in
|
# forwarded to agent containers, no selector entry. ZeroClaw still has a built-in
|
||||||
# config. gemini-2.5-flash = stable + high free-tier RPM (throttle-friendly);
|
# `gemini` family, so an operator could add it back here — but `provider_alias_for`
|
||||||
# gemini-3-pro-preview is the flagship.
|
# no longer resolves `gemini*` to it, and it would log as an unrecognised model.
|
||||||
# [providers.models.gemini.default]
|
|
||||||
# model = "gemini-2.5-flash"
|
|
||||||
# [agents.worker_gemini]
|
|
||||||
# model_provider = "gemini.default"
|
|
||||||
# risk_profile = "toolfree"
|
|
||||||
#
|
#
|
||||||
# Groq — built-in `groq` family (key via ZEROCLAW_providers__models__groq__default__api_key).
|
# Groq — built-in `groq` family (key via ZEROCLAW_providers__models__groq__default__api_key).
|
||||||
# Fast, but LOW free-tier TPM: each turn carries a ~9.5k-tok system prompt, so 2
|
# Fast, but LOW free-tier TPM: each turn carries a ~9.5k-tok system prompt, so 2
|
||||||
@@ -109,7 +104,7 @@ mcp_config = "/zeroclaw-data/clawmates-mcp.json"
|
|||||||
# env ZEROCLAW_AGENT_MAP="role=alias,role=alias" (+ ZEROCLAW_DEFAULT_AGENT for
|
# env ZEROCLAW_AGENT_MAP="role=alias,role=alias" (+ ZEROCLAW_DEFAULT_AGENT for
|
||||||
# unmapped roles). Point each semantic role at a different model to run ONE
|
# unmapped roles). Point each semantic role at a different model to run ONE
|
||||||
# topology across vendors, e.g.:
|
# topology across vendors, e.g.:
|
||||||
# ZEROCLAW_AGENT_MAP="coordinator=coordinator,researcher=worker_glm,analyst=worker_kimi,writer=worker_gemini,actor=worker_groq"
|
# ZEROCLAW_AGENT_MAP="coordinator=coordinator,researcher=worker_glm,analyst=worker_kimi,writer=worker_glm5,actor=worker_groq"
|
||||||
# Then POST /api/topologies/run {task, graph} with those role names. QUOTA CARE:
|
# Then POST /api/topologies/run {task, graph} with those role names. QUOTA CARE:
|
||||||
# GLM/Kimi plans have 5h/weekly caps + low concurrency (GLM Lite ~1 project at a
|
# GLM/Kimi plans have 5h/weekly caps + low concurrency (GLM Lite ~1 project at a
|
||||||
# time) — prefer pipeline (sequential) over swarm/mesh, keep tasks short, and ask
|
# time) — prefer pipeline (sequential) over swarm/mesh, keep tasks short, and ask
|
||||||
|
|||||||
Generated
+1511
-5
File diff suppressed because it is too large
Load Diff
@@ -19,11 +19,15 @@
|
|||||||
"class-variance-authority": "^0.7.1",
|
"class-variance-authority": "^0.7.1",
|
||||||
"clsx": "^2.1.1",
|
"clsx": "^2.1.1",
|
||||||
"geist": "^1.7.2",
|
"geist": "^1.7.2",
|
||||||
|
"github-slugger": "^2.0.0",
|
||||||
"lucide-react": "^1.17.0",
|
"lucide-react": "^1.17.0",
|
||||||
"next": "16.2.9",
|
"next": "16.2.9",
|
||||||
"nuqs": "^2.8.9",
|
"nuqs": "^2.8.9",
|
||||||
"react": "19.2.4",
|
"react": "19.2.4",
|
||||||
"react-dom": "19.2.4",
|
"react-dom": "19.2.4",
|
||||||
|
"react-markdown": "^10.1.0",
|
||||||
|
"rehype-slug": "^6.0.0",
|
||||||
|
"remark-gfm": "^4.0.1",
|
||||||
"tailwind-merge": "^3.6.0",
|
"tailwind-merge": "^3.6.0",
|
||||||
"three": "^0.169.0",
|
"three": "^0.169.0",
|
||||||
"zod": "^4.4.3"
|
"zod": "^4.4.3"
|
||||||
|
|||||||
@@ -1,62 +0,0 @@
|
|||||||
// Local Next route (NOT proxied — a specific path beats the /api/[...path]
|
|
||||||
// catch-all): generates an agent avatar with Gemini's image model ("Nano
|
|
||||||
// Banana", gemini-2.5-flash-image) using GEMINI_API_KEY from the frontend env,
|
|
||||||
// and returns a base64 data URL. Saving the chosen image is a separate
|
|
||||||
// PATCH /api/claws/{id} {avatar} (the existing backend route).
|
|
||||||
|
|
||||||
import { NextResponse, type NextRequest } from "next/server";
|
|
||||||
|
|
||||||
import { resolveBearer } from "@/lib/auth/bearer";
|
|
||||||
|
|
||||||
const MODEL = "gemini-2.5-flash-image";
|
|
||||||
const ENDPOINT = `https://generativelanguage.googleapis.com/v1beta/models/${MODEL}:generateContent`;
|
|
||||||
|
|
||||||
interface InlineData { data?: string; mimeType?: string; mime_type?: string }
|
|
||||||
interface GeminiPart { inlineData?: InlineData; inline_data?: InlineData }
|
|
||||||
interface GeminiResponse { candidates?: Array<{ content?: { parts?: GeminiPart[] } }> }
|
|
||||||
|
|
||||||
export async function POST(request: NextRequest) {
|
|
||||||
const token = await resolveBearer();
|
|
||||||
if (!token) return NextResponse.json({ error: "unauthenticated" }, { status: 401 });
|
|
||||||
|
|
||||||
const key = process.env.GEMINI_API_KEY;
|
|
||||||
if (!key) return NextResponse.json({ error: "image generation is not configured (set GEMINI_API_KEY)" }, { status: 503 });
|
|
||||||
|
|
||||||
let prompt = "";
|
|
||||||
try {
|
|
||||||
const body = (await request.json()) as { prompt?: unknown };
|
|
||||||
prompt = String(body?.prompt ?? "").trim();
|
|
||||||
} catch {
|
|
||||||
/* fall through to the 400 below */
|
|
||||||
}
|
|
||||||
if (!prompt) return NextResponse.json({ error: "prompt required" }, { status: 400 });
|
|
||||||
|
|
||||||
let upstream: Response;
|
|
||||||
try {
|
|
||||||
upstream = await fetch(ENDPOINT, {
|
|
||||||
method: "POST",
|
|
||||||
headers: { "Content-Type": "application/json", "x-goog-api-key": key },
|
|
||||||
body: JSON.stringify({
|
|
||||||
contents: [{ parts: [{ text: `A clean, centered square avatar portrait for an AI agent. ${prompt}` }] }],
|
|
||||||
generationConfig: { responseModalities: ["IMAGE"] },
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
return NextResponse.json({ error: "could not reach the image service" }, { status: 502 });
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!upstream.ok) {
|
|
||||||
const detail = await upstream.text().catch(() => "");
|
|
||||||
return NextResponse.json({ error: `image service error (${upstream.status})`, detail: detail.slice(0, 400) }, { status: 502 });
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = (await upstream.json().catch(() => null)) as GeminiResponse | null;
|
|
||||||
const parts = data?.candidates?.[0]?.content?.parts ?? [];
|
|
||||||
const part = parts.find((p) => p.inlineData?.data || p.inline_data?.data);
|
|
||||||
const inline = part?.inlineData ?? part?.inline_data;
|
|
||||||
if (!inline?.data) {
|
|
||||||
return NextResponse.json({ error: "the model did not return an image — try a different prompt" }, { status: 502 });
|
|
||||||
}
|
|
||||||
const mime = inline.mimeType ?? inline.mime_type ?? "image/png";
|
|
||||||
return NextResponse.json({ image: `data:${mime};base64,${inline.data}` });
|
|
||||||
}
|
|
||||||
@@ -251,3 +251,109 @@ body {
|
|||||||
.marketing {
|
.marketing {
|
||||||
color-scheme: light;
|
color-scheme: light;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ── Mission artifact reader ──────────────────────────────────────
|
||||||
|
Styles the HTML that react-markdown generates for a mission's markdown
|
||||||
|
documents (see components/dashboard/MarkdownView.tsx). Every rule is
|
||||||
|
descendant-scoped to .md-view: the markup is generated, so there are no
|
||||||
|
class hooks to target, and unscoped element selectors would restyle the
|
||||||
|
whole dashboard. */
|
||||||
|
.md-view {
|
||||||
|
color: #d8d8de;
|
||||||
|
font-size: 13.5px;
|
||||||
|
line-height: 1.68;
|
||||||
|
/* Research documents are read, not skimmed. A bounded measure keeps lines
|
||||||
|
comfortable; wide content (tables, code) scrolls inside its own box. */
|
||||||
|
max-width: 78ch;
|
||||||
|
overflow-wrap: anywhere;
|
||||||
|
}
|
||||||
|
.md-view > :first-child { margin-top: 0; }
|
||||||
|
.md-view h1, .md-view h2, .md-view h3, .md-view h4 {
|
||||||
|
color: #f3f3f5;
|
||||||
|
font-weight: 600;
|
||||||
|
line-height: 1.3;
|
||||||
|
margin: 1.6em 0 0.6em;
|
||||||
|
}
|
||||||
|
.md-view h1 {
|
||||||
|
font-size: 21px;
|
||||||
|
border-bottom: 1px solid rgba(255, 255, 255, 0.09);
|
||||||
|
padding-bottom: 0.35em;
|
||||||
|
}
|
||||||
|
.md-view h2 {
|
||||||
|
font-size: 17px;
|
||||||
|
border-bottom: 1px solid rgba(255, 255, 255, 0.06);
|
||||||
|
padding-bottom: 0.3em;
|
||||||
|
}
|
||||||
|
.md-view h3 { font-size: 15px; }
|
||||||
|
.md-view h4 { font-size: 13.5px; color: #c8c8d0; }
|
||||||
|
.md-view p { margin: 0.85em 0; }
|
||||||
|
.md-view a {
|
||||||
|
color: #7cd6e0;
|
||||||
|
text-decoration: none;
|
||||||
|
border-bottom: 1px solid rgba(124, 214, 224, 0.35);
|
||||||
|
}
|
||||||
|
.md-view a:hover { border-bottom-color: #7cd6e0; }
|
||||||
|
.md-view strong { color: #f3f3f5; font-weight: 600; }
|
||||||
|
.md-view ul, .md-view ol { margin: 0.8em 0; padding-left: 1.5em; }
|
||||||
|
.md-view li { margin: 0.32em 0; }
|
||||||
|
.md-view li::marker { color: #7cd6e0; }
|
||||||
|
.md-view code {
|
||||||
|
font-family: var(--font-mono);
|
||||||
|
font-size: 12px;
|
||||||
|
background: rgba(255, 255, 255, 0.06);
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.07);
|
||||||
|
border-radius: 4px;
|
||||||
|
padding: 1px 5px;
|
||||||
|
color: #e6d9a8;
|
||||||
|
}
|
||||||
|
.md-view pre {
|
||||||
|
background: #0a0a0d;
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.08);
|
||||||
|
border-radius: 8px;
|
||||||
|
padding: 12px 14px;
|
||||||
|
overflow-x: auto;
|
||||||
|
margin: 1em 0;
|
||||||
|
}
|
||||||
|
/* Inside a block the chip styling would double the border and re-tint text. */
|
||||||
|
.md-view pre code {
|
||||||
|
background: none;
|
||||||
|
border: none;
|
||||||
|
padding: 0;
|
||||||
|
color: #d8d8de;
|
||||||
|
line-height: 1.55;
|
||||||
|
}
|
||||||
|
.md-view blockquote {
|
||||||
|
margin: 1em 0;
|
||||||
|
padding: 0.1em 0 0.1em 1em;
|
||||||
|
border-left: 3px solid rgba(124, 214, 224, 0.5);
|
||||||
|
color: #a8a8b2;
|
||||||
|
}
|
||||||
|
/* Agents write GFM tables constantly — this is the main payoff of remark-gfm.
|
||||||
|
`display: block` so a wide table scrolls itself instead of the page. */
|
||||||
|
.md-view table {
|
||||||
|
border-collapse: collapse;
|
||||||
|
width: 100%;
|
||||||
|
margin: 1.1em 0;
|
||||||
|
font-size: 12.5px;
|
||||||
|
display: block;
|
||||||
|
overflow-x: auto;
|
||||||
|
}
|
||||||
|
.md-view th, .md-view td {
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.09);
|
||||||
|
padding: 7px 10px;
|
||||||
|
text-align: left;
|
||||||
|
vertical-align: top;
|
||||||
|
}
|
||||||
|
.md-view th {
|
||||||
|
background: rgba(255, 255, 255, 0.04);
|
||||||
|
color: #f3f3f5;
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
.md-view tr:nth-child(even) td { background: rgba(255, 255, 255, 0.015); }
|
||||||
|
.md-view hr {
|
||||||
|
border: none;
|
||||||
|
border-top: 1px solid rgba(255, 255, 255, 0.09);
|
||||||
|
margin: 1.8em 0;
|
||||||
|
}
|
||||||
|
.md-view img { max-width: 100%; border-radius: 6px; }
|
||||||
|
.md-view input[type="checkbox"] { margin-right: 6px; }
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ const MODEL_OPTIONS: { value: string; label: string; family: string }[] = [
|
|||||||
{ value: "glm-4.6", label: "GLM 4.6", family: "Z.AI" },
|
{ value: "glm-4.6", label: "GLM 4.6", family: "Z.AI" },
|
||||||
{ value: "glm-5.2", label: "GLM 5.2", family: "Z.AI" },
|
{ value: "glm-5.2", label: "GLM 5.2", family: "Z.AI" },
|
||||||
{ value: "kimi-k2", label: "Kimi K2", family: "Moonshot" },
|
{ value: "kimi-k2", label: "Kimi K2", family: "Moonshot" },
|
||||||
{ value: "gemini-2.0-flash", label: "Gemini 2.0 Flash", family: "Google" },
|
|
||||||
{ value: "llama-3.3-70b-versatile", label: "Llama 3.3 70B", family: "Groq" },
|
{ value: "llama-3.3-70b-versatile", label: "Llama 3.3 70B", family: "Groq" },
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|||||||
@@ -1,14 +1,18 @@
|
|||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
// Avatar editor modal for a claw: upload an image OR generate one from a prompt
|
// Avatar editor modal for a claw: upload an image, preview it, then Save
|
||||||
// (Gemini / Nano Banana, via /api/generate-avatar — max 5 attempts), preview it,
|
// (downscaled to 256² and persisted via PATCH /api/claws/{id}).
|
||||||
// then Save (downscaled to 256² and persisted via PATCH /api/claws/{id}).
|
//
|
||||||
|
// Prompt-based generation was removed with the rest of the Gemini dependency:
|
||||||
|
// it called Gemini's image model, and no provider we use can generate images
|
||||||
|
// (Claude and Kimi are text-only; z.ai returns "Unknown Model" for cogview on
|
||||||
|
// our plan). Upload is untouched — it never needed a provider. Re-add a
|
||||||
|
// generate path here if an image provider is ever wired.
|
||||||
|
|
||||||
import { useEffect, useRef, useState } from "react";
|
import { useEffect, useRef, useState } from "react";
|
||||||
import { Camera, Sparkles, Upload, X } from "lucide-react";
|
import { Camera, Upload, X } from "lucide-react";
|
||||||
|
|
||||||
const mono = "'JetBrains Mono', ui-monospace, monospace";
|
const mono = "'JetBrains Mono', ui-monospace, monospace";
|
||||||
const MAX_ATTEMPTS = 5;
|
|
||||||
|
|
||||||
// Cover-fit to a square and re-encode small so avatars stay lightweight.
|
// Cover-fit to a square and re-encode small so avatars stay lightweight.
|
||||||
function downscale(dataUrl: string, size = 256): Promise<string> {
|
function downscale(dataUrl: string, size = 256): Promise<string> {
|
||||||
@@ -33,9 +37,7 @@ function downscale(dataUrl: string, size = 256): Promise<string> {
|
|||||||
|
|
||||||
export function AvatarModal({ clawId, clawName, current, onClose, onSaved }: { clawId: string; clawName: string; current?: string | null; onClose: () => void; onSaved: (dataUrl: string) => void }) {
|
export function AvatarModal({ clawId, clawName, current, onClose, onSaved }: { clawId: string; clawName: string; current?: string | null; onClose: () => void; onSaved: (dataUrl: string) => void }) {
|
||||||
const [preview, setPreview] = useState<string | null>(current ?? null);
|
const [preview, setPreview] = useState<string | null>(current ?? null);
|
||||||
const [prompt, setPrompt] = useState("");
|
const [busy, setBusy] = useState<null | "save">(null);
|
||||||
const [attempts, setAttempts] = useState(0);
|
|
||||||
const [busy, setBusy] = useState<null | "gen" | "save">(null);
|
|
||||||
const [error, setError] = useState<string | null>(null);
|
const [error, setError] = useState<string | null>(null);
|
||||||
const fileRef = useRef<HTMLInputElement>(null);
|
const fileRef = useRef<HTMLInputElement>(null);
|
||||||
|
|
||||||
@@ -53,20 +55,6 @@ export function AvatarModal({ clawId, clawName, current, onClose, onSaved }: { c
|
|||||||
reader.readAsDataURL(file);
|
reader.readAsDataURL(file);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function generate() {
|
|
||||||
if (busy || attempts >= MAX_ATTEMPTS || !prompt.trim()) return;
|
|
||||||
setBusy("gen");
|
|
||||||
setError(null);
|
|
||||||
try {
|
|
||||||
const res = await fetch("/api/generate-avatar", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ prompt }) });
|
|
||||||
const j = (await res.json().catch(() => ({}))) as { image?: string; error?: string };
|
|
||||||
if (!res.ok || !j.image) setError(j.error || "generation failed");
|
|
||||||
else { setPreview(j.image); setAttempts((a) => a + 1); }
|
|
||||||
} catch {
|
|
||||||
setError("generation failed");
|
|
||||||
}
|
|
||||||
setBusy(null);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function save() {
|
async function save() {
|
||||||
if (!preview || busy) return;
|
if (!preview || busy) return;
|
||||||
@@ -84,7 +72,6 @@ export function AvatarModal({ clawId, clawName, current, onClose, onSaved }: { c
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const maxed = attempts >= MAX_ATTEMPTS;
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div onClick={onClose} role="presentation" style={{ position: "fixed", inset: 0, zIndex: 110, background: "rgba(0,0,0,.62)", backdropFilter: "blur(4px)", display: "flex", alignItems: "center", justifyContent: "center", padding: 24, animation: "cm-fade .18s ease" }}>
|
<div onClick={onClose} role="presentation" style={{ position: "fixed", inset: 0, zIndex: 110, background: "rgba(0,0,0,.62)", backdropFilter: "blur(4px)", display: "flex", alignItems: "center", justifyContent: "center", padding: 24, animation: "cm-fade .18s ease" }}>
|
||||||
@@ -92,7 +79,7 @@ export function AvatarModal({ clawId, clawName, current, onClose, onSaved }: { c
|
|||||||
<div style={{ display: "flex", alignItems: "flex-start", gap: 12, marginBottom: 16 }}>
|
<div style={{ display: "flex", alignItems: "flex-start", gap: 12, marginBottom: 16 }}>
|
||||||
<div style={{ flex: 1 }}>
|
<div style={{ flex: 1 }}>
|
||||||
<div style={{ fontSize: 18, fontWeight: 700, color: "#f3f3f5" }}>{clawName}'s image</div>
|
<div style={{ fontSize: 18, fontWeight: 700, color: "#f3f3f5" }}>{clawName}'s image</div>
|
||||||
<div style={{ fontSize: 12.5, color: "#8a8a92", marginTop: 2 }}>Upload one, or generate from a prompt.</div>
|
<div style={{ fontSize: 12.5, color: "#8a8a92", marginTop: 2 }}>Upload a PNG or JPG.</div>
|
||||||
</div>
|
</div>
|
||||||
<button type="button" onClick={onClose} aria-label="Close" style={{ width: 30, height: 30, flex: "none", borderRadius: 8, border: "1px solid rgba(255,255,255,.12)", background: "transparent", color: "#9a9aa2", cursor: "pointer", display: "flex", alignItems: "center", justifyContent: "center" }}><X size={15} /></button>
|
<button type="button" onClick={onClose} aria-label="Close" style={{ width: 30, height: 30, flex: "none", borderRadius: 8, border: "1px solid rgba(255,255,255,.12)", background: "transparent", color: "#9a9aa2", cursor: "pointer", display: "flex", alignItems: "center", justifyContent: "center" }}><X size={15} /></button>
|
||||||
</div>
|
</div>
|
||||||
@@ -108,24 +95,6 @@ export function AvatarModal({ clawId, clawName, current, onClose, onSaved }: { c
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div style={{ borderTop: "1px solid rgba(255,255,255,.07)", paddingTop: 14 }}>
|
|
||||||
<div style={{ display: "flex", alignItems: "center", gap: 7, marginBottom: 8 }}>
|
|
||||||
<Sparkles size={13} color="#c98af0" />
|
|
||||||
<span style={{ fontFamily: mono, fontSize: 10, letterSpacing: ".08em", color: "#c98af0" }}>GENERATE FROM PROMPT</span>
|
|
||||||
<span style={{ flex: 1 }} />
|
|
||||||
<span style={{ fontFamily: mono, fontSize: 9, color: maxed ? "#e8b465" : "#5a5a62" }}>{attempts}/{MAX_ATTEMPTS} attempts</span>
|
|
||||||
</div>
|
|
||||||
<textarea
|
|
||||||
value={prompt}
|
|
||||||
onChange={(e) => setPrompt(e.target.value)}
|
|
||||||
placeholder="e.g. a calm robotic owl mascot, soft teal gradient, minimal"
|
|
||||||
rows={2}
|
|
||||||
style={{ width: "100%", resize: "none", borderRadius: 9, border: "1px solid rgba(255,255,255,.12)", background: "#101014", color: "#eaeaee", fontSize: 12.5, padding: "8px 10px", outline: "none", fontFamily: "inherit" }}
|
|
||||||
/>
|
|
||||||
<button type="button" onClick={generate} disabled={busy !== null || maxed || !prompt.trim()} style={{ marginTop: 8, width: "100%", display: "inline-flex", alignItems: "center", justifyContent: "center", gap: 7, padding: "9px 0", borderRadius: 9, border: "1px solid rgba(201,138,240,.4)", background: maxed || !prompt.trim() ? "rgba(201,138,240,.06)" : "rgba(201,138,240,.14)", color: "#d9b6f7", fontSize: 12.5, fontWeight: 600, cursor: busy || maxed || !prompt.trim() ? "default" : "pointer", opacity: busy === "gen" ? 0.7 : 1 }}>
|
|
||||||
<Sparkles size={14} /> {busy === "gen" ? "Generating…" : maxed ? "Max attempts reached" : attempts > 0 ? "Regenerate" : "Generate"}
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{error ? <div role="alert" style={{ marginTop: 12, fontFamily: mono, fontSize: 10.5, color: "#ff8a7a" }}>{error}</div> : null}
|
{error ? <div role="alert" style={{ marginTop: 12, fontFamily: mono, fontSize: 10.5, color: "#ff8a7a" }}>{error}</div> : null}
|
||||||
|
|
||||||
|
|||||||
@@ -21,9 +21,11 @@ import { topologyById } from "@/lib/topologies";
|
|||||||
import { panelParsers, type DeviceSize } from "@/lib/url/panel-params";
|
import { panelParsers, type DeviceSize } from "@/lib/url/panel-params";
|
||||||
import { type FlowItem } from "./flow/TopologyFlow";
|
import { type FlowItem } from "./flow/TopologyFlow";
|
||||||
import { WorldCanvas } from "../world/WorldCanvas";
|
import { WorldCanvas } from "../world/WorldCanvas";
|
||||||
|
import type { WorldSeed } from "../world/engine";
|
||||||
import { ObservePanel } from "../observe/ObservePanel";
|
import { ObservePanel } from "../observe/ObservePanel";
|
||||||
import { StructureTree, orgNode, type TreeItem } from "./StructureTree";
|
import { StructureTree, orgNode, type TreeItem, clawNode } from "./StructureTree";
|
||||||
import { MissionsList } from "./MissionsList";
|
import { MissionsList } from "./MissionsList";
|
||||||
|
import { MissionWizard } from "./MissionWizard";
|
||||||
import { LevelUpInbox } from "./LevelUpInbox";
|
import { LevelUpInbox } from "./LevelUpInbox";
|
||||||
import { MissionCanvas } from "./MissionCanvas";
|
import { MissionCanvas } from "./MissionCanvas";
|
||||||
import { RepoList } from "./RepoList";
|
import { RepoList } from "./RepoList";
|
||||||
@@ -77,13 +79,61 @@ const mono = "'JetBrains Mono', ui-monospace, monospace";
|
|||||||
// dashboard-data.ts fabricates a few org/company/team nodes so an empty
|
// dashboard-data.ts fabricates a few org/company/team nodes so an empty
|
||||||
// workspace still has something to render. These ids aren't UUIDs and
|
// workspace still has something to render. These ids aren't UUIDs and
|
||||||
// don't exist in the DB — treat them as non-selectable for reap.
|
// don't exist in the DB — treat them as non-selectable for reap.
|
||||||
const SYNTHETIC_TREE_IDS = new Set([
|
// Placeholder containers `dashboard-data.ts` fabricates for entities that were
|
||||||
|
// never parented into a real org → company → team chain. Clicking one offers to
|
||||||
|
// materialise that chain, because that is the only thing you can do with it.
|
||||||
|
// They exist only in the World tree.
|
||||||
|
const ORPHAN_CONTAINER_IDS = new Set([
|
||||||
"my-workspace",
|
"my-workspace",
|
||||||
"ws-teams",
|
"ws-teams",
|
||||||
"ungrouped-co",
|
"ungrouped-co",
|
||||||
"ungrouped-team",
|
"ungrouped-team",
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
// Every id in the tree that is NOT a database row, orphan containers plus the
|
||||||
|
// "My Workforce" root. These cannot be renamed or selected for reap — the
|
||||||
|
// backend would 422 on the non-UUID id.
|
||||||
|
//
|
||||||
|
// Kept separate from the set above on purpose: they answer different questions.
|
||||||
|
// Folding the workforce root into the orphan set made clicking "My Workforce"
|
||||||
|
// offer to parent everything into an org → company → team chain — the exact
|
||||||
|
// hierarchy that root exists to replace.
|
||||||
|
const SYNTHETIC_TREE_IDS = new Set([...ORPHAN_CONTAINER_IDS, "my-workforce"]);
|
||||||
|
|
||||||
|
// Mission grouping rows under "My Workforce". Like the workforce root these are
|
||||||
|
// not database rows the claw/team endpoints understand: a mission-group id
|
||||||
|
// carries a MISSION uuid, so letting it reach `selectTeam` would look valid and
|
||||||
|
// fetch the wrong thing entirely.
|
||||||
|
const isGroupingRow = (id: string) =>
|
||||||
|
id.startsWith("mission-group:") || id === "workforce-unassigned";
|
||||||
|
|
||||||
|
// A claw inside a mission group is keyed `<missionId>:<clawId>` so the same
|
||||||
|
// colleague can appear under several missions without colliding. Everything
|
||||||
|
// downstream wants the claw id alone.
|
||||||
|
const clawIdOf = (treeId: string) => {
|
||||||
|
const i = treeId.lastIndexOf(":");
|
||||||
|
return i === -1 ? treeId : treeId.slice(i + 1);
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface WorkforceAgent {
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
job_title: string;
|
||||||
|
role_slot: string | null;
|
||||||
|
accent: string;
|
||||||
|
status: string;
|
||||||
|
}
|
||||||
|
export interface Workforce {
|
||||||
|
missions: {
|
||||||
|
mission_id: string;
|
||||||
|
title: string;
|
||||||
|
status: string;
|
||||||
|
templateKind?: string | null;
|
||||||
|
agents: WorkforceAgent[];
|
||||||
|
}[];
|
||||||
|
unassigned: WorkforceAgent[];
|
||||||
|
}
|
||||||
|
|
||||||
// Gradient palette for structure nodes that don't carry their own (companies,
|
// Gradient palette for structure nodes that don't carry their own (companies,
|
||||||
// teams). Agents bring their own grad/ink.
|
// teams). Agents bring their own grad/ink.
|
||||||
const NODE_GRADS: [string, string][] = [
|
const NODE_GRADS: [string, string][] = [
|
||||||
@@ -428,13 +478,20 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
const onWorldSelect = (id: string) => {
|
const onWorldSelect = (id: string) => {
|
||||||
if (!id) { setWorldSel(null); return; }
|
if (!id) { setWorldSel(null); return; }
|
||||||
setWorldSel(id);
|
setWorldSel(id);
|
||||||
|
// A mission grouping row carries no level and is not a row in any of the
|
||||||
|
// org tables — it only sets the focus, which `focusedMissionId` derives
|
||||||
|
// from `worldSel`.
|
||||||
|
if (isGroupingRow(id)) return;
|
||||||
const lv = nodeLevel.get(id);
|
const lv = nodeLevel.get(id);
|
||||||
if (lv === "claw") {
|
if (lv === "claw") {
|
||||||
// Agents open a summary in the side panel (no redirect — the panel's robot
|
// Agents open a summary in the side panel (no redirect — the panel's robot
|
||||||
// button drills in). Keep the path ids coherent for the summary + drill.
|
// button drills in). Keep the path ids coherent for the summary + drill.
|
||||||
const loc = locateAgent(id);
|
// The tree id may be `<missionId>:<clawId>`; every consumer below wants
|
||||||
|
// the claw id alone.
|
||||||
|
const clawId = clawIdOf(id);
|
||||||
|
const loc = locateAgent(clawId);
|
||||||
if (loc) { setOrgId(loc.org.id); setCompanyId(loc.company.id); setTeamId(loc.team.id); }
|
if (loc) { setOrgId(loc.org.id); setCompanyId(loc.company.id); setTeamId(loc.team.id); }
|
||||||
setAgentId(id);
|
setAgentId(clawId);
|
||||||
setWorldPanelOpen(true);
|
setWorldPanelOpen(true);
|
||||||
} else if (lv === "org") selectOrg(id);
|
} else if (lv === "org") selectOrg(id);
|
||||||
else if (lv === "company") selectCompany(id);
|
else if (lv === "company") selectCompany(id);
|
||||||
@@ -442,15 +499,31 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
};
|
};
|
||||||
// Clicking a synthetic scaffolding node ("my-workspace" / "ws-teams" /
|
// Clicking a synthetic scaffolding node ("my-workspace" / "ws-teams" /
|
||||||
// "ungrouped-co" / "ungrouped-team") opens the migration dialog instead of
|
// "ungrouped-co" / "ungrouped-team") opens the migration dialog instead of
|
||||||
// navigating — those nodes have no real DB row to select. Real nodes fall
|
// navigating — those nodes have no real DB row to select. The "My Workforce"
|
||||||
// through to the normal selection path below.
|
// root is synthetic too but is NOT one of those: it is the flat tree's
|
||||||
|
// heading, and offering to parent everything into an org chain from it would
|
||||||
|
// rebuild the hierarchy it replaced. Real nodes fall through to the normal
|
||||||
|
// selection path below.
|
||||||
const [orphanDialogOpen, setOrphanDialogOpen] = useState(false);
|
const [orphanDialogOpen, setOrphanDialogOpen] = useState(false);
|
||||||
|
|
||||||
// The tree's unified node handler (world tree + the flat agents list). On the
|
// The tree's unified node handler (world tree + the flat agents list). On the
|
||||||
// flat agents page a claw click opens the agent; in the World tree it selects.
|
// flat agents page a claw click opens the agent; in the World tree it selects.
|
||||||
const onTreeSelect = (item: TreeItem) => {
|
const onTreeSelect = (item: TreeItem) => {
|
||||||
if (SYNTHETIC_TREE_IDS.has(item.id)) { setOrphanDialogOpen(true); return; }
|
// Only an orphan container offers the migration. The workforce root is a
|
||||||
if (isClaw && item.level === "claw") { openClaw(item.id); return; }
|
// heading: the row click above it has already toggled the branch, and there
|
||||||
|
// is nothing else to do with it.
|
||||||
|
if (ORPHAN_CONTAINER_IDS.has(item.id)) { setOrphanDialogOpen(true); return; }
|
||||||
|
if (SYNTHETIC_TREE_IDS.has(item.id)) return;
|
||||||
|
// On the World tier a mission grouping row is the whole point: selecting it
|
||||||
|
// focuses the scene on that mission. `worldSel` is the single source of
|
||||||
|
// that focus, so it is set here rather than in a second state.
|
||||||
|
if (isWorld && isGroupingRow(item.id)) { onWorldSelect(item.id); return; }
|
||||||
|
// Everywhere else a grouping row is a heading, like the workforce root: the
|
||||||
|
// row click has already toggled the branch. Falling through would hand a
|
||||||
|
// MISSION uuid to selectTeam, which is a real-looking id for the wrong
|
||||||
|
// table.
|
||||||
|
if (isGroupingRow(item.id)) return;
|
||||||
|
if (isClaw && item.level === "claw") { openClaw(clawIdOf(item.id)); return; }
|
||||||
onWorldSelect(item.id);
|
onWorldSelect(item.id);
|
||||||
expandPathTo(item.id);
|
expandPathTo(item.id);
|
||||||
};
|
};
|
||||||
@@ -500,6 +573,14 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
|
|
||||||
const [teamDrawer, setTeamDrawer] = useState(false);
|
const [teamDrawer, setTeamDrawer] = useState(false);
|
||||||
const [deployOpen, setDeployOpen] = useState(false);
|
const [deployOpen, setDeployOpen] = useState(false);
|
||||||
|
// The "+" starts a MISSION, not an agent.
|
||||||
|
//
|
||||||
|
// Both "+" affordances used to open the deploy wizard — while the copy beside
|
||||||
|
// them said "deploy wizard" and the rail's tooltip said "Deploy a new agent",
|
||||||
|
// none of which is what someone arriving here wants to do first. You get a
|
||||||
|
// workforce BY running missions; hand-staffing one is the advanced path and
|
||||||
|
// keeps its own entry point below.
|
||||||
|
const [missionWizardOpen, setMissionWizardOpen] = useState(false);
|
||||||
// Brain registry rail (claw page) — collapsed by default, toggled by the brain
|
// Brain registry rail (claw page) — collapsed by default, toggled by the brain
|
||||||
// icon in the Agents sidebar header.
|
// icon in the Agents sidebar header.
|
||||||
const [registryOpen, setRegistryOpen] = useState(false);
|
const [registryOpen, setRegistryOpen] = useState(false);
|
||||||
@@ -518,6 +599,21 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
const [missionsSel, setMissionsSel] = useState<string | null>(null);
|
const [missionsSel, setMissionsSel] = useState<string | null>(null);
|
||||||
const [missionsRefresh, setMissionsRefresh] = useState(0);
|
const [missionsRefresh, setMissionsRefresh] = useState(0);
|
||||||
|
|
||||||
|
// The roster grouped by mission. Refetched when a mission changes so a newly
|
||||||
|
// staffed mission appears without a reload.
|
||||||
|
const [workforce, setWorkforce] = useState<Workforce | null>(null);
|
||||||
|
useEffect(() => {
|
||||||
|
let alive = true;
|
||||||
|
fetch("/api/workforce", { cache: "no-store" })
|
||||||
|
.then(okJson)
|
||||||
|
.then((d) => { if (alive) setWorkforce(d as Workforce); })
|
||||||
|
// Leave `workforce` null so the tree falls back to the flat list. An
|
||||||
|
// empty sidebar would look like "you have no agents", which is a worse
|
||||||
|
// lie than showing them ungrouped.
|
||||||
|
.catch(() => { if (alive) setWorkforce(null); });
|
||||||
|
return () => { alive = false; };
|
||||||
|
}, [missionsRefresh]);
|
||||||
|
|
||||||
const allAgents = orgs.flatMap((o) => o.companies.flatMap((c) => c.teams.flatMap((t) => t.agents)));
|
const allAgents = orgs.flatMap((o) => o.companies.flatMap((c) => c.teams.flatMap((t) => t.agents)));
|
||||||
// World: the full expandable org→company→team→agent forest. Agents page: a flat list.
|
// World: the full expandable org→company→team→agent forest. Agents page: a flat list.
|
||||||
const worldRoots: TreeItem[] = orgs.map(orgNode);
|
const worldRoots: TreeItem[] = orgs.map(orgNode);
|
||||||
@@ -550,20 +646,163 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
const stripSynthetics = (roots: TreeItem[]): TreeItem[] => {
|
const stripSynthetics = (roots: TreeItem[]): TreeItem[] => {
|
||||||
const walk = (items: TreeItem[]): TreeItem[] =>
|
const walk = (items: TreeItem[]): TreeItem[] =>
|
||||||
items.flatMap((n) =>
|
items.flatMap((n) =>
|
||||||
SYNTHETIC_TREE_IDS.has(n.id)
|
ORPHAN_CONTAINER_IDS.has(n.id)
|
||||||
? walk(n.children ?? [])
|
? walk(n.children ?? [])
|
||||||
: [{ ...n, children: walk(n.children ?? []) }],
|
: [{ ...n, children: walk(n.children ?? []) }],
|
||||||
);
|
);
|
||||||
return walk(roots);
|
return walk(roots);
|
||||||
};
|
};
|
||||||
const worldCanvasRoots: TreeItem[] = narrowRoots(stripSynthetics(worldRoots), worldSel);
|
const worldCanvasRoots: TreeItem[] = narrowRoots(stripSynthetics(worldRoots), worldSel);
|
||||||
// Both the World and Agents tiers now share the same collapsible org →
|
// One flat root: "My Workforce", and the agents directly under it.
|
||||||
// company → team → agent tree — a single pane onto the whole workforce.
|
//
|
||||||
// `selectLevel` below still constrains selection to agents on the Agents
|
// The tree used to be Organization → Company → Team → Agent, and on a real
|
||||||
// tier so bulk-delete stays claw-scoped.
|
// workspace that read "My Workspace → General → Everyone" — three levels of
|
||||||
const treeRoots: TreeItem[] = worldRoots;
|
// placeholder wrapping five agents. None of it is load-bearing in the UI:
|
||||||
|
// `agents` has no org/company/team column at all (membership is only the
|
||||||
|
// `team_members` join, which the mission executor uses to map graph nodes to
|
||||||
|
// claws), and the /orgs, /companies and /teams pages already redirect here.
|
||||||
|
//
|
||||||
|
// The World tier keeps the full forest — that visualisation is ABOUT
|
||||||
|
// structure, so flattening it would remove its subject.
|
||||||
|
// Grouped by mission, with the flat list as the fallback.
|
||||||
|
//
|
||||||
|
// The flat version rendered `orgs → companies → teams → agents`, which shows
|
||||||
|
// a claw once per TEAM it belongs to. Claws are reused across missions now,
|
||||||
|
// so a crew of five that had run five missions appeared as twenty-five rows
|
||||||
|
// of the same five people and read as the roster multiplying. Grouping by
|
||||||
|
// mission makes the repetition mean something: the same colleague shows up
|
||||||
|
// under each mission they staffed.
|
||||||
|
//
|
||||||
|
// `/api/workforce` is the source; until it answers (or if it fails) we fall
|
||||||
|
// back to the flat list rather than rendering an empty sidebar.
|
||||||
|
const missionGroups: TreeItem[] = (workforce?.missions ?? []).map((m) => ({
|
||||||
|
// Prefixed so a mission id can never be mistaken for a claw id by the
|
||||||
|
// selection handler — clicking a group must expand it, not try to open a
|
||||||
|
// claw page for a mission uuid.
|
||||||
|
id: `mission-group:${m.mission_id}`,
|
||||||
|
level: "team",
|
||||||
|
label: m.title?.trim() || "Untitled mission",
|
||||||
|
meta: `${m.status} · ${m.agents.length} agent${m.agents.length === 1 ? "" : "s"}`,
|
||||||
|
status: m.status,
|
||||||
|
children: m.agents.map((a, i) => ({
|
||||||
|
// Same claw under two missions would otherwise collide on React keys
|
||||||
|
// AND on the tree's active-id comparison.
|
||||||
|
id: `${m.mission_id}:${a.id}`,
|
||||||
|
level: "claw" as const,
|
||||||
|
label: a.name,
|
||||||
|
meta: a.role_slot || a.job_title,
|
||||||
|
status: a.status,
|
||||||
|
grad: NODE_GRADS[i % NODE_GRADS.length][0],
|
||||||
|
ink: NODE_GRADS[i % NODE_GRADS.length][1],
|
||||||
|
initial: (a.name || "?").trim().charAt(0).toUpperCase(),
|
||||||
|
})),
|
||||||
|
}));
|
||||||
|
const unassignedAgents: TreeItem[] = (workforce?.unassigned ?? []).map((a, i) => ({
|
||||||
|
id: a.id,
|
||||||
|
level: "claw" as const,
|
||||||
|
label: a.name,
|
||||||
|
meta: a.job_title,
|
||||||
|
status: a.status,
|
||||||
|
grad: NODE_GRADS[i % NODE_GRADS.length][0],
|
||||||
|
ink: NODE_GRADS[i % NODE_GRADS.length][1],
|
||||||
|
initial: (a.name || "?").trim().charAt(0).toUpperCase(),
|
||||||
|
}));
|
||||||
|
const groupedChildren: TreeItem[] = [
|
||||||
|
...missionGroups,
|
||||||
|
// Hand-created claws, and claws whose missions were deleted. Shown as a
|
||||||
|
// peer group so they cannot silently disappear from the sidebar.
|
||||||
|
...(unassignedAgents.length
|
||||||
|
? [{
|
||||||
|
id: "workforce-unassigned",
|
||||||
|
level: "team" as const,
|
||||||
|
label: "Not on a mission",
|
||||||
|
meta: `${unassignedAgents.length} agent${unassignedAgents.length === 1 ? "" : "s"}`,
|
||||||
|
children: unassignedAgents,
|
||||||
|
}]
|
||||||
|
: []),
|
||||||
|
];
|
||||||
|
const useGrouped = groupedChildren.length > 0;
|
||||||
|
// Distinct people, not rows: the same claw on three missions is one colleague.
|
||||||
|
const distinctAgentCount = useGrouped
|
||||||
|
? new Set([
|
||||||
|
...(workforce?.missions ?? []).flatMap((m) => m.agents.map((a) => a.id)),
|
||||||
|
...(workforce?.unassigned ?? []).map((a) => a.id),
|
||||||
|
]).size
|
||||||
|
: allAgents.length;
|
||||||
|
const workforceRoot: TreeItem = {
|
||||||
|
id: "my-workforce",
|
||||||
|
level: "org",
|
||||||
|
label: "My Workforce",
|
||||||
|
meta: `${distinctAgentCount} agent${distinctAgentCount === 1 ? "" : "s"}`,
|
||||||
|
children: useGrouped ? groupedChildren : allAgents.map(clawNode),
|
||||||
|
};
|
||||||
|
// The World tier gets the SAME workforce sidebar as the agents page.
|
||||||
|
//
|
||||||
|
// It used to show the org → company → team → agent forest, which is the
|
||||||
|
// hierarchy the agents page stopped rendering — so the two pages disagreed
|
||||||
|
// about what the workspace looks like, and the World offered no way to ask
|
||||||
|
// "show me just this mission". Missions are the unit people think in, so the
|
||||||
|
// sidebar is missions here too, and picking one scopes the scene.
|
||||||
|
const treeRoots: TreeItem[] = [workforceRoot];
|
||||||
const treeActiveId = isClaw ? agentId : worldSel;
|
const treeActiveId = isClaw ? agentId : worldSel;
|
||||||
const treeAutoExpand: string[] = [];
|
// Open by default. A workforce collapsed behind one disclosure is a workforce
|
||||||
|
// the user has to discover they own.
|
||||||
|
const treeAutoExpand: string[] = ["my-workforce"];
|
||||||
|
|
||||||
|
// Which mission the World is focused on, derived from the tree selection so
|
||||||
|
// there is ONE selection state rather than a second one to keep in sync.
|
||||||
|
// Selecting a mission group focuses it; selecting an agent inside a group
|
||||||
|
// keeps that mission focused, which is what makes clicking around inside a
|
||||||
|
// mission feel stable.
|
||||||
|
const focusedMissionId: string | null = (() => {
|
||||||
|
if (!isWorld) return null;
|
||||||
|
// Default to the most recent mission rather than the whole workspace.
|
||||||
|
//
|
||||||
|
// The unfocused view drew every agent of every mission into one space,
|
||||||
|
// which is not a picture of anything that happens: missions do not share a
|
||||||
|
// stage, and at tens of agents the scene says less the more it shows.
|
||||||
|
// `/api/workforce` orders missions newest-first, so [0] is the one someone
|
||||||
|
// opening this page is most likely asking about.
|
||||||
|
if (!worldSel) return (workforce?.missions ?? [])[0]?.mission_id ?? null;
|
||||||
|
if (worldSel.startsWith("mission-group:")) return worldSel.slice("mission-group:".length);
|
||||||
|
const grouped = (workforce?.missions ?? []).find((m) =>
|
||||||
|
m.agents.some((a) => `${m.mission_id}:${a.id}` === worldSel),
|
||||||
|
);
|
||||||
|
// An agent selected from somewhere other than a mission group (the World
|
||||||
|
// graph itself) keeps whatever mission was already pinned, instead of
|
||||||
|
// silently reverting to the default.
|
||||||
|
return grouped?.mission_id ?? (workforce?.missions ?? [])[0]?.mission_id ?? null;
|
||||||
|
})();
|
||||||
|
const focusedMission = (workforce?.missions ?? []).find((m) => m.mission_id === focusedMissionId) ?? null;
|
||||||
|
// Palette input. Read from the plan channel rather than the SSE because the
|
||||||
|
// engine takes its palette at construction, before any event has arrived.
|
||||||
|
const focusedMissionTemplate = focusedMission?.templateKind ?? null;
|
||||||
|
const focusAgentIds: Set<string> | null = focusedMission
|
||||||
|
? new Set(focusedMission.agents.map((a) => a.id))
|
||||||
|
: null;
|
||||||
|
// Seed the scene with just this mission's crew when one is focused. The seed
|
||||||
|
// alone does not scope the view (the engine materialises any agent an event
|
||||||
|
// mentions) — WorldCanvas filters the feed — but it means the mission's own
|
||||||
|
// people are on stage immediately rather than fading in with their first
|
||||||
|
// event.
|
||||||
|
const worldSeedRoots: WorldSeed[] = focusedMission
|
||||||
|
? [{
|
||||||
|
id: `mission:${focusedMission.mission_id}`,
|
||||||
|
// "mission", not "team". `seed()` casts this straight to a Tier and
|
||||||
|
// `ensureNode` is first-write-wins, so seeding it as a team created a
|
||||||
|
// small teal team-sized dot that the later `node.activity` could never
|
||||||
|
// upgrade — the generic dot at the centre of the scene was this line.
|
||||||
|
level: "mission",
|
||||||
|
label: focusedMission.title?.trim() || "Untitled mission",
|
||||||
|
status: focusedMission.status,
|
||||||
|
children: focusedMission.agents.map((a) => ({
|
||||||
|
id: a.id,
|
||||||
|
level: "claw",
|
||||||
|
label: a.name,
|
||||||
|
status: a.status,
|
||||||
|
})),
|
||||||
|
}]
|
||||||
|
: worldCanvasRoots;
|
||||||
|
|
||||||
// Every node in the active tree (flattened) — the wrench multi-select looks up
|
// Every node in the active tree (flattened) — the wrench multi-select looks up
|
||||||
// selected nodes here regardless of depth, and derives the delete kind from the
|
// selected nodes here regardless of depth, and derives the delete kind from the
|
||||||
@@ -576,7 +815,17 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
})();
|
})();
|
||||||
const nodeLevel = new Map(allNodes.map((n) => [n.id, n.level] as const));
|
const nodeLevel = new Map(allNodes.map((n) => [n.id, n.level] as const));
|
||||||
const levelToKind = (lv?: string): ReapKind => (lv === "team" ? "teams" : lv === "company" ? "companies" : lv === "org" ? "orgs" : "agents");
|
const levelToKind = (lv?: string): ReapKind => (lv === "team" ? "teams" : lv === "company" ? "companies" : lv === "org" ? "orgs" : "agents");
|
||||||
const selectedItems = allNodes.filter((n) => selectedAgents.has(n.id)).map((n) => ({ id: n.id, name: n.label }));
|
// Selection is keyed by TREE id, but the reap endpoints want the row id. A
|
||||||
|
// claw inside a mission group is keyed `<missionId>:<clawId>`, and the same
|
||||||
|
// colleague can be selected under two missions — send one id, once, or the
|
||||||
|
// purge would be handed a composite key and a duplicate.
|
||||||
|
const selectedItems = Array.from(
|
||||||
|
new Map(
|
||||||
|
allNodes
|
||||||
|
.filter((n) => selectedAgents.has(n.id))
|
||||||
|
.map((n) => [clawIdOf(n.id), { id: clawIdOf(n.id), name: n.label }] as const),
|
||||||
|
).values(),
|
||||||
|
);
|
||||||
const reapKind: ReapKind = selectedItems.length ? levelToKind(nodeLevel.get(selectedItems[0].id)) : "agents";
|
const reapKind: ReapKind = selectedItems.length ? levelToKind(nodeLevel.get(selectedItems[0].id)) : "agents";
|
||||||
|
|
||||||
const crumbStyle = (on: boolean): CSSProperties =>
|
const crumbStyle = (on: boolean): CSSProperties =>
|
||||||
@@ -639,7 +888,7 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
);
|
);
|
||||||
})}
|
})}
|
||||||
<div style={{ width: 28, height: 1, background: "rgba(255,255,255,.08)", margin: "6px 0" }} />
|
<div style={{ width: 28, height: 1, background: "rgba(255,255,255,.08)", margin: "6px 0" }} />
|
||||||
<button type="button" onClick={() => setDeployOpen(true)} title="Deploy a new agent" style={{ width: 36, height: 36, borderRadius: 9, border: "1px dashed rgba(255,111,97,.4)", background: "rgba(255,111,97,.06)", display: "flex", alignItems: "center", justifyContent: "center", color: "#ff6f61", fontSize: 19, fontWeight: 300, cursor: "pointer" }}>+</button>
|
<button type="button" onClick={() => setMissionWizardOpen(true)} title="New mission" style={{ width: 36, height: 36, borderRadius: 9, border: "1px dashed rgba(255,111,97,.4)", background: "rgba(255,111,97,.06)", display: "flex", alignItems: "center", justifyContent: "center", color: "#ff6f61", fontSize: 19, fontWeight: 300, cursor: "pointer" }}>+</button>
|
||||||
</div>
|
</div>
|
||||||
<div style={{ flex: 1 }} />
|
<div style={{ flex: 1 }} />
|
||||||
</div>
|
</div>
|
||||||
@@ -676,7 +925,10 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
{isWorld ? (
|
{isWorld ? (
|
||||||
<div style={{ padding: "16px 16px 12px", borderBottom: "1px solid rgba(255,255,255,.06)", display: "flex", alignItems: "flex-start", gap: 8 }}>
|
<div style={{ padding: "16px 16px 12px", borderBottom: "1px solid rgba(255,255,255,.06)", display: "flex", alignItems: "flex-start", gap: 8 }}>
|
||||||
<div style={{ flex: 1, minWidth: 0 }}>
|
<div style={{ flex: 1, minWidth: 0 }}>
|
||||||
<div style={{ fontFamily: mono, fontSize: 10, letterSpacing: ".12em", color: "#5a5a62", marginBottom: 6 }}>{orgs.length} ORG{orgs.length === 1 ? "" : "S"} · {allAgents.length} AGENTS</div>
|
{/* Counts the things this sidebar actually lists. It used to
|
||||||
|
read "N ORGS", which described the org→company→team forest
|
||||||
|
this tier no longer shows. */}
|
||||||
|
<div style={{ fontFamily: mono, fontSize: 10, letterSpacing: ".12em", color: "#5a5a62", marginBottom: 6 }}>{(workforce?.missions ?? []).length} MISSION{(workforce?.missions ?? []).length === 1 ? "" : "S"} · {distinctAgentCount} AGENT{distinctAgentCount === 1 ? "" : "S"}</div>
|
||||||
<div style={{ fontSize: 18, fontWeight: 700, color: "#f3f3f5", letterSpacing: "-.01em" }}>Visualizations</div>
|
<div style={{ fontSize: 18, fontWeight: 700, color: "#f3f3f5", letterSpacing: "-.01em" }}>Visualizations</div>
|
||||||
</div>
|
</div>
|
||||||
<button type="button" onClick={() => { setSelectMode((v) => { if (v) setSelectedAgents(new Set()); return !v; }); }} title="Select to manage" aria-label="Select to manage" style={{ flex: "none", width: 34, height: 34, borderRadius: 9, border: `1px solid ${selectMode ? "rgba(255,111,97,.5)" : "rgba(255,255,255,.12)"}`, background: selectMode ? "rgba(255,111,97,.12)" : "transparent", color: selectMode ? "#ff6f61" : "#9a9aa2", cursor: "pointer", display: "inline-flex", alignItems: "center", justifyContent: "center" }}><Wrench aria-hidden size={16} /></button>
|
<button type="button" onClick={() => { setSelectMode((v) => { if (v) setSelectedAgents(new Set()); return !v; }); }} title="Select to manage" aria-label="Select to manage" style={{ flex: "none", width: 34, height: 34, borderRadius: 9, border: `1px solid ${selectMode ? "rgba(255,111,97,.5)" : "rgba(255,255,255,.12)"}`, background: selectMode ? "rgba(255,111,97,.12)" : "transparent", color: selectMode ? "#ff6f61" : "#9a9aa2", cursor: "pointer", display: "inline-flex", alignItems: "center", justifyContent: "center" }}><Wrench aria-hidden size={16} /></button>
|
||||||
@@ -750,7 +1002,7 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
<div style={{ flex: 1, minHeight: 0, display: "flex", flexDirection: "column", alignItems: "center", justifyContent: "center", padding: "24px 20px", gap: 10, textAlign: "center" }}>
|
<div style={{ flex: 1, minHeight: 0, display: "flex", flexDirection: "column", alignItems: "center", justifyContent: "center", padding: "24px 20px", gap: 10, textAlign: "center" }}>
|
||||||
<span style={{ fontFamily: mono, fontSize: 10.5, letterSpacing: ".14em", color: "#5a5a62" }}>NOTHING TO SHOW</span>
|
<span style={{ fontFamily: mono, fontSize: 10.5, letterSpacing: ".14em", color: "#5a5a62" }}>NOTHING TO SHOW</span>
|
||||||
<span style={{ fontSize: 13, color: "#cfcfd5", lineHeight: 1.55 }}>Your workforce is empty.</span>
|
<span style={{ fontSize: 13, color: "#cfcfd5", lineHeight: 1.55 }}>Your workforce is empty.</span>
|
||||||
<span style={{ fontFamily: mono, fontSize: 11, color: "#8a8a92", lineHeight: 1.6, maxWidth: 200 }}>Hit the <span style={{ color: "#ff8a7a" }}>+</span> on the rail (or in the canvas) to add a new agent, team, company, or organization.</span>
|
<span style={{ fontFamily: mono, fontSize: 11, color: "#8a8a92", lineHeight: 1.6, maxWidth: 200 }}>Hit the <span style={{ color: "#ff8a7a" }}>+</span> to start a mission — the agents it needs join your workforce.</span>
|
||||||
</div>
|
</div>
|
||||||
) : (
|
) : (
|
||||||
<>
|
<>
|
||||||
@@ -768,12 +1020,20 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
// "ungrouped-co", "ungrouped-team") aren't real DB rows — the
|
// "ungrouped-co", "ungrouped-team") aren't real DB rows — the
|
||||||
// backend would 422 on the non-UUID id. Silently ignore taps.
|
// backend would 422 on the non-UUID id. Silently ignore taps.
|
||||||
if (SYNTHETIC_TREE_IDS.has(id)) return;
|
if (SYNTHETIC_TREE_IDS.has(id)) return;
|
||||||
|
// Mission groups are headings, not rows. Their id carries a
|
||||||
|
// MISSION uuid, so a reap would target the wrong table with a
|
||||||
|
// perfectly valid-looking id.
|
||||||
|
if (isGroupingRow(id)) return;
|
||||||
setSelectedAgents((prev) => { const next = new Set(prev); if (next.has(id)) { next.delete(id); return next; } const lv = nodeLevel.get(id); const curLv = prev.size ? nodeLevel.get([...prev][0]) : lv; if (lv !== curLv) return new Set([id]); next.add(id); return next; });
|
setSelectedAgents((prev) => { const next = new Set(prev); if (next.has(id)) { next.delete(id); return next; } const lv = nodeLevel.get(id); const curLv = prev.size ? nodeLevel.get([...prev][0]) : lv; if (lv !== curLv) return new Set([id]); next.add(id); return next; });
|
||||||
}}
|
}}
|
||||||
// Only real (non-synthetic, non-claw) nodes accept an inline
|
// Only real (non-synthetic, non-claw) nodes accept an inline
|
||||||
// rename. Synthetic scaffolding gets swapped for real rows in the
|
// rename. Synthetic scaffolding gets swapped for real rows in the
|
||||||
// next commit (wizard auto-materialize + migration dialog).
|
// next commit (wizard auto-materialize + migration dialog).
|
||||||
canRename={(it) => it.level !== "claw" && !SYNTHETIC_TREE_IDS.has(it.id)}
|
// Mission groups render at team level but are NOT teams — renaming
|
||||||
|
// one would PATCH /api/teams/<missionId>/name: a well-formed uuid
|
||||||
|
// pointing at the wrong table, which fails as a silent no-op rather
|
||||||
|
// than an error.
|
||||||
|
canRename={(it) => it.level !== "claw" && !SYNTHETIC_TREE_IDS.has(it.id) && !isGroupingRow(it.id)}
|
||||||
onRename={async (id, level, newLabel) => {
|
onRename={async (id, level, newLabel) => {
|
||||||
const path = level === "org" ? "orgs" : level === "company" ? "companies" : level === "team" ? "teams" : null;
|
const path = level === "org" ? "orgs" : level === "company" ? "companies" : level === "team" ? "teams" : null;
|
||||||
if (!path) return;
|
if (!path) return;
|
||||||
@@ -832,7 +1092,18 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
<>
|
<>
|
||||||
{/* Graph stage — condensed by the right slide-out's width. */}
|
{/* Graph stage — condensed by the right slide-out's width. */}
|
||||||
<div style={{ position: "absolute", top: 0, bottom: 0, left: 0, right: "var(--world-width, 0px)", background: "radial-gradient(120% 90% at 55% 38%, #0e0e13 0%, #08080a 70%)", transition: "right var(--duration-normal) var(--ease-app)" }}>
|
<div style={{ position: "absolute", top: 0, bottom: 0, left: 0, right: "var(--world-width, 0px)", background: "radial-gradient(120% 90% at 55% 38%, #0e0e13 0%, #08080a 70%)", transition: "right var(--duration-normal) var(--ease-app)" }}>
|
||||||
<WorldCanvas roots={worldCanvasRoots} expanded={expanded} onToggleExpand={toggleExpand} selectedId={worldSel} onSelect={onWorldSelect} onOpenRuns={() => setRunsOpen(true)} />
|
<WorldCanvas
|
||||||
|
roots={worldSeedRoots}
|
||||||
|
expanded={expanded}
|
||||||
|
onToggleExpand={toggleExpand}
|
||||||
|
selectedId={worldSel}
|
||||||
|
onSelect={onWorldSelect}
|
||||||
|
onOpenRuns={() => setRunsOpen(true)}
|
||||||
|
focusAgents={focusAgentIds}
|
||||||
|
focusMissionId={focusedMissionId}
|
||||||
|
focusLabel={focusedMission?.title ?? null}
|
||||||
|
templateKind={focusedMissionTemplate}
|
||||||
|
/>
|
||||||
{/* Open the slide-out (top-right) when it's closed. */}
|
{/* Open the slide-out (top-right) when it's closed. */}
|
||||||
{!worldPanelOpen ? (
|
{!worldPanelOpen ? (
|
||||||
<button type="button" aria-label="Open panel" title="Panel" onClick={() => setWorldPanelOpen(true)} style={{ position: "absolute", top: 14, right: 16, zIndex: 50, width: 38, height: 38, borderRadius: "50%", border: "1px solid rgba(255,111,97,.4)", background: "rgba(255,111,97,.08)", color: "#ff6f61", cursor: "pointer", display: "flex", alignItems: "center", justifyContent: "center" }}><PanelRight aria-hidden size={19} /></button>
|
<button type="button" aria-label="Open panel" title="Panel" onClick={() => setWorldPanelOpen(true)} style={{ position: "absolute", top: 14, right: 16, zIndex: 50, width: 38, height: 38, borderRadius: "50%", border: "1px solid rgba(255,111,97,.4)", background: "rgba(255,111,97,.08)", color: "#ff6f61", cursor: "pointer", display: "flex", alignItems: "center", justifyContent: "center" }}><PanelRight aria-hidden size={19} /></button>
|
||||||
@@ -849,8 +1120,12 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
<button type="button" aria-label="Close panel" onClick={() => setWorldPanelOpen(false)} style={{ width: 30, height: 30, flex: "none", borderRadius: 8, border: "1px solid rgba(255,255,255,.12)", background: "transparent", color: "#cfcfd5", cursor: "pointer", display: "flex", alignItems: "center", justifyContent: "center" }}><X aria-hidden size={16} /></button>
|
<button type="button" aria-label="Close panel" onClick={() => setWorldPanelOpen(false)} style={{ width: 30, height: 30, flex: "none", borderRadius: 8, border: "1px solid rgba(255,255,255,.12)", background: "transparent", color: "#cfcfd5", cursor: "pointer", display: "flex", alignItems: "center", justifyContent: "center" }}><X aria-hidden size={16} /></button>
|
||||||
</div>
|
</div>
|
||||||
{(() => {
|
{(() => {
|
||||||
const loc = locateAgent(worldSel);
|
// `worldSel` may be a mission-group id or a composite
|
||||||
if (!loc) return <ObservePanel focusAgent={worldSel} onClearFocus={() => setWorldSel(null)} />;
|
// `<missionId>:<clawId>`; the panel wants a claw id, and a
|
||||||
|
// grouping row has none.
|
||||||
|
const selClawId = worldSel && !isGroupingRow(worldSel) ? clawIdOf(worldSel) : null;
|
||||||
|
const loc = locateAgent(selClawId);
|
||||||
|
if (!loc) return <ObservePanel focusAgent={selClawId} onClearFocus={() => setWorldSel(null)} />;
|
||||||
const { org: o, company: co, team: t, agent: a } = loc;
|
const { org: o, company: co, team: t, agent: a } = loc;
|
||||||
const statusCol = a.status === "running" ? "#5ec8d8" : a.status === "online" ? "#5fd08a" : "#5a5a62";
|
const statusCol = a.status === "running" ? "#5ec8d8" : a.status === "online" ? "#5fd08a" : "#5a5a62";
|
||||||
const chip = (text: string, col = "#9a9aa2") => <span style={{ fontFamily: mono, fontSize: 10, color: col, padding: "3px 9px", borderRadius: 999, background: `${col}1a`, border: `1px solid ${col}33` }}>{text}</span>;
|
const chip = (text: string, col = "#9a9aa2") => <span style={{ fontFamily: mono, fontSize: 10, color: col, padding: "3px 9px", borderRadius: 999, background: `${col}1a`, border: `1px solid ${col}33` }}>{text}</span>;
|
||||||
@@ -1004,7 +1279,10 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
</div>
|
</div>
|
||||||
</>
|
</>
|
||||||
) : (
|
) : (
|
||||||
<EmptyRosterStage onAdd={() => setDeployOpen(true)} />
|
<EmptyRosterStage
|
||||||
|
onAdd={() => setMissionWizardOpen(true)}
|
||||||
|
onCreateAgent={() => setDeployOpen(true)}
|
||||||
|
/>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -1014,6 +1292,18 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
{/* MASTER PLANNER — the "+" opens a chat with Opus 4.8 that proposes and
|
{/* MASTER PLANNER — the "+" opens a chat with Opus 4.8 that proposes and
|
||||||
scaffolds a whole team of agents. */}
|
scaffolds a whole team of agents. */}
|
||||||
{deployOpen ? <MasterPlannerModal onClose={() => setDeployOpen(false)} /> : null}
|
{deployOpen ? <MasterPlannerModal onClose={() => setDeployOpen(false)} /> : null}
|
||||||
|
|
||||||
|
{/* The mission wizard, behind every "+" on the dashboard. */}
|
||||||
|
{missionWizardOpen ? (
|
||||||
|
<MissionWizard
|
||||||
|
onClose={() => setMissionWizardOpen(false)}
|
||||||
|
onCreated={() => {
|
||||||
|
setMissionWizardOpen(false);
|
||||||
|
setTier("missions");
|
||||||
|
router.refresh();
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
) : null}
|
||||||
{orphanDialogOpen ? (
|
{orphanDialogOpen ? (
|
||||||
<OrphanMigrationDialog
|
<OrphanMigrationDialog
|
||||||
onClose={() => setOrphanDialogOpen(false)}
|
onClose={() => setOrphanDialogOpen(false)}
|
||||||
@@ -1085,23 +1375,33 @@ export function Dashboard({ user, orgs, claws }: { user?: { display_name?: strin
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Canvas empty state for a workspace with no orgs/companies/teams/agents.
|
/** Canvas empty state for a workspace with no agents yet.
|
||||||
* The + button opens the same MasterPlannerModal the rail's + does. */
|
*
|
||||||
function EmptyRosterStage({ onAdd }: { onAdd: () => void }) {
|
* The primary action starts a MISSION, because that is how a workforce comes
|
||||||
|
* to exist — a mission mints the agents it needs and they stay. Hand-staffing
|
||||||
|
* one is the advanced path and gets a quieter secondary link. */
|
||||||
|
function EmptyRosterStage({
|
||||||
|
onAdd,
|
||||||
|
onCreateAgent,
|
||||||
|
}: {
|
||||||
|
onAdd: () => void;
|
||||||
|
onCreateAgent: () => void;
|
||||||
|
}) {
|
||||||
return (
|
return (
|
||||||
<div style={{ position: "absolute", inset: 0, display: "flex", flexDirection: "column", alignItems: "center", justifyContent: "center", gap: 22, background: "radial-gradient(120% 90% at 50% 42%, #100e13 0%, #08080a 70%)" }}>
|
<div style={{ position: "absolute", inset: 0, display: "flex", flexDirection: "column", alignItems: "center", justifyContent: "center", gap: 22, background: "radial-gradient(120% 90% at 50% 42%, #100e13 0%, #08080a 70%)" }}>
|
||||||
<div style={{ fontFamily: mono, fontSize: 11, letterSpacing: ".18em", color: "#5a5a62" }}>YOUR WORKFORCE IS EMPTY</div>
|
<div style={{ fontFamily: mono, fontSize: 11, letterSpacing: ".18em", color: "#5a5a62" }}>YOUR WORKFORCE IS EMPTY</div>
|
||||||
<div style={{ fontSize: 22, fontWeight: 700, color: "#f3f3f5", letterSpacing: "-.015em", maxWidth: 480, textAlign: "center", lineHeight: 1.25 }}>
|
<div style={{ fontSize: 22, fontWeight: 700, color: "#f3f3f5", letterSpacing: "-.015em", maxWidth: 480, textAlign: "center", lineHeight: 1.25 }}>
|
||||||
Add a new agent, team, company, or organization
|
Create your agent workforce
|
||||||
</div>
|
</div>
|
||||||
<div style={{ fontFamily: mono, fontSize: 12, color: "#8a8a92", maxWidth: 460, textAlign: "center", lineHeight: 1.6 }}>
|
<div style={{ fontFamily: mono, fontSize: 12, color: "#8a8a92", maxWidth: 460, textAlign: "center", lineHeight: 1.6 }}>
|
||||||
The <span style={{ color: "#ff8a7a" }}>+</span> below opens the deploy wizard — pitch it a team spec and it scaffolds a whole workforce in one pass.
|
Start a mission and the agents it needs are hired for it — they stay in
|
||||||
|
your workforce afterwards.
|
||||||
</div>
|
</div>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
onClick={onAdd}
|
onClick={onAdd}
|
||||||
aria-label="Deploy a new agent"
|
aria-label="Start a mission"
|
||||||
title="Deploy — opens the same wizard as the rail's +"
|
title="Start a mission"
|
||||||
style={{
|
style={{
|
||||||
width: 74,
|
width: 74,
|
||||||
height: 74,
|
height: 74,
|
||||||
@@ -1123,6 +1423,25 @@ function EmptyRosterStage({ onAdd }: { onAdd: () => void }) {
|
|||||||
>
|
>
|
||||||
+
|
+
|
||||||
</button>
|
</button>
|
||||||
|
{/* The old primary action, demoted rather than removed: staffing and
|
||||||
|
upskilling a workforce by hand is a real thing to want, just not the
|
||||||
|
first thing. */}
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={onCreateAgent}
|
||||||
|
style={{
|
||||||
|
background: "transparent",
|
||||||
|
border: 0,
|
||||||
|
color: "#8a8a92",
|
||||||
|
fontFamily: mono,
|
||||||
|
fontSize: 11.5,
|
||||||
|
cursor: "pointer",
|
||||||
|
textDecoration: "underline",
|
||||||
|
textUnderlineOffset: 3,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
or create an agent yourself
|
||||||
|
</button>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,65 @@
|
|||||||
|
import { render, screen } from "@testing-library/react";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
|
import { MarkdownBlock, outlineOf } from "./MarkdownBlock";
|
||||||
|
|
||||||
|
describe("MarkdownBlock", () => {
|
||||||
|
/**
|
||||||
|
* The reason this renderer was replaced. Agent research briefs are largely GFM
|
||||||
|
* pipe tables; the previous zero-dep renderer had no table support and showed
|
||||||
|
* them as literal pipe characters — the least readable form of the most
|
||||||
|
* structured content in the document.
|
||||||
|
*/
|
||||||
|
it("renders GFM tables as tables, not as pipe characters", () => {
|
||||||
|
render(
|
||||||
|
<MarkdownBlock
|
||||||
|
source={[
|
||||||
|
"| Section | What it adds |",
|
||||||
|
"| --- | --- |",
|
||||||
|
"| SIMD | Fletcher32 |",
|
||||||
|
].join("\n")}
|
||||||
|
/>,
|
||||||
|
);
|
||||||
|
expect(screen.getByRole("table")).toBeInTheDocument();
|
||||||
|
expect(screen.getByRole("columnheader", { name: "Section" })).toBeInTheDocument();
|
||||||
|
expect(screen.getByRole("cell", { name: "Fletcher32" })).toBeInTheDocument();
|
||||||
|
expect(document.body.textContent).not.toContain("| Section |");
|
||||||
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The outline rail scrolls to `#id`. If `outlineOf` and the rendered headings
|
||||||
|
* disagree by even one, every link below the divergence lands on the wrong
|
||||||
|
* section — and it fails silently, because scrolling to a missing anchor just
|
||||||
|
* does nothing.
|
||||||
|
*/
|
||||||
|
it("gives the outline the same ids as the rendered headings", () => {
|
||||||
|
const md = [
|
||||||
|
"# Overview",
|
||||||
|
"text",
|
||||||
|
"## Details",
|
||||||
|
"more",
|
||||||
|
"## Details",
|
||||||
|
"duplicate heading on purpose",
|
||||||
|
].join("\n");
|
||||||
|
|
||||||
|
const { container } = render(<MarkdownBlock source={md} />);
|
||||||
|
const renderedIds = Array.from(container.querySelectorAll("h1, h2, h3")).map(
|
||||||
|
(h) => h.id,
|
||||||
|
);
|
||||||
|
expect(outlineOf(md).map((h) => h.id)).toEqual(renderedIds);
|
||||||
|
// And duplicates must be distinguished, or both links go to the first one.
|
||||||
|
expect(new Set(renderedIds).size).toBe(renderedIds.length);
|
||||||
|
});
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A `# comment` inside a fenced block is code. Counting it would put shell
|
||||||
|
* comments in the outline AND shift every later duplicate suffix, breaking the
|
||||||
|
* ids the test above pins.
|
||||||
|
*/
|
||||||
|
it("does not treat comments inside code fences as headings", () => {
|
||||||
|
const md = ["# Real", "```sh", "# not a heading", "```", "## Also real"].join(
|
||||||
|
"\n",
|
||||||
|
);
|
||||||
|
expect(outlineOf(md).map((h) => h.text)).toEqual(["Real", "Also real"]);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,313 +1,90 @@
|
|||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
// MarkdownBlock — tiny zero-dep Markdown renderer. Handles the subset
|
// MarkdownBlock — the mission reading surface's markdown renderer.
|
||||||
// the refiner emits: h1/h2/h3 headings, - / * bullets, 1. numbered
|
//
|
||||||
// lists, `**bold**`, `` `code` ``, blank-line-separated paragraphs.
|
// This was a deliberately tiny zero-dep renderer covering "the subset the
|
||||||
// Not a general-purpose renderer — deliberately small to avoid a
|
// refiner emits": h1-h3, bullets, `**bold**`, `` `code` ``, fenced blocks. That
|
||||||
// react-markdown dep for one canvas surface.
|
// subset stopped matching reality. Agent research output is full of GFM PIPE
|
||||||
|
// TABLES — the ClawHDF5 briefs were largely tables ("| Section | What it adds |")
|
||||||
|
// — and a renderer without table support shows them as literal pipe characters,
|
||||||
|
// which is the least readable possible form of the most structured content.
|
||||||
|
//
|
||||||
|
// It now delegates to react-markdown + remark-gfm: tables, task lists,
|
||||||
|
// strikethrough and autolinks, plus correct handling of the nesting cases (a
|
||||||
|
// list inside a blockquote, emphasis inside a heading) that a line-oriented
|
||||||
|
// parser gets wrong.
|
||||||
|
//
|
||||||
|
// `outlineOf` keeps its signature — MissionOutputReader's outline rail depends
|
||||||
|
// on it. `headingId` is gone: ids now come from rehype-slug, because the outline
|
||||||
|
// and the rendered headings MUST agree and hand-counting ordinals across two
|
||||||
|
// code paths is exactly how they drift.
|
||||||
|
//
|
||||||
|
// Visual styling lives in globals.css under `.md-view`: this markup is
|
||||||
|
// generated, so there are no class hooks to target, and the project has no
|
||||||
|
// styled-jsx registry (next/dist/docs/01-app/02-guides/css-in-js.md).
|
||||||
|
|
||||||
import React from "react";
|
import GithubSlugger from "github-slugger";
|
||||||
|
import ReactMarkdown from "react-markdown";
|
||||||
|
import rehypeSlug from "rehype-slug";
|
||||||
|
import remarkGfm from "remark-gfm";
|
||||||
|
|
||||||
const mono =
|
/**
|
||||||
"ui-monospace, SFMono-Regular, SF Mono, Menlo, Monaco, Consolas, monospace";
|
* The h1-h3 headings of a document, in order, for the outline rail.
|
||||||
|
*
|
||||||
type Block =
|
* Slugged with the same GithubSlugger that rehype-slug uses on the rendered
|
||||||
| { kind: "h1" | "h2" | "h3"; text: string; id?: string }
|
* headings, so `#h-id` anchors resolve. Two details keep the two in step:
|
||||||
| { kind: "p"; text: string }
|
*
|
||||||
| { kind: "ul"; items: string[] }
|
* - **Fence-aware.** A `# comment` inside a ```sh block is code, not a
|
||||||
| { kind: "ol"; items: string[] }
|
* heading. rehype-slug never sees it because it is a code node, so counting
|
||||||
| { kind: "code"; lang: string; text: string };
|
* it here would desynchronise every following duplicate suffix.
|
||||||
|
* - **All levels feed the slugger, only h1-h3 are returned.** GithubSlugger's
|
||||||
/** Stable slug for a heading, so the reader's outline can scroll to it. */
|
* duplicate counter is per document ("intro", "intro-1"); if an h4 shares a
|
||||||
export function headingId(text: string, ordinal: number): string {
|
* title with an h3 and only one side counted it, the suffixes would diverge.
|
||||||
const slug = text
|
*/
|
||||||
.toLowerCase()
|
|
||||||
.replace(/[^a-z0-9]+/g, "-")
|
|
||||||
.replace(/^-+|-+$/g, "")
|
|
||||||
.slice(0, 60);
|
|
||||||
return `h-${ordinal}-${slug || "section"}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** The headings of a document, for an outline rail. */
|
|
||||||
export function outlineOf(
|
export function outlineOf(
|
||||||
md: string,
|
md: string,
|
||||||
): Array<{ id: string; text: string; level: 1 | 2 | 3 }> {
|
): Array<{ id: string; text: string; level: 1 | 2 | 3 }> {
|
||||||
return parse(md).flatMap((b, idx) =>
|
const slugger = new GithubSlugger();
|
||||||
b.kind === "h1" || b.kind === "h2" || b.kind === "h3"
|
const out: Array<{ id: string; text: string; level: 1 | 2 | 3 }> = [];
|
||||||
? [
|
let inFence = false;
|
||||||
{
|
for (const raw of md.replace(/\r\n/g, "\n").split("\n")) {
|
||||||
id: b.id ?? headingId(b.text, idx),
|
const line = raw.trim();
|
||||||
text: b.text,
|
if (/^```/.test(line)) {
|
||||||
level: Number(b.kind.slice(1)) as 1 | 2 | 3,
|
inFence = !inFence;
|
||||||
},
|
|
||||||
]
|
|
||||||
: [],
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
function parse(md: string): Block[] {
|
|
||||||
const lines = md.replace(/\r\n/g, "\n").split("\n");
|
|
||||||
const blocks: Block[] = [];
|
|
||||||
let i = 0;
|
|
||||||
while (i < lines.length) {
|
|
||||||
const line = lines[i];
|
|
||||||
const trimmed = line.trim();
|
|
||||||
if (!trimmed) {
|
|
||||||
i++;
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// Fenced code block. Agent output is full of ```rust / ```toml
|
if (inFence) continue;
|
||||||
// blocks; without this they render as mangled paragraphs.
|
const h = /^(#{1,6})\s+(.+)$/.exec(line);
|
||||||
const fence = /^```([A-Za-z0-9_+-]*)\s*$/.exec(trimmed);
|
if (!h) continue;
|
||||||
if (fence) {
|
// Strip inline markup so the rail shows "Crate graph", not "**Crate graph**"
|
||||||
const lang = fence[1] ?? "";
|
// — and so the slug matches what rehype-slug computes from rendered text.
|
||||||
const body: string[] = [];
|
const text = h[2].replace(/[*_`]/g, "").trim();
|
||||||
i++;
|
const id = slugger.slug(text);
|
||||||
while (i < lines.length && !/^```\s*$/.test(lines[i].trim())) {
|
const level = h[1].length;
|
||||||
body.push(lines[i]);
|
if (level <= 3) out.push({ id, text, level: level as 1 | 2 | 3 });
|
||||||
i++;
|
|
||||||
}
|
|
||||||
i++; // consume the closing fence (or run off the end on an unclosed block)
|
|
||||||
blocks.push({ kind: "code", lang, text: body.join("\n") });
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// Headings
|
|
||||||
const h = /^(#{1,6})\s+(.*)$/.exec(trimmed);
|
|
||||||
if (h) {
|
|
||||||
// h4-h6 are rare in agent output; render them as h3 rather than
|
|
||||||
// dropping the text into a paragraph.
|
|
||||||
const level = Math.min(h[1].length, 3) as 1 | 2 | 3;
|
|
||||||
const text = h[2];
|
|
||||||
blocks.push({
|
|
||||||
kind: (`h${level}` as "h1" | "h2" | "h3"),
|
|
||||||
text,
|
|
||||||
id: headingId(text, blocks.length),
|
|
||||||
});
|
|
||||||
i++;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// Bullet list
|
|
||||||
if (/^[-*]\s+/.test(trimmed)) {
|
|
||||||
const items: string[] = [];
|
|
||||||
while (i < lines.length && /^[-*]\s+/.test(lines[i].trim())) {
|
|
||||||
items.push(lines[i].trim().replace(/^[-*]\s+/, ""));
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
blocks.push({ kind: "ul", items });
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// Numbered list
|
|
||||||
if (/^\d+\.\s+/.test(trimmed)) {
|
|
||||||
const items: string[] = [];
|
|
||||||
while (i < lines.length && /^\d+\.\s+/.test(lines[i].trim())) {
|
|
||||||
items.push(lines[i].trim().replace(/^\d+\.\s+/, ""));
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
blocks.push({ kind: "ol", items });
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
// Paragraph — greedily accumulate until blank line or block boundary
|
|
||||||
const paraLines: string[] = [];
|
|
||||||
while (
|
|
||||||
i < lines.length &&
|
|
||||||
lines[i].trim() &&
|
|
||||||
!/^(#{1,6})\s+/.test(lines[i].trim()) &&
|
|
||||||
!/^```/.test(lines[i].trim()) &&
|
|
||||||
!/^[-*]\s+/.test(lines[i].trim()) &&
|
|
||||||
!/^\d+\.\s+/.test(lines[i].trim())
|
|
||||||
) {
|
|
||||||
paraLines.push(lines[i].trim());
|
|
||||||
i++;
|
|
||||||
}
|
|
||||||
if (paraLines.length) blocks.push({ kind: "p", text: paraLines.join(" ") });
|
|
||||||
}
|
}
|
||||||
return blocks;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Inline: **bold**, `code`. Simple sequential scan.
|
|
||||||
function renderInline(text: string): React.ReactNode[] {
|
|
||||||
const out: React.ReactNode[] = [];
|
|
||||||
const re = /(\*\*[^*]+\*\*|`[^`]+`)/g;
|
|
||||||
let last = 0;
|
|
||||||
let m: RegExpExecArray | null;
|
|
||||||
let key = 0;
|
|
||||||
while ((m = re.exec(text)) !== null) {
|
|
||||||
if (m.index > last) out.push(text.slice(last, m.index));
|
|
||||||
const tok = m[0];
|
|
||||||
if (tok.startsWith("**")) {
|
|
||||||
out.push(
|
|
||||||
<strong key={key++} style={{ color: "#f3f3f5" }}>
|
|
||||||
{tok.slice(2, -2)}
|
|
||||||
</strong>,
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
out.push(
|
|
||||||
<code
|
|
||||||
key={key++}
|
|
||||||
style={{
|
|
||||||
fontFamily: mono,
|
|
||||||
fontSize: 11.5,
|
|
||||||
padding: "1px 5px",
|
|
||||||
borderRadius: 4,
|
|
||||||
background: "rgba(255,255,255,.06)",
|
|
||||||
color: "#ffb44a",
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
{tok.slice(1, -1)}
|
|
||||||
</code>,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
last = m.index + tok.length;
|
|
||||||
}
|
|
||||||
if (last < text.length) out.push(text.slice(last));
|
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function MarkdownBlock({ source }: { source: string }) {
|
export function MarkdownBlock({ source }: { source: string }) {
|
||||||
const blocks = React.useMemo(() => parse(source), [source]);
|
|
||||||
return (
|
return (
|
||||||
<div
|
<div className="md-view">
|
||||||
style={{
|
<ReactMarkdown
|
||||||
display: "flex",
|
remarkPlugins={[remarkGfm]}
|
||||||
flexDirection: "column",
|
rehypePlugins={[rehypeSlug]}
|
||||||
gap: 10,
|
components={{
|
||||||
color: "#cfcfd5",
|
// Agent briefs cite sources. Opening in-tab would lose the mission
|
||||||
fontSize: 13,
|
// view; `noopener` because a bare `target=_blank` hands the opened
|
||||||
lineHeight: 1.55,
|
// page a handle back to this one.
|
||||||
}}
|
a: ({ href, children }) => (
|
||||||
>
|
<a href={href} target="_blank" rel="noreferrer noopener">
|
||||||
{blocks.map((b, idx) => {
|
{children}
|
||||||
if (b.kind === "h1")
|
</a>
|
||||||
return (
|
),
|
||||||
<h1
|
}}
|
||||||
key={idx}
|
>
|
||||||
id={b.id}
|
{source}
|
||||||
style={{
|
</ReactMarkdown>
|
||||||
margin: "8px 0 2px",
|
|
||||||
fontSize: 17,
|
|
||||||
color: "#f3f3f5",
|
|
||||||
fontWeight: 600,
|
|
||||||
letterSpacing: ".01em",
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
{renderInline(b.text)}
|
|
||||||
</h1>
|
|
||||||
);
|
|
||||||
if (b.kind === "h2")
|
|
||||||
return (
|
|
||||||
<h2
|
|
||||||
key={idx}
|
|
||||||
id={b.id}
|
|
||||||
style={{
|
|
||||||
margin: "10px 0 -2px",
|
|
||||||
fontSize: 12,
|
|
||||||
color: "#7cd6e0",
|
|
||||||
fontFamily: mono,
|
|
||||||
letterSpacing: ".14em",
|
|
||||||
textTransform: "uppercase",
|
|
||||||
fontWeight: 600,
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
{renderInline(b.text)}
|
|
||||||
</h2>
|
|
||||||
);
|
|
||||||
if (b.kind === "h3")
|
|
||||||
return (
|
|
||||||
<h3
|
|
||||||
key={idx}
|
|
||||||
id={b.id}
|
|
||||||
style={{
|
|
||||||
margin: "6px 0 -4px",
|
|
||||||
fontSize: 11.5,
|
|
||||||
color: "#a0a0a8",
|
|
||||||
fontFamily: mono,
|
|
||||||
letterSpacing: ".10em",
|
|
||||||
textTransform: "uppercase",
|
|
||||||
fontWeight: 500,
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
{renderInline(b.text)}
|
|
||||||
</h3>
|
|
||||||
);
|
|
||||||
if (b.kind === "code")
|
|
||||||
return (
|
|
||||||
<pre
|
|
||||||
key={idx}
|
|
||||||
style={{
|
|
||||||
margin: 0,
|
|
||||||
padding: "10px 12px",
|
|
||||||
borderRadius: 8,
|
|
||||||
border: "1px solid rgba(255,255,255,.07)",
|
|
||||||
background: "rgba(0,0,0,.45)",
|
|
||||||
color: "#e0e0e5",
|
|
||||||
fontFamily: mono,
|
|
||||||
fontSize: 11.5,
|
|
||||||
lineHeight: 1.5,
|
|
||||||
// Code is the one thing that may scroll sideways; the
|
|
||||||
// page itself must never scroll horizontally.
|
|
||||||
overflowX: "auto",
|
|
||||||
whiteSpace: "pre",
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
{b.lang && (
|
|
||||||
<span
|
|
||||||
style={{
|
|
||||||
display: "block",
|
|
||||||
marginBottom: 6,
|
|
||||||
fontSize: 9.5,
|
|
||||||
letterSpacing: ".12em",
|
|
||||||
textTransform: "uppercase",
|
|
||||||
color: "#6a6a72",
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
{b.lang}
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
<code>{b.text}</code>
|
|
||||||
</pre>
|
|
||||||
);
|
|
||||||
if (b.kind === "p")
|
|
||||||
return (
|
|
||||||
<p key={idx} style={{ margin: 0 }}>
|
|
||||||
{renderInline(b.text)}
|
|
||||||
</p>
|
|
||||||
);
|
|
||||||
if (b.kind === "ul")
|
|
||||||
return (
|
|
||||||
<ul
|
|
||||||
key={idx}
|
|
||||||
style={{
|
|
||||||
margin: 0,
|
|
||||||
paddingLeft: 18,
|
|
||||||
display: "flex",
|
|
||||||
flexDirection: "column",
|
|
||||||
gap: 4,
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
{b.items.map((it, i) => (
|
|
||||||
<li key={i}>{renderInline(it)}</li>
|
|
||||||
))}
|
|
||||||
</ul>
|
|
||||||
);
|
|
||||||
if (b.kind === "ol")
|
|
||||||
return (
|
|
||||||
<ol
|
|
||||||
key={idx}
|
|
||||||
style={{
|
|
||||||
margin: 0,
|
|
||||||
paddingLeft: 20,
|
|
||||||
display: "flex",
|
|
||||||
flexDirection: "column",
|
|
||||||
gap: 4,
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
{b.items.map((it, i) => (
|
|
||||||
<li key={i}>{renderInline(it)}</li>
|
|
||||||
))}
|
|
||||||
</ol>
|
|
||||||
);
|
|
||||||
return null;
|
|
||||||
})}
|
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,232 @@
|
|||||||
|
"use client";
|
||||||
|
|
||||||
|
// MissionArtifacts — the mission's captured FILES, and a reader for them.
|
||||||
|
//
|
||||||
|
// Distinct from MissionOutputReader, which shows agent turn output: that is the
|
||||||
|
// agent's ACCOUNT of the work, this is the work. For a repo-less research
|
||||||
|
// mission these artifacts are the only durable result — see
|
||||||
|
// `cm-api/src/mission_outputs.rs`.
|
||||||
|
//
|
||||||
|
// Bodies are fetched on demand. The mission detail carries paths only, and a
|
||||||
|
// research phase can leave dozens of documents.
|
||||||
|
|
||||||
|
import { useCallback, useState } from "react";
|
||||||
|
import { Download, FileText, GitMerge } from "lucide-react";
|
||||||
|
|
||||||
|
import {
|
||||||
|
getArtifactContent,
|
||||||
|
mergeMissionBranch,
|
||||||
|
type ArtifactContent,
|
||||||
|
type MergeResult,
|
||||||
|
type MissionDetail,
|
||||||
|
} from "@/lib/api/missions";
|
||||||
|
import { MarkdownBlock } from "./MarkdownBlock";
|
||||||
|
|
||||||
|
const mono =
|
||||||
|
"ui-monospace, SFMono-Regular, SF Mono, Menlo, Monaco, Consolas, monospace";
|
||||||
|
|
||||||
|
export function MissionArtifacts({
|
||||||
|
mission,
|
||||||
|
secondaryBtn,
|
||||||
|
}: {
|
||||||
|
mission: MissionDetail;
|
||||||
|
secondaryBtn: React.CSSProperties;
|
||||||
|
}) {
|
||||||
|
const missionId = mission.id;
|
||||||
|
const artifacts = mission.artifacts;
|
||||||
|
const [merging, setMerging] = useState(false);
|
||||||
|
const [merge, setMerge] = useState<MergeResult | null>(null);
|
||||||
|
const [openId, setOpenId] = useState<string | null>(null);
|
||||||
|
const [text, setText] = useState<ArtifactContent | null>(null);
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Open one artifact, fetching its text. Clicking the open one closes it.
|
||||||
|
*
|
||||||
|
* Errors surface in place rather than being swallowed: a file the server
|
||||||
|
* refuses to read (outside `_outputs`, or reaped with its mission) has to say
|
||||||
|
* so, or the panel sits empty and reads as a slow load that never finishes.
|
||||||
|
*/
|
||||||
|
const open = useCallback(
|
||||||
|
async (artifactId: string) => {
|
||||||
|
if (openId === artifactId) {
|
||||||
|
setOpenId(null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setOpenId(artifactId);
|
||||||
|
setText(null);
|
||||||
|
setError(null);
|
||||||
|
setLoading(true);
|
||||||
|
try {
|
||||||
|
setText(await getArtifactContent(missionId, artifactId));
|
||||||
|
} catch (e) {
|
||||||
|
setError(e instanceof Error ? e.message : "could not read this artifact");
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
[missionId, openId],
|
||||||
|
);
|
||||||
|
|
||||||
|
// The branch delivery actually pushed, read from the artifact that recorded
|
||||||
|
// it — not rebuilt from the mission id, so a phase that never pushed shows no
|
||||||
|
// button rather than a button that cannot work.
|
||||||
|
const branch = artifacts
|
||||||
|
.map((a) => (a.metadata ?? {}) as Record<string, unknown>)
|
||||||
|
.filter((m) => m.pushed === true)
|
||||||
|
.map((m) => (typeof m.branch === "string" ? m.branch : null))
|
||||||
|
.filter(Boolean)
|
||||||
|
.pop() as string | null;
|
||||||
|
|
||||||
|
const doMerge = async () => {
|
||||||
|
if (merging) return;
|
||||||
|
setMerging(true);
|
||||||
|
setMerge(null);
|
||||||
|
try {
|
||||||
|
setMerge(await mergeMissionBranch(mission.id));
|
||||||
|
} catch (e) {
|
||||||
|
setMerge({
|
||||||
|
merged: false,
|
||||||
|
reason: e instanceof Error ? e.message : "merge request failed",
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
setMerging(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (artifacts.length === 0) {
|
||||||
|
return (
|
||||||
|
<div style={{ fontFamily: mono, fontSize: 11.5, color: "#8a8a92", padding: 12 }}>
|
||||||
|
no artifacts yet — phases produce them as they run
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||||
|
{branch && (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
padding: 11,
|
||||||
|
borderRadius: 10,
|
||||||
|
border: "1px solid rgba(124,214,224,.22)",
|
||||||
|
background: "rgba(124,214,224,.05)",
|
||||||
|
display: "flex",
|
||||||
|
alignItems: "center",
|
||||||
|
gap: 10,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<GitMerge size={16} style={{ color: "#7cd6e0", flex: "none" }} />
|
||||||
|
<div style={{ flex: 1, minWidth: 0 }}>
|
||||||
|
<div style={{ fontSize: 13, color: "#f3f3f5", fontWeight: 500 }}>
|
||||||
|
Merge this mission’s work
|
||||||
|
</div>
|
||||||
|
<div style={{ fontFamily: mono, fontSize: 10.5, color: "#8a8a92" }}>
|
||||||
|
{merge ? merge.reason : `${branch} → default branch`}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={doMerge}
|
||||||
|
disabled={merging || merge?.merged === true}
|
||||||
|
style={{
|
||||||
|
...secondaryBtn,
|
||||||
|
padding: "5px 10px",
|
||||||
|
fontSize: 11,
|
||||||
|
opacity: merging || merge?.merged ? 0.6 : 1,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{merging ? "Merging…" : merge?.merged ? "Merged" : "Merge to main"}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{artifacts.map((a) => {
|
||||||
|
const isOpen = openId === a.id;
|
||||||
|
return (
|
||||||
|
<div key={a.id} style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
padding: 11,
|
||||||
|
borderRadius: 10,
|
||||||
|
border: "1px solid rgba(255,255,255,.07)",
|
||||||
|
background: "#101014",
|
||||||
|
display: "flex",
|
||||||
|
alignItems: "center",
|
||||||
|
gap: 10,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<FileText size={16} style={{ color: "#7cd6e0", flex: "none" }} />
|
||||||
|
<div style={{ flex: 1, minWidth: 0 }}>
|
||||||
|
<div style={{ fontSize: 13, color: "#f3f3f5", fontWeight: 500 }}>
|
||||||
|
{a.title ?? a.path.split("/").pop() ?? a.path}
|
||||||
|
</div>
|
||||||
|
<div style={{ fontFamily: mono, fontSize: 10.5, color: "#8a8a92" }}>
|
||||||
|
{a.kind} · {a.path}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => open(a.id)}
|
||||||
|
style={{ ...secondaryBtn, padding: "5px 10px", fontSize: 11 }}
|
||||||
|
>
|
||||||
|
{isOpen ? "Hide" : "Read"}
|
||||||
|
</button>
|
||||||
|
{/* A plain link, not a fetch-and-Blob.
|
||||||
|
The read endpoint caps at 2 MiB and decodes as UTF-8, so
|
||||||
|
building the download from what "Read" already fetched would
|
||||||
|
inherit both limits and silently produce a truncated or
|
||||||
|
undownloadable file for exactly the artifacts worth
|
||||||
|
downloading. This hits the streaming route instead. */}
|
||||||
|
<a
|
||||||
|
href={`/api/missions/${missionId}/artifacts/${a.id}/download`}
|
||||||
|
download
|
||||||
|
style={{
|
||||||
|
...secondaryBtn,
|
||||||
|
padding: "5px 10px",
|
||||||
|
fontSize: 11,
|
||||||
|
textDecoration: "none",
|
||||||
|
display: "inline-flex",
|
||||||
|
alignItems: "center",
|
||||||
|
gap: 5,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<Download aria-hidden size={12} />
|
||||||
|
Download
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
{isOpen && (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
border: "1px solid rgba(255,255,255,.06)",
|
||||||
|
borderRadius: 8,
|
||||||
|
background: "#0a0a0d",
|
||||||
|
padding: 16,
|
||||||
|
maxHeight: 640,
|
||||||
|
overflowY: "auto",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{loading ? (
|
||||||
|
<div style={{ fontFamily: mono, fontSize: 11.5, color: "#8a8a92" }}>
|
||||||
|
loading…
|
||||||
|
</div>
|
||||||
|
) : error ? (
|
||||||
|
<div style={{ fontFamily: mono, fontSize: 11.5, color: "#ff8a7a" }}>
|
||||||
|
{error}
|
||||||
|
</div>
|
||||||
|
) : text?.truncated ? (
|
||||||
|
<div style={{ fontFamily: mono, fontSize: 11.5, color: "#8a8a92" }}>
|
||||||
|
too large to display inline ({text.bytes.toLocaleString()}{" "}
|
||||||
|
bytes) — use Download
|
||||||
|
</div>
|
||||||
|
) : text ? (
|
||||||
|
<MarkdownBlock source={text.content} />
|
||||||
|
) : null}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -12,7 +12,6 @@ import React, { useCallback, useEffect, useMemo, useState } from "react";
|
|||||||
import {
|
import {
|
||||||
ChevronDown,
|
ChevronDown,
|
||||||
ChevronUp,
|
ChevronUp,
|
||||||
FileText,
|
|
||||||
Pencil,
|
Pencil,
|
||||||
Play,
|
Play,
|
||||||
Plus,
|
Plus,
|
||||||
@@ -42,6 +41,7 @@ import {
|
|||||||
} from "@/lib/api/missions";
|
} from "@/lib/api/missions";
|
||||||
import { EditMissionModal } from "./EditMissionModal";
|
import { EditMissionModal } from "./EditMissionModal";
|
||||||
import { MarkdownBlock } from "./MarkdownBlock";
|
import { MarkdownBlock } from "./MarkdownBlock";
|
||||||
|
import { MissionArtifacts } from "./MissionArtifacts";
|
||||||
import { MissionLiveEvents } from "./MissionLiveEvents";
|
import { MissionLiveEvents } from "./MissionLiveEvents";
|
||||||
import { MissionLivePane } from "./MissionLivePane";
|
import { MissionLivePane } from "./MissionLivePane";
|
||||||
import { MissionOutputReader } from "./MissionOutputReader";
|
import { MissionOutputReader } from "./MissionOutputReader";
|
||||||
@@ -135,7 +135,7 @@ export function MissionCanvas({
|
|||||||
const [refineDiff, setRefineDiff] = useState<RefineResult | null>(null);
|
const [refineDiff, setRefineDiff] = useState<RefineResult | null>(null);
|
||||||
const [accepting, setAccepting] = useState(false);
|
const [accepting, setAccepting] = useState(false);
|
||||||
const [phaseBusy, setPhaseBusy] = useState<string | null>(null);
|
const [phaseBusy, setPhaseBusy] = useState<string | null>(null);
|
||||||
const [pdfPreviewId, setPdfPreviewId] = useState<string | null>(null);
|
|
||||||
const [wizardOpen, setWizardOpen] = useState(false);
|
const [wizardOpen, setWizardOpen] = useState(false);
|
||||||
const [editOpen, setEditOpen] = useState(false);
|
const [editOpen, setEditOpen] = useState(false);
|
||||||
const [deleteBusy, setDeleteBusy] = useState(false);
|
const [deleteBusy, setDeleteBusy] = useState(false);
|
||||||
@@ -1016,101 +1016,8 @@ export function MissionCanvas({
|
|||||||
)}
|
)}
|
||||||
|
|
||||||
{tab === "output" && outputSub === "artifacts" && (
|
{tab === "output" && outputSub === "artifacts" && (
|
||||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
<MissionArtifacts mission={mission} secondaryBtn={secondaryBtn} />
|
||||||
{mission.artifacts.length === 0 ? (
|
|
||||||
<Empty label="no artifacts yet — phases produce them as they run" />
|
|
||||||
) : (
|
|
||||||
mission.artifacts.map((a) => {
|
|
||||||
const isPreviewing = pdfPreviewId === a.id;
|
|
||||||
return (
|
|
||||||
<div
|
|
||||||
key={a.id}
|
|
||||||
style={{ display: "flex", flexDirection: "column", gap: 8 }}
|
|
||||||
>
|
|
||||||
<div
|
|
||||||
style={{
|
|
||||||
padding: 11,
|
|
||||||
borderRadius: 10,
|
|
||||||
border: "1px solid rgba(255,255,255,.07)",
|
|
||||||
background: "#101014",
|
|
||||||
display: "flex",
|
|
||||||
alignItems: "center",
|
|
||||||
gap: 10,
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
<FileText size={16} style={{ color: "#7cd6e0", flex: "none" }} />
|
|
||||||
<div style={{ flex: 1, minWidth: 0 }}>
|
|
||||||
<div style={{ fontSize: 13, color: "#f3f3f5", fontWeight: 500 }}>
|
|
||||||
{a.title ?? a.path.split("/").pop() ?? a.path}
|
|
||||||
</div>
|
|
||||||
<div style={{ fontFamily: mono, fontSize: 10.5, color: "#8a8a92" }}>
|
|
||||||
{a.kind} · {a.path}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
{a.rendered_pdf_path ? (
|
|
||||||
<>
|
|
||||||
<button
|
|
||||||
type="button"
|
|
||||||
onClick={() =>
|
|
||||||
setPdfPreviewId(isPreviewing ? null : a.id)
|
|
||||||
}
|
|
||||||
style={{
|
|
||||||
...secondaryBtn,
|
|
||||||
padding: "5px 10px",
|
|
||||||
fontSize: 11,
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
{isPreviewing ? "Hide" : "Preview"}
|
|
||||||
</button>
|
|
||||||
<a
|
|
||||||
href={a.rendered_pdf_path}
|
|
||||||
target="_blank"
|
|
||||||
rel="noreferrer"
|
|
||||||
style={{
|
|
||||||
...secondaryBtn,
|
|
||||||
padding: "5px 10px",
|
|
||||||
fontSize: 11,
|
|
||||||
textDecoration: "none",
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
Open
|
|
||||||
</a>
|
|
||||||
</>
|
|
||||||
) : a.render_pdf_status !== "skip" ? (
|
|
||||||
<span
|
|
||||||
style={{
|
|
||||||
fontFamily: mono,
|
|
||||||
fontSize: 10.5,
|
|
||||||
color:
|
|
||||||
a.render_pdf_status === "failed"
|
|
||||||
? "#ff8a7a"
|
|
||||||
: "#8a8a92",
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
pdf: {a.render_pdf_status}
|
|
||||||
</span>
|
|
||||||
) : null}
|
|
||||||
</div>
|
|
||||||
{isPreviewing && a.rendered_pdf_path && (
|
|
||||||
<iframe
|
|
||||||
src={a.rendered_pdf_path}
|
|
||||||
title={a.title ?? a.path}
|
|
||||||
style={{
|
|
||||||
width: "100%",
|
|
||||||
height: 640,
|
|
||||||
border: "1px solid rgba(255,255,255,.06)",
|
|
||||||
borderRadius: 8,
|
|
||||||
background: "#0a0a0d",
|
|
||||||
}}
|
|
||||||
/>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
})
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
)}
|
)}
|
||||||
|
|
||||||
{tab === "output" && outputSub === "benchmarks" && (
|
{tab === "output" && outputSub === "benchmarks" && (
|
||||||
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
<div style={{ display: "flex", flexDirection: "column", gap: 8 }}>
|
||||||
{mission.benchmarks.length === 0 ? (
|
{mission.benchmarks.length === 0 ? (
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user