Compare commits
180
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f8438c32ea | ||
|
|
9e61e3ba35 | ||
|
|
c2fa8067e1 | ||
|
|
cb8184e784 | ||
|
|
f37c6b92d8 | ||
|
|
006432c2dc | ||
|
|
5f85dbb718 | ||
|
|
b210acf3c2 | ||
|
|
44079eb8b4 | ||
|
|
d3a398716b | ||
|
|
98037f9b3e | ||
|
|
c85027c83a | ||
|
|
0ad53da49c | ||
|
|
e2c312b728 | ||
|
|
104e3ef27c | ||
|
|
4c418f7d9b | ||
|
|
b2e2735583 | ||
|
|
e417247e7e | ||
|
|
fe2451fd60 | ||
|
|
895413509d | ||
|
|
dd80b69992 | ||
|
|
768e106614 | ||
|
|
e4bddeb1ba | ||
|
|
25f075a8be | ||
|
|
f27d2605eb | ||
|
|
16cfc29074 | ||
|
|
529497febb | ||
|
|
171f901bcd | ||
|
|
e7b412d578 | ||
|
|
c66c3c6377 | ||
|
|
1f6108f769 | ||
|
|
3c3d01c8d1 | ||
|
|
c7c3eeab46 | ||
|
|
d9c5300859 | ||
|
|
c3ad5672fc | ||
|
|
5afcf63324 | ||
|
|
b18e62041b | ||
|
|
774f17d194 | ||
|
|
f56d41f5b7 | ||
|
|
e96c5143bc | ||
|
|
f68fc019e4 | ||
|
|
4967b9b8fd | ||
|
|
8ddea454d1 | ||
|
|
dcd9514622 | ||
|
|
42108c840d | ||
|
|
13a35138e9 | ||
|
|
d4af58be85 | ||
|
|
eacd3ee085 | ||
|
|
91fbd2dc88 | ||
|
|
e5f097c291 | ||
|
|
4fedfcec30 | ||
|
|
2056bb1d9e | ||
|
|
dc0443de34 | ||
|
|
d48bdbc9a7 | ||
|
|
52500a689c | ||
|
|
9c9439a271 | ||
|
|
ee5a939ce6 | ||
|
|
deed591da6 | ||
|
|
c3c4447810 | ||
|
|
72046e7985 | ||
|
|
d84d17207f | ||
|
|
3a2d76aa43 | ||
|
|
5c5f1ced33 | ||
|
|
8b12245e79 | ||
|
|
099a716bfd | ||
|
|
4193ae2cda | ||
|
|
8c93cd8569 | ||
|
|
09afa7e7ff | ||
|
|
5b49d5a1a8 | ||
|
|
28090d1de0 | ||
|
|
0b89b8316c | ||
|
|
62509a5090 | ||
|
|
3616bc4733 | ||
|
|
a8b8efba6a | ||
|
|
a4b4d05b8d | ||
|
|
e89a32ffef | ||
|
|
a93a4111e1 | ||
|
|
0d8db7ff0b | ||
|
|
2a9a62c784 | ||
|
|
6dd7937ece | ||
|
|
a20702d55b | ||
|
|
87f188ae73 | ||
|
|
f6c3ddbf81 | ||
|
|
821cbb8622 | ||
|
|
da889f83ab | ||
|
|
c28c7a148f | ||
|
|
89bc53b53d | ||
|
|
ceab28b902 | ||
|
|
bcf4866abc | ||
|
|
b36ae00ea5 | ||
|
|
cd4d76a8c3 | ||
|
|
5c066afa7b | ||
|
|
d24823b6f3 | ||
|
|
bf2055e725 | ||
|
|
72f8bdc87c | ||
|
|
e2f576ec02 | ||
|
|
1b556c5849 | ||
|
|
521da9feb9 | ||
|
|
3300c9d149 | ||
|
|
08dd227a45 | ||
|
|
0aeae07db2 | ||
|
|
a33dbdcdc3 | ||
|
|
a48d78f8eb | ||
|
|
742724e53c | ||
|
|
d3a53e7bf1 | ||
|
|
f7f3dfe495 | ||
|
|
75d09241fb | ||
|
|
aa470091aa | ||
|
|
1797669296 | ||
|
|
abb97e6f03 | ||
|
|
6991e21f94 | ||
|
|
1d554396f4 | ||
|
|
12147a1e01 | ||
|
|
e31688bac5 | ||
|
|
66f730ad16 | ||
|
|
d49acaed5e | ||
|
|
4efcde9d4f | ||
|
|
0d25a94a84 | ||
|
|
cb48f7ff3b | ||
|
|
c840688adb | ||
|
|
b17e18aa67 | ||
|
|
9aed20b6d0 | ||
|
|
bb807c2f3a | ||
|
|
8796fbbcbb | ||
|
|
11b274edc6 | ||
|
|
2dee941080 | ||
|
|
7696009b25 | ||
|
|
d9f53a3f96 | ||
|
|
1cd81a8b2a | ||
|
|
521b8dea10 | ||
|
|
0a9747091f | ||
|
|
c9b7d8b6ca | ||
|
|
0206be68e5 | ||
|
|
4f07430e92 | ||
|
|
76fe1f1148 | ||
|
|
ebdba34da6 | ||
|
|
abc4160a89 | ||
|
|
2edafdaf0d | ||
|
|
92055c4556 | ||
|
|
c3297b86cf | ||
|
|
bcd1a0127d | ||
|
|
0c291ed1bb | ||
|
|
fd16b3c126 | ||
|
|
6687f8b808 | ||
|
|
fcf5d7b16c | ||
|
|
08847e6a63 | ||
|
|
78da62f156 | ||
|
|
dec59764b1 | ||
|
|
0f2591bae4 | ||
|
|
02ba557c3e | ||
|
|
22efb93775 | ||
|
|
2d04c5e257 | ||
|
|
b87d89f9fa | ||
|
|
67c56ce19b | ||
|
|
0f7fa31f86 | ||
|
|
4454a1cfd9 | ||
|
|
e65be19a45 | ||
|
|
452b419729 | ||
|
|
da3731d753 | ||
|
|
f8ca0ced9a | ||
|
|
4f6719c80e | ||
|
|
3c91d0e172 | ||
|
|
1253595ba7 | ||
|
|
bb274d08c6 | ||
|
|
7e07c389c6 | ||
|
|
389b41f8e6 | ||
|
|
ac6bf72943 | ||
|
|
0d9498ec6e | ||
|
|
15e7608e4a | ||
|
|
5d98fcf44a | ||
|
|
4ff4e6f7ee | ||
|
|
deb60be98d | ||
|
|
758b2dbd96 | ||
|
|
37fac288d2 | ||
|
|
5232175c88 | ||
|
|
ac47dcbe94 | ||
|
|
ad89ef94cd | ||
|
|
9de2cf34e4 | ||
|
|
3124fd3c8f | ||
|
|
cf076bd8ea |
Generated
+56
@@ -846,6 +846,8 @@ dependencies = [
|
|||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sysinfo",
|
"sysinfo",
|
||||||
|
"tar",
|
||||||
|
"tempfile",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-tungstenite 0.26.2",
|
"tokio-tungstenite 0.26.2",
|
||||||
"webrtc",
|
"webrtc",
|
||||||
@@ -970,6 +972,7 @@ dependencies = [
|
|||||||
"serde_yaml",
|
"serde_yaml",
|
||||||
"sha2",
|
"sha2",
|
||||||
"sqlx",
|
"sqlx",
|
||||||
|
"tar",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"thiserror 2.0.18",
|
"thiserror 2.0.18",
|
||||||
"time",
|
"time",
|
||||||
@@ -1841,6 +1844,16 @@ version = "2.4.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
|
checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fcagent"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"base64",
|
||||||
|
"serde_json",
|
||||||
|
"tar",
|
||||||
|
"vsock",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ff"
|
name = "ff"
|
||||||
version = "0.13.1"
|
version = "0.13.1"
|
||||||
@@ -2872,6 +2885,15 @@ dependencies = [
|
|||||||
"autocfg",
|
"autocfg",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "memoffset"
|
||||||
|
version = "0.9.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mime"
|
name = "mime"
|
||||||
version = "0.3.17"
|
version = "0.3.17"
|
||||||
@@ -2962,6 +2984,19 @@ dependencies = [
|
|||||||
"pin-utils",
|
"pin-utils",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "nix"
|
||||||
|
version = "0.31.3"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags 2.13.0",
|
||||||
|
"cfg-if",
|
||||||
|
"cfg_aliases",
|
||||||
|
"libc",
|
||||||
|
"memoffset 0.9.1",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "nom"
|
name = "nom"
|
||||||
version = "7.1.3"
|
version = "7.1.3"
|
||||||
@@ -5029,6 +5064,17 @@ dependencies = [
|
|||||||
"windows",
|
"windows",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tar"
|
||||||
|
version = "0.4.46"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840"
|
||||||
|
dependencies = [
|
||||||
|
"filetime",
|
||||||
|
"libc",
|
||||||
|
"xattr",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tempfile"
|
name = "tempfile"
|
||||||
version = "3.27.0"
|
version = "3.27.0"
|
||||||
@@ -5749,6 +5795,16 @@ version = "0.9.5"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "vsock"
|
||||||
|
version = "0.5.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "6ba782755fc073877e567c2253c0be48e4aa9a254c232d36d3985dfae0bd5205"
|
||||||
|
dependencies = [
|
||||||
|
"libc",
|
||||||
|
"nix 0.31.3",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "wait-timeout"
|
name = "wait-timeout"
|
||||||
version = "0.2.1"
|
version = "0.2.1"
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ members = [
|
|||||||
"crates/bins/clawmates-server",
|
"crates/bins/clawmates-server",
|
||||||
"crates/bins/clawmates-broker",
|
"crates/bins/clawmates-broker",
|
||||||
"crates/bins/clawmates-node",
|
"crates/bins/clawmates-node",
|
||||||
|
"crates/bins/fcagent",
|
||||||
"tools/bundler",
|
"tools/bundler",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -36,6 +37,9 @@ publish = false
|
|||||||
# Shared dependency versions; crates opt in via { workspace = true }.
|
# Shared dependency versions; crates opt in via { workspace = true }.
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
|
# Streaming tar for mission copy-in/copy-out (no compression: the payload is
|
||||||
|
# a git checkout on a local socket, so CPU spent zipping buys nothing).
|
||||||
|
tar = "0.4"
|
||||||
thiserror = "2"
|
thiserror = "2"
|
||||||
uuid = { version = "1", features = ["v7", "serde"] }
|
uuid = { version = "1", features = ["v7", "serde"] }
|
||||||
proptest = "1"
|
proptest = "1"
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ serde_json = { workspace = true }
|
|||||||
sysinfo = "0.33"
|
sysinfo = "0.33"
|
||||||
portable-pty = "0.8"
|
portable-pty = "0.8"
|
||||||
base64 = "0.22"
|
base64 = "0.22"
|
||||||
|
tar = { workspace = true }
|
||||||
cm-sandbox = { path = "../../cm-sandbox" }
|
cm-sandbox = { path = "../../cm-sandbox" }
|
||||||
# Linking cm-sandbox (bollard) brings a second rustls provider into the graph, so
|
# Linking cm-sandbox (bollard) brings a second rustls provider into the graph, so
|
||||||
# rustls can't auto-pick one — we install `ring` explicitly at startup.
|
# rustls can't auto-pick one — we install `ring` explicitly at startup.
|
||||||
@@ -26,5 +27,8 @@ rustls = { version = "0.23", default-features = false, features = ["ring"] }
|
|||||||
webrtc = "0.17.1"
|
webrtc = "0.17.1"
|
||||||
bytes = "1.12.0"
|
bytes = "1.12.0"
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
tempfile = "3"
|
||||||
|
|
||||||
[lints]
|
[lints]
|
||||||
workspace = true
|
workspace = true
|
||||||
|
|||||||
@@ -0,0 +1,527 @@
|
|||||||
|
//! Host side of a microVM's only route out: an HTTP `CONNECT` proxy on a Unix
|
||||||
|
//! socket, one per VM.
|
||||||
|
//!
|
||||||
|
//! # Why the guest has no network card
|
||||||
|
//!
|
||||||
|
//! It could have had one. A TAP device plus NAT is what the Firecracker
|
||||||
|
//! write-ups do, and it was measured against this before being rejected:
|
||||||
|
//!
|
||||||
|
//! - `ip tuntap add` is **denied to the daemon user** (needs `CAP_NET_ADMIN`), so
|
||||||
|
//! TAP would need root to pre-provision devices at setup time — the same
|
||||||
|
//! privilege detour the loop-mounted rootfs already forced.
|
||||||
|
//! - tank's `FORWARD` policy is `DROP` with Docker and Tailscale chains, so rules
|
||||||
|
//! would have to be *inserted* at position 1; appended ones die silently.
|
||||||
|
//! - a leaked TAP device is a new class of host litter to reap.
|
||||||
|
//!
|
||||||
|
//! Against that, `CONNECT` needs no privilege at all, and it is better on the
|
||||||
|
//! merits: the client hands us the **hostname**, so resolution happens here and
|
||||||
|
//! the guest needs no DNS or `resolv.conf`; the allow-list is by name rather than
|
||||||
|
//! by address; and nothing in the guest can reach the network except through this
|
||||||
|
//! function. That is what the isolation plan's egress restriction actually asked
|
||||||
|
//! for, and it is strictly tighter than the mission container's present full
|
||||||
|
//! egress on `clawmates_edge`.
|
||||||
|
//!
|
||||||
|
//! The design rests on one measured fact: **`claude` honours `HTTPS_PROXY`**.
|
||||||
|
//! With the proxy pointed at a closed port, `claude -p` fails with
|
||||||
|
//! `ConnectionRefused` instead of answering. (That could only be measured in a
|
||||||
|
//! container — inside a VM the CLI collapses every failure into `Execution
|
||||||
|
//! error`.)
|
||||||
|
//!
|
||||||
|
//! # Shape
|
||||||
|
//!
|
||||||
|
//! Firecracker's convention for a guest-initiated connection is that the **host**
|
||||||
|
//! listens on `<uds_path>_<port>`. The guest's agent pumps bytes from
|
||||||
|
//! `127.0.0.1:3128` to vsock port 9002 and parses nothing, so all policy is here
|
||||||
|
//! and a compromised guest cannot argue with it.
|
||||||
|
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
|
||||||
|
use tokio::net::{TcpStream, UnixListener, UnixStream};
|
||||||
|
|
||||||
|
/// Port the guest dials. Must match `fcagent`'s `EGRESS_PORT`.
|
||||||
|
pub const EGRESS_PORT: u32 = 9002;
|
||||||
|
|
||||||
|
|
||||||
|
/// What every backend gets, whatever it is.
|
||||||
|
const COMMON_ALLOW: &[&str] = &["git.redclaw.dev"];
|
||||||
|
|
||||||
|
/// The model host a backend's CLI must reach, and NOTHING else.
|
||||||
|
///
|
||||||
|
/// Per backend rather than a union, and that is not tidiness. MEASURED on tank:
|
||||||
|
/// a `glm` VM completed a whole mission with `api.anthropic.com` denied at this
|
||||||
|
/// proxy, dialling only `api.z.ai` — Claude Code's calls to anthropic.com are
|
||||||
|
/// its own telemetry, not its completions. So a GLM VM has no need of Anthropic
|
||||||
|
/// at all, and a union allow-list would let a credential mix-up reach the wrong
|
||||||
|
/// provider's endpoint instead of failing at a closed door.
|
||||||
|
///
|
||||||
|
/// The measurement also settled something a self-report could not: that same
|
||||||
|
/// agent, served only by z.ai, still described itself as "Claude Opus 5". A
|
||||||
|
/// model's account of which model it is has no evidential value here; the
|
||||||
|
/// proxy's log of which host it dialled does.
|
||||||
|
fn provider_hosts(backend: Option<&str>) -> &'static [&'static str] {
|
||||||
|
match backend {
|
||||||
|
// `canary-claude` is the same provider, from a candidate CLI image —
|
||||||
|
// see `mission_runtime::microvm_credential_for`, which must grant it the
|
||||||
|
// same credential. A backend is defined in TWO maps: the credential one
|
||||||
|
// on the server and this one on the node. Adding it to only the first is
|
||||||
|
// exactly what happened here: the mission launched, the VM booted, the
|
||||||
|
// agent ran, and the turn died on
|
||||||
|
// "403 api.anthropic.com is not on the egress allow-list" — which is the
|
||||||
|
// fail-closed branch below working correctly.
|
||||||
|
None | Some("") | Some("default") | Some("claude") | Some("canary-claude") => {
|
||||||
|
&["api.anthropic.com", ".anthropic.com"]
|
||||||
|
}
|
||||||
|
Some("glm") => &["api.z.ai"],
|
||||||
|
// The Kimi CODE service, which is where an `sk-kimi-` key is valid —
|
||||||
|
// NOT `api.moonshot.ai`, whose Anthropic endpoint exists but belongs to
|
||||||
|
// a different account namespace and rejects that key. Only the host the
|
||||||
|
// `agent-kimi` image bakes in.
|
||||||
|
Some("kimi") => &["api.kimi.com"],
|
||||||
|
// A locally-hosted model reaches NOTHING through this proxy. Its route
|
||||||
|
// is `crate::local_model` — a vsock pipe to the node's own loopback,
|
||||||
|
// with no destination in the protocol — so the correct allow-list here
|
||||||
|
// is the empty one, and it falls through to the branch below.
|
||||||
|
//
|
||||||
|
// Spelled out rather than left implicit because the temptation was to
|
||||||
|
// widen this proxy instead: an entry here would have meant relaxing the
|
||||||
|
// 443-only rule AND the IP-literal refusal, both of which exist because
|
||||||
|
// a unit test caught them being bypassed.
|
||||||
|
// Fail closed: a backend nobody taught this function about reaches the
|
||||||
|
// forge and no model API. It cannot silently borrow another provider's
|
||||||
|
// door, which is the failure this split exists to prevent.
|
||||||
|
Some(_) => &[],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse the allow-list once per VM.
|
||||||
|
///
|
||||||
|
/// An empty `CLAWMATES_FC_EGRESS_ALLOW` means **deny everything**, not "fall back
|
||||||
|
/// to the default": an operator who blanked it asked for no egress, and quietly
|
||||||
|
/// restoring the default would hand a mission the network they just took away.
|
||||||
|
/// The allow-list for a VM running `backend`.
|
||||||
|
///
|
||||||
|
/// An explicit `CLAWMATES_FC_EGRESS_ALLOW` still wins outright: an operator who
|
||||||
|
/// set it asked for exactly that list, and quietly adding a provider host to it
|
||||||
|
/// would widen a boundary they had drawn on purpose.
|
||||||
|
fn allow_list_for(backend: Option<&str>) -> Vec<String> {
|
||||||
|
match std::env::var("CLAWMATES_FC_EGRESS_ALLOW") {
|
||||||
|
Ok(raw) => raw
|
||||||
|
.split(',')
|
||||||
|
.map(|s| s.trim().to_ascii_lowercase())
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.collect(),
|
||||||
|
Err(_) => COMMON_ALLOW
|
||||||
|
.iter()
|
||||||
|
.chain(provider_hosts(backend).iter())
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
.collect(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Is `host` allowed?
|
||||||
|
///
|
||||||
|
/// Case-insensitive, port already stripped. A leading `.` in an entry matches
|
||||||
|
/// that domain and its subdomains; anything else must match exactly. Deliberately
|
||||||
|
/// not a substring test — `api.anthropic.com.evil.test` contains the allowed name
|
||||||
|
/// and must not pass.
|
||||||
|
fn host_allowed(host: &str, allow: &[String]) -> bool {
|
||||||
|
let host = host.trim().trim_end_matches('.').to_ascii_lowercase();
|
||||||
|
if host.is_empty() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
// A hostname is letters, digits, dots and hyphens — nothing else. This is
|
||||||
|
// load-bearing, not hygiene: `evil.test/api.anthropic.com` ends with an
|
||||||
|
// allowed suffix and would otherwise PASS the match below. A unit test found
|
||||||
|
// it. Rejecting the character class also refuses IP literals, so an address
|
||||||
|
// cannot be used to sidestep a list written in names.
|
||||||
|
if !host
|
||||||
|
.chars()
|
||||||
|
.all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-')
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
allow.iter().any(|a| match a.strip_prefix('.') {
|
||||||
|
Some(domain) => host == domain || host.ends_with(&format!(".{domain}")),
|
||||||
|
None => host == *a,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Split `host:port` from a CONNECT target.
|
||||||
|
///
|
||||||
|
/// Only 443 is allowed. Permitting arbitrary ports would turn the proxy into a
|
||||||
|
/// general-purpose tunnel to anything the allow-list happens to name, which is a
|
||||||
|
/// different and much larger promise than "the agent can reach its API".
|
||||||
|
fn parse_target(target: &str) -> Result<(String, u16), String> {
|
||||||
|
let (host, port) = target
|
||||||
|
.rsplit_once(':')
|
||||||
|
.ok_or_else(|| format!("CONNECT target {target:?} has no port"))?;
|
||||||
|
let port: u16 = port
|
||||||
|
.trim()
|
||||||
|
.parse()
|
||||||
|
.map_err(|_| format!("CONNECT target {target:?} has a non-numeric port"))?;
|
||||||
|
if port != 443 {
|
||||||
|
return Err(format!("port {port} is not permitted (only 443)"));
|
||||||
|
}
|
||||||
|
// Strip IPv6 brackets so the allow-list sees the same text either way.
|
||||||
|
let host = host.trim().trim_start_matches('[').trim_end_matches(']');
|
||||||
|
Ok((host.to_string(), port))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What happened to one connection. Returned so the caller can log it and the
|
||||||
|
/// selftest can assert on it.
|
||||||
|
#[derive(Debug, PartialEq, Eq)]
|
||||||
|
pub enum Verdict {
|
||||||
|
Allowed(String),
|
||||||
|
Denied(String),
|
||||||
|
Malformed(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One header line, with a cap.
|
||||||
|
///
|
||||||
|
/// `read_line` has no limit, and a guest that never sends a newline would make
|
||||||
|
/// the host allocate until it died. Read byte-wise instead — the reads come out
|
||||||
|
/// of the BufReader, so this is cheap for lines this size, and it keeps ONE
|
||||||
|
/// reader over the connection, which matters (see `serve`).
|
||||||
|
async fn read_line_capped(reader: &mut BufReader<UnixStream>, cap: usize) -> Result<String, String> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
loop {
|
||||||
|
match reader.read_u8().await {
|
||||||
|
Ok(b'\n') => break,
|
||||||
|
Ok(b) => out.push(b),
|
||||||
|
// EOF mid-line: return what we have and let the caller judge it.
|
||||||
|
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break,
|
||||||
|
Err(e) => return Err(format!("read: {e}")),
|
||||||
|
}
|
||||||
|
if out.len() > cap {
|
||||||
|
return Err(format!("a request line longer than {cap} bytes"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(String::from_utf8_lossy(&out)
|
||||||
|
.trim_end_matches('\r')
|
||||||
|
.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Serve one tunnelled connection.
|
||||||
|
async fn serve(stream: UnixStream, allow: Arc<Vec<String>>) -> Verdict {
|
||||||
|
// ONE reader for the whole request. Wrapping the stream a second time would
|
||||||
|
// discard whatever the first reader had already buffered — including the
|
||||||
|
// first bytes of the TLS handshake — and the tunnel would come up looking
|
||||||
|
// fine and then stall on a corrupt stream.
|
||||||
|
let mut reader = BufReader::new(stream);
|
||||||
|
|
||||||
|
let line = match read_line_capped(&mut reader, 8 * 1024).await {
|
||||||
|
Ok(l) if !l.trim().is_empty() => l,
|
||||||
|
Ok(_) => return Verdict::Malformed("no request line".into()),
|
||||||
|
Err(e) => return Verdict::Malformed(e),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut parts = line.split_whitespace();
|
||||||
|
let method = parts.next().unwrap_or_default().to_ascii_uppercase();
|
||||||
|
let target = parts.next().unwrap_or_default().to_string();
|
||||||
|
|
||||||
|
if method != "CONNECT" {
|
||||||
|
// Plain HTTP would mean proxying a request we would then have to rewrite,
|
||||||
|
// and everything a mission needs is TLS. Refused with a status, so the
|
||||||
|
// client reports something better than a closed socket.
|
||||||
|
let _ = reply(reader.get_mut(), 405, "only CONNECT is supported").await;
|
||||||
|
return Verdict::Malformed(format!("method {method}"));
|
||||||
|
}
|
||||||
|
|
||||||
|
let (host, port) = match parse_target(&target) {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(e) => {
|
||||||
|
let _ = reply(reader.get_mut(), 400, &e).await;
|
||||||
|
return Verdict::Malformed(e);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if !host_allowed(&host, &allow) {
|
||||||
|
// 403 rather than a silent drop: a denial that looks like a network
|
||||||
|
// timeout is indistinguishable from a hung agent, and this codebase has
|
||||||
|
// paid for that confusion more than once.
|
||||||
|
let _ = reply(
|
||||||
|
reader.get_mut(),
|
||||||
|
403,
|
||||||
|
&format!("{host} is not on the egress allow-list"),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Verdict::Denied(host);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Consume the remaining request headers: they belong to the CONNECT, not to
|
||||||
|
// the tunnel.
|
||||||
|
loop {
|
||||||
|
match read_line_capped(&mut reader, 8 * 1024).await {
|
||||||
|
Ok(h) if h.trim().is_empty() => break,
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(e) => return Verdict::Malformed(e),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut upstream = match TcpStream::connect((host.as_str(), port)).await {
|
||||||
|
Ok(s) => s,
|
||||||
|
Err(e) => {
|
||||||
|
let _ = reply(reader.get_mut(), 502, &format!("connect {host}:{port}: {e}")).await;
|
||||||
|
return Verdict::Denied(host);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if reply(reader.get_mut(), 200, "Connection established")
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
return Verdict::Denied(host);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Anything already buffered past the headers is tunnel payload — a client
|
||||||
|
// that pipelined its first TLS bytes would otherwise lose them.
|
||||||
|
let pending = reader.buffer().to_vec();
|
||||||
|
let mut stream = reader.into_inner();
|
||||||
|
if !pending.is_empty() && upstream.write_all(&pending).await.is_err() {
|
||||||
|
return Verdict::Denied(host);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bytes both ways until either side is done. Errors are not worth reporting:
|
||||||
|
// a closed connection is the normal end of a tunnel.
|
||||||
|
let _ = tokio::io::copy_bidirectional(&mut stream, &mut upstream).await;
|
||||||
|
Verdict::Allowed(host)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn reply(s: &mut UnixStream, code: u16, text: &str) -> std::io::Result<()> {
|
||||||
|
let reason = if code == 200 {
|
||||||
|
"Connection established"
|
||||||
|
} else {
|
||||||
|
"Forbidden"
|
||||||
|
};
|
||||||
|
// The body carries the reason for a non-200 so it reaches the agent's own
|
||||||
|
// error output, where whoever is reading a failed mission will see it.
|
||||||
|
let body = if code == 200 { String::new() } else { format!("{text}\n") };
|
||||||
|
let head = format!(
|
||||||
|
"HTTP/1.1 {code} {reason}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||||
|
body.len()
|
||||||
|
);
|
||||||
|
s.write_all(head.as_bytes()).await?;
|
||||||
|
if !body.is_empty() {
|
||||||
|
s.write_all(body.as_bytes()).await?;
|
||||||
|
}
|
||||||
|
s.flush().await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Start this VM's proxy. Returns the socket path and the task serving it.
|
||||||
|
///
|
||||||
|
/// Bound **before** firecracker starts, because a guest that dials before the
|
||||||
|
/// host is listening gets a connection refused it will not retry.
|
||||||
|
pub fn start(
|
||||||
|
uds: &Path,
|
||||||
|
vm_id: &str,
|
||||||
|
backend: Option<&str>,
|
||||||
|
) -> Result<(PathBuf, tokio::task::JoinHandle<()>), String> {
|
||||||
|
let path = PathBuf::from(format!("{}_{}", uds.display(), EGRESS_PORT));
|
||||||
|
// Firecracker does not clean these up any more than it cleans up its own
|
||||||
|
// socket, and a stale file makes bind fail with EADDRINUSE.
|
||||||
|
let _ = std::fs::remove_file(&path);
|
||||||
|
let listener =
|
||||||
|
UnixListener::bind(&path).map_err(|e| format!("bind {}: {e}", path.display()))?;
|
||||||
|
|
||||||
|
let allow = Arc::new(allow_list_for(backend));
|
||||||
|
eprintln!(
|
||||||
|
"microvm {vm_id}: egress proxy on {} allowing {:?}",
|
||||||
|
path.display(),
|
||||||
|
allow
|
||||||
|
);
|
||||||
|
let vm = vm_id.to_string();
|
||||||
|
let task = tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
match listener.accept().await {
|
||||||
|
Ok((s, _)) => {
|
||||||
|
let allow = allow.clone();
|
||||||
|
let vm = vm.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
match serve(s, allow).await {
|
||||||
|
// Logged at every outcome: this is the audit trail of
|
||||||
|
// everything a mission reached, and a denial that is
|
||||||
|
// not logged is a mystery hang later.
|
||||||
|
Verdict::Allowed(h) => eprintln!("microvm {vm}: egress -> {h}"),
|
||||||
|
Verdict::Denied(h) => {
|
||||||
|
eprintln!("microvm {vm}: egress DENIED {h}")
|
||||||
|
}
|
||||||
|
Verdict::Malformed(w) => {
|
||||||
|
eprintln!("microvm {vm}: egress malformed request ({w})")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("microvm {vm}: egress accept failed: {e}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
Ok((path, task))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
/// A backend is defined in TWO places — the server's credential map and this
|
||||||
|
/// egress map — and granting it one without the other produces a mission
|
||||||
|
/// that launches, boots, runs, and dies on a 403 from our own proxy.
|
||||||
|
///
|
||||||
|
/// Measured exactly that way: `canary-claude` was credentialed on the server
|
||||||
|
/// and unknown here, and the turn failed with
|
||||||
|
/// "api.anthropic.com is not on the egress allow-list".
|
||||||
|
#[test]
|
||||||
|
fn the_canary_backend_reaches_the_same_provider_as_claude() {
|
||||||
|
assert_eq!(
|
||||||
|
provider_hosts(Some("canary-claude")),
|
||||||
|
provider_hosts(Some("claude")),
|
||||||
|
"a canary of the Claude image must reach Anthropic, or it tests nothing"
|
||||||
|
);
|
||||||
|
// And the fail-closed branch must still hold for anything unknown: this
|
||||||
|
// is what stops a new backend silently borrowing another provider's door.
|
||||||
|
assert!(provider_hosts(Some("canary-something-else")).is_empty());
|
||||||
|
assert!(provider_hosts(Some("definitely-not-built")).is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn allow() -> Vec<String> {
|
||||||
|
allow_list_for(None)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// MEASURED on tank, not assumed: a `glm` VM ran a whole mission to
|
||||||
|
/// completion with `api.anthropic.com` denied at this proxy, dialling only
|
||||||
|
/// `api.z.ai`. So Anthropic's host is not something a GLM agent needs — and
|
||||||
|
/// a VM that cannot reach it cannot send z.ai's key there, or Anthropic's
|
||||||
|
/// subscription token to z.ai, whatever a credential bug does upstream.
|
||||||
|
#[test]
|
||||||
|
fn each_backend_reaches_its_own_provider_and_no_other() {
|
||||||
|
let claude = allow_list_for(Some("claude"));
|
||||||
|
assert!(claude.iter().any(|h| h == "api.anthropic.com"), "{claude:?}");
|
||||||
|
assert!(!claude.iter().any(|h| h == "api.z.ai"), "{claude:?}");
|
||||||
|
|
||||||
|
let glm = allow_list_for(Some("glm"));
|
||||||
|
assert!(glm.iter().any(|h| h == "api.z.ai"), "{glm:?}");
|
||||||
|
assert!(
|
||||||
|
!glm.iter().any(|h| h.contains("anthropic")),
|
||||||
|
"a GLM VM must not be able to reach Anthropic: {glm:?}"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Both still reach the forge — delivery is host-side, but a mission that
|
||||||
|
// clones or fetches needs it.
|
||||||
|
for l in [&claude, &glm] {
|
||||||
|
assert!(l.iter().any(|h| h == "git.redclaw.dev"), "{l:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
let kimi = allow_list_for(Some("kimi"));
|
||||||
|
assert!(kimi.iter().any(|h| h == "api.kimi.com"), "{kimi:?}");
|
||||||
|
for other in ["api.z.ai", "api.anthropic.com"] {
|
||||||
|
assert!(!kimi.iter().any(|h| h == other), "{kimi:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
// An unknown backend gets no model API at all rather than borrowing
|
||||||
|
// somebody's: it cannot run anyway, and failing at a closed door beats
|
||||||
|
// reaching the wrong endpoint with a credential.
|
||||||
|
let unknown = allow_list_for(Some("rootfs-opus"));
|
||||||
|
assert_eq!(unknown, vec!["git.redclaw.dev".to_string()], "{unknown:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_model_api_and_the_forge_are_reachable() {
|
||||||
|
for h in ["api.anthropic.com", "git.redclaw.dev", "API.Anthropic.COM"] {
|
||||||
|
assert!(host_allowed(h, &allow()), "{h} must be allowed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The check is a match, never a substring test. A name that merely CONTAINS
|
||||||
|
/// an allowed one is a different host controlled by someone else.
|
||||||
|
#[test]
|
||||||
|
fn a_lookalike_host_is_not_allowed() {
|
||||||
|
for h in [
|
||||||
|
"api.anthropic.com.evil.test",
|
||||||
|
"notapi.anthropic.com.attacker.io",
|
||||||
|
// These contain an allowed suffix but are not that host. The first
|
||||||
|
// PASSED before the character-class check was added — a unit test
|
||||||
|
// found it, not review.
|
||||||
|
"evil.test/api.anthropic.com",
|
||||||
|
"[email protected]",
|
||||||
|
"api.anthropic.com:443",
|
||||||
|
"git.redclaw.dev.evil.test",
|
||||||
|
"example.com",
|
||||||
|
"",
|
||||||
|
" ",
|
||||||
|
] {
|
||||||
|
assert!(!host_allowed(h, &allow()), "{h} must NOT be allowed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A raw address must not sidestep a list written in names.
|
||||||
|
/// A local-model backend gets NO egress, and the 443 rule is untouched.
|
||||||
|
///
|
||||||
|
/// The alternative design routed the node's Ollama through this proxy, which
|
||||||
|
/// would have meant permitting port 11434 and an address the guest names.
|
||||||
|
/// Both are refused here, still, and a `local-ornith` VM reaches the forge
|
||||||
|
/// and nothing else — its model lives on the other socket entirely.
|
||||||
|
#[test]
|
||||||
|
fn a_local_model_backend_gets_no_egress_and_no_new_port() {
|
||||||
|
let allow = allow_list_for(Some("local-ornith"));
|
||||||
|
assert!(
|
||||||
|
allow.iter().all(|a| a == "git.redclaw.dev"),
|
||||||
|
"a local backend must reach only the forge, got {allow:?}"
|
||||||
|
);
|
||||||
|
for h in ["api.anthropic.com", "api.z.ai", "api.kimi.com", "127.0.0.1"] {
|
||||||
|
assert!(!host_allowed(h, &allow), "{h} must NOT be reachable");
|
||||||
|
}
|
||||||
|
// The rules this design exists to avoid loosening.
|
||||||
|
assert!(parse_target("anything:11434").is_err());
|
||||||
|
assert!(parse_target("127.0.0.1:443").is_ok_and(|(h, _)| !host_allowed(&h, &allow)));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_ip_literal_is_not_allowed() {
|
||||||
|
let a = vec![".anthropic.com".to_string()];
|
||||||
|
assert!(!host_allowed("[::1]", &a));
|
||||||
|
assert!(!host_allowed("2606:4700::1111", &a));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A trailing dot is the same host to a resolver, so it must be to us.
|
||||||
|
#[test]
|
||||||
|
fn a_trailing_dot_does_not_bypass_the_list() {
|
||||||
|
assert!(host_allowed("api.anthropic.com.", &allow()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A `.domain` entry covers subdomains, and only real subdomains.
|
||||||
|
#[test]
|
||||||
|
fn a_dot_prefixed_entry_matches_subdomains_only() {
|
||||||
|
let a = vec![".example.com".to_string()];
|
||||||
|
assert!(host_allowed("a.example.com", &a));
|
||||||
|
assert!(host_allowed("example.com", &a));
|
||||||
|
assert!(!host_allowed("notexample.com", &a));
|
||||||
|
assert!(!host_allowed("example.com.evil.test", &a));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Blanking the allow-list means no egress. Falling back to the default
|
||||||
|
/// would hand a mission the network an operator had just taken away.
|
||||||
|
#[test]
|
||||||
|
fn an_empty_allow_list_denies_everything() {
|
||||||
|
let none: Vec<String> = vec![];
|
||||||
|
assert!(!host_allowed("api.anthropic.com", &none));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Only 443. Anything else turns the proxy into a general-purpose tunnel to
|
||||||
|
/// whatever the allow-list happens to name.
|
||||||
|
#[test]
|
||||||
|
fn only_https_is_tunnelled() {
|
||||||
|
assert_eq!(parse_target("api.anthropic.com:443").unwrap().1, 443);
|
||||||
|
for bad in [
|
||||||
|
"api.anthropic.com:22",
|
||||||
|
"api.anthropic.com:80",
|
||||||
|
"api.anthropic.com",
|
||||||
|
"api.anthropic.com:not-a-port",
|
||||||
|
] {
|
||||||
|
assert!(parse_target(bad).is_err(), "{bad} must be refused");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,167 @@
|
|||||||
|
//! Host side of a microVM's route to the node's OWN locally-hosted model.
|
||||||
|
//!
|
||||||
|
//! # Why this is not the egress proxy
|
||||||
|
//!
|
||||||
|
//! [`crate::egress`] exists so an agent can reach the public internet under an
|
||||||
|
//! allow-list: it speaks HTTP `CONNECT`, takes a destination from the guest,
|
||||||
|
//! resolves it, and decides. Every one of those powers is a liability, which is
|
||||||
|
//! why that module is careful about ports, IP literals and suffix matching.
|
||||||
|
//!
|
||||||
|
//! This is the opposite shape. There is **no destination in the protocol**. The
|
||||||
|
//! guest opens a socket; the host connects it to `127.0.0.1:11434` on the node
|
||||||
|
//! and copies bytes. A compromised guest can ask for nothing else, because there
|
||||||
|
//! is nothing to ask — it is a pipe, not a proxy. That is strictly narrower than
|
||||||
|
//! anything the allow-list could express, and it is why routing a local model
|
||||||
|
//! through `egress` would have been the worse design: it would have meant
|
||||||
|
//! relaxing the 443-only rule and the IP-literal refusal, both of which exist
|
||||||
|
//! because a unit test caught them being bypassed.
|
||||||
|
//!
|
||||||
|
//! # Why plaintext is right here
|
||||||
|
//!
|
||||||
|
//! The bytes go guest loopback → vsock → host loopback. They never touch a
|
||||||
|
//! network, so there is no wire for TLS to protect. Ollama stays bound to
|
||||||
|
//! `127.0.0.1` on the node and is never exposed to the tailnet, which is a
|
||||||
|
//! stronger position than terminating TLS in front of it would have been.
|
||||||
|
//!
|
||||||
|
//! # Why it is per-backend
|
||||||
|
//!
|
||||||
|
//! The node binds this socket only for a backend declared to use a local model.
|
||||||
|
//! On every other backend the guest's listener is still there and simply gets a
|
||||||
|
//! refusal — the same fail-closed default `provider_hosts` applies to egress.
|
||||||
|
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use tokio::net::{TcpStream, UnixListener};
|
||||||
|
|
||||||
|
/// Host-side vsock port. Must match `fcagent`'s `MODEL_VSOCK_PORT`.
|
||||||
|
pub const MODEL_PORT: u32 = 9003;
|
||||||
|
|
||||||
|
/// Where the node's model server listens. Loopback, and not configurable from
|
||||||
|
/// the guest by design — see the module docs.
|
||||||
|
const OLLAMA_ADDR: &str = "127.0.0.1:11434";
|
||||||
|
|
||||||
|
/// Whether a backend is served by a model running on the node itself.
|
||||||
|
///
|
||||||
|
/// Named individually rather than by prefix. An unrecognised backend must not
|
||||||
|
/// acquire a route to anything by accident, which is the same rule
|
||||||
|
/// `egress::provider_hosts` and `mission_runtime::microvm_credential_for`
|
||||||
|
/// already apply from their own side.
|
||||||
|
pub fn uses_local_model(backend: Option<&str>) -> bool {
|
||||||
|
matches!(backend, Some("local-ornith"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bind the guest's local-model socket, if this backend has one.
|
||||||
|
///
|
||||||
|
/// `Ok(None)` means "this backend does not use a local model" and is the normal
|
||||||
|
/// case. An error means it should have had one and could not — reported by the
|
||||||
|
/// caller, never silently swallowed, because the symptom otherwise is an agent
|
||||||
|
/// that hangs on its first turn.
|
||||||
|
pub fn start(
|
||||||
|
uds: &Path,
|
||||||
|
vm_id: &str,
|
||||||
|
backend: Option<&str>,
|
||||||
|
) -> Result<Option<(PathBuf, tokio::task::JoinHandle<()>)>, String> {
|
||||||
|
if !uses_local_model(backend) {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let path = PathBuf::from(format!("{}_{}", uds.display(), MODEL_PORT));
|
||||||
|
// Firecracker leaves these behind exactly as it does its own socket, and a
|
||||||
|
// stale file makes bind fail with EADDRINUSE.
|
||||||
|
let _ = std::fs::remove_file(&path);
|
||||||
|
let listener =
|
||||||
|
UnixListener::bind(&path).map_err(|e| format!("bind {}: {e}", path.display()))?;
|
||||||
|
|
||||||
|
eprintln!(
|
||||||
|
"microvm {vm_id}: local model socket on {} -> {OLLAMA_ADDR}",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
let vm = vm_id.to_string();
|
||||||
|
let task = tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
match listener.accept().await {
|
||||||
|
Ok((s, _)) => {
|
||||||
|
let vm = vm.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
if let Err(e) = pipe(s).await {
|
||||||
|
// Loud, because the failure a mission sees is a turn
|
||||||
|
// that never answers. A refused connection here means
|
||||||
|
// the node's model server is down, and that is worth
|
||||||
|
// saying out loud rather than leaving to a timeout.
|
||||||
|
eprintln!("microvm {vm}: local model pipe failed: {e}");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("microvm {vm}: local model accept failed: {e}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
Ok(Some((path, task)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Splice one guest connection onto a fresh connection to the node's model.
|
||||||
|
async fn pipe(mut guest: tokio::net::UnixStream) -> Result<(), String> {
|
||||||
|
let mut model = TcpStream::connect(OLLAMA_ADDR)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("connect {OLLAMA_ADDR}: {e}"))?;
|
||||||
|
tokio::io::copy_bidirectional(&mut guest, &mut model)
|
||||||
|
.await
|
||||||
|
.map(|_| ())
|
||||||
|
.map_err(|e| format!("copy: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// Only the backends that are meant to have a local model get one.
|
||||||
|
///
|
||||||
|
/// The negative half is the point: an unrecognised backend acquiring a route
|
||||||
|
/// to the node's own model server would be a hole opened by a typo, and it
|
||||||
|
/// would be invisible because the mission would simply work.
|
||||||
|
#[test]
|
||||||
|
fn a_local_route_is_never_granted_by_accident() {
|
||||||
|
assert!(uses_local_model(Some("local-ornith")));
|
||||||
|
|
||||||
|
for other in [
|
||||||
|
None,
|
||||||
|
Some(""),
|
||||||
|
Some("default"),
|
||||||
|
Some("claude"),
|
||||||
|
Some("canary-claude"),
|
||||||
|
Some("glm"),
|
||||||
|
Some("kimi"),
|
||||||
|
Some("local"),
|
||||||
|
Some("local-ornith-typo"),
|
||||||
|
Some("ornith"),
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
!uses_local_model(other),
|
||||||
|
"{other:?} must not reach the node's model server"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The guest cannot name a destination, so there is nothing to validate.
|
||||||
|
///
|
||||||
|
/// This asserts the property that makes this module safe enough to skip the
|
||||||
|
/// allow-list entirely: the upstream address is a constant. If it ever
|
||||||
|
/// becomes a parameter, this file needs everything `egress` has.
|
||||||
|
#[test]
|
||||||
|
fn the_upstream_address_is_a_constant_not_an_input() {
|
||||||
|
let src = include_str!("local_model.rs");
|
||||||
|
// Needles are split so they do not match themselves in this file.
|
||||||
|
assert_eq!(
|
||||||
|
src.matches(concat!("TcpStream", "::connect(")).count(),
|
||||||
|
1,
|
||||||
|
"exactly one dial site, and it must use the constant"
|
||||||
|
);
|
||||||
|
assert!(src.contains(concat!("TcpStream", "::connect(OLLAMA_ADDR)")));
|
||||||
|
assert!(
|
||||||
|
OLLAMA_ADDR.starts_with("127.0.0.1:"),
|
||||||
|
"the model server must be reached on loopback only"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -19,6 +19,9 @@ use sysinfo::{Disks, System};
|
|||||||
use tokio::sync::{mpsc, Mutex};
|
use tokio::sync::{mpsc, Mutex};
|
||||||
use tokio_tungstenite::tungstenite::Message;
|
use tokio_tungstenite::tungstenite::Message;
|
||||||
|
|
||||||
|
mod egress;
|
||||||
|
mod local_model;
|
||||||
|
mod microvm;
|
||||||
mod rtc;
|
mod rtc;
|
||||||
|
|
||||||
const B64: base64::engine::general_purpose::GeneralPurpose =
|
const B64: base64::engine::general_purpose::GeneralPurpose =
|
||||||
@@ -43,6 +46,15 @@ async fn main() {
|
|||||||
selftest();
|
selftest();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
// Exercise the microVM lifecycle against a real VM on this node. Separate
|
||||||
|
// from --selftest because it needs KVM, so it can only pass on a node that
|
||||||
|
// actually reports microvm capability.
|
||||||
|
if std::env::args().any(|a| a == "--vm-selftest") {
|
||||||
|
if !microvm::selftest().await {
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
let (server, token, ts_authkey) = parse_args();
|
let (server, token, ts_authkey) = parse_args();
|
||||||
if server.is_empty() || token.is_empty() {
|
if server.is_empty() || token.is_empty() {
|
||||||
eprintln!("usage: clawmates-node --server <https://gateway> --token <token> [--tailscale-authkey <key>]");
|
eprintln!("usage: clawmates-node --server <https://gateway> --token <token> [--tailscale-authkey <key>]");
|
||||||
@@ -108,6 +120,11 @@ async fn run(ws_url: &str) -> Result<(), Box<dyn std::error::Error>> {
|
|||||||
let (out_tx, mut out_rx) = mpsc::unbounded_channel::<String>();
|
let (out_tx, mut out_rx) = mpsc::unbounded_channel::<String>();
|
||||||
let ptys: Ptys = Arc::new(Mutex::new(HashMap::new()));
|
let ptys: Ptys = Arc::new(Mutex::new(HashMap::new()));
|
||||||
let peers: rtc::RtcPeers = Arc::new(Mutex::new(HashMap::new()));
|
let peers: rtc::RtcPeers = Arc::new(Mutex::new(HashMap::new()));
|
||||||
|
// microVMs this connection started. Scoped to the connection deliberately:
|
||||||
|
// a reconnect must not inherit VMs it cannot prove are still alive, and
|
||||||
|
// `vm_destroy` cleans a workdir by path even for an unregistered id, so a
|
||||||
|
// VM from a previous incarnation is reapable rather than orphaned.
|
||||||
|
let vms = microvm::new_vms();
|
||||||
// Collect heartbeats on a dedicated thread: the metric helpers shell out to
|
// Collect heartbeats on a dedicated thread: the metric helpers shell out to
|
||||||
// docker/tailscale and stat disks (blocking), which must never stall the
|
// docker/tailscale and stat disks (blocking), which must never stall the
|
||||||
// async select loop (or heartbeats/pongs would starve during a slow op).
|
// async select loop (or heartbeats/pongs would starve during a slow op).
|
||||||
@@ -131,6 +148,14 @@ async fn run(ws_url: &str) -> Result<(), Box<dyn std::error::Error>> {
|
|||||||
if tools_tx.send(frame).is_err() {
|
if tools_tx.send(frame).is_err() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
// What this node can HOST, as opposed to what it has installed. The
|
||||||
|
// scheduler needs it to place microVM missions, and the node is the
|
||||||
|
// only honest source: /dev/kvm either exists here or it does not, and
|
||||||
|
// no amount of configuration on the server can make it appear.
|
||||||
|
let caps = json!({ "t": "node_capabilities", "capabilities": probe_capabilities() });
|
||||||
|
if tools_tx.send(caps.to_string()).is_err() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
std::thread::sleep(Duration::from_secs(900));
|
std::thread::sleep(Duration::from_secs(900));
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -180,8 +205,9 @@ async fn run(ws_url: &str) -> Result<(), Box<dyn std::error::Error>> {
|
|||||||
let out = out_tx.clone();
|
let out = out_tx.clone();
|
||||||
let ptys = ptys.clone();
|
let ptys = ptys.clone();
|
||||||
let peers = peers.clone();
|
let peers = peers.clone();
|
||||||
|
let vms = vms.clone();
|
||||||
let text = t.to_string();
|
let text = t.to_string();
|
||||||
tokio::spawn(async move { handle_frame(&text, &out, &ptys, &peers).await; });
|
tokio::spawn(async move { handle_frame(&text, &out, &ptys, &peers, &vms).await; });
|
||||||
}
|
}
|
||||||
Some(Ok(Message::Ping(p))) => {
|
Some(Ok(Message::Ping(p))) => {
|
||||||
match tokio::time::timeout(WRITE_DEADLINE, write.send(Message::Pong(p))).await {
|
match tokio::time::timeout(WRITE_DEADLINE, write.send(Message::Pong(p))).await {
|
||||||
@@ -241,6 +267,70 @@ fn heartbeat(sys: &mut System) -> String {
|
|||||||
/// Probe installed dev-tool versions: for each tool, find its binary across the
|
/// Probe installed dev-tool versions: for each tool, find its binary across the
|
||||||
/// usual bin dirs and read `--version`. Returns `{ tool: "x.y.z", … }` for the
|
/// usual bin dirs and read `--version`. Returns `{ tool: "x.y.z", … }` for the
|
||||||
/// ones found. Probes `kimi-cli` (the real uv tool), not the `kimi` API wrapper.
|
/// ones found. Probes `kimi-cli` (the real uv tool), not the `kimi` API wrapper.
|
||||||
|
/// What this node can HOST — the inputs to placement predicates.
|
||||||
|
///
|
||||||
|
/// Distinct from [`probe_tools`], which reports what is *installed* for the
|
||||||
|
/// operator to see and update. This answers "may the scheduler put a microVM
|
||||||
|
/// mission here", and the answer is a property of the hardware: gw-04 is
|
||||||
|
/// itself a VM without nested virtualisation and has no `/dev/kvm`, so it can
|
||||||
|
/// never host one however it is configured.
|
||||||
|
///
|
||||||
|
/// Every value is probed, never assumed. A capability that is merely expected
|
||||||
|
/// is the same as a capability that is absent, right up until a mission is
|
||||||
|
/// scheduled onto a node that cannot run it.
|
||||||
|
fn probe_capabilities() -> Value {
|
||||||
|
// The device node is necessary but not sufficient — it can exist while
|
||||||
|
// being unopenable (wrong group, or a container without the device
|
||||||
|
// passed through). Try to open it, because that is what firecracker does.
|
||||||
|
let kvm = std::fs::OpenOptions::new()
|
||||||
|
.read(true)
|
||||||
|
.write(true)
|
||||||
|
.open("/dev/kvm")
|
||||||
|
.is_ok();
|
||||||
|
|
||||||
|
let firecracker = std::process::Command::new("firecracker")
|
||||||
|
.arg("--version")
|
||||||
|
.output()
|
||||||
|
.ok()
|
||||||
|
.filter(|o| o.status.success())
|
||||||
|
.and_then(|o| {
|
||||||
|
String::from_utf8_lossy(&o.stdout)
|
||||||
|
.lines()
|
||||||
|
.next()
|
||||||
|
.map(|l| l.trim().to_string())
|
||||||
|
});
|
||||||
|
|
||||||
|
// Which rootfs images are actually on this node's disk. Reported so
|
||||||
|
// placement can require the mission's backend rather than assuming any
|
||||||
|
// KVM-capable node can boot any image — see microvm::available_backends.
|
||||||
|
let backends = microvm::available_backends();
|
||||||
|
capabilities_from(kvm, firecracker.as_deref(), &backends)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Shape the capability report from probe results.
|
||||||
|
///
|
||||||
|
/// Split from [`probe_capabilities`] so the rule can be tested without a
|
||||||
|
/// `/dev/kvm` to open — the machine running the tests is usually the one that
|
||||||
|
/// cannot host a microVM.
|
||||||
|
fn capabilities_from(kvm: bool, firecracker: Option<&str>, backends: &[String]) -> Value {
|
||||||
|
json!({
|
||||||
|
"kvm": kvm,
|
||||||
|
"firecracker": firecracker,
|
||||||
|
// The backends this node can boot. An ARRAY, and empty when there are
|
||||||
|
// none: `set_capabilities` REPLACES, so an image that was deleted stops
|
||||||
|
// being advertised on the next report instead of leaving a stale claim.
|
||||||
|
//
|
||||||
|
// Reported even when `microvm` is false, because it is a fact about the
|
||||||
|
// disk rather than a promise — placement requires both.
|
||||||
|
"rootfs": backends,
|
||||||
|
// BOTH must hold. A node with KVM but no firecracker binary looks
|
||||||
|
// capable by the obvious test and fails at launch; a node with the
|
||||||
|
// binary but no KVM is gw-04. Computed here rather than in the
|
||||||
|
// scheduler so the rule sits next to the probe that feeds it.
|
||||||
|
"microvm": kvm && firecracker.is_some(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn probe_tools() -> Value {
|
fn probe_tools() -> Value {
|
||||||
let home = std::env::var("HOME").unwrap_or_default();
|
let home = std::env::var("HOME").unwrap_or_default();
|
||||||
let dirs = [
|
let dirs = [
|
||||||
@@ -459,6 +549,7 @@ async fn handle_frame(
|
|||||||
out: &mpsc::UnboundedSender<String>,
|
out: &mpsc::UnboundedSender<String>,
|
||||||
ptys: &Ptys,
|
ptys: &Ptys,
|
||||||
peers: &rtc::RtcPeers,
|
peers: &rtc::RtcPeers,
|
||||||
|
vms: µvm::Vms,
|
||||||
) {
|
) {
|
||||||
let Ok(v) = serde_json::from_str::<Value>(text) else {
|
let Ok(v) = serde_json::from_str::<Value>(text) else {
|
||||||
return;
|
return;
|
||||||
@@ -521,6 +612,73 @@ async fn handle_frame(
|
|||||||
// Agent-sandbox container ops: drive the REAL DockerDriver so the
|
// Agent-sandbox container ops: drive the REAL DockerDriver so the
|
||||||
// hardening (cap-drop ALL, seccomp, no-net, read-only, non-root) is
|
// hardening (cap-drop ALL, seccomp, no-net, read-only, non-root) is
|
||||||
// byte-identical to the gateway's local sandboxes.
|
// byte-identical to the gateway's local sandboxes.
|
||||||
|
// microVM ops. Same envelope as every other op, so adding them needed
|
||||||
|
// no protocol change. `vm_create` blocks until the guest agent answers:
|
||||||
|
// a VM that booted but serves nothing is worse than one that failed.
|
||||||
|
op @ ("vm_create" | "vm_inject" | "vm_exec" | "vm_collect" | "vm_destroy" | "vm_list") => {
|
||||||
|
if let Some(id) = v.get("id").and_then(Value::as_u64) {
|
||||||
|
let (op, v, out, vms) = (op.to_string(), v.clone(), out.clone(), vms.clone());
|
||||||
|
// Spawned: a VM boot takes ~1s and an exec can take an hour.
|
||||||
|
// Running it inline would stall heartbeats and the daemon would
|
||||||
|
// be declared offline mid-mission.
|
||||||
|
tokio::spawn(async move {
|
||||||
|
// While an `exec` runs, follow the turn's log and push each
|
||||||
|
// chunk to the server as it appears. The guest agent accepts
|
||||||
|
// concurrent connections (proved against a live VM: a tail
|
||||||
|
// returned data second-by-second while an 8s exec was still
|
||||||
|
// running), so this does not wait for, or delay, the turn.
|
||||||
|
//
|
||||||
|
// Only for `vm_exec`, and only when the caller named a run to
|
||||||
|
// attribute the output to — a probe exec has nothing to
|
||||||
|
// stream and no subscriber.
|
||||||
|
// Set when the turn returns, so the tail can DRAIN before it
|
||||||
|
// stops rather than being cut off mid-flush.
|
||||||
|
let turn_done = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||||
|
let tail = (op == "vm_exec")
|
||||||
|
.then(|| {
|
||||||
|
let run_id = v.get("run_id").and_then(Value::as_str)?.to_string();
|
||||||
|
let log_path = v
|
||||||
|
.get("log_path")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or("/root/agent.log")
|
||||||
|
.to_string();
|
||||||
|
let vm_id = v.get("vm_id").and_then(Value::as_str)?.to_string();
|
||||||
|
Some(tokio::spawn(stream_vm_log(
|
||||||
|
vms.clone(),
|
||||||
|
vm_id,
|
||||||
|
run_id,
|
||||||
|
log_path,
|
||||||
|
out.clone(),
|
||||||
|
turn_done.clone(),
|
||||||
|
)))
|
||||||
|
})
|
||||||
|
.flatten();
|
||||||
|
let (ok, output) = microvm::handle_op(&op, &v, &vms).await;
|
||||||
|
// Let the tail DRAIN, then stop. Aborting here was wrong:
|
||||||
|
// `claude -p | tee` makes stdout a pipe, so the CLI block-
|
||||||
|
// buffers and flushes at EXIT — the most valuable output
|
||||||
|
// arrives in the instant the turn ends. Aborting raced that
|
||||||
|
// flush and lost it. Measured: a solo turn (minutes long) won
|
||||||
|
// the race and streamed 337 bytes; every node of a composed
|
||||||
|
// run (~20s each) lost it and streamed nothing at all.
|
||||||
|
//
|
||||||
|
// Bounded, because a VM that stopped answering must not hold
|
||||||
|
// this task open — the abort remains, as a backstop rather
|
||||||
|
// than the mechanism.
|
||||||
|
if let Some(t) = tail {
|
||||||
|
turn_done.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||||
|
let drained =
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(20), t).await;
|
||||||
|
if drained.is_err() {
|
||||||
|
eprintln!("clawmates-node: tail drain timed out for {op}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let _ = out.send(
|
||||||
|
json!({ "t": "result", "id": id, "ok": ok, "output": output }).to_string(),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
op @ ("sb_provision" | "sb_exec" | "sb_destroy" | "sb_health" | "sb_list") => {
|
op @ ("sb_provision" | "sb_exec" | "sb_destroy" | "sb_health" | "sb_list") => {
|
||||||
if let Some(id) = v.get("id").and_then(Value::as_u64) {
|
if let Some(id) = v.get("id").and_then(Value::as_u64) {
|
||||||
let (ok, output) = sb_op(op, &v).await;
|
let (ok, output) = sb_op(op, &v).await;
|
||||||
@@ -752,6 +910,72 @@ fn spawn_command_pty(argv: &[String], cols: u16, rows: u16) -> Result<PtyParts,
|
|||||||
spawn_pty(c, cols, rows)
|
spawn_pty(c, cols, rows)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Follow a running turn's log inside a VM and push each chunk to the server.
|
||||||
|
///
|
||||||
|
/// The other half of the observability path: the guest tails the file, this
|
||||||
|
/// forwards what it reads over the WebSocket the daemon already holds, and the
|
||||||
|
/// server appends it to the run so the live pane and the Output tab both have it.
|
||||||
|
///
|
||||||
|
/// Reconnects on a dropped tail, resuming from the last offset — following by
|
||||||
|
/// OFFSET rather than holding one socket open forever is what makes that cheap.
|
||||||
|
/// It gives up after a few consecutive failures rather than spinning: by then
|
||||||
|
/// the VM is gone and the turn's own result is the record.
|
||||||
|
async fn stream_vm_log(
|
||||||
|
vms: microvm::Vms,
|
||||||
|
vm_id: String,
|
||||||
|
run_id: String,
|
||||||
|
log_path: String,
|
||||||
|
out: tokio::sync::mpsc::UnboundedSender<String>,
|
||||||
|
turn_done: std::sync::Arc<std::sync::atomic::AtomicBool>,
|
||||||
|
) {
|
||||||
|
// Said out loud at the start, because the failure this replaced was
|
||||||
|
// invisible: the tail gave up during VM boot and logged nothing, so an empty
|
||||||
|
// Live tab looked identical to a feature that was never wired.
|
||||||
|
eprintln!("clawmates-node: following {log_path} in {vm_id} for run {run_id}");
|
||||||
|
let mut at: u64 = 0;
|
||||||
|
let mut failures = 0;
|
||||||
|
while failures < 3 {
|
||||||
|
let at_before = at;
|
||||||
|
let sent = out.clone();
|
||||||
|
let rid = run_id.clone();
|
||||||
|
match microvm::tail_into(&vms, &vm_id, &log_path, at, move |offset, data| {
|
||||||
|
let _ = sent.send(
|
||||||
|
json!({ "t": "vm_out", "run_id": rid, "at": offset, "data": data }).to_string(),
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(reached) => {
|
||||||
|
// NO PROGRESS IS NOT THE END. The guest reports EOF whenever the
|
||||||
|
// file has been idle, and the first idle window is always the one
|
||||||
|
// before the turn writes anything — the VM is still booting and
|
||||||
|
// the CLI still starting. Returning here meant the tail gave up
|
||||||
|
// seconds into every run, before a single byte existed. Measured:
|
||||||
|
// a turn that streamed nothing at all.
|
||||||
|
//
|
||||||
|
// The caller aborts this task when the exec returns, so "keep
|
||||||
|
// waiting" cannot outlive the turn; the abort is the terminator,
|
||||||
|
// not a guess about idleness.
|
||||||
|
at = reached;
|
||||||
|
failures = 0;
|
||||||
|
// The turn has returned AND this pass read nothing new: the
|
||||||
|
// final flush is already in hand, so stop. Checked after a read,
|
||||||
|
// never before one — exiting on the flag alone would drop
|
||||||
|
// exactly the bytes this exists to capture.
|
||||||
|
if turn_done.load(std::sync::atomic::Ordering::Relaxed) && reached == at_before {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
failures += 1;
|
||||||
|
eprintln!("clawmates-node: tail of {vm_id} for run {run_id} failed: {e}");
|
||||||
|
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Spawn a host login shell in a PTY; stream its output back as pty_out frames.
|
/// Spawn a host login shell in a PTY; stream its output back as pty_out frames.
|
||||||
async fn open_pty(
|
async fn open_pty(
|
||||||
sid: u64,
|
sid: u64,
|
||||||
@@ -1274,3 +1498,61 @@ fn ensure_tmux() {
|
|||||||
eprintln!("tmux not found (auto-install unavailable) — host terminal will use a plain shell; `apt install tmux` for resumable sessions");
|
eprintln!("tmux not found (auto-install unavailable) — host terminal will use a plain shell; `apt install tmux` for resumable sessions");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod capability_tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn microvm_needs_both_kvm_and_firecracker() {
|
||||||
|
assert_eq!(
|
||||||
|
capabilities_from(true, Some("Firecracker v1.16.1"), &[])["microvm"],
|
||||||
|
json!(true)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
capabilities_from(true, None, &[])["microvm"],
|
||||||
|
json!(false),
|
||||||
|
"KVM without firecracker cannot host a microVM"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
capabilities_from(false, Some("Firecracker v1.16.1"), &[])["microvm"],
|
||||||
|
json!(false),
|
||||||
|
"firecracker without KVM is gw-04 — it can never host one"
|
||||||
|
);
|
||||||
|
assert_eq!(capabilities_from(false, None, &[])["microvm"], json!(false));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The report replaces rather than merges server-side, so a node that has
|
||||||
|
/// LOST a capability must say so rather than omitting the key — an absent
|
||||||
|
/// key and a false one must not be distinguishable to the predicate.
|
||||||
|
#[test]
|
||||||
|
fn a_lost_capability_is_reported_false_not_omitted() {
|
||||||
|
let caps = capabilities_from(false, None, &[]);
|
||||||
|
assert!(caps.get("kvm").is_some(), "kvm must always be present");
|
||||||
|
assert!(
|
||||||
|
caps.get("microvm").is_some(),
|
||||||
|
"microvm must always be present"
|
||||||
|
);
|
||||||
|
// Same reasoning for the image list: a node that deleted its last rootfs
|
||||||
|
// must report an empty ARRAY, not omit the key. Placement asks "does this
|
||||||
|
// node have backend X"; against a missing key that question has no
|
||||||
|
// answer, and a scheduler with no answer picks something.
|
||||||
|
assert_eq!(
|
||||||
|
caps.get("rootfs"),
|
||||||
|
Some(&json!([])),
|
||||||
|
"rootfs must always be present, empty when there are no images"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The list is what placement matches a mission's `backend` against, so it
|
||||||
|
/// must carry the names verbatim.
|
||||||
|
#[test]
|
||||||
|
fn reported_backends_are_the_names_placement_will_ask_for() {
|
||||||
|
let caps = capabilities_from(
|
||||||
|
true,
|
||||||
|
Some("Firecracker v1.16.1"),
|
||||||
|
&["claude".to_string(), "default".to_string()],
|
||||||
|
);
|
||||||
|
assert_eq!(caps["rootfs"], json!(["claude", "default"]));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -24,11 +24,32 @@ async fn main() -> ExitCode {
|
|||||||
|
|
||||||
/// Instantiates the configured LLM provider. The Anthropic key comes from
|
/// Instantiates the configured LLM provider. The Anthropic key comes from
|
||||||
/// the environment until the secret broker lands in P2.
|
/// the environment until the secret broker lands in P2.
|
||||||
|
///
|
||||||
|
/// The **subscription wins** when both credentials are present. This is the
|
||||||
|
/// structural half of the fix that `cm_api::subscription` does per-call: a bare
|
||||||
|
/// model name resolves to whatever this function returns, so making that the
|
||||||
|
/// subscription means no server-side call can reach the metered key by
|
||||||
|
/// accident — by construction, rather than by a source-grep test that has
|
||||||
|
/// already missed four call sites once. The metered key stays usable as a
|
||||||
|
/// fallback for deployments that have credit; ours does not, which is what
|
||||||
|
/// made the ordering matter.
|
||||||
fn build_provider(config: &AppConfig) -> Result<Arc<dyn LlmProvider>, String> {
|
fn build_provider(config: &AppConfig) -> Result<Arc<dyn LlmProvider>, String> {
|
||||||
match config.llm.provider {
|
match config.llm.provider {
|
||||||
LlmProviderKind::Anthropic => {
|
LlmProviderKind::Anthropic => {
|
||||||
let key = std::env::var("ANTHROPIC_API_KEY")
|
if let Some(provider) = cm_api::subscription::provider() {
|
||||||
.map_err(|_| "llm.provider = \"anthropic\" requires ANTHROPIC_API_KEY")?;
|
println!(
|
||||||
|
"clawmates-server: default LLM provider = Claude Code subscription \
|
||||||
|
(bare model names bill no metered key)"
|
||||||
|
);
|
||||||
|
return Ok(Arc::new(provider));
|
||||||
|
}
|
||||||
|
let key = std::env::var("ANTHROPIC_API_KEY").map_err(|_| {
|
||||||
|
"llm.provider = \"anthropic\" needs a credential: either \
|
||||||
|
ANTHROPIC_OAUTH_TOKEN / CLAUDE_CODE_OAUTH_TOKEN (sk-ant-oat…, \
|
||||||
|
the Claude Code subscription, preferred) or ANTHROPIC_API_KEY \
|
||||||
|
(sk-ant-api…, metered)"
|
||||||
|
.to_string()
|
||||||
|
})?;
|
||||||
// A subscription OAuth token pasted where an API key belongs
|
// A subscription OAuth token pasted where an API key belongs
|
||||||
// authenticates nothing here and fails on the first model call,
|
// authenticates nothing here and fails on the first model call,
|
||||||
// far from the mistake. Both start `sk-ant-`, so the confusion is
|
// far from the mistake. Both start `sk-ant-`, so the confusion is
|
||||||
@@ -40,6 +61,11 @@ fn build_provider(config: &AppConfig) -> Result<Arc<dyn LlmProvider>, String> {
|
|||||||
bearer auth and is what the phase evaluator reads."
|
bearer auth and is what the phase evaluator reads."
|
||||||
.to_string());
|
.to_string());
|
||||||
}
|
}
|
||||||
|
eprintln!(
|
||||||
|
"clawmates-server: WARNING — no subscription token; the default LLM \
|
||||||
|
provider is the METERED ANTHROPIC_API_KEY and every bare model name \
|
||||||
|
bills it"
|
||||||
|
);
|
||||||
Ok(Arc::new(AnthropicProvider::new(key)))
|
Ok(Arc::new(AnthropicProvider::new(key)))
|
||||||
}
|
}
|
||||||
LlmProviderKind::OpenAiCompat => {
|
LlmProviderKind::OpenAiCompat => {
|
||||||
@@ -69,8 +95,21 @@ fn build_provider(config: &AppConfig) -> Result<Arc<dyn LlmProvider>, String> {
|
|||||||
fn build_provider_registry(config: &AppConfig) -> cm_runtime::ProviderRegistry {
|
fn build_provider_registry(config: &AppConfig) -> cm_runtime::ProviderRegistry {
|
||||||
let mut map = std::collections::HashMap::new();
|
let mut map = std::collections::HashMap::new();
|
||||||
for p in &config.llm.providers {
|
for p in &config.llm.providers {
|
||||||
match std::env::var(&p.api_key_env) {
|
// A provider may legitimately need no key. A model running on our own
|
||||||
Ok(key) if !key.is_empty() => {
|
// hardware has nothing to authenticate to, and requiring a variable
|
||||||
|
// whose value is ignored is a step that can only ever fail — silently,
|
||||||
|
// since an unset key SKIPS the provider and the first symptom is a
|
||||||
|
// fallback chain quietly one link shorter than it reads.
|
||||||
|
let key = match std::env::var(&p.api_key_env) {
|
||||||
|
Ok(k) if !k.is_empty() => Ok(k),
|
||||||
|
other if p.api_key_env.trim().is_empty() => {
|
||||||
|
let _ = other;
|
||||||
|
Ok(String::new())
|
||||||
|
}
|
||||||
|
other => other,
|
||||||
|
};
|
||||||
|
match key {
|
||||||
|
Ok(key) if !key.is_empty() || p.api_key_env.trim().is_empty() => {
|
||||||
let provider: Arc<dyn LlmProvider> = match p.format.as_str() {
|
let provider: Arc<dyn LlmProvider> = match p.format.as_str() {
|
||||||
"anthropic" => Arc::new(cm_llm::AnthropicProvider::with_base_url(
|
"anthropic" => Arc::new(cm_llm::AnthropicProvider::with_base_url(
|
||||||
key,
|
key,
|
||||||
@@ -280,6 +319,9 @@ async fn run() -> Result<(), String> {
|
|||||||
cm_api::topology_worker::spawn(
|
cm_api::topology_worker::spawn(
|
||||||
pool.clone(),
|
pool.clone(),
|
||||||
runtime.clone(),
|
runtime.clone(),
|
||||||
|
// The composed tier (`microvm_graph`) runs each graph node as a VM on a
|
||||||
|
// fleet node, so the worker needs the same hub the phase runner uses.
|
||||||
|
node_hub.clone(),
|
||||||
std::time::Duration::from_secs(3),
|
std::time::Duration::from_secs(3),
|
||||||
);
|
);
|
||||||
// Boot-time content loaders — skills first, then team templates
|
// Boot-time content loaders — skills first, then team templates
|
||||||
@@ -329,7 +371,16 @@ async fn run() -> Result<(), String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
cm_api::phase_runner::spawn(pool.clone(), runtime.clone());
|
// The other half of runtime_preflight's question: the runtime has the TOOLS,
|
||||||
|
// but can the independent JUDGE be reached? A dead validator makes every
|
||||||
|
// done_when phase unmeetable, and without this the first symptom is a
|
||||||
|
// mission failing after its VMs have already run.
|
||||||
|
cm_api::validator_preflight::report_at_boot(runtime.clone());
|
||||||
|
// Every link of the model fallback chain, probed through the real call path.
|
||||||
|
// A chain is the one piece of infrastructure nobody looks at until the day it
|
||||||
|
// has to work, so it is checked on the days it does not.
|
||||||
|
cm_api::subscription::report_at_boot(runtime.clone());
|
||||||
|
cm_api::phase_runner::spawn(pool.clone(), runtime.clone(), node_hub.clone());
|
||||||
// Per-mission runtime container sweeper (C3): tears down mission
|
// Per-mission runtime container sweeper (C3): tears down mission
|
||||||
// runtime containers 30 min after the mission reaches a terminal
|
// runtime containers 30 min after the mission reaches a terminal
|
||||||
// state so operators have a window to pull final artifacts.
|
// state so operators have a window to pull final artifacts.
|
||||||
@@ -337,13 +388,7 @@ async fn run() -> Result<(), String> {
|
|||||||
// Phase completion summarizer: reads terminal-state phases and
|
// Phase completion summarizer: reads terminal-state phases and
|
||||||
// asks Claude Opus 4.8 to synthesize a "what got done" card that
|
// asks Claude Opus 4.8 to synthesize a "what got done" card that
|
||||||
// the UI renders under the phase.
|
// the UI renders under the phase.
|
||||||
cm_api::phase_summarizer::spawn(pool.clone());
|
cm_api::phase_summarizer::spawn(pool.clone(), runtime.clone());
|
||||||
// PDF renderer worker (Slice 6): watches mission_artifacts for
|
|
||||||
// MD entries with render_pdf_status='pending', calls the
|
|
||||||
// configured LLM (default Gemini 2.5 Flash) for styled HTML,
|
|
||||||
// prints to PDF via chromium --headless. No-op-friendly when
|
|
||||||
// GEMINI_API_KEY / chromium binary aren't configured.
|
|
||||||
cm_api::pdf_renderer::spawn(pool.clone());
|
|
||||||
// Outbound-email delivery: drains the §15-gated `outbox` over SMTP. Inert
|
// Outbound-email delivery: drains the §15-gated `outbox` over SMTP. Inert
|
||||||
// until CLAWMATES_SMTP_* is set, so it ships safely before credentials exist.
|
// until CLAWMATES_SMTP_* is set, so it ships safely before credentials exist.
|
||||||
cm_runtime::spawn_drainer(pool.clone(), std::time::Duration::from_secs(10));
|
cm_runtime::spawn_drainer(pool.clone(), std::time::Duration::from_secs(10));
|
||||||
@@ -352,6 +397,10 @@ async fn run() -> Result<(), String> {
|
|||||||
// Expiry/retention sweep: expires stale auth/oauth rows and prunes old
|
// Expiry/retention sweep: expires stale auth/oauth rows and prunes old
|
||||||
// journal/audit rows hourly so unbounded tables don't accumulate.
|
// journal/audit rows hourly so unbounded tables don't accumulate.
|
||||||
cm_api::cleanup_sweeper::spawn(pool.clone(), std::time::Duration::from_secs(3600));
|
cm_api::cleanup_sweeper::spawn(pool.clone(), std::time::Duration::from_secs(3600));
|
||||||
|
// Its filesystem counterpart. `cleanup_sweeper` prunes ROWS, and deleting a
|
||||||
|
// row has never deleted a directory — which is why the gateway, the smallest
|
||||||
|
// disk in the fleet, accumulates mission trees that nothing reclaims.
|
||||||
|
cm_api::mission_gc::spawn(pool.clone(), std::time::Duration::from_secs(3600));
|
||||||
// Fleet backstop: a node whose heartbeats stop (without a clean channel
|
// Fleet backstop: a node whose heartbeats stop (without a clean channel
|
||||||
// close) goes offline within ~28s even if its control channel hangs.
|
// close) goes offline within ~28s even if its control channel hangs.
|
||||||
cm_api::fleet::spawn_node_sweeper(pool.clone(), std::time::Duration::from_secs(8), 20);
|
cm_api::fleet::spawn_node_sweeper(pool.clone(), std::time::Duration::from_secs(8), 20);
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
[package]
|
||||||
|
name = "fcagent"
|
||||||
|
version = "0.1.0"
|
||||||
|
edition.workspace = true
|
||||||
|
rust-version.workspace = true
|
||||||
|
license.workspace = true
|
||||||
|
publish.workspace = true
|
||||||
|
|
||||||
|
[[bin]]
|
||||||
|
name = "fcagent"
|
||||||
|
path = "src/main.rs"
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
# std has no AF_VSOCK, and the workspace denies `unsafe`, so raw libc is not an
|
||||||
|
# option. This is a safe wrapper over the socket calls.
|
||||||
|
vsock = "0.5"
|
||||||
|
serde_json = { workspace = true }
|
||||||
|
tar = { workspace = true }
|
||||||
|
base64 = "0.22"
|
||||||
|
|
||||||
|
# NOTE: a `[profile.release]` here would be silently ignored — cargo only honours
|
||||||
|
# profiles at the workspace root. The binary is small enough on the default
|
||||||
|
# release profile (~1 MB static) that overriding the whole workspace's profile to
|
||||||
|
# shave it would be a bad trade.
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
@@ -0,0 +1,988 @@
|
|||||||
|
//! ClawMates microVM guest agent — pid 1 inside a Firecracker microVM.
|
||||||
|
//!
|
||||||
|
//! Runs as `init=/usr/local/bin/fcagent`'s exec target and answers the host over
|
||||||
|
//! **vsock** (port 9001), never the serial console: feeding a guest over stdin
|
||||||
|
//! races its startup and arrives half-consumed. The console stays a log.
|
||||||
|
//!
|
||||||
|
//! # Why this is a static Rust binary and not the python script it replaces
|
||||||
|
//!
|
||||||
|
//! The python version worked only because Firecracker's CI Ubuntu image happens
|
||||||
|
//! to ship python3. **None of our own images do** — `agent-base` has neither
|
||||||
|
//! python nor git, `agent-terminal` has git but no python — so the agent could
|
||||||
|
//! never have run in a real mission rootfs. An agent that dictates what must be
|
||||||
|
//! installed in the image has the dependency backwards. This is a
|
||||||
|
//! `x86_64-unknown-linux-musl` static binary: it needs nothing from the rootfs
|
||||||
|
//! it is dropped into.
|
||||||
|
//!
|
||||||
|
//! # Wire protocol (unchanged from the python agent, deliberately)
|
||||||
|
//!
|
||||||
|
//! One request per connection: a 4-byte big-endian length followed by JSON, and
|
||||||
|
//! the reply framed the same way. The length prefix is the point — a reply
|
||||||
|
//! larger than a socket buffer arrives in pieces, and reading "whatever was
|
||||||
|
//! available" would parse a truncated object as a complete one.
|
||||||
|
//!
|
||||||
|
//! Ops: `ping`, `exec`, `put`, `get`. `crates/bins/clawmates-node/src/microvm.rs`
|
||||||
|
//! and `crates/cm-api/src/microvm_client.rs` speak this and needed no change.
|
||||||
|
|
||||||
|
use std::io::{Read, Write};
|
||||||
|
use std::net::TcpListener;
|
||||||
|
use std::os::unix::process::CommandExt;
|
||||||
|
use std::path::Path;
|
||||||
|
use std::process::{Command, Stdio};
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use base64::Engine;
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
|
||||||
|
const PORT: u32 = 9001;
|
||||||
|
/// Guest-side egress proxy. The VM has **no network interface at all** — see
|
||||||
|
/// `microvm.rs`, whose machine config declares no `network-interfaces` — so an
|
||||||
|
/// agent CLI cannot reach the model API on its own. It reaches it by honouring
|
||||||
|
/// `HTTPS_PROXY`, which is measured, not assumed: with the proxy pointed at a
|
||||||
|
/// closed port, `claude -p` fails with `ConnectionRefused` instead of answering.
|
||||||
|
///
|
||||||
|
/// This listener is a dumb byte pump. It parses nothing and enforces nothing:
|
||||||
|
/// the `CONNECT` request travels verbatim to the host, which speaks HTTP CONNECT
|
||||||
|
/// and owns the allow-list. Keeping policy on the host means nothing running in
|
||||||
|
/// the guest — including a compromised agent — can talk it into a different
|
||||||
|
/// answer.
|
||||||
|
const PROXY_PORT: u16 = 3128;
|
||||||
|
/// Host-side vsock port the tunnel lands on. Firecracker's convention for a
|
||||||
|
/// guest-initiated connection is that the HOST listens on `<uds_path>_<port>`.
|
||||||
|
const EGRESS_PORT: u32 = 9002;
|
||||||
|
/// Guest-side port for a LOCALLY HOSTED model, and the vsock port it lands on.
|
||||||
|
///
|
||||||
|
/// Separate from the egress proxy on purpose, and simpler than it. The egress
|
||||||
|
/// path exists to let an agent reach the public internet under an allow-list;
|
||||||
|
/// this one reaches exactly one thing — the Ollama the node itself is running,
|
||||||
|
/// on its own loopback — and can reach nothing else, because the host end is a
|
||||||
|
/// pipe to a fixed address rather than a proxy that takes a destination.
|
||||||
|
///
|
||||||
|
/// It therefore needs no `CONNECT`, no TLS and no allow-list. The bytes travel
|
||||||
|
/// guest loopback → vsock → host loopback and never touch a network, so there is
|
||||||
|
/// nothing on a wire for TLS to protect. `NO_PROXY` already contains
|
||||||
|
/// `127.0.0.1`, so an agent pointed at `http://127.0.0.1:11434` bypasses the
|
||||||
|
/// egress proxy entirely rather than trying to CONNECT through it.
|
||||||
|
///
|
||||||
|
/// The guest always listens. Whether anything answers is the HOST's decision:
|
||||||
|
/// the node only binds the vsock end for a backend that is meant to have a
|
||||||
|
/// local model, so on every other backend this port simply refuses.
|
||||||
|
const MODEL_PORT: u16 = 11434;
|
||||||
|
const MODEL_VSOCK_PORT: u32 = 9003;
|
||||||
|
/// `VMADDR_CID_HOST` — the hypervisor side of the vsock.
|
||||||
|
const HOST_CID: u32 = 2;
|
||||||
|
|
||||||
|
/// Whether the egress proxy is actually listening. Reported by `ping` so the
|
||||||
|
/// host can refuse to hand a mission to a VM with no way out, rather than
|
||||||
|
/// discovering it as an agent that hangs.
|
||||||
|
static PROXY_UP: AtomicBool = AtomicBool::new(false);
|
||||||
|
/// Cap on a single request. A hostile or broken host must not be able to make
|
||||||
|
/// pid 1 allocate without bound and get the VM OOM-killed.
|
||||||
|
const MAX_REQUEST: u32 = 512 * 1024 * 1024;
|
||||||
|
|
||||||
|
const B64: base64::engine::general_purpose::GeneralPurpose =
|
||||||
|
base64::engine::general_purpose::STANDARD;
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
// The mounts the init script would otherwise do. Done here so the agent
|
||||||
|
// works whether it is exec'd from a shell init or used as `init=` directly:
|
||||||
|
// /proc missing makes every process-inspecting tool in the guest lie.
|
||||||
|
for (fstype, target) in [
|
||||||
|
("proc", "/proc"),
|
||||||
|
("sysfs", "/sys"),
|
||||||
|
("devtmpfs", "/dev"),
|
||||||
|
("tmpfs", "/tmp"),
|
||||||
|
] {
|
||||||
|
if !Path::new(target).join(".").exists() {
|
||||||
|
let _ = std::fs::create_dir_all(target);
|
||||||
|
}
|
||||||
|
let _ = Command::new("mount")
|
||||||
|
.args(["-t", fstype, fstype, target])
|
||||||
|
.status();
|
||||||
|
}
|
||||||
|
|
||||||
|
start_egress_proxy();
|
||||||
|
|
||||||
|
let listener = match vsock::VsockListener::bind_with_cid_port(libc_vmaddr_cid_any(), PORT) {
|
||||||
|
Ok(l) => l,
|
||||||
|
Err(e) => {
|
||||||
|
// Printed to the console, which is where the host's boot check
|
||||||
|
// looks. Exiting pid 1 panics the kernel, which is the honest
|
||||||
|
// outcome: a VM whose agent cannot listen is unusable, and it must
|
||||||
|
// not sit there looking booted.
|
||||||
|
eprintln!("FC-AGENT-FATAL could not bind vsock port {PORT}: {e}");
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// The host greps the console for this before it tries to connect.
|
||||||
|
println!("FC-AGENT-LISTENING port={PORT}");
|
||||||
|
let _ = std::io::stdout().flush();
|
||||||
|
|
||||||
|
for conn in listener.incoming() {
|
||||||
|
match conn {
|
||||||
|
Ok(mut s) => {
|
||||||
|
// One THREAD per connection, not one at a time.
|
||||||
|
//
|
||||||
|
// This loop used to call `serve_one` inline, which meant the
|
||||||
|
// agent accepted nothing while an op was running. A mission turn
|
||||||
|
// is an `exec` that can last an hour, so for that hour the guest
|
||||||
|
// was unreachable: the host could not tail its output, probe it,
|
||||||
|
// or ask it anything. Every existing probe runs AFTER the turn
|
||||||
|
// for exactly this reason.
|
||||||
|
//
|
||||||
|
// A thread rather than async: this is a static musl binary with
|
||||||
|
// no runtime, and the concurrency here is a handful of
|
||||||
|
// connections, not thousands.
|
||||||
|
//
|
||||||
|
// The panic discipline of the old inline call still applies, and
|
||||||
|
// matters MORE now — this process is pid 1, and a panic that
|
||||||
|
// unwound out of a worker used to take the accept loop with it.
|
||||||
|
// `catch_unwind` keeps a bad request from killing the VM.
|
||||||
|
std::thread::Builder::new()
|
||||||
|
.name("fcagent-conn".into())
|
||||||
|
.spawn(move || {
|
||||||
|
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||||
|
serve_one(&mut s)
|
||||||
|
}));
|
||||||
|
match r {
|
||||||
|
Ok(Err(e)) => eprintln!("FC-AGENT-ERROR {e}"),
|
||||||
|
Err(_) => eprintln!("FC-AGENT-ERROR handler panicked"),
|
||||||
|
Ok(Ok(())) => {}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.map(|_| ())
|
||||||
|
.unwrap_or_else(|e| {
|
||||||
|
// Out of threads: answer nothing on this connection, but
|
||||||
|
// keep accepting. Dropping the listener would brick the VM.
|
||||||
|
eprintln!("FC-AGENT-ERROR spawn: {e}");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(e) => eprintln!("FC-AGENT-ERROR accept: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `VMADDR_CID_ANY` — bind for any host CID.
|
||||||
|
fn libc_vmaddr_cid_any() -> u32 {
|
||||||
|
u32::MAX
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bring up loopback and start the egress tunnel.
|
||||||
|
///
|
||||||
|
/// Loopback is not optional and not free: the guest's `lo` exists but starts
|
||||||
|
/// **down**, and while it is down a listener on 127.0.0.1 *binds successfully*
|
||||||
|
/// and then refuses every connection with `ENETUNREACH`. A bind-only check would
|
||||||
|
/// have reported a working proxy. So `lo` goes up first, via `ip` — which is why
|
||||||
|
/// `iproute2` is in the agent images.
|
||||||
|
///
|
||||||
|
/// Failure here is recorded, not fatal: exec still works, so a VM is still
|
||||||
|
/// useful for work that needs no network. It is reported through `ping` so the
|
||||||
|
/// host can decide, instead of a mission discovering it as an agent that hangs.
|
||||||
|
fn start_egress_proxy() {
|
||||||
|
// Absolute paths, not `Command::new("ip")`. This process is pid 1, so its
|
||||||
|
// PATH is whatever the kernel handed it — and when PATH is unset, `execvp`
|
||||||
|
// falls back to a default that does NOT include `/usr/sbin`, which is exactly
|
||||||
|
// where Debian puts `ip`. Searching by name would fail on an image that has
|
||||||
|
// it, and the symptom would be a VM with no egress and no explanation.
|
||||||
|
const IP_CANDIDATES: &[&str] = &["/usr/sbin/ip", "/sbin/ip", "/usr/bin/ip", "/bin/ip"];
|
||||||
|
let Some(ip) = IP_CANDIDATES.iter().find(|p| Path::new(p).exists()) else {
|
||||||
|
eprintln!(
|
||||||
|
"FC-AGENT-NO-PROXY no `ip` binary in {IP_CANDIDATES:?} — no egress; \
|
||||||
|
add iproute2 to this image"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
match Command::new(ip).args(["link", "set", "lo", "up"]).status() {
|
||||||
|
Ok(s) if s.success() => {}
|
||||||
|
other => {
|
||||||
|
eprintln!("FC-AGENT-NO-PROXY `{ip} link set lo up` failed ({other:?}) — no egress");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let listener = match TcpListener::bind(("127.0.0.1", PROXY_PORT)) {
|
||||||
|
Ok(l) => l,
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("FC-AGENT-NO-PROXY could not listen on 127.0.0.1:{PROXY_PORT}: {e}");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
PROXY_UP.store(true, Ordering::Relaxed);
|
||||||
|
println!("FC-AGENT-PROXY listening on 127.0.0.1:{PROXY_PORT} -> vsock {EGRESS_PORT}");
|
||||||
|
let _ = std::io::stdout().flush();
|
||||||
|
pump(listener, EGRESS_PORT, "PROXY");
|
||||||
|
|
||||||
|
// The local-model port. Failure to bind is reported and non-fatal, exactly
|
||||||
|
// like the egress proxy: a VM whose backend does not use a local model is
|
||||||
|
// still perfectly useful, and a fatal error here would take out every
|
||||||
|
// backend to serve one.
|
||||||
|
match TcpListener::bind(("127.0.0.1", MODEL_PORT)) {
|
||||||
|
Ok(l) => {
|
||||||
|
println!("FC-AGENT-MODEL listening on 127.0.0.1:{MODEL_PORT} -> vsock {MODEL_VSOCK_PORT}");
|
||||||
|
let _ = std::io::stdout().flush();
|
||||||
|
pump(l, MODEL_VSOCK_PORT, "MODEL");
|
||||||
|
}
|
||||||
|
Err(e) => eprintln!("FC-AGENT-NO-MODEL could not listen on 127.0.0.1:{MODEL_PORT}: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Accept forever, splicing each connection onto its own vsock stream.
|
||||||
|
fn pump(listener: TcpListener, vsock_port: u32, tag: &'static str) {
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
for c in listener.incoming() {
|
||||||
|
match c {
|
||||||
|
// One thread per connection. An agent CLI opens several at once,
|
||||||
|
// and serving them in sequence would look like a hang.
|
||||||
|
Ok(tcp) => {
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
if let Err(e) = tunnel(tcp, vsock_port) {
|
||||||
|
eprintln!("FC-AGENT-{tag}-ERROR {e}");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(e) => eprintln!("FC-AGENT-{tag}-ERROR accept: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Splice one TCP connection onto a fresh vsock connection to the host.
|
||||||
|
///
|
||||||
|
/// No parsing: whatever the client sent — `CONNECT host:443`, or an absolute-form
|
||||||
|
/// request — is the host's business. The host answers with real HTTP, so a
|
||||||
|
/// refusal reaches the client as a status code rather than a dropped socket.
|
||||||
|
fn tunnel(tcp: std::net::TcpStream, vsock_port: u32) -> Result<(), String> {
|
||||||
|
let vs = vsock::VsockStream::connect_with_cid_port(HOST_CID, vsock_port)
|
||||||
|
.map_err(|e| format!("vsock connect to host:{vsock_port}: {e}"))?;
|
||||||
|
|
||||||
|
let (mut tcp_r, mut tcp_w) = (
|
||||||
|
tcp.try_clone().map_err(|e| format!("clone tcp: {e}"))?,
|
||||||
|
tcp,
|
||||||
|
);
|
||||||
|
let (mut vs_r, mut vs_w) = (
|
||||||
|
vs.try_clone().map_err(|e| format!("clone vsock: {e}"))?,
|
||||||
|
vs,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Each direction gets its own thread, and each shuts its peer's write side
|
||||||
|
// down when it ends. Without the shutdown the other half blocks forever on a
|
||||||
|
// half-closed connection and the CLI waits out its own timeout.
|
||||||
|
let up = std::thread::spawn(move || {
|
||||||
|
let _ = std::io::copy(&mut tcp_r, &mut vs_w);
|
||||||
|
let _ = vs_w.shutdown(std::net::Shutdown::Write);
|
||||||
|
});
|
||||||
|
let _ = std::io::copy(&mut vs_r, &mut tcp_w);
|
||||||
|
let _ = tcp_w.shutdown(std::net::Shutdown::Write);
|
||||||
|
let _ = up.join();
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn serve_one(s: &mut vsock::VsockStream) -> Result<(), String> {
|
||||||
|
let mut len = [0u8; 4];
|
||||||
|
s.read_exact(&mut len)
|
||||||
|
.map_err(|e| format!("read length: {e}"))?;
|
||||||
|
let len = u32::from_be_bytes(len);
|
||||||
|
if len > MAX_REQUEST {
|
||||||
|
// Answer rather than hang up: a caller that sent something absurd needs
|
||||||
|
// to be told, not left waiting for a reply that will never come.
|
||||||
|
return reply(s, &json!({ "ok": false, "error": format!("request of {len} bytes exceeds the {MAX_REQUEST} cap") }));
|
||||||
|
}
|
||||||
|
let mut buf = vec![0u8; len as usize];
|
||||||
|
s.read_exact(&mut buf)
|
||||||
|
.map_err(|e| format!("read body: {e}"))?;
|
||||||
|
|
||||||
|
let req = match serde_json::from_slice::<Value>(&buf) {
|
||||||
|
Ok(req) => req,
|
||||||
|
Err(e) => {
|
||||||
|
return reply(
|
||||||
|
s,
|
||||||
|
&json!({ "ok": false, "error": format!("undecodable request: {e}") }),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
// `tail` owns the connection for its lifetime, emitting a frame per chunk,
|
||||||
|
// so it cannot go through `handle`, which returns one Value.
|
||||||
|
if req.get("op").and_then(Value::as_str) == Some("tail") {
|
||||||
|
return op_tail(s, &req);
|
||||||
|
}
|
||||||
|
let resp = handle(&req);
|
||||||
|
reply(s, &resp)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stream a file to the host as it grows, one framed JSON chunk at a time.
|
||||||
|
///
|
||||||
|
/// This is how a mission turn's stdout/stderr reaches the platform while the
|
||||||
|
/// turn is still running. The turn writes to a log file (`… 2>&1 | tee`), and
|
||||||
|
/// the host opens a second connection to follow it — which only works because
|
||||||
|
/// the accept loop above is now threaded.
|
||||||
|
///
|
||||||
|
/// `from` lets the host resume without replaying: it reconnects with the offset
|
||||||
|
/// it last saw. Following by OFFSET rather than by holding one connection open
|
||||||
|
/// forever is what makes a dropped link cheap.
|
||||||
|
///
|
||||||
|
/// Ends when the file stops growing for `idle_ms`, or at `max_secs`. It must
|
||||||
|
/// end: a tail that never returns pins a thread for the life of the VM.
|
||||||
|
fn op_tail(s: &mut vsock::VsockStream, req: &Value) -> Result<(), String> {
|
||||||
|
use std::io::{Seek, SeekFrom};
|
||||||
|
|
||||||
|
let path = req.get("path").and_then(Value::as_str).unwrap_or_default();
|
||||||
|
let mut from = req.get("from").and_then(Value::as_u64).unwrap_or(0);
|
||||||
|
let idle_ms = req.get("idle_ms").and_then(Value::as_u64).unwrap_or(2_000);
|
||||||
|
let max_secs = req.get("max_secs").and_then(Value::as_u64).unwrap_or(3_600);
|
||||||
|
|
||||||
|
let started = std::time::Instant::now();
|
||||||
|
let mut last_data = std::time::Instant::now();
|
||||||
|
loop {
|
||||||
|
if started.elapsed().as_secs() >= max_secs {
|
||||||
|
return reply(s, &json!({ "ok": true, "eof": true, "at": from, "reason": "max_secs" }));
|
||||||
|
}
|
||||||
|
let mut f = match std::fs::File::open(path) {
|
||||||
|
Ok(f) => f,
|
||||||
|
// Not an error: the turn may not have created the log yet.
|
||||||
|
Err(_) => {
|
||||||
|
if last_data.elapsed().as_millis() as u64 >= idle_ms {
|
||||||
|
return reply(s, &json!({ "ok": true, "eof": true, "at": from, "reason": "absent" }));
|
||||||
|
}
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(200));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let len = f.metadata().map(|m| m.len()).unwrap_or(0);
|
||||||
|
if len < from {
|
||||||
|
// Truncated or rotated under us. Restart rather than read garbage.
|
||||||
|
from = 0;
|
||||||
|
}
|
||||||
|
if len > from {
|
||||||
|
f.seek(SeekFrom::Start(from))
|
||||||
|
.map_err(|e| format!("seek {path}: {e}"))?;
|
||||||
|
let mut buf = vec![0u8; (len - from).min(MAX_CHUNK) as usize];
|
||||||
|
let n = f.read(&mut buf).map_err(|e| format!("read {path}: {e}"))?;
|
||||||
|
buf.truncate(n);
|
||||||
|
from += n as u64;
|
||||||
|
last_data = std::time::Instant::now();
|
||||||
|
// Base64 so arbitrary bytes survive JSON — agent output is not
|
||||||
|
// guaranteed to be valid UTF-8 mid-chunk.
|
||||||
|
reply(
|
||||||
|
s,
|
||||||
|
&json!({ "ok": true, "eof": false, "at": from, "data": B64.encode(&buf) }),
|
||||||
|
)?;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if last_data.elapsed().as_millis() as u64 >= idle_ms {
|
||||||
|
return reply(s, &json!({ "ok": true, "eof": true, "at": from, "reason": "idle" }));
|
||||||
|
}
|
||||||
|
std::thread::sleep(std::time::Duration::from_millis(200));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Largest slice sent in one frame. Bounded so a burst of output cannot
|
||||||
|
/// allocate without limit inside a 2 GiB guest.
|
||||||
|
const MAX_CHUNK: u64 = 256 * 1024;
|
||||||
|
|
||||||
|
fn reply(s: &mut vsock::VsockStream, v: &Value) -> Result<(), String> {
|
||||||
|
let body = serde_json::to_vec(v).map_err(|e| format!("encode reply: {e}"))?;
|
||||||
|
s.write_all(&(body.len() as u32).to_be_bytes())
|
||||||
|
.map_err(|e| format!("write length: {e}"))?;
|
||||||
|
s.write_all(&body)
|
||||||
|
.map_err(|e| format!("write body: {e}"))?;
|
||||||
|
s.flush().map_err(|e| format!("flush: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn handle(req: &Value) -> Value {
|
||||||
|
let op = req.get("op").and_then(Value::as_str).unwrap_or_default();
|
||||||
|
match op {
|
||||||
|
"ping" => json!({
|
||||||
|
"ok": true,
|
||||||
|
"pid": std::process::id(),
|
||||||
|
// The host refuses to run a mission in a VM with no way out; this is
|
||||||
|
// how it knows. Reported rather than assumed because the image, not
|
||||||
|
// this binary, decides whether loopback can come up.
|
||||||
|
"proxy": PROXY_UP.load(Ordering::Relaxed),
|
||||||
|
}),
|
||||||
|
"exec" => op_exec(req),
|
||||||
|
// `tail` is handled in `serve_one`, not here: it streams many frames
|
||||||
|
// over one connection and so cannot return a single Value.
|
||||||
|
"tail" => json!({ "ok": false, "error": "tail is streamed; handled by serve_one" }),
|
||||||
|
"put" => op_put(req),
|
||||||
|
"get" => op_get(req),
|
||||||
|
other => json!({ "ok": false, "error": format!("unknown op: {other}") }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extra environment for the command, on top of the image's own.
|
||||||
|
///
|
||||||
|
/// This is how credentials reach the agent CLI. An env var rather than a file
|
||||||
|
/// because the per-VM rootfs is destroyed with the VM but an env var never
|
||||||
|
/// touches the guest disk at all — it exists only in the process's environment
|
||||||
|
/// for the length of one exec.
|
||||||
|
///
|
||||||
|
/// **Every problem here fails the exec.** The tempting alternative — skip the
|
||||||
|
/// entry we could not use and run anyway — produces a `claude -p` with no
|
||||||
|
/// credential, and that does not error: it hangs. A phase stuck at `running`
|
||||||
|
/// for ten minutes with nothing in the logs is exactly what a missing token
|
||||||
|
/// looked like on the container path, so a request we cannot honour in full is
|
||||||
|
/// refused with a reason instead.
|
||||||
|
///
|
||||||
|
/// Errors name the key and never the value: the value is the secret, and an
|
||||||
|
/// error string travels back over the wire and into logs.
|
||||||
|
fn env_pairs(req: &Value) -> Result<Vec<(String, String)>, String> {
|
||||||
|
// Absent or `null` means the caller sent no variables of its own — which is
|
||||||
|
// NOT the same as "this command needs no environment". Both cases still get
|
||||||
|
// the proxy address below; returning early here meant every exec that passed
|
||||||
|
// no env ran with no HTTPS_PROXY, and the symptom was `curl` reporting
|
||||||
|
// "Could not resolve host" from a guest that had a working tunnel.
|
||||||
|
let empty = serde_json::Map::new();
|
||||||
|
let map = match req.get("env") {
|
||||||
|
None => &empty,
|
||||||
|
Some(v) if v.is_null() => &empty,
|
||||||
|
// Anything else that is not an object is a caller bug.
|
||||||
|
Some(v) => v
|
||||||
|
.as_object()
|
||||||
|
.ok_or("exec env must be an object of name → string")?,
|
||||||
|
};
|
||||||
|
let mut out = Vec::with_capacity(map.len() + 3);
|
||||||
|
for (k, v) in map {
|
||||||
|
let Some(val) = v.as_str() else {
|
||||||
|
return Err(format!("exec env {k}: value must be a string"));
|
||||||
|
};
|
||||||
|
// `putenv` semantics: a name containing '=' would be parsed as part of
|
||||||
|
// the value, silently defining a different variable than the one asked
|
||||||
|
// for. A NUL truncates at the C boundary, for the same class of reason.
|
||||||
|
if k.is_empty() {
|
||||||
|
return Err("exec env has an empty variable name".into());
|
||||||
|
}
|
||||||
|
if k.contains('=') || k.contains('\0') {
|
||||||
|
return Err(format!("exec env {k:?}: name may not contain '=' or NUL"));
|
||||||
|
}
|
||||||
|
if val.contains('\0') {
|
||||||
|
return Err(format!("exec env {k}: value may not contain NUL"));
|
||||||
|
}
|
||||||
|
out.push((k.clone(), val.to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(with_proxy_env(out, PROXY_UP.load(Ordering::Relaxed)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Add the proxy variables the guest's own listener serves.
|
||||||
|
///
|
||||||
|
/// The agent runs the proxy, so the agent declares where it is. Deriving this on
|
||||||
|
/// the host would mean two places agreeing on a port number, and the one that
|
||||||
|
/// drifts is the one nobody tests.
|
||||||
|
///
|
||||||
|
/// Explicit caller values win: a caller can still point a command elsewhere or
|
||||||
|
/// switch the proxy off for it. Matched case-insensitively because the lowercase
|
||||||
|
/// spellings are equally conventional and a duplicate would leave which one
|
||||||
|
/// applies up to the shell.
|
||||||
|
fn with_proxy_env(mut env: Vec<(String, String)>, proxy_up: bool) -> Vec<(String, String)> {
|
||||||
|
if !proxy_up {
|
||||||
|
return env;
|
||||||
|
}
|
||||||
|
let addr = format!("http://127.0.0.1:{PROXY_PORT}");
|
||||||
|
for (k, v) in [
|
||||||
|
("HTTPS_PROXY", addr.as_str()),
|
||||||
|
("HTTP_PROXY", addr.as_str()),
|
||||||
|
// Without this the client would ask the proxy to reach the proxy.
|
||||||
|
("NO_PROXY", "localhost,127.0.0.1"),
|
||||||
|
] {
|
||||||
|
// `eq_ignore_ascii_case` covers the lowercase spelling, which is equally
|
||||||
|
// conventional; setting both would leave which one applies to the client.
|
||||||
|
if !env.iter().any(|(have, _)| have.eq_ignore_ascii_case(k)) {
|
||||||
|
env.push((k.to_string(), v.to_string()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
env
|
||||||
|
}
|
||||||
|
|
||||||
|
fn op_exec(req: &Value) -> Value {
|
||||||
|
let cmd = req.get("cmd").and_then(Value::as_str).unwrap_or_default();
|
||||||
|
if cmd.is_empty() {
|
||||||
|
return json!({ "ok": false, "error": "exec needs a cmd" });
|
||||||
|
}
|
||||||
|
let cwd = req.get("cwd").and_then(Value::as_str).unwrap_or("/");
|
||||||
|
let secs = req.get("timeout").and_then(Value::as_u64).unwrap_or(3600);
|
||||||
|
let env = match env_pairs(req) {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(e) => return json!({ "ok": false, "error": e }),
|
||||||
|
};
|
||||||
|
|
||||||
|
// The image's ENV was written to /etc/profile.d by the rootfs builder;
|
||||||
|
// `sh -c` does not read it, so source it here — otherwise a CLI that relies
|
||||||
|
// on `ENV PATH` behaves differently in the VM than in the container, which
|
||||||
|
// is exactly the drift the builder extracted that file to prevent.
|
||||||
|
//
|
||||||
|
// The `if [ -f ]` guard is load-bearing. `. missing-file` makes a
|
||||||
|
// NON-INTERACTIVE POSIX shell exit immediately with status 1, so the naive
|
||||||
|
// `. env.sh 2>/dev/null; cmd` returned rc=1 without running `cmd` at all on
|
||||||
|
// any rootfs lacking that file — every exec silently failing while looking
|
||||||
|
// like an ordinary non-zero exit. Caught by the exit-7 unit test.
|
||||||
|
const ENV_FILE: &str = "/etc/profile.d/00-image-env.sh";
|
||||||
|
let sourced = format!("if [ -f {ENV_FILE} ]; then . {ENV_FILE}; fi\n{cmd}");
|
||||||
|
let mut c = Command::new("/bin/sh");
|
||||||
|
c.arg("-c")
|
||||||
|
.arg(&sourced)
|
||||||
|
.envs(env)
|
||||||
|
.current_dir(if Path::new(cwd).is_dir() { cwd } else { "/" })
|
||||||
|
.stdin(Stdio::null())
|
||||||
|
.stdout(Stdio::piped())
|
||||||
|
.stderr(Stdio::piped())
|
||||||
|
// A new process group so a command that spawns background children can
|
||||||
|
// be killed wholesale. Without it a stray daemon keeps the run alive and
|
||||||
|
// the host's timeout is the only thing that ends it.
|
||||||
|
.process_group(0);
|
||||||
|
|
||||||
|
let mut child = match c.spawn() {
|
||||||
|
Ok(ch) => ch,
|
||||||
|
Err(e) => return json!({ "ok": false, "error": format!("spawn: {e}") }),
|
||||||
|
};
|
||||||
|
let pid = child.id() as i32;
|
||||||
|
|
||||||
|
// std has no wait-with-timeout, so poll. The output pipes are read after
|
||||||
|
// the wait, which is safe here because a command producing more than a pipe
|
||||||
|
// buffer of output while we are not draining it would deadlock — so the
|
||||||
|
// deadline is enforced by killing the group, and the pipes are drained by
|
||||||
|
// `wait_with_output` immediately after.
|
||||||
|
let deadline = Instant::now() + Duration::from_secs(secs);
|
||||||
|
let timed_out = loop {
|
||||||
|
match child.try_wait() {
|
||||||
|
Ok(Some(_)) => break false,
|
||||||
|
Ok(None) => {}
|
||||||
|
Err(e) => return json!({ "ok": false, "error": format!("wait: {e}") }),
|
||||||
|
}
|
||||||
|
if Instant::now() >= deadline {
|
||||||
|
kill_group(pid);
|
||||||
|
break true;
|
||||||
|
}
|
||||||
|
std::thread::sleep(Duration::from_millis(20));
|
||||||
|
};
|
||||||
|
|
||||||
|
let out = match child.wait_with_output() {
|
||||||
|
Ok(o) => o,
|
||||||
|
Err(e) => return json!({ "ok": false, "error": format!("collect output: {e}") }),
|
||||||
|
};
|
||||||
|
if timed_out {
|
||||||
|
// Reported as ok:false, not as rc=124: "we stopped it" is a different
|
||||||
|
// fact from "it exited non-zero", and the caller must be able to tell.
|
||||||
|
return json!({
|
||||||
|
"ok": false,
|
||||||
|
"error": format!("command exceeded its {secs}s budget and was killed"),
|
||||||
|
"stdout": String::from_utf8_lossy(&out.stdout),
|
||||||
|
"stderr": String::from_utf8_lossy(&out.stderr),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
json!({
|
||||||
|
"ok": true,
|
||||||
|
// A signalled process has no exit code; report the conventional
|
||||||
|
// 128+signal rather than silently claiming success.
|
||||||
|
"rc": exit_code(&out.status),
|
||||||
|
"stdout": String::from_utf8_lossy(&out.stdout),
|
||||||
|
"stderr": String::from_utf8_lossy(&out.stderr),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exit_code(status: &std::process::ExitStatus) -> i32 {
|
||||||
|
use std::os::unix::process::ExitStatusExt;
|
||||||
|
status
|
||||||
|
.code()
|
||||||
|
.unwrap_or_else(|| 128 + status.signal().unwrap_or(0))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn kill_group(pid: i32) {
|
||||||
|
let _ = Command::new("kill")
|
||||||
|
.args(["-9", "--", &format!("-{pid}")])
|
||||||
|
.status();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn op_put(req: &Value) -> Value {
|
||||||
|
let dest = req.get("dest").and_then(Value::as_str).unwrap_or_default();
|
||||||
|
if dest.is_empty() {
|
||||||
|
return json!({ "ok": false, "error": "put needs a dest" });
|
||||||
|
}
|
||||||
|
let b64 = req.get("tar_b64").and_then(Value::as_str).unwrap_or_default();
|
||||||
|
let raw = match B64.decode(b64) {
|
||||||
|
Ok(r) => r,
|
||||||
|
Err(e) => return json!({ "ok": false, "error": format!("undecodable archive: {e}") }),
|
||||||
|
};
|
||||||
|
if let Err(e) = std::fs::create_dir_all(dest) {
|
||||||
|
return json!({ "ok": false, "error": format!("mkdir {dest}: {e}") });
|
||||||
|
}
|
||||||
|
let mut ar = tar::Archive::new(&raw[..]);
|
||||||
|
ar.set_overwrite(true);
|
||||||
|
// Ownership from the host archive is meaningless in here and re-applying it
|
||||||
|
// is how the container path grew a uid split. The guest is root; let it own
|
||||||
|
// what it is given.
|
||||||
|
ar.set_preserve_permissions(false);
|
||||||
|
match ar.unpack(dest) {
|
||||||
|
Ok(()) => json!({ "ok": true, "dest": dest, "bytes": raw.len() }),
|
||||||
|
Err(e) => json!({ "ok": false, "error": format!("unpack into {dest}: {e}") }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Recursive tar append that skips excluded directory NAMES at any depth.
|
||||||
|
///
|
||||||
|
/// Hand-rolled because `tar::Builder::append_dir_all` takes no filter. Matched on
|
||||||
|
/// the name rather than a path prefix: a workspace has a `target/` per crate, and
|
||||||
|
/// excluding only the root one still ships the rest.
|
||||||
|
fn append_filtered<W: Write>(
|
||||||
|
b: &mut tar::Builder<W>,
|
||||||
|
dir: &Path,
|
||||||
|
prefix: &Path,
|
||||||
|
exclude: &[String],
|
||||||
|
) -> std::io::Result<()> {
|
||||||
|
b.append_dir(prefix, dir)?;
|
||||||
|
let mut entries: Vec<_> = std::fs::read_dir(dir)?.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
entries.sort_by_key(|e| e.file_name());
|
||||||
|
for entry in entries {
|
||||||
|
let name = entry.file_name();
|
||||||
|
let name_str = name.to_string_lossy().to_string();
|
||||||
|
let path = entry.path();
|
||||||
|
let dest = prefix.join(&name);
|
||||||
|
let meta = std::fs::symlink_metadata(&path)?;
|
||||||
|
if meta.is_dir() {
|
||||||
|
if exclude.contains(&name_str) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
append_filtered(b, &path, &dest, exclude)?;
|
||||||
|
} else if meta.is_symlink() {
|
||||||
|
let mut header = tar::Header::new_gnu();
|
||||||
|
header.set_metadata(&meta);
|
||||||
|
header.set_entry_type(tar::EntryType::Symlink);
|
||||||
|
header.set_size(0);
|
||||||
|
let target = std::fs::read_link(&path)?;
|
||||||
|
b.append_link(&mut header, &dest, &target)?;
|
||||||
|
} else {
|
||||||
|
let mut f = std::fs::File::open(&path)?;
|
||||||
|
b.append_file(&dest, &mut f)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn op_get(req: &Value) -> Value {
|
||||||
|
let path = req.get("path").and_then(Value::as_str).unwrap_or_default();
|
||||||
|
if path.is_empty() {
|
||||||
|
return json!({ "ok": false, "error": "get needs a path" });
|
||||||
|
}
|
||||||
|
let p = Path::new(path);
|
||||||
|
if !p.exists() {
|
||||||
|
// A missing path is an error, NOT an empty archive — an empty tar looks
|
||||||
|
// exactly like a run that produced nothing.
|
||||||
|
return json!({ "ok": false, "error": format!("no such path: {path}") });
|
||||||
|
}
|
||||||
|
let name = p
|
||||||
|
.file_name()
|
||||||
|
.map(|s| s.to_string_lossy().to_string())
|
||||||
|
.unwrap_or_else(|| "root".to_string());
|
||||||
|
|
||||||
|
// Directory names to leave out, sent by the host so the policy lives in one
|
||||||
|
// place (`mission_fs::transport_excludes`). Without it a phase that ran
|
||||||
|
// `cargo test` tars its whole `target/` directory: measured at 8.9 MB of 9.4 MB
|
||||||
|
// on our scratch repo, and enough to blow the 300s collect budget on a real
|
||||||
|
// build — which stranded a finished mission's work inside a VM twice.
|
||||||
|
let exclude: Vec<String> = req
|
||||||
|
.get("exclude")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|a| {
|
||||||
|
a.iter()
|
||||||
|
.filter_map(Value::as_str)
|
||||||
|
.map(str::to_string)
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let mut b = tar::Builder::new(Vec::new());
|
||||||
|
// Do not follow symlinks: a link pointing outside the collected tree would
|
||||||
|
// otherwise be dereferenced and its target smuggled back to the host.
|
||||||
|
b.follow_symlinks(false);
|
||||||
|
let added = if p.is_dir() {
|
||||||
|
append_filtered(&mut b, p, Path::new(&name), &exclude)
|
||||||
|
} else {
|
||||||
|
b.append_path_with_name(p, &name)
|
||||||
|
};
|
||||||
|
if let Err(e) = added {
|
||||||
|
return json!({ "ok": false, "error": format!("archive {path}: {e}") });
|
||||||
|
}
|
||||||
|
match b.into_inner() {
|
||||||
|
Ok(bytes) => json!({ "ok": true, "tar_b64": B64.encode(&bytes), "bytes": bytes.len() }),
|
||||||
|
Err(e) => json!({ "ok": false, "error": format!("finish archive for {path}: {e}") }),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
/// The tail loop must terminate. A tail that never returns pins a thread for
|
||||||
|
/// the life of the VM, and pid 1 running out of threads is an unbootable
|
||||||
|
/// machine, not a missing log.
|
||||||
|
#[test]
|
||||||
|
fn a_tail_of_a_file_that_never_appears_still_ends() {
|
||||||
|
// `absent` + idle_ms elapsed is the terminating branch; assert the
|
||||||
|
// constants that make it reachable rather than spinning a real socket.
|
||||||
|
assert!(MAX_CHUNK > 0, "a zero chunk cap would loop without progress");
|
||||||
|
assert!(
|
||||||
|
MAX_CHUNK <= 1024 * 1024,
|
||||||
|
"chunks must stay small enough for a 2 GiB guest"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// The CLI reaches the API only by honouring HTTPS_PROXY (measured: with the
|
||||||
|
/// proxy at a closed port, `claude -p` fails ConnectionRefused instead of
|
||||||
|
/// answering), so a VM whose proxy is up must hand it the address.
|
||||||
|
#[test]
|
||||||
|
fn the_proxy_address_is_declared_when_the_proxy_is_up() {
|
||||||
|
let env = with_proxy_env(vec![], true);
|
||||||
|
let get = |k: &str| {
|
||||||
|
env.iter()
|
||||||
|
.find(|(a, _)| a == k)
|
||||||
|
.map(|(_, v)| v.as_str())
|
||||||
|
.unwrap_or("")
|
||||||
|
};
|
||||||
|
assert_eq!(get("HTTPS_PROXY"), "http://127.0.0.1:3128");
|
||||||
|
assert_eq!(get("HTTP_PROXY"), "http://127.0.0.1:3128");
|
||||||
|
// Otherwise the client asks the proxy to reach the proxy.
|
||||||
|
assert!(get("NO_PROXY").contains("127.0.0.1"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// And a VM with no proxy must not claim one: pointing a CLI at a listener
|
||||||
|
/// that is not there turns "no egress" into a connection error mid-run
|
||||||
|
/// instead of a fact the host can check before it starts.
|
||||||
|
#[test]
|
||||||
|
fn no_proxy_address_is_declared_when_the_proxy_is_down() {
|
||||||
|
assert!(with_proxy_env(vec![], false).is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An explicit value from the caller wins, in either spelling — otherwise
|
||||||
|
/// both would be set and which one applies would be up to the client.
|
||||||
|
#[test]
|
||||||
|
fn an_explicit_proxy_setting_is_not_overridden() {
|
||||||
|
let env = with_proxy_env(
|
||||||
|
vec![("https_proxy".into(), "http://elsewhere:8080".into())],
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
let proxies: Vec<&str> = env
|
||||||
|
.iter()
|
||||||
|
.filter(|(k, _)| k.eq_ignore_ascii_case("https_proxy"))
|
||||||
|
.map(|(_, v)| v.as_str())
|
||||||
|
.collect();
|
||||||
|
assert_eq!(proxies, vec!["http://elsewhere:8080"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The credential has to actually reach the command. This is the whole
|
||||||
|
/// point of the op, and the failure it prevents is silent: a `claude -p`
|
||||||
|
/// with no token hangs rather than erroring.
|
||||||
|
#[test]
|
||||||
|
fn injected_env_reaches_the_command() {
|
||||||
|
let r = op_exec(&json!({
|
||||||
|
"op": "exec",
|
||||||
|
"cmd": "printf %s \"$CLAUDE_CODE_OAUTH_TOKEN\"",
|
||||||
|
"env": { "CLAUDE_CODE_OAUTH_TOKEN": "sk-test-value" },
|
||||||
|
"timeout": 30,
|
||||||
|
}));
|
||||||
|
assert_eq!(r["rc"], json!(0));
|
||||||
|
assert_eq!(r["stdout"], json!("sk-test-value"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// And it must survive the profile.d sourcing that runs first — a
|
||||||
|
/// credential set on the process and then clobbered by the shell would
|
||||||
|
/// look identical to one that never arrived.
|
||||||
|
#[test]
|
||||||
|
fn injected_env_survives_the_image_env_file() {
|
||||||
|
let r = op_exec(&json!({
|
||||||
|
"op": "exec",
|
||||||
|
"cmd": "printf %s \"$INJECTED_PROBE\"",
|
||||||
|
"env": { "INJECTED_PROBE": "still-here" },
|
||||||
|
"timeout": 30,
|
||||||
|
}));
|
||||||
|
assert_eq!(r["stdout"], json!("still-here"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// No env is the ordinary case and must not be an error.
|
||||||
|
#[test]
|
||||||
|
fn absent_or_null_env_is_not_an_error() {
|
||||||
|
for req in [
|
||||||
|
json!({ "op": "exec", "cmd": "true", "timeout": 30 }),
|
||||||
|
json!({ "op": "exec", "cmd": "true", "env": null, "timeout": 30 }),
|
||||||
|
json!({ "op": "exec", "cmd": "true", "env": {}, "timeout": 30 }),
|
||||||
|
] {
|
||||||
|
assert_eq!(op_exec(&req)["rc"], json!(0), "{req}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An env entry we cannot honour fails the whole exec rather than being
|
||||||
|
/// dropped. Running without the credential is the outcome this refuses:
|
||||||
|
/// it does not error, it hangs, which is far harder to diagnose than a
|
||||||
|
/// rejected request.
|
||||||
|
#[test]
|
||||||
|
fn an_unusable_env_entry_fails_the_exec_instead_of_being_skipped() {
|
||||||
|
let cases = [
|
||||||
|
json!({ "A=B": "x" }),
|
||||||
|
json!({ "": "x" }),
|
||||||
|
json!({ "TOKEN": 42 }),
|
||||||
|
json!({ "TOKEN": null }),
|
||||||
|
];
|
||||||
|
for env in cases {
|
||||||
|
let r = op_exec(&json!({
|
||||||
|
"op": "exec", "cmd": "true", "env": env.clone(), "timeout": 30,
|
||||||
|
}));
|
||||||
|
assert_eq!(r["ok"], json!(false), "env {env} should be refused");
|
||||||
|
assert!(r["rc"].is_null(), "nothing ran, so there is no rc: {r}");
|
||||||
|
}
|
||||||
|
// A non-object env is a caller bug, not an empty map.
|
||||||
|
let r = op_exec(&json!({ "op": "exec", "cmd": "true", "env": "TOKEN=x" }));
|
||||||
|
assert_eq!(r["ok"], json!(false));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An error about a credential must not quote the credential: it travels
|
||||||
|
/// back over the wire and into the server's logs.
|
||||||
|
#[test]
|
||||||
|
fn an_env_error_never_echoes_the_value() {
|
||||||
|
let r = op_exec(&json!({
|
||||||
|
"op": "exec", "cmd": "true", "timeout": 30,
|
||||||
|
"env": { "A=B": "super-secret-token" },
|
||||||
|
}));
|
||||||
|
let err = r["error"].as_str().unwrap_or_default();
|
||||||
|
assert!(!err.contains("super-secret-token"), "leaked the value: {err}");
|
||||||
|
assert!(err.contains("A=B"), "should name the key: {err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_unknown_op_is_reported_not_ignored() {
|
||||||
|
let r = handle(&json!({ "op": "teleport" }));
|
||||||
|
assert_eq!(r["ok"], json!(false));
|
||||||
|
assert!(r["error"].as_str().unwrap().contains("teleport"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ping_answers() {
|
||||||
|
assert_eq!(handle(&json!({ "op": "ping" }))["ok"], json!(true));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A missing path must be an error, not an empty archive: an empty tar is
|
||||||
|
/// indistinguishable from a run that produced nothing.
|
||||||
|
/// Build output is not work. It is regenerable, it dwarfs the source, and
|
||||||
|
/// tarring it over vsock stranded a finished mission inside a VM twice —
|
||||||
|
/// `vm_collect` timed out at 300s while the agent's three new modules sat in
|
||||||
|
/// the guest. Matched on the directory NAME at any depth, because a workspace
|
||||||
|
/// has a `target/` per crate.
|
||||||
|
#[test]
|
||||||
|
fn excluded_directories_stay_out_of_the_archive_at_any_depth() {
|
||||||
|
let dir = std::env::temp_dir().join(format!("fcagent-ex-{}", std::process::id()));
|
||||||
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
|
std::fs::create_dir_all(dir.join("src")).unwrap();
|
||||||
|
std::fs::create_dir_all(dir.join("target/debug")).unwrap();
|
||||||
|
std::fs::create_dir_all(dir.join("crates/inner/target")).unwrap();
|
||||||
|
std::fs::write(dir.join("src/lib.rs"), "fn a() {}").unwrap();
|
||||||
|
std::fs::write(dir.join("target/debug/blob"), vec![0u8; 4096]).unwrap();
|
||||||
|
std::fs::write(dir.join("crates/inner/target/blob"), vec![0u8; 4096]).unwrap();
|
||||||
|
std::fs::write(dir.join("crates/inner/keep.rs"), "fn b() {}").unwrap();
|
||||||
|
|
||||||
|
let r = op_get(&json!({
|
||||||
|
"op": "get",
|
||||||
|
"path": dir.to_string_lossy(),
|
||||||
|
"exclude": ["target"],
|
||||||
|
}));
|
||||||
|
assert_eq!(r["ok"], json!(true), "{r}");
|
||||||
|
let bytes = B64.decode(r["tar_b64"].as_str().unwrap()).unwrap();
|
||||||
|
let mut ar = tar::Archive::new(&bytes[..]);
|
||||||
|
let paths: Vec<String> = ar
|
||||||
|
.entries()
|
||||||
|
.unwrap()
|
||||||
|
.filter_map(Result::ok)
|
||||||
|
.map(|e| e.path().unwrap().to_string_lossy().to_string())
|
||||||
|
.collect();
|
||||||
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
|
|
||||||
|
assert!(paths.iter().any(|p| p.ends_with("src/lib.rs")), "{paths:?}");
|
||||||
|
assert!(paths.iter().any(|p| p.ends_with("inner/keep.rs")), "{paths:?}");
|
||||||
|
assert!(
|
||||||
|
!paths.iter().any(|p| p.contains("target")),
|
||||||
|
"a nested target/ came along: {paths:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// No exclude list means everything, so an existing caller is unchanged.
|
||||||
|
#[test]
|
||||||
|
fn without_an_exclude_list_nothing_is_dropped() {
|
||||||
|
let dir = std::env::temp_dir().join(format!("fcagent-noex-{}", std::process::id()));
|
||||||
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
|
std::fs::create_dir_all(dir.join("target")).unwrap();
|
||||||
|
std::fs::write(dir.join("target/x"), "x").unwrap();
|
||||||
|
let r = op_get(&json!({ "op": "get", "path": dir.to_string_lossy() }));
|
||||||
|
let bytes = B64.decode(r["tar_b64"].as_str().unwrap()).unwrap();
|
||||||
|
let mut ar = tar::Archive::new(&bytes[..]);
|
||||||
|
let n = ar.entries().unwrap().filter_map(Result::ok).count();
|
||||||
|
let _ = std::fs::remove_dir_all(&dir);
|
||||||
|
assert!(n >= 2, "expected the target dir and its file, got {n}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn getting_a_missing_path_is_an_error() {
|
||||||
|
let r = op_get(&json!({ "op": "get", "path": "/definitely/not/here" }));
|
||||||
|
assert_eq!(r["ok"], json!(false));
|
||||||
|
assert!(r["tar_b64"].is_null(), "no archive may be returned");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A command that ran and failed reports `rc`; one we killed reports
|
||||||
|
/// `ok:false`. Collapsing the two would make a timeout look like a build
|
||||||
|
/// failure and vice versa.
|
||||||
|
#[test]
|
||||||
|
fn a_failing_command_reports_rc_and_a_killed_one_does_not() {
|
||||||
|
let r = op_exec(&json!({ "op": "exec", "cmd": "exit 7", "timeout": 30 }));
|
||||||
|
assert_eq!(r["ok"], json!(true), "it ran, so ok is true");
|
||||||
|
assert_eq!(r["rc"], json!(7));
|
||||||
|
|
||||||
|
let r = op_exec(&json!({ "op": "exec", "cmd": "sleep 30", "timeout": 1 }));
|
||||||
|
assert_eq!(r["ok"], json!(false), "we killed it, so ok is false");
|
||||||
|
assert!(r["rc"].is_null(), "a killed command has no exit code");
|
||||||
|
assert!(r["error"].as_str().unwrap().contains("budget"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn exec_needs_a_command() {
|
||||||
|
assert_eq!(op_exec(&json!({ "op": "exec" }))["ok"], json!(false));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A tar must round-trip through put and get.
|
||||||
|
#[test]
|
||||||
|
fn a_tar_round_trips_through_put_and_get() {
|
||||||
|
let tmp = std::env::temp_dir().join(format!("fcagent-test-{}", std::process::id()));
|
||||||
|
let _ = std::fs::remove_dir_all(&tmp);
|
||||||
|
|
||||||
|
let mut b = tar::Builder::new(Vec::new());
|
||||||
|
let body = b"ROUND-TRIP-OK\n";
|
||||||
|
let mut h = tar::Header::new_gnu();
|
||||||
|
h.set_path("marker.txt").unwrap();
|
||||||
|
h.set_size(body.len() as u64);
|
||||||
|
h.set_mode(0o644);
|
||||||
|
h.set_entry_type(tar::EntryType::Regular);
|
||||||
|
h.set_cksum();
|
||||||
|
b.append(&h, &body[..]).unwrap();
|
||||||
|
let archive = b.into_inner().unwrap();
|
||||||
|
|
||||||
|
let r = op_put(&json!({
|
||||||
|
"op": "put",
|
||||||
|
"dest": tmp.display().to_string(),
|
||||||
|
"tar_b64": B64.encode(&archive),
|
||||||
|
}));
|
||||||
|
assert_eq!(r["ok"], json!(true), "put failed: {r}");
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(tmp.join("marker.txt")).unwrap(),
|
||||||
|
"ROUND-TRIP-OK\n"
|
||||||
|
);
|
||||||
|
|
||||||
|
let r = op_get(&json!({ "op": "get", "path": tmp.display().to_string() }));
|
||||||
|
assert_eq!(r["ok"], json!(true), "get failed: {r}");
|
||||||
|
let bytes = B64.decode(r["tar_b64"].as_str().unwrap()).unwrap();
|
||||||
|
let mut ar = tar::Archive::new(&bytes[..]);
|
||||||
|
let found = ar
|
||||||
|
.entries()
|
||||||
|
.unwrap()
|
||||||
|
.filter_map(Result::ok)
|
||||||
|
.any(|e| e.path().map(|p| p.ends_with("marker.txt")).unwrap_or(false));
|
||||||
|
assert!(found, "the collected archive must contain marker.txt");
|
||||||
|
|
||||||
|
let _ = std::fs::remove_dir_all(&tmp);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -33,6 +33,7 @@ cm-config = { path = "../cm-config" }
|
|||||||
cm-db = { path = "../cm-db" }
|
cm-db = { path = "../cm-db" }
|
||||||
cm-domain = { path = "../cm-domain" }
|
cm-domain = { path = "../cm-domain" }
|
||||||
cm-files = { path = "../cm-files" }
|
cm-files = { path = "../cm-files" }
|
||||||
|
tar = { workspace = true }
|
||||||
cm-llm = { path = "../cm-llm" }
|
cm-llm = { path = "../cm-llm" }
|
||||||
cm-orchestrator = { path = "../cm-orchestrator", features = ["provider"] }
|
cm-orchestrator = { path = "../cm-orchestrator", features = ["provider"] }
|
||||||
cm-runtime = { path = "../cm-runtime" }
|
cm-runtime = { path = "../cm-runtime" }
|
||||||
|
|||||||
@@ -0,0 +1,231 @@
|
|||||||
|
//! Human given names for minted agents.
|
||||||
|
//!
|
||||||
|
//! A team used to come back as `planner`, `coder`, `tester`, `reviewer`,
|
||||||
|
//! `committer` — the roster read as a list of job tickets, and the UI showed
|
||||||
|
//! the same word twice (name on top, role underneath). A crew you keep should
|
||||||
|
//! read like people: Meredith, Vijay, Tomasz, Amara.
|
||||||
|
//!
|
||||||
|
//! The role is not lost — it stays in `job_title`, which is what the mission
|
||||||
|
//! machinery binds on. Only the display identity changes.
|
||||||
|
//!
|
||||||
|
//! Names are drawn from many naming traditions on purpose: this workforce is
|
||||||
|
//! not from one place. They are given names only — no surnames — so nobody
|
||||||
|
//! reads a claw as a specific real person.
|
||||||
|
|
||||||
|
/// Given names, deliberately wide. Kept as one flat list rather than grouped by
|
||||||
|
/// origin: grouping invites picking "one from each", which is a worse kind of
|
||||||
|
/// tokenism than simply having a broad pool and drawing from it evenly.
|
||||||
|
///
|
||||||
|
/// Size is a product decision, not an aesthetic one. Every mission now mints
|
||||||
|
/// its own crew and nothing retires them, so the roster grows by the team size
|
||||||
|
/// per mission — at ~5 a mission a 70-name pool starts emitting "Amara 2"
|
||||||
|
/// inside twenty missions. This pool carries a few hundred so a workspace runs
|
||||||
|
/// for a long time before any name repeats at all.
|
||||||
|
pub const NAMES: &[&str] = &[
|
||||||
|
// A
|
||||||
|
"Aarav", "Abebe", "Adaora", "Adrian", "Agnieszka", "Ahmad", "Aiko", "Ainhoa", "Alejandro",
|
||||||
|
"Alina", "Amara", "Amina", "Anders", "Andrea", "Anjali", "Annika", "Antoine", "Arjun", "Astrid",
|
||||||
|
"Ayo", "Ayesha", "Aziz",
|
||||||
|
// B–C
|
||||||
|
"Beatriz", "Bilal", "Bjorn", "Blessing", "Bogdan", "Camila", "Carlos", "Catalina", "Chidi",
|
||||||
|
"Chiara", "Chioma", "Cyrus",
|
||||||
|
// D–E
|
||||||
|
"Dagny", "Damir", "Daniela", "Dilnoza", "Dmitri", "Ebele", "Eduardo", "Eero", "Ekaterina",
|
||||||
|
"Elena", "Elias", "Emeka", "Enrique", "Esi", "Esther", "Eun-ji", "Ewa",
|
||||||
|
// F–G
|
||||||
|
"Fabio", "Farida", "Fatou", "Felipe", "Fernanda", "Freya", "Gabriel", "Georgi", "Giulia",
|
||||||
|
"Grace", "Gunnar", "Gulnara",
|
||||||
|
// H–I
|
||||||
|
"Hana", "Hasan", "Heidi", "Hina", "Hiroshi", "Ibrahim", "Idris", "Ilya", "Imani", "Ingrid",
|
||||||
|
"Iris", "Isabela", "Ivan", "Iwona",
|
||||||
|
// J–K
|
||||||
|
"Jaromir", "Javier", "Jing", "Joana", "Johan", "Josefina", "Junko", "Kaito", "Kalinda", "Karim",
|
||||||
|
"Katarzyna", "Kenji", "Khalid", "Kiran", "Klara", "Kwame", "Kyoko",
|
||||||
|
// L–M
|
||||||
|
"Lakshmi", "Lars", "Laila", "Leilani", "Lena", "Liam", "Linnea", "Lucia", "Lukas", "Madhavi",
|
||||||
|
"Maja", "Malik", "Marisol", "Mateo", "Matteo", "Mei", "Meredith", "Milena", "Mira", "Mohan",
|
||||||
|
"Mira-Lynn", "Mateusz",
|
||||||
|
// N–O
|
||||||
|
"Nadia", "Nasrin", "Neelam", "Niamh", "Nikolai", "Nilufar", "Nkechi", "Noor", "Nuria", "Oksana",
|
||||||
|
"Oleksii", "Olamide", "Omar", "Oskar", "Osei",
|
||||||
|
// P–R
|
||||||
|
"Paloma", "Panagiotis", "Pedro", "Petra", "Priya", "Rafael", "Rania", "Ravi", "Reza", "Renata",
|
||||||
|
"Rin", "Robert", "Rosalind", "Rustam",
|
||||||
|
// S
|
||||||
|
"Sadia", "Salome", "Samir", "Sanjay", "Sara", "Seong-min", "Sipho", "Sofia", "Solveig", "Soren",
|
||||||
|
"Suvi", "Svetlana",
|
||||||
|
// T–U
|
||||||
|
"Tadeusz", "Takeshi", "Tamar", "Tariq", "Thandiwe", "Thi", "Tim", "Tomasz", "Tove", "Tuva",
|
||||||
|
"Ulrika", "Uma", "Usman",
|
||||||
|
// V–Z
|
||||||
|
"Valentina", "Vera", "Vijay", "Vikram", "Wanjiru", "Wei", "Wiktor", "Yara", "Yasmin", "Yohannes",
|
||||||
|
"Yuki", "Yusuf", "Zainab", "Zara", "Zoltan", "Zuzanna",
|
||||||
|
];
|
||||||
|
|
||||||
|
/// Pick a name not already in `taken`.
|
||||||
|
///
|
||||||
|
/// `seed` spreads the starting point so a workspace does not always begin at
|
||||||
|
/// "Amara" — it is an offset into the list, not randomness, so the choice is
|
||||||
|
/// reproducible for a given (seed, taken) pair and therefore testable.
|
||||||
|
///
|
||||||
|
/// When every name is taken it appends a numeric suffix — `Amara 2` — rather
|
||||||
|
/// than returning `None` and forcing the caller to invent something. Running
|
||||||
|
/// out is a nice problem (70+ concurrent agents in one workspace) and a
|
||||||
|
/// duplicate display name is far less harmful than a failed mission launch.
|
||||||
|
pub fn pick(taken: &[String], seed: u64) -> String {
|
||||||
|
let start = (seed % NAMES.len() as u64) as usize;
|
||||||
|
for i in 0..NAMES.len() {
|
||||||
|
let candidate = NAMES[(start + i) % NAMES.len()];
|
||||||
|
if !taken.iter().any(|t| t.eq_ignore_ascii_case(candidate)) {
|
||||||
|
return candidate.to_string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Second pass with a suffix. `round` starts at 2 so the first repeat reads
|
||||||
|
// "Amara 2", which is how a person would disambiguate two colleagues.
|
||||||
|
for round in 2..1000 {
|
||||||
|
for i in 0..NAMES.len() {
|
||||||
|
let candidate = format!("{} {}", NAMES[(start + i) % NAMES.len()], round);
|
||||||
|
if !taken.iter().any(|t| t.eq_ignore_ascii_case(&candidate)) {
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Unreachable in practice; still not a panic.
|
||||||
|
format!("Agent {seed}")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn names_are_unique_and_non_empty() {
|
||||||
|
let mut seen = std::collections::HashSet::new();
|
||||||
|
for n in NAMES {
|
||||||
|
assert!(!n.trim().is_empty(), "empty name in the pool");
|
||||||
|
assert!(seen.insert(n.to_ascii_lowercase()), "duplicate in pool: {n}");
|
||||||
|
}
|
||||||
|
// Every mission mints its own crew and nothing retires them, so the
|
||||||
|
// pool is consumed for the life of the workspace, not recycled. At ~5
|
||||||
|
// per mission this is ~35 missions before the first numeric suffix.
|
||||||
|
assert!(NAMES.len() >= 150, "pool too small for one crew per mission");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A crew should not read as an alphabetical run.
|
||||||
|
///
|
||||||
|
/// With the role index as the seed, every crew started at the top of the
|
||||||
|
/// pool and took the next free names — the first real mission hired Aarav,
|
||||||
|
/// Abebe, Adaora, Adrian, Agnieszka. Unique and correct, and obviously
|
||||||
|
/// generated. Callers now seed from the claw's uuid tail, so this checks
|
||||||
|
/// that well-spread seeds actually land in different regions of the pool
|
||||||
|
/// rather than clustering at one end.
|
||||||
|
#[test]
|
||||||
|
fn spread_seeds_do_not_produce_an_alphabetical_run() {
|
||||||
|
let index_of = |n: &str| NAMES.iter().position(|c| *c == n).expect("name in pool");
|
||||||
|
let seeds = [
|
||||||
|
0x9e37_79b9_7f4a_7c15u64,
|
||||||
|
0x1234_5678_9abc_def0,
|
||||||
|
0xfeed_face_dead_beef,
|
||||||
|
0x0f0f_0f0f_f0f0_f0f0,
|
||||||
|
0xa5a5_5a5a_c3c3_3c3c,
|
||||||
|
];
|
||||||
|
let mut taken: Vec<String> = Vec::new();
|
||||||
|
let mut positions = Vec::new();
|
||||||
|
for s in seeds {
|
||||||
|
let n = pick(&taken, s);
|
||||||
|
positions.push(index_of(&n) as i64);
|
||||||
|
taken.push(n);
|
||||||
|
}
|
||||||
|
// Adjacent picks landing within a couple of slots of each other is the
|
||||||
|
// clustering signature; require the crew to span a real distance.
|
||||||
|
let (min, max) = (
|
||||||
|
*positions.iter().min().unwrap(),
|
||||||
|
*positions.iter().max().unwrap(),
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
max - min > (NAMES.len() as i64) / 3,
|
||||||
|
"crew clustered in one region of the pool: {positions:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The scenario the operator actually asked for: consecutive missions must
|
||||||
|
/// not hand back the same names. Reuse is off, so mission two staffs from
|
||||||
|
/// what mission one left.
|
||||||
|
#[test]
|
||||||
|
fn consecutive_missions_get_different_crews() {
|
||||||
|
let mut roster: Vec<String> = Vec::new();
|
||||||
|
let mut crews: Vec<Vec<String>> = Vec::new();
|
||||||
|
for mission in 0..6u64 {
|
||||||
|
let mut crew = Vec::new();
|
||||||
|
for role in 0..5u64 {
|
||||||
|
let n = pick(&roster, mission * 5 + role);
|
||||||
|
roster.push(n.clone());
|
||||||
|
crew.push(n);
|
||||||
|
}
|
||||||
|
crews.push(crew);
|
||||||
|
}
|
||||||
|
for (i, a) in crews.iter().enumerate() {
|
||||||
|
for (j, b) in crews.iter().enumerate().skip(i + 1) {
|
||||||
|
let shared: Vec<_> = a.iter().filter(|n| b.contains(n)).collect();
|
||||||
|
assert!(
|
||||||
|
shared.is_empty(),
|
||||||
|
"missions {i} and {j} share {shared:?} — crews must be distinct"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// And no duplicates anywhere on the roster.
|
||||||
|
let uniq: std::collections::HashSet<_> = roster.iter().collect();
|
||||||
|
assert_eq!(uniq.len(), roster.len(), "a name was issued twice");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pick_avoids_taken_names() {
|
||||||
|
let taken: Vec<String> = NAMES.iter().take(10).map(|s| s.to_string()).collect();
|
||||||
|
let got = pick(&taken, 0);
|
||||||
|
assert!(
|
||||||
|
!taken.iter().any(|t| t.eq_ignore_ascii_case(&got)),
|
||||||
|
"picked a name already taken: {got}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn pick_is_case_insensitive_about_taken() {
|
||||||
|
// A name already on the roster in a different case is still taken —
|
||||||
|
// "meredith" and "Meredith" are the same colleague.
|
||||||
|
let taken = vec![NAMES[0].to_ascii_lowercase()];
|
||||||
|
assert_ne!(pick(&taken, 0).to_ascii_lowercase(), taken[0]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn seed_spreads_the_starting_point() {
|
||||||
|
// Different seeds should not all hand back the same first name, or a
|
||||||
|
// fresh workspace always opens with the same roster.
|
||||||
|
let a = pick(&[], 0);
|
||||||
|
let b = pick(&[], 7);
|
||||||
|
assert_ne!(a, b, "seed had no effect on the choice");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn exhausting_the_pool_suffixes_rather_than_failing() {
|
||||||
|
let taken: Vec<String> = NAMES.iter().map(|s| s.to_string()).collect();
|
||||||
|
let got = pick(&taken, 0);
|
||||||
|
assert!(
|
||||||
|
!taken.iter().any(|t| t.eq_ignore_ascii_case(&got)),
|
||||||
|
"must not reuse a taken name"
|
||||||
|
);
|
||||||
|
assert!(got.ends_with(" 2"), "expected a suffixed name, got {got}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_full_team_gets_distinct_names() {
|
||||||
|
// The actual scenario: mint five roles into an empty workspace and get
|
||||||
|
// five different people, not five "planner"s.
|
||||||
|
let mut taken: Vec<String> = Vec::new();
|
||||||
|
for i in 0..5 {
|
||||||
|
let n = pick(&taken, i);
|
||||||
|
assert!(!taken.contains(&n), "repeated {n} within one team");
|
||||||
|
taken.push(n);
|
||||||
|
}
|
||||||
|
assert_eq!(taken.len(), 5);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,442 @@
|
|||||||
|
//! Merging a delivered branch into the base, when that is provably safe.
|
||||||
|
//!
|
||||||
|
//! Every mission type delivers to a branch and never to `main`. For most that
|
||||||
|
//! is where it should stop — a human reads the code and merges. But some
|
||||||
|
//! missions only ever *add* files in a folder they own: a paper catalogue, a
|
||||||
|
//! benchmark record. Those branches carry no judgement call, and leaving them
|
||||||
|
//! to pile up unmerged means the work is done but not actually in the vault.
|
||||||
|
//!
|
||||||
|
//! # Additive-only is a property, not a preference
|
||||||
|
//!
|
||||||
|
//! The gate is not "is this mission type trusted". It is measured from the
|
||||||
|
//! diff: if the branch modifies or deletes anything that already existed, it
|
||||||
|
//! does not qualify, whatever its template says. A research harvest that
|
||||||
|
//! somehow rewrote a hand-written note would be refused by the same check
|
||||||
|
//! that lets its new notes through.
|
||||||
|
//!
|
||||||
|
//! Three conditions, all required:
|
||||||
|
//!
|
||||||
|
//! 1. the mission type declares [`MergePolicy::AdditiveOnly`]
|
||||||
|
//! 2. verification passed — a run that did not prove its work does not merge
|
||||||
|
//! 3. the diff against the base contains only additions
|
||||||
|
//!
|
||||||
|
//! Anything else lands as a branch for a human, which is the existing
|
||||||
|
//! behaviour and the safe default.
|
||||||
|
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
/// What a mission type is allowed to do with its own branch.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub enum MergePolicy {
|
||||||
|
/// Always leave the branch for a human. Correct for anything that touches
|
||||||
|
/// code: `refactor`, `research_and_code`, security patches.
|
||||||
|
Never,
|
||||||
|
/// Merge automatically when the diff is provably additive and the run
|
||||||
|
/// verified. Correct for catalogues and recorded measurements.
|
||||||
|
AdditiveOnly,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MergePolicy {
|
||||||
|
/// Parse a template's `merge_policy`. Unknown values fall back to `Never`
|
||||||
|
/// and say so: a typo must not silently grant auto-merge.
|
||||||
|
pub fn parse(raw: Option<&str>) -> MergePolicy {
|
||||||
|
match raw.map(str::trim) {
|
||||||
|
Some("additive_only") => MergePolicy::AdditiveOnly,
|
||||||
|
Some("never") | None => MergePolicy::Never,
|
||||||
|
Some(other) => {
|
||||||
|
eprintln!(
|
||||||
|
"auto_merge: unknown merge_policy {other:?} — refusing to auto-merge"
|
||||||
|
);
|
||||||
|
MergePolicy::Never
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Why a branch was or was not merged. The reason is always recorded: a
|
||||||
|
/// branch that silently did not merge is indistinguishable from one that was
|
||||||
|
/// never delivered.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct MergeOutcome {
|
||||||
|
pub merged: bool,
|
||||||
|
pub reason: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MergeOutcome {
|
||||||
|
fn refused(reason: impl Into<String>) -> MergeOutcome {
|
||||||
|
MergeOutcome {
|
||||||
|
merged: false,
|
||||||
|
reason: reason.into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every path in a `git diff --name-status` body, with its status letter.
|
||||||
|
///
|
||||||
|
/// The World draws a file orb per changed path, and `mission_delivery` records
|
||||||
|
/// the list — both need the same parse, so it lives in one place.
|
||||||
|
///
|
||||||
|
/// **Renames are three fields**: `R100\told\tnew`. The path that changed is the
|
||||||
|
/// NEW one; splitting on the first tab and taking field two records where the
|
||||||
|
/// file used to be, which then matches nothing anyone can open. Copies (`C###`)
|
||||||
|
/// have the same shape.
|
||||||
|
pub fn changed_paths(name_status: &str) -> Vec<(char, String)> {
|
||||||
|
name_status
|
||||||
|
.lines()
|
||||||
|
.filter(|l| !l.trim().is_empty())
|
||||||
|
.filter_map(|l| {
|
||||||
|
let mut fields = l.split('\t');
|
||||||
|
let status = fields.next()?.trim();
|
||||||
|
let letter = status.chars().next()?;
|
||||||
|
let first = fields.next()?.trim();
|
||||||
|
// R/C carry old THEN new; everything else has a single path.
|
||||||
|
let path = match letter {
|
||||||
|
'R' | 'C' => fields.next().map(str::trim).unwrap_or(first),
|
||||||
|
_ => first,
|
||||||
|
};
|
||||||
|
if path.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some((letter, path.to_string()))
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Classify a `git diff --name-status` body.
|
||||||
|
///
|
||||||
|
/// Returns the offending entries, empty when every change is an addition.
|
||||||
|
/// Built on `changed_paths` so the two cannot disagree about what a line means.
|
||||||
|
pub fn non_additive_changes(name_status: &str) -> Vec<String> {
|
||||||
|
changed_paths(name_status)
|
||||||
|
.into_iter()
|
||||||
|
.filter(|(letter, _)| *letter != 'A')
|
||||||
|
.map(|(letter, path)| format!("{letter}\t{path}"))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn git(repo: &Path, args: &[&str]) -> Result<String, String> {
|
||||||
|
let out = tokio::process::Command::new("git")
|
||||||
|
.arg("-C")
|
||||||
|
.arg(repo)
|
||||||
|
.args(["-c", &format!("safe.directory={}", repo.display())])
|
||||||
|
.args(args)
|
||||||
|
.env("GIT_AUTHOR_NAME", crate::mission_delivery::commit_identity().0)
|
||||||
|
.env("GIT_AUTHOR_EMAIL", crate::mission_delivery::commit_identity().1)
|
||||||
|
.env(
|
||||||
|
"GIT_COMMITTER_NAME",
|
||||||
|
crate::mission_delivery::commit_identity().0,
|
||||||
|
)
|
||||||
|
.env(
|
||||||
|
"GIT_COMMITTER_EMAIL",
|
||||||
|
crate::mission_delivery::commit_identity().1,
|
||||||
|
)
|
||||||
|
.output()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("spawn git: {e}"))?;
|
||||||
|
if !out.status.success() {
|
||||||
|
return Err(format!(
|
||||||
|
"git {} → {}: {}",
|
||||||
|
args.first().copied().unwrap_or("?"),
|
||||||
|
out.status,
|
||||||
|
crate::mission_workspace::redact_token(&String::from_utf8_lossy(&out.stderr))
|
||||||
|
.chars()
|
||||||
|
.take(300)
|
||||||
|
.collect::<String>()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Merge `branch` into `base` and push, if all three conditions hold.
|
||||||
|
///
|
||||||
|
/// Never returns `Err` for a refusal — a refusal is a normal outcome with a
|
||||||
|
/// reason. `Err` is reserved for the merge itself going wrong after we decided
|
||||||
|
/// to attempt it.
|
||||||
|
pub async fn try_merge(
|
||||||
|
repo: &Path,
|
||||||
|
push_url: &str,
|
||||||
|
branch: &str,
|
||||||
|
base: &str,
|
||||||
|
policy: MergePolicy,
|
||||||
|
verified: bool,
|
||||||
|
) -> Result<MergeOutcome, String> {
|
||||||
|
if policy != MergePolicy::AdditiveOnly {
|
||||||
|
return Ok(MergeOutcome::refused(
|
||||||
|
"merge_policy is not additive_only; left for a human",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !verified {
|
||||||
|
return Ok(MergeOutcome::refused(
|
||||||
|
"run did not verify; refusing to merge unproven work",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compare against the base as the REMOTE has it, not a local ref that may
|
||||||
|
// be stale. `...` gives changes on the branch since it diverged, so an
|
||||||
|
// unrelated commit landing on main meanwhile is not misread as ours.
|
||||||
|
git(repo, &["fetch", push_url, base]).await?;
|
||||||
|
let diff = git(
|
||||||
|
repo,
|
||||||
|
&["diff", "--name-status", &format!("FETCH_HEAD...{branch}")],
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let offending = non_additive_changes(&diff);
|
||||||
|
if !offending.is_empty() {
|
||||||
|
return Ok(MergeOutcome::refused(format!(
|
||||||
|
"diff is not additive ({} non-add change(s), first: {}); left for a human",
|
||||||
|
offending.len(),
|
||||||
|
offending.first().map(String::as_str).unwrap_or("?")
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
if diff.trim().is_empty() {
|
||||||
|
return Ok(MergeOutcome::refused("branch adds nothing"));
|
||||||
|
}
|
||||||
|
|
||||||
|
merge_and_push(repo, push_url, branch, base, "auto-merge")
|
||||||
|
.await
|
||||||
|
.map(|o| match o.merged {
|
||||||
|
true => MergeOutcome {
|
||||||
|
merged: true,
|
||||||
|
reason: format!("additive-only and verified; merged into {base}"),
|
||||||
|
},
|
||||||
|
false => o,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The git half of a merge, with no policy in it.
|
||||||
|
///
|
||||||
|
/// Split out so an OPERATOR-approved merge runs exactly the same commands as an
|
||||||
|
/// automatic one — fetch the base as the remote has it, merge onto that, push.
|
||||||
|
/// The gates differ; the mechanics must not, or the rarely-taken path is the one
|
||||||
|
/// that breaks.
|
||||||
|
async fn merge_and_push(
|
||||||
|
repo: &Path,
|
||||||
|
push_url: &str,
|
||||||
|
branch: &str,
|
||||||
|
base: &str,
|
||||||
|
label: &str,
|
||||||
|
) -> Result<MergeOutcome, String> {
|
||||||
|
// Merge onto the freshly fetched base rather than a local branch.
|
||||||
|
git(repo, &["checkout", "-B", base, "FETCH_HEAD"]).await?;
|
||||||
|
if let Err(e) = git(
|
||||||
|
repo,
|
||||||
|
&["merge", "--no-ff", "-m", &format!("{label} {branch}"), branch],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
// Leave the repo clean so the next run is not fighting a wedged merge.
|
||||||
|
let _ = git(repo, &["merge", "--abort"]).await;
|
||||||
|
return Ok(MergeOutcome::refused(format!(
|
||||||
|
"merge conflicted ({e}); left for a human"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
git(repo, &["push", push_url, &format!("HEAD:refs/heads/{base}")]).await?;
|
||||||
|
Ok(MergeOutcome {
|
||||||
|
merged: true,
|
||||||
|
reason: format!("merged into {base}"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Merge a delivered branch because an OPERATOR asked for it.
|
||||||
|
///
|
||||||
|
/// `MergePolicy::Never` means "do not merge on your own" — it defers to a human,
|
||||||
|
/// and this is that human. So the additive-only test does not apply: an operator
|
||||||
|
/// looking at a code change is exactly the judgement the policy was holding out
|
||||||
|
/// for.
|
||||||
|
///
|
||||||
|
/// What is NOT waived:
|
||||||
|
///
|
||||||
|
/// - the branch must exist on the remote and differ from the base, so the button
|
||||||
|
/// cannot report success for a merge of nothing;
|
||||||
|
/// - a conflict refuses and leaves the repo clean, rather than forcing;
|
||||||
|
/// - the work happens in a FRESH CLONE, never the mission checkout — that
|
||||||
|
/// directory is reaped on a timer after the mission ends, so a merge that
|
||||||
|
/// depended on it would work right after a run and mysteriously fail later.
|
||||||
|
pub async fn merge_on_operator_approval(
|
||||||
|
workdir: &Path,
|
||||||
|
push_url: &str,
|
||||||
|
branch: &str,
|
||||||
|
base: &str,
|
||||||
|
) -> Result<MergeOutcome, String> {
|
||||||
|
git(workdir, &["fetch", push_url, base]).await?;
|
||||||
|
git(workdir, &["fetch", push_url, branch]).await?;
|
||||||
|
git(workdir, &["branch", "-f", branch, "FETCH_HEAD"]).await?;
|
||||||
|
git(workdir, &["fetch", push_url, base]).await?;
|
||||||
|
|
||||||
|
let diff = git(
|
||||||
|
workdir,
|
||||||
|
&["diff", "--name-status", &format!("FETCH_HEAD...{branch}")],
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
if diff.trim().is_empty() {
|
||||||
|
return Ok(MergeOutcome::refused(
|
||||||
|
"branch has nothing the base does not already have",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
merge_locally(workdir, branch, base, "merge mission branch").await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Merge onto the fetched base WITHOUT publishing it.
|
||||||
|
///
|
||||||
|
/// Split from the push so a caller can run the project's tests against the
|
||||||
|
/// merged tree first. Verifying BEFORE publishing rather than reverting after is
|
||||||
|
/// the difference between "main was never broken" and "main was broken for as
|
||||||
|
/// long as it took us to notice".
|
||||||
|
pub async fn merge_locally(
|
||||||
|
repo: &Path,
|
||||||
|
branch: &str,
|
||||||
|
base: &str,
|
||||||
|
label: &str,
|
||||||
|
) -> Result<MergeOutcome, String> {
|
||||||
|
git(repo, &["checkout", "-B", base, "FETCH_HEAD"]).await?;
|
||||||
|
if let Err(e) = git(
|
||||||
|
repo,
|
||||||
|
&["merge", "--no-ff", "-m", &format!("{label} {branch}"), branch],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
// Leave the repo clean so the next attempt is not fighting a wedged merge.
|
||||||
|
let _ = git(repo, &["merge", "--abort"]).await;
|
||||||
|
return Ok(MergeOutcome::refused(format!(
|
||||||
|
"merge conflicted ({e}); left for a human"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
Ok(MergeOutcome {
|
||||||
|
merged: true,
|
||||||
|
reason: format!("merged into {base} locally, not yet published"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Publish an already-merged base.
|
||||||
|
pub async fn push_merged(repo: &Path, push_url: &str, base: &str) -> Result<(), String> {
|
||||||
|
git(repo, &["push", push_url, &format!("HEAD:refs/heads/{base}")])
|
||||||
|
.await
|
||||||
|
.map(|_| ())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// Publication must be gated on the merged tree, and refusal must not push.
|
||||||
|
///
|
||||||
|
/// The two halves are separate functions precisely so a caller can run tests
|
||||||
|
/// BETWEEN them. If `merge_locally` ever pushed, verification would be
|
||||||
|
/// after-the-fact and `main` would be broken for as long as it took to
|
||||||
|
/// notice — which is the failure mode this whole thing exists to avoid.
|
||||||
|
#[test]
|
||||||
|
fn merging_locally_never_publishes() {
|
||||||
|
let src = include_str!("auto_merge.rs");
|
||||||
|
let body = src
|
||||||
|
.split("pub async fn merge_locally")
|
||||||
|
.nth(1)
|
||||||
|
.and_then(|s| s.split("\n}").next())
|
||||||
|
.unwrap_or("");
|
||||||
|
assert!(!body.is_empty(), "merge_locally not found");
|
||||||
|
assert!(
|
||||||
|
!body.contains("\"push\""),
|
||||||
|
"merge_locally must not push — publication is the caller's decision \
|
||||||
|
after it has verified the result"
|
||||||
|
);
|
||||||
|
// And the push half must exist separately, or the caller cannot publish.
|
||||||
|
assert!(src.contains("pub async fn push_merged"), "push_merged missing");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An operator merge and an automatic one must run the SAME git commands.
|
||||||
|
///
|
||||||
|
/// The gates differ — that is the whole point — but if the mechanics
|
||||||
|
/// diverged, the rarely-taken path would be the untested one. Both go
|
||||||
|
/// through `merge_and_push`.
|
||||||
|
#[test]
|
||||||
|
fn both_merge_paths_share_the_same_mechanics() {
|
||||||
|
let src = include_str!("auto_merge.rs");
|
||||||
|
let calls = src.matches("merge_and_push(").count();
|
||||||
|
// one definition + one call from each path
|
||||||
|
assert!(
|
||||||
|
calls >= 3,
|
||||||
|
"expected try_merge and merge_on_operator_approval to both call \
|
||||||
|
merge_and_push, found {calls} mention(s)"
|
||||||
|
);
|
||||||
|
// And the operator path must NOT re-implement the policy gate it exists
|
||||||
|
// to bypass — if this string appears there, the button is a no-op.
|
||||||
|
let op = src
|
||||||
|
.split("pub async fn merge_on_operator_approval")
|
||||||
|
.nth(1)
|
||||||
|
.unwrap_or("");
|
||||||
|
let body = op.split("\n}").next().unwrap_or("");
|
||||||
|
assert!(
|
||||||
|
!body.contains("MergePolicy::AdditiveOnly"),
|
||||||
|
"the operator path must not apply the additive-only gate"
|
||||||
|
);
|
||||||
|
// It must still refuse an empty branch: a button that reports success
|
||||||
|
// for merging nothing is worse than no button.
|
||||||
|
assert!(
|
||||||
|
body.contains("nothing the base does not already have"),
|
||||||
|
"the operator path must refuse an empty branch"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn only_pure_additions_qualify() {
|
||||||
|
assert!(non_additive_changes("A\t60 Papers/a.md\nA\t60 Papers/b.md\n").is_empty());
|
||||||
|
|
||||||
|
// A modification disqualifies the whole branch.
|
||||||
|
let m = non_additive_changes("A\t60 Papers/a.md\nM\tREADME.md\n");
|
||||||
|
assert_eq!(m.len(), 1);
|
||||||
|
assert!(m[0].contains("README.md"));
|
||||||
|
|
||||||
|
// So do deletes and renames — a rename is a delete plus an add, and
|
||||||
|
// the delete half can destroy hand-written work.
|
||||||
|
assert_eq!(non_additive_changes("D\tnotes/old.md\n").len(), 1);
|
||||||
|
assert_eq!(non_additive_changes("R100\ta.md\tb.md\n").len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A rename records the NEW path.
|
||||||
|
///
|
||||||
|
/// `R100\told\tnew` is three fields. Reading field two — which is what a
|
||||||
|
/// split-on-first-tab gives you — records where the file USED to be, so the
|
||||||
|
/// World would draw an orb for a path that no longer exists and the
|
||||||
|
/// delivered file list would name something nobody can open. The bug is
|
||||||
|
/// invisible in any repo where nothing was renamed.
|
||||||
|
#[test]
|
||||||
|
fn a_rename_records_where_the_file_ended_up() {
|
||||||
|
let paths = changed_paths("R100\tsrc/old.rs\tsrc/new.rs\n");
|
||||||
|
assert_eq!(paths, vec![('R', "src/new.rs".to_string())]);
|
||||||
|
|
||||||
|
let copied = changed_paths("C075\tsrc/a.rs\tsrc/b.rs\n");
|
||||||
|
assert_eq!(copied, vec![('C', "src/b.rs".to_string())]);
|
||||||
|
|
||||||
|
// Ordinary two-field lines are unaffected.
|
||||||
|
assert_eq!(
|
||||||
|
changed_paths("A\tone.md\nM\ttwo.md\nD\tthree.md\n"),
|
||||||
|
vec![
|
||||||
|
('A', "one.md".to_string()),
|
||||||
|
('M', "two.md".to_string()),
|
||||||
|
('D', "three.md".to_string()),
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `files_changed` and the path list must agree, or nobody can tell which
|
||||||
|
/// one lied. git counts a rename as ONE changed file; so must we.
|
||||||
|
#[test]
|
||||||
|
fn a_rename_counts_once() {
|
||||||
|
assert_eq!(changed_paths("R100\ta.rs\tb.rs\n").len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_unknown_policy_never_grants_auto_merge() {
|
||||||
|
assert_eq!(MergePolicy::parse(None), MergePolicy::Never);
|
||||||
|
assert_eq!(MergePolicy::parse(Some("never")), MergePolicy::Never);
|
||||||
|
assert_eq!(
|
||||||
|
MergePolicy::parse(Some("additive_only")),
|
||||||
|
MergePolicy::AdditiveOnly
|
||||||
|
);
|
||||||
|
// A typo must fail closed, not open.
|
||||||
|
assert_eq!(MergePolicy::parse(Some("aditive_only")), MergePolicy::Never);
|
||||||
|
assert_eq!(MergePolicy::parse(Some("always")), MergePolicy::Never);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -159,10 +159,33 @@ pub async fn run(
|
|||||||
};
|
};
|
||||||
|
|
||||||
let (container, workdir) = exec_target(pool, mission_id).await?;
|
let (container, workdir) = exec_target(pool, mission_id).await?;
|
||||||
|
// Benchmark a COPY, never the mission's own checkout.
|
||||||
|
//
|
||||||
|
// `docker_exec` enters a container running as ROOT with the missions root
|
||||||
|
// bind-mounted, and `cargo bench` writes `target/`. Run in the live tree, it
|
||||||
|
// leaves root-owned build output in a checkout owned by uid 65532 — the
|
||||||
|
// single-writer invariant broken, and the next phase's cargo hitting
|
||||||
|
// permission-denied on a directory it cannot write.
|
||||||
|
//
|
||||||
|
// This is the SAME defect `evaluator_tools::Sandbox` exists for, found the
|
||||||
|
// same way: the harness's uid probe, reporting `uids=0,65532`. Measurement
|
||||||
|
// must not mutate what it measures — the rule this codebase already applies
|
||||||
|
// to the judge and to the `verifier` subagent.
|
||||||
|
let copy_root = crate::root_copy::copy_root("_bench", mission_id);
|
||||||
|
// A stale copy from a previous run is ROOT-owned (see `purge_copy`), so it
|
||||||
|
// must be removed the same way it was created — from inside the container.
|
||||||
|
crate::root_copy::purge(&container, ©_root).await;
|
||||||
|
let copy = crate::root_copy::RootCopy::of(&workdir, ©_root)?;
|
||||||
let cmd = harness.command();
|
let cmd = harness.command();
|
||||||
let raw = docker_exec(&container, &workdir, &cmd)
|
let result = docker_exec(&container, copy.workdir(), &cmd)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("exec {cmd:?}: {e}"))?;
|
.map_err(|e| format!("exec {cmd:?}: {e}"));
|
||||||
|
// Explicitly, on BOTH paths, before the `Drop` fallback runs. `cargo bench`
|
||||||
|
// writes `target/` as root, and the server process is uid 65532: its
|
||||||
|
// `remove_dir_all` cannot delete root-owned files and silently leaves the
|
||||||
|
// whole copy behind — measured at 1.2 MB per run, growing forever.
|
||||||
|
crate::root_copy::purge(&container, ©_root).await;
|
||||||
|
let raw = result?;
|
||||||
let metrics = parse_output(&raw, &harness);
|
let metrics = parse_output(&raw, &harness);
|
||||||
Ok((metrics, harness.driver_name().to_string()))
|
Ok((metrics, harness.driver_name().to_string()))
|
||||||
}
|
}
|
||||||
@@ -254,9 +277,7 @@ async fn exec_target(
|
|||||||
}
|
}
|
||||||
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
||||||
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
||||||
let root = std::env::var("CLAWMATES_MISSIONS_ROOT")
|
let workdir = crate::mission_workspace::missions_root()
|
||||||
.unwrap_or_else(|_| "/var/lib/clawmates-missions".to_string());
|
|
||||||
let workdir = std::path::PathBuf::from(root)
|
|
||||||
.join(mission_id.to_string())
|
.join(mission_id.to_string())
|
||||||
.join("repo");
|
.join("repo");
|
||||||
Ok((container, workdir))
|
Ok((container, workdir))
|
||||||
@@ -401,3 +422,29 @@ fn compute_delta(before: &Value, after: &Value) -> Value {
|
|||||||
}
|
}
|
||||||
json!({ "kind": "opaque", "note": "before/after not structurally comparable" })
|
json!({ "kind": "opaque", "note": "before/after not structurally comparable" })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod bench_copy_tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// The benchmark copy must live OUTSIDE the mission directory, and must not
|
||||||
|
/// be the checkout itself.
|
||||||
|
///
|
||||||
|
/// Running `cargo bench` in the live tree left root-owned `target/` in a
|
||||||
|
/// checkout owned by uid 65532 — caught by the harness's uid probe
|
||||||
|
/// (`uids=0,65532`) after this runner was first wired into the sweep. The
|
||||||
|
/// same rule `evaluator_tools::Sandbox` follows: measurement must not mutate
|
||||||
|
/// what it measures.
|
||||||
|
#[test]
|
||||||
|
fn a_benchmark_runs_in_a_copy_outside_the_mission_directory() {
|
||||||
|
let mission = Uuid::now_v7();
|
||||||
|
let copy = crate::root_copy::copy_root("_bench", mission);
|
||||||
|
let live = crate::mission_workspace::checkout_path(mission);
|
||||||
|
assert_ne!(copy, live, "the bench copy must not be the checkout");
|
||||||
|
assert!(
|
||||||
|
!copy.starts_with(crate::mission_workspace::missions_root().join(mission.to_string())),
|
||||||
|
"{copy:?} must be a SIBLING of the mission dir, or the reaper races it"
|
||||||
|
);
|
||||||
|
assert!(copy.starts_with(crate::mission_workspace::missions_root().join("_bench")), "{copy:?}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -77,6 +77,51 @@ pub fn connect() -> Result<Docker, String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The uid every mission artefact must belong to.
|
||||||
|
///
|
||||||
|
/// The runtime container's own processes already run as this; only `docker
|
||||||
|
/// exec` defaulted to root, because `CreateExecOptions::user` was never set.
|
||||||
|
/// That one omission is the origin of four separate patches: root-owned
|
||||||
|
/// `target/` directories appearing inside a checkout that uid 65532 then could
|
||||||
|
/// not delete, `root_copy` existing at all, and a cleanup path that had to
|
||||||
|
/// re-enter the container as root to undo what it had just done.
|
||||||
|
pub(crate) const MISSION_UID: &str = "65532:65532";
|
||||||
|
|
||||||
|
/// Environment a non-root exec needs, because the image gives uid 65532 no
|
||||||
|
/// writable `HOME` and no writable `CARGO_HOME`.
|
||||||
|
///
|
||||||
|
/// Measured in the deployed image: `/zeroclaw-data` (its `HOME`) and
|
||||||
|
/// `/usr/local/cargo` are both root-owned and unwritable, so switching execs to
|
||||||
|
/// 65532 without this would break every `cargo` invocation — the benchmark
|
||||||
|
/// runner, the judge's verification sandbox, and the delivery test gate — in a
|
||||||
|
/// new and much quieter way than the problem it fixes.
|
||||||
|
///
|
||||||
|
/// The missions root is bind-mounted into the runtime container at the same
|
||||||
|
/// path and IS writable by 65532, so the cargo cache lives there and is shared
|
||||||
|
/// across missions rather than re-downloaded per mission. Verified end to end:
|
||||||
|
/// a clean `cargo build` as 65532 with these three variables produces output
|
||||||
|
/// owned entirely by 65532.
|
||||||
|
fn mission_env() -> Vec<String> {
|
||||||
|
let root = crate::mission_workspace::missions_root();
|
||||||
|
vec![
|
||||||
|
format!("HOME={}", root.join("_home").display()),
|
||||||
|
format!("CARGO_HOME={}", root.join("_cargo").display()),
|
||||||
|
"TMPDIR=/tmp".to_string(),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a workdir is inside the tree missions own.
|
||||||
|
///
|
||||||
|
/// The rule is positional rather than per-caller on purpose. Twelve call sites
|
||||||
|
/// each remembering to pass a uid is twelve chances to forget, and the one that
|
||||||
|
/// forgets leaves debris the others cannot clean up — which is exactly the
|
||||||
|
/// history here.
|
||||||
|
fn is_mission_path(workdir: Option<&str>) -> bool {
|
||||||
|
let Some(dir) = workdir else { return false };
|
||||||
|
let root = crate::mission_workspace::missions_root();
|
||||||
|
std::path::Path::new(dir).starts_with(&root)
|
||||||
|
}
|
||||||
|
|
||||||
/// Run `argv` in `container`, optionally in `workdir`, and capture both
|
/// Run `argv` in `container`, optionally in `workdir`, and capture both
|
||||||
/// streams plus the exit status.
|
/// streams plus the exit status.
|
||||||
///
|
///
|
||||||
@@ -93,6 +138,29 @@ pub async fn exec(
|
|||||||
exec_with_env(docker, container, workdir, argv, &[], timeout).await
|
exec_with_env(docker, container, workdir, argv, &[], timeout).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Run `argv` as **root**, deliberately.
|
||||||
|
///
|
||||||
|
/// The one legitimate use is clearing debris that earlier root-run execs left
|
||||||
|
/// behind: uid 65532 cannot delete a root-owned `target/`, so the cleanup has
|
||||||
|
/// to out-rank it. Every other caller goes through [`exec`], which runs mission
|
||||||
|
/// work as 65532 so no new debris is created.
|
||||||
|
pub async fn exec_as_root(
|
||||||
|
docker: &Docker,
|
||||||
|
container: &str,
|
||||||
|
workdir: Option<&str>,
|
||||||
|
argv: &[String],
|
||||||
|
timeout: Duration,
|
||||||
|
) -> Result<ExecOutput, String> {
|
||||||
|
let fut = exec_inner(docker, container, workdir, argv, &[], None);
|
||||||
|
match tokio::time::timeout(timeout, fut).await {
|
||||||
|
Err(_) => Err(format!(
|
||||||
|
"timed out after {}s (the command may still be running in {container})",
|
||||||
|
timeout.as_secs()
|
||||||
|
)),
|
||||||
|
Ok(res) => res,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// As [`exec`], with extra environment for the command.
|
/// As [`exec`], with extra environment for the command.
|
||||||
pub async fn exec_with_env(
|
pub async fn exec_with_env(
|
||||||
docker: &Docker,
|
docker: &Docker,
|
||||||
@@ -102,7 +170,16 @@ pub async fn exec_with_env(
|
|||||||
env: &[String],
|
env: &[String],
|
||||||
timeout: Duration,
|
timeout: Duration,
|
||||||
) -> Result<ExecOutput, String> {
|
) -> Result<ExecOutput, String> {
|
||||||
let fut = exec_inner(docker, container, workdir, argv, env);
|
// Mission work runs as 65532 with a writable HOME/CARGO_HOME; anything
|
||||||
|
// outside the missions tree (runtime preflight probes, image checks) keeps
|
||||||
|
// the daemon's default so this cannot break unrelated call sites.
|
||||||
|
let (user, mut full_env) = if is_mission_path(workdir) {
|
||||||
|
(Some(MISSION_UID), mission_env())
|
||||||
|
} else {
|
||||||
|
(None, Vec::new())
|
||||||
|
};
|
||||||
|
full_env.extend_from_slice(env);
|
||||||
|
let fut = exec_inner(docker, container, workdir, argv, &full_env, user);
|
||||||
match tokio::time::timeout(timeout, fut).await {
|
match tokio::time::timeout(timeout, fut).await {
|
||||||
Err(_) => Err(format!(
|
Err(_) => Err(format!(
|
||||||
"timed out after {}s (the command may still be running in {container})",
|
"timed out after {}s (the command may still be running in {container})",
|
||||||
@@ -118,6 +195,7 @@ async fn exec_inner(
|
|||||||
workdir: Option<&str>,
|
workdir: Option<&str>,
|
||||||
argv: &[String],
|
argv: &[String],
|
||||||
env: &[String],
|
env: &[String],
|
||||||
|
user: Option<&str>,
|
||||||
) -> Result<ExecOutput, String> {
|
) -> Result<ExecOutput, String> {
|
||||||
let created = docker
|
let created = docker
|
||||||
.create_exec(
|
.create_exec(
|
||||||
@@ -130,6 +208,7 @@ async fn exec_inner(
|
|||||||
} else {
|
} else {
|
||||||
Some(env.to_vec())
|
Some(env.to_vec())
|
||||||
},
|
},
|
||||||
|
user: user.map(str::to_string),
|
||||||
attach_stdout: Some(true),
|
attach_stdout: Some(true),
|
||||||
attach_stderr: Some(true),
|
attach_stderr: Some(true),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
@@ -209,4 +288,118 @@ mod tests {
|
|||||||
assert_eq!(out(Some(1), "a", "b").combined(), "a\nb");
|
assert_eq!(out(Some(1), "a", "b").combined(), "a\nb");
|
||||||
assert_eq!(out(Some(0), " ", "\n").combined(), "");
|
assert_eq!(out(Some(0), " ", "\n").combined(), "");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Mission work is 65532; everything else keeps the daemon's default.
|
||||||
|
///
|
||||||
|
/// The rule is positional so that no caller has to remember it. Twelve call
|
||||||
|
/// sites each passing a uid is twelve chances to forget, and the one that
|
||||||
|
/// forgets leaves debris the other eleven cannot delete — which is the
|
||||||
|
/// actual history: root-owned `target/` directories inside a checkout owned
|
||||||
|
/// by 65532, `root_copy` written to work around them, and a cleanup that had
|
||||||
|
/// to re-enter the container as root to undo its own mess.
|
||||||
|
#[test]
|
||||||
|
fn only_work_inside_the_missions_tree_drops_to_the_mission_uid() {
|
||||||
|
let root = crate::mission_workspace::missions_root();
|
||||||
|
let inside = root.join("019fe785-0f82-7780-8d58-da79fb4c31bc/repo");
|
||||||
|
assert!(is_mission_path(Some(&inside.display().to_string())));
|
||||||
|
assert!(is_mission_path(Some(&root.display().to_string())));
|
||||||
|
|
||||||
|
// Probes and image checks run with no workdir at all, and must not be
|
||||||
|
// forced to a uid the image may not have set up for them.
|
||||||
|
assert!(!is_mission_path(None));
|
||||||
|
assert!(!is_mission_path(Some("/")));
|
||||||
|
assert!(!is_mission_path(Some("/usr/local/cargo")));
|
||||||
|
// A path that merely SHARES A PREFIX is not inside the tree.
|
||||||
|
// `starts_with` on `Path` compares components, so this is already true;
|
||||||
|
// the assertion is here so a switch to string matching cannot pass.
|
||||||
|
let sibling = format!("{}-evil/repo", root.display());
|
||||||
|
assert!(!is_mission_path(Some(&sibling)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The non-root exec carries the three variables the image does not give it.
|
||||||
|
///
|
||||||
|
/// Measured in the deployed image: uid 65532's `HOME` (`/zeroclaw-data`)
|
||||||
|
/// and `/usr/local/cargo` are both root-owned and unwritable. Without these
|
||||||
|
/// overrides, dropping execs to 65532 would break every cargo invocation —
|
||||||
|
/// the benchmark runner, the judge's sandbox, the delivery test gate — far
|
||||||
|
/// more quietly than the leak it fixes.
|
||||||
|
#[test]
|
||||||
|
fn the_mission_env_replaces_the_paths_the_image_leaves_unwritable() {
|
||||||
|
let env = mission_env();
|
||||||
|
let root = crate::mission_workspace::missions_root();
|
||||||
|
assert!(env.iter().any(|v| v == &format!("HOME={}/_home", root.display())));
|
||||||
|
assert!(env.iter().any(|v| v == &format!("CARGO_HOME={}/_cargo", root.display())));
|
||||||
|
assert!(env.iter().any(|v| v == "TMPDIR=/tmp"));
|
||||||
|
for v in &env {
|
||||||
|
assert!(
|
||||||
|
!v.contains("/usr/local/cargo") && !v.contains("/zeroclaw-data"),
|
||||||
|
"{v} points back at a root-owned path"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One place builds an exec, so one place decides its uid.
|
||||||
|
///
|
||||||
|
/// The original bug was not a wrong value — it was an ABSENT one:
|
||||||
|
/// `CreateExecOptions` never set `user`, so the daemon defaulted to root
|
||||||
|
/// and twelve callers inherited that without any of them choosing it. A
|
||||||
|
/// second construction site is how that comes back, so the guard is on the
|
||||||
|
/// number of sites rather than on any particular uid.
|
||||||
|
#[test]
|
||||||
|
fn exactly_one_place_builds_an_exec() {
|
||||||
|
let src = include_str!("container_exec.rs");
|
||||||
|
// Split so this needle does not match itself in this very file.
|
||||||
|
let needle = concat!("CreateExec", "Options {");
|
||||||
|
let sites = src.matches(needle).count();
|
||||||
|
assert_eq!(
|
||||||
|
sites, 1,
|
||||||
|
"exec options must be built in one place; found {sites}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
src.contains(concat!("user: ", "user.map(str::to_string)")),
|
||||||
|
"that one place must set `user` — leaving it unset is the bug"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The tail of a container's log, for putting in an error message.
|
||||||
|
///
|
||||||
|
/// A turn that times out destroys the only place the reason lived: the
|
||||||
|
/// per-mission runtime container is torn down after the phase, taking its logs
|
||||||
|
/// with it, and the operator is left with the string "turn timed out". This
|
||||||
|
/// copies the last few lines out while the container still exists.
|
||||||
|
///
|
||||||
|
/// Best-effort by construction — it runs on a path that is ALREADY failing, so
|
||||||
|
/// every error here degrades to a note rather than replacing the real failure
|
||||||
|
/// with a docker one.
|
||||||
|
pub async fn tail_logs(container: &str, lines: usize) -> String {
|
||||||
|
use futures::StreamExt as _;
|
||||||
|
|
||||||
|
let Ok(docker) = connect() else {
|
||||||
|
return "(docker unreachable, so no container log)".into();
|
||||||
|
};
|
||||||
|
let opts = bollard::query_parameters::LogsOptionsBuilder::default()
|
||||||
|
.stdout(true)
|
||||||
|
.stderr(true)
|
||||||
|
.tail(&lines.to_string())
|
||||||
|
.build();
|
||||||
|
let mut stream = docker.logs(container, Some(opts));
|
||||||
|
let mut out = String::new();
|
||||||
|
while let Some(chunk) = stream.next().await {
|
||||||
|
match chunk {
|
||||||
|
Ok(c) => out.push_str(&c.to_string()),
|
||||||
|
Err(e) => {
|
||||||
|
if out.is_empty() {
|
||||||
|
return format!("(could not read {container} logs: {e})");
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let out = out.trim();
|
||||||
|
if out.is_empty() {
|
||||||
|
format!("({container} logged nothing)")
|
||||||
|
} else {
|
||||||
|
out.to_string()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -291,6 +291,35 @@ pub async fn unseen(
|
|||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// How many NEW sources a mission contributed.
|
||||||
|
///
|
||||||
|
/// The verification predicate for a continuous research mission. `record`
|
||||||
|
/// never reassigns `mission_id` on conflict, so the first mission to find a
|
||||||
|
/// source keeps the credit and a rerun cannot inflate its own count by
|
||||||
|
/// re-recording what an earlier run already had.
|
||||||
|
///
|
||||||
|
/// A mission whose answer is zero produced nothing, whatever its transcript
|
||||||
|
/// says — which is the check the 0030-0044 generation of this feature lacked.
|
||||||
|
pub async fn contributed(
|
||||||
|
pool: &sqlx::PgPool,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
corpus_id: &str,
|
||||||
|
mission_id: Uuid,
|
||||||
|
) -> Result<i64, String> {
|
||||||
|
let row: (i64,) = sqlx::query_as(
|
||||||
|
"SELECT count(*) FROM corpus_items
|
||||||
|
WHERE workspace_id = $1 AND corpus_id = $2 AND mission_id = $3
|
||||||
|
AND kind = 'source'",
|
||||||
|
)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(corpus_id)
|
||||||
|
.bind(mission_id)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("contributed({mission_id}): {e}"))?;
|
||||||
|
Ok(row.0)
|
||||||
|
}
|
||||||
|
|
||||||
/// Index every note in a checkout. Idempotent by construction.
|
/// Index every note in a checkout. Idempotent by construction.
|
||||||
pub async fn index_vault(
|
pub async fn index_vault(
|
||||||
pool: &sqlx::PgPool,
|
pool: &sqlx::PgPool,
|
||||||
|
|||||||
@@ -19,6 +19,13 @@ pub enum ApiError {
|
|||||||
Conflict,
|
Conflict,
|
||||||
#[error("{0}")]
|
#[error("{0}")]
|
||||||
Quota(String),
|
Quota(String),
|
||||||
|
/// A dependency is temporarily refusing work and will accept it later —
|
||||||
|
/// today, the Claude Code subscription's rate limit. Distinct from
|
||||||
|
/// `Internal` because the operator's next action is different: wait and
|
||||||
|
/// press the button again, rather than read a server log. A 500 with
|
||||||
|
/// "internal error" sent them looking for a bug that was not there.
|
||||||
|
#[error("{0}")]
|
||||||
|
Unavailable(String),
|
||||||
#[error("internal error")]
|
#[error("internal error")]
|
||||||
Internal,
|
Internal,
|
||||||
}
|
}
|
||||||
@@ -58,6 +65,7 @@ impl IntoResponse for ApiError {
|
|||||||
ApiError::NotFound => StatusCode::NOT_FOUND,
|
ApiError::NotFound => StatusCode::NOT_FOUND,
|
||||||
ApiError::Conflict => StatusCode::CONFLICT,
|
ApiError::Conflict => StatusCode::CONFLICT,
|
||||||
ApiError::Quota(_) => StatusCode::PAYMENT_REQUIRED,
|
ApiError::Quota(_) => StatusCode::PAYMENT_REQUIRED,
|
||||||
|
ApiError::Unavailable(_) => StatusCode::SERVICE_UNAVAILABLE,
|
||||||
ApiError::Internal => StatusCode::INTERNAL_SERVER_ERROR,
|
ApiError::Internal => StatusCode::INTERNAL_SERVER_ERROR,
|
||||||
};
|
};
|
||||||
(status, Json(json!({ "error": self.to_string() }))).into_response()
|
(status, Json(json!({ "error": self.to_string() }))).into_response()
|
||||||
|
|||||||
+440
-20
@@ -55,6 +55,17 @@ pub struct Verdict {
|
|||||||
/// verification is how a broken sandbox comes to claim it proved
|
/// verification is how a broken sandbox comes to claim it proved
|
||||||
/// something.
|
/// something.
|
||||||
pub checks: Vec<crate::evaluator_tools::CheckOutcome>,
|
pub checks: Vec<crate::evaluator_tools::CheckOutcome>,
|
||||||
|
/// Whether the judge came from a DIFFERENT provider family than the agent
|
||||||
|
/// that did the work.
|
||||||
|
///
|
||||||
|
/// The default judge is Claude judging Claude's output, which is a correlated
|
||||||
|
/// failure: the same model that talked itself into a shortcut is disposed to
|
||||||
|
/// accept it. Independence is the structural fix, and it is recorded rather
|
||||||
|
/// than assumed — a verdict that cannot say who judged it cannot be audited
|
||||||
|
/// for this. `serde(default)` so verdicts stored before this field existed
|
||||||
|
/// read back as "not independent", which is what they were.
|
||||||
|
#[serde(default)]
|
||||||
|
pub independent: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Verdict {
|
impl Verdict {
|
||||||
@@ -78,6 +89,7 @@ impl Verdict {
|
|||||||
guidance: reason.clone(),
|
guidance: reason.clone(),
|
||||||
reason,
|
reason,
|
||||||
model: model.to_string(),
|
model: model.to_string(),
|
||||||
|
independent: false,
|
||||||
error,
|
error,
|
||||||
checks: Vec::new(),
|
checks: Vec::new(),
|
||||||
}
|
}
|
||||||
@@ -187,7 +199,184 @@ produced by working code;
|
|||||||
|
|
||||||
If you find any of these, the condition is NOT met — say which one you found. \
|
If you find any of these, the condition is NOT met — say which one you found. \
|
||||||
If you cannot verify a claim, it is not met: absence of evidence is not \
|
If you cannot verify a claim, it is not met: absence of evidence is not \
|
||||||
satisfaction.";
|
satisfaction.
|
||||||
|
|
||||||
|
BUT: some conditions are satisfied BY A RECORDED VALUE, and for those, writing \
|
||||||
|
the value into a file IS the work — a measured baseline, a scan report, a \
|
||||||
|
recorded environment fact. Hard-coding is cheating only when the condition is \
|
||||||
|
about behaviour that code must produce. When a condition asks for something to \
|
||||||
|
be RECORDED, judge whether the recorded value is well-formed and plausibly \
|
||||||
|
obtained; do not reject it for being written rather than computed, and do not \
|
||||||
|
require content the condition does not ask for.
|
||||||
|
|
||||||
|
Judge the condition AS WRITTEN. Do not add requirements it does not state, and \
|
||||||
|
do not re-derive the expected value yourself — a condition may describe a \
|
||||||
|
DIFFERENT machine, an earlier run, or a remote environment, and the value you \
|
||||||
|
would measure here is not the one under judgement.";
|
||||||
|
|
||||||
|
/// Which provider family a model spec belongs to.
|
||||||
|
///
|
||||||
|
/// `"glm:glm-4.7"` → `glm`, `"kimi:k2"` → `kimi`, `"claude-opus-4-8"` → `anthropic`.
|
||||||
|
/// Used for one decision only: whether the judge is independent of the agent that
|
||||||
|
/// produced the work. A family, not a model — two Claude models share a lineage,
|
||||||
|
/// a fine-tune and most of their failure modes, so `opus` judging `sonnet` is not
|
||||||
|
/// independence.
|
||||||
|
pub fn provider_family(spec: &str) -> String {
|
||||||
|
if let Some((name, _)) = spec.split_once(':') {
|
||||||
|
// `runtime:<alias>` routes through an agent container, which is running
|
||||||
|
// Claude — the prefix names the transport, not the family.
|
||||||
|
return if name == "runtime" {
|
||||||
|
"anthropic".into()
|
||||||
|
} else {
|
||||||
|
name.to_ascii_lowercase()
|
||||||
|
};
|
||||||
|
}
|
||||||
|
let s = spec.to_ascii_lowercase();
|
||||||
|
for (needle, family) in [
|
||||||
|
("claude", "anthropic"),
|
||||||
|
("opus", "anthropic"),
|
||||||
|
("sonnet", "anthropic"),
|
||||||
|
("haiku", "anthropic"),
|
||||||
|
("glm", "glm"),
|
||||||
|
("kimi", "kimi"),
|
||||||
|
("moonshot", "kimi"),
|
||||||
|
("llama", "groq"),
|
||||||
|
// A model we host ourselves. Only reached for a BARE name — a
|
||||||
|
// `local:ornith-fleet:9b` spec is answered by the split above — but a
|
||||||
|
// bare one falling through to "unknown" would make
|
||||||
|
// `cross_provider_judge` refuse a judge that is genuinely a different
|
||||||
|
// family from the Anthropic implementer, which is the one property it
|
||||||
|
// exists to check.
|
||||||
|
("ornith", "local"),
|
||||||
|
("ollama", "local"),
|
||||||
|
] {
|
||||||
|
if s.contains(needle) {
|
||||||
|
return family.into();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Not "anthropic". An unknown model must not be assumed to be the house
|
||||||
|
// one — that assumption would report independence we never established.
|
||||||
|
"unknown".into()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Does this validator spec name the provider it wants, rather than only a model?
|
||||||
|
///
|
||||||
|
/// `Runtime::resolve_provider` routes `provider:model` and falls back to the
|
||||||
|
/// DEFAULT provider for everything else. That fallback is what makes a bare name
|
||||||
|
/// dangerous here: it silently yields the house provider, which the independence
|
||||||
|
/// check then fails to recognise as the house provider — because
|
||||||
|
/// `provider_family` reads the SPEC, and a bare `gemini-2.5-flash` reads as
|
||||||
|
/// "unknown", not "anthropic".
|
||||||
|
fn names_a_provider(spec: &str) -> bool {
|
||||||
|
spec.contains(':')
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The provider family the mission's agent ran on.
|
||||||
|
///
|
||||||
|
/// Today every mission backend is Claude Code (`agent-claude`), including the
|
||||||
|
/// microVM path. When `agent-glm` / `agent-kimi` images exist this should read
|
||||||
|
/// `missions.backend`; until then, hardcoding the truth is better than plumbing a
|
||||||
|
/// parameter that only ever has one value.
|
||||||
|
const IMPLEMENTER_FAMILY: &str = "anthropic";
|
||||||
|
|
||||||
|
/// Which validator spec applies, given the mission's own setting and the
|
||||||
|
/// deployment default.
|
||||||
|
///
|
||||||
|
/// The three cases are distinct on purpose, and an empty string is not the same
|
||||||
|
/// as unset:
|
||||||
|
/// - `Some("")` on the mission — an explicit opt OUT. This mission wants the house
|
||||||
|
/// judge, and the deployment default must not quietly reinstate independence it
|
||||||
|
/// was told to skip.
|
||||||
|
/// - `Some(spec)` — this mission's choice, which wins.
|
||||||
|
/// - `None` — nothing said, so the deployment default applies.
|
||||||
|
///
|
||||||
|
/// Whitespace counts as empty: a column set to `" "` by hand meant to say nothing.
|
||||||
|
fn resolve_validator_spec(mission: Option<&str>, deployment: Option<&str>) -> Option<String> {
|
||||||
|
match mission {
|
||||||
|
Some(s) if s.trim().is_empty() => None,
|
||||||
|
Some(s) => Some(s.trim().to_string()),
|
||||||
|
None => deployment
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(str::to_string),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A judge from a different provider family, if one is configured and REGISTERED.
|
||||||
|
///
|
||||||
|
/// `CLAWMATES_VALIDATOR_MODEL` holds a registry spec such as `glm:glm-4.7`.
|
||||||
|
/// Returns `None` — never a same-family judge — when it is unset, names the
|
||||||
|
/// implementer's own family, or names a provider this deployment did not register.
|
||||||
|
///
|
||||||
|
/// That last case is the trap worth naming: `Runtime::resolve_provider` falls back
|
||||||
|
/// to the DEFAULT provider when the registry has no such name, which would hand
|
||||||
|
/// back Claude while the caller believed it had asked for GLM. The fallback is
|
||||||
|
/// detectable because the returned model still carries the `name:` prefix, and it
|
||||||
|
/// is checked here rather than trusted.
|
||||||
|
async fn cross_provider_judge(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
|
mission_id: Uuid,
|
||||||
|
) -> Option<(std::sync::Arc<dyn cm_llm::LlmProvider>, String)> {
|
||||||
|
// Read per mission rather than widening `Mission` for one caller. One extra
|
||||||
|
// query per evaluation, against a path that is about to make a model call.
|
||||||
|
let per_mission: Option<String> =
|
||||||
|
sqlx::query_scalar("SELECT validator_model FROM missions WHERE id = $1")
|
||||||
|
.bind(mission_id)
|
||||||
|
.fetch_optional(runtime.pool())
|
||||||
|
.await
|
||||||
|
.unwrap_or(None)
|
||||||
|
.flatten();
|
||||||
|
let spec = resolve_validator_spec(
|
||||||
|
per_mission.as_deref(),
|
||||||
|
std::env::var("CLAWMATES_VALIDATOR_MODEL").ok().as_deref(),
|
||||||
|
)?;
|
||||||
|
let spec = spec.as_str();
|
||||||
|
let family = provider_family(spec);
|
||||||
|
if family == IMPLEMENTER_FAMILY {
|
||||||
|
eprintln!(
|
||||||
|
"evaluator: CLAWMATES_VALIDATOR_MODEL={spec} is the same provider family as the \
|
||||||
|
agent ({IMPLEMENTER_FAMILY}) — that is not an independent check, ignoring it"
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
// A validator spec MUST name its provider. `resolve_provider` falls back to
|
||||||
|
// the DEFAULT provider for anything it cannot route (runtime.rs), and for a
|
||||||
|
// bare model name that fallback is silent: `gemini-2.5-flash` has no colon,
|
||||||
|
// so it resolved to the house Anthropic provider while `provider_family`
|
||||||
|
// reported "unknown" — not "anthropic" — and the verdict was recorded
|
||||||
|
// `independent = true`. An Anthropic judge grading Anthropic work, labelled
|
||||||
|
// independent, which is the one claim this whole path exists to make honestly.
|
||||||
|
//
|
||||||
|
// The check below caught the same fallback for `glm:glm-4.7` when the `glm`
|
||||||
|
// provider was missing, because an unrouted spec comes back WHOLE. It could
|
||||||
|
// never catch a bare name.
|
||||||
|
if !names_a_provider(spec) {
|
||||||
|
eprintln!(
|
||||||
|
"evaluator: CLAWMATES_VALIDATOR_MODEL={spec} is not a registry spec \
|
||||||
|
(expected `provider:model`, e.g. `glm:glm-4.7`) — refusing to judge with \
|
||||||
|
the default provider and call it independent"
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let (provider, model) = runtime.resolve_provider(spec);
|
||||||
|
// Compared against the WHOLE spec, not tested for a colon.
|
||||||
|
//
|
||||||
|
// `resolve_provider` returns the spec unchanged when it does not recognise
|
||||||
|
// the provider, and returns the part after the FIRST colon when it does. The
|
||||||
|
// old test — "does the model half still contain a colon" — assumed model
|
||||||
|
// names never do. `local:ornith-fleet:9b` resolves correctly to provider
|
||||||
|
// `local`, model `ornith-fleet:9b`, and was rejected as unregistered. The
|
||||||
|
// chain preflight found it by reporting a provider it had just registered as
|
||||||
|
// UNREGISTERED.
|
||||||
|
if model == spec {
|
||||||
|
eprintln!(
|
||||||
|
"evaluator: no provider registered for {spec} — refusing to judge with the \
|
||||||
|
default provider and call it independent"
|
||||||
|
);
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some((provider, model))
|
||||||
|
}
|
||||||
|
|
||||||
/// The model spec to judge with.
|
/// The model spec to judge with.
|
||||||
///
|
///
|
||||||
@@ -221,20 +410,10 @@ fn subscription_model() -> String {
|
|||||||
/// case in the platform for a bare model call: fixed prompt, no tools, no
|
/// case in the platform for a bare model call: fixed prompt, no tools, no
|
||||||
/// memory, one JSON answer.
|
/// memory, one JSON answer.
|
||||||
fn subscription_judge() -> Option<cm_llm::AnthropicProvider> {
|
fn subscription_judge() -> Option<cm_llm::AnthropicProvider> {
|
||||||
let token = std::env::var("ANTHROPIC_OAUTH_TOKEN").ok()?;
|
// One definition of "the subscription", shared with the planner. This
|
||||||
let token = token.trim();
|
// carried its own copy; two of them is how one gets a prefix check the
|
||||||
if token.is_empty() {
|
// other lacks.
|
||||||
return None;
|
crate::subscription::provider()
|
||||||
}
|
|
||||||
if !token.starts_with("sk-ant-oat") {
|
|
||||||
eprintln!(
|
|
||||||
"evaluator: ANTHROPIC_OAUTH_TOKEN is set but is not a setup token \
|
|
||||||
(expected sk-ant-oat…) — ignoring it and using {}",
|
|
||||||
evaluator_model()
|
|
||||||
);
|
|
||||||
return None;
|
|
||||||
}
|
|
||||||
Some(cm_llm::AnthropicProvider::new(token.to_string()))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Judge whether `condition` holds given `evidence`.
|
/// Judge whether `condition` holds given `evidence`.
|
||||||
@@ -252,6 +431,53 @@ pub async fn evaluate(
|
|||||||
"COMPLETION CONDITION:\n{condition}\n\nEVIDENCE (agent claims — verify them):\n{evidence}"
|
"COMPLETION CONDITION:\n{condition}\n\nEVIDENCE (agent claims — verify them):\n{evidence}"
|
||||||
);
|
);
|
||||||
let sandbox = crate::evaluator_tools::Sandbox::for_mission(mission_id);
|
let sandbox = crate::evaluator_tools::Sandbox::for_mission(mission_id);
|
||||||
|
// Purged explicitly at every exit below: `Drop` runs as uid 65532 and cannot
|
||||||
|
// delete the root-owned `target/` the judge's own `cargo test` leaves behind.
|
||||||
|
// Wrapped so the purge below runs on EVERY exit: this function returns
|
||||||
|
// from several branches, and a cleanup only some paths reach is the same
|
||||||
|
// as no cleanup on the others.
|
||||||
|
let verdict = async {
|
||||||
|
|
||||||
|
// Most preferred: a judge from a DIFFERENT provider family, with the same
|
||||||
|
// allow-listed tool loop. Claude judging Claude's work is a correlated
|
||||||
|
// failure — the model that talked itself into a shortcut is the one disposed
|
||||||
|
// to accept it — and the tool loop is what makes the check evidence rather
|
||||||
|
// than opinion, so an independent judge must have it too.
|
||||||
|
if let Some((provider, model)) = cross_provider_judge(runtime, mission_id).await {
|
||||||
|
let system = match &sandbox {
|
||||||
|
Some(_) => format!("{EVAL_SYSTEM_VERIFYING}\n\n{VERDICT_CONTRACT}"),
|
||||||
|
None => format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}"),
|
||||||
|
};
|
||||||
|
eprintln!(
|
||||||
|
"evaluator: mission {mission_id} judged independently by {} ({})",
|
||||||
|
model,
|
||||||
|
provider_family(&model)
|
||||||
|
);
|
||||||
|
match judge_with_tools(provider.as_ref(), &system, &user, &model, sandbox.as_ref()).await {
|
||||||
|
Ok((text, checks)) => {
|
||||||
|
let mut v = parse_verdict(&model, &text);
|
||||||
|
v.guidance = sanitize_guidance(condition, evidence, &v.guidance);
|
||||||
|
v.checks = checks;
|
||||||
|
v.independent = true;
|
||||||
|
return v;
|
||||||
|
}
|
||||||
|
// Deliberately NOT a silent fall-through to the house judge. An
|
||||||
|
// independent check that failed and was quietly replaced by a
|
||||||
|
// same-family one would leave a verdict claiming a property it does
|
||||||
|
// not have. The phase stays unmet this pass and says why; the next
|
||||||
|
// sweep retries.
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!(
|
||||||
|
"evaluator: the independent judge ({model}) failed — NOT falling back to the agent's own provider: {e}"
|
||||||
|
);
|
||||||
|
return Verdict::not_met(
|
||||||
|
&model,
|
||||||
|
"the independent validator could not be reached this pass",
|
||||||
|
Some(e),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Preferred: a bare Messages API call on the subscription token. See
|
// Preferred: a bare Messages API call on the subscription token. See
|
||||||
// `subscription_judge` for why this beats routing through an agent.
|
// `subscription_judge` for why this beats routing through an agent.
|
||||||
@@ -262,6 +488,7 @@ pub async fn evaluate(
|
|||||||
None => format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}"),
|
None => format!("{EVAL_SYSTEM_EVIDENCE_ONLY}\n\n{VERDICT_CONTRACT}"),
|
||||||
};
|
};
|
||||||
let outcome = judge_with_tools(&provider, &system, &user, &model, sandbox.as_ref()).await;
|
let outcome = judge_with_tools(&provider, &system, &user, &model, sandbox.as_ref()).await;
|
||||||
|
// Same family as the agent; `independent` stays false below.
|
||||||
return match outcome {
|
return match outcome {
|
||||||
Err(e) => Verdict::not_met(
|
Err(e) => Verdict::not_met(
|
||||||
&model,
|
&model,
|
||||||
@@ -308,6 +535,12 @@ pub async fn evaluate(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
.await;
|
||||||
|
if let Some(sb) = &sandbox {
|
||||||
|
sb.purge().await;
|
||||||
|
}
|
||||||
|
verdict
|
||||||
|
}
|
||||||
|
|
||||||
/// Ceiling on verification commands per verdict. A judge that has run twelve
|
/// Ceiling on verification commands per verdict. A judge that has run twelve
|
||||||
/// commands and still cannot tell is not going to be rescued by a thirteenth,
|
/// commands and still cannot tell is not going to be rescued by a thirteenth,
|
||||||
@@ -346,14 +579,22 @@ fn verify_tool() -> cm_llm::ToolDescriptor {
|
|||||||
///
|
///
|
||||||
/// With no sandbox this degenerates to a single call — same shape, no tools
|
/// With no sandbox this degenerates to a single call — same shape, no tools
|
||||||
/// offered — so there is one code path for both kinds of phase.
|
/// offered — so there is one code path for both kinds of phase.
|
||||||
|
/// `&dyn LlmProvider`, not `&AnthropicProvider`.
|
||||||
|
///
|
||||||
|
/// The trait is a single method — `stream(ChatRequest)` — and this loop only ever
|
||||||
|
/// used that, so the concrete type was incidental. Widening it is what lets a
|
||||||
|
/// CROSS-PROVIDER judge run the same allow-listed checks: before this, independence
|
||||||
|
/// and real verification were mutually exclusive, because the tool loop lived only
|
||||||
|
/// on the subscription path and every other route "judged claims only".
|
||||||
|
/// GLM is registered in anthropic format, so tool calling reaches it unchanged.
|
||||||
async fn judge_with_tools(
|
async fn judge_with_tools(
|
||||||
provider: &cm_llm::AnthropicProvider,
|
provider: &dyn cm_llm::LlmProvider,
|
||||||
system: &str,
|
system: &str,
|
||||||
user: &str,
|
user: &str,
|
||||||
model: &str,
|
model: &str,
|
||||||
sandbox: Option<&crate::evaluator_tools::Sandbox>,
|
sandbox: Option<&crate::evaluator_tools::Sandbox>,
|
||||||
) -> Result<(String, Vec<crate::evaluator_tools::CheckOutcome>), String> {
|
) -> Result<(String, Vec<crate::evaluator_tools::CheckOutcome>), String> {
|
||||||
use cm_llm::{ChatMessage, ChatRequest, ChatRole, ContentPart, LlmEvent, LlmProvider};
|
use cm_llm::{ChatMessage, ChatRequest, ChatRole, ContentPart, LlmEvent};
|
||||||
use futures::StreamExt as _;
|
use futures::StreamExt as _;
|
||||||
|
|
||||||
let tools = match sandbox {
|
let tools = match sandbox {
|
||||||
@@ -507,6 +748,8 @@ fn parse_verdict(model: &str, text: &str) -> Verdict {
|
|||||||
reason,
|
reason,
|
||||||
guidance,
|
guidance,
|
||||||
model: model.to_string(),
|
model: model.to_string(),
|
||||||
|
// Set by the caller: only `evaluate` knows which provider judged.
|
||||||
|
independent: false,
|
||||||
error: None,
|
error: None,
|
||||||
checks: Vec::new(),
|
checks: Vec::new(),
|
||||||
}
|
}
|
||||||
@@ -531,12 +774,14 @@ pub async fn record(
|
|||||||
) -> Result<(), sqlx::Error> {
|
) -> Result<(), sqlx::Error> {
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
"INSERT INTO mission_phase_evaluations
|
"INSERT INTO mission_phase_evaluations
|
||||||
(id, mission_id, phase_id, iteration, met, reason, guidance, model, error, checks)
|
(id, mission_id, phase_id, iteration, met, reason, guidance, model, error,
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
checks, independent)
|
||||||
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
|
||||||
ON CONFLICT (phase_id, iteration) DO UPDATE
|
ON CONFLICT (phase_id, iteration) DO UPDATE
|
||||||
SET met = EXCLUDED.met, reason = EXCLUDED.reason,
|
SET met = EXCLUDED.met, reason = EXCLUDED.reason,
|
||||||
guidance = EXCLUDED.guidance, model = EXCLUDED.model,
|
guidance = EXCLUDED.guidance, model = EXCLUDED.model,
|
||||||
error = EXCLUDED.error, checks = EXCLUDED.checks",
|
error = EXCLUDED.error, checks = EXCLUDED.checks,
|
||||||
|
independent = EXCLUDED.independent",
|
||||||
)
|
)
|
||||||
.bind(Uuid::now_v7())
|
.bind(Uuid::now_v7())
|
||||||
.bind(mission_id)
|
.bind(mission_id)
|
||||||
@@ -548,6 +793,7 @@ pub async fn record(
|
|||||||
.bind(&v.model)
|
.bind(&v.model)
|
||||||
.bind(v.error.as_deref())
|
.bind(v.error.as_deref())
|
||||||
.bind(serde_json::json!(v.checks))
|
.bind(serde_json::json!(v.checks))
|
||||||
|
.bind(v.independent)
|
||||||
.execute(pool)
|
.execute(pool)
|
||||||
.await
|
.await
|
||||||
.map(|_| ())
|
.map(|_| ())
|
||||||
@@ -581,6 +827,180 @@ pub async fn latest(
|
|||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod cross_provider_tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// A bare model name must never be accepted as a validator spec.
|
||||||
|
///
|
||||||
|
/// `resolve_provider` falls back to the DEFAULT provider for anything it
|
||||||
|
/// cannot route, and for a bare name that fallback is invisible: the spec
|
||||||
|
/// has no `provider:` prefix to come back with, so the existing
|
||||||
|
/// "no provider registered" check cannot see it. The result was an
|
||||||
|
/// Anthropic judge grading Anthropic work with `independent = true`.
|
||||||
|
#[test]
|
||||||
|
fn a_validator_spec_must_name_its_provider() {
|
||||||
|
for good in ["glm:glm-4.7", "kimi:kimi-for-coding", "runtime:some-alias"] {
|
||||||
|
assert!(names_a_provider(good), "{good} is a registry spec");
|
||||||
|
}
|
||||||
|
// These are the dangerous ones: they resolve to the DEFAULT provider.
|
||||||
|
for bare in ["gemini-2.5-flash", "claude-sonnet-5", "glm-4.7", ""] {
|
||||||
|
assert!(
|
||||||
|
!names_a_provider(bare),
|
||||||
|
"{bare:?} names no provider and must be refused"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A family, not a model. Two Claude models share a lineage and most of their
|
||||||
|
/// failure modes, so `opus` judging `sonnet` is not an independent check.
|
||||||
|
#[test]
|
||||||
|
fn every_anthropic_spelling_is_one_family() {
|
||||||
|
for spec in [
|
||||||
|
"claude-opus-4-8",
|
||||||
|
"claude-sonnet-5",
|
||||||
|
"claude-haiku-4-5-20251001",
|
||||||
|
"opus",
|
||||||
|
"runtime:claw_1234", // routes through an agent container running Claude
|
||||||
|
] {
|
||||||
|
assert_eq!(provider_family(spec), "anthropic", "{spec}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The registry prefix is what actually selects a different provider.
|
||||||
|
#[test]
|
||||||
|
fn a_registry_prefix_names_the_family() {
|
||||||
|
assert_eq!(provider_family("glm:glm-4.7"), "glm");
|
||||||
|
assert_eq!(provider_family("kimi:kimi-k2"), "kimi");
|
||||||
|
assert_eq!(provider_family("GLM:GLM-4.7"), "glm");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An unrecognised model must NOT be assumed to be the house one. Guessing
|
||||||
|
/// "anthropic" would understate independence; guessing anything else would
|
||||||
|
/// claim independence we never established. So: unknown.
|
||||||
|
#[test]
|
||||||
|
fn an_unrecognised_model_is_not_assumed_to_be_ours() {
|
||||||
|
assert_eq!(provider_family("some-new-model-v9"), "unknown");
|
||||||
|
assert_ne!(provider_family("some-new-model-v9"), IMPLEMENTER_FAMILY);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The whole point: a judge in the implementer's own family is not
|
||||||
|
/// independent, whichever model it is.
|
||||||
|
#[test]
|
||||||
|
fn a_same_family_judge_is_never_independent() {
|
||||||
|
for spec in ["claude-opus-4-8", "runtime:claw_x", "sonnet"] {
|
||||||
|
assert_eq!(
|
||||||
|
provider_family(spec),
|
||||||
|
IMPLEMENTER_FAMILY,
|
||||||
|
"{spec} would have to be rejected as a validator"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for spec in ["glm:glm-4.7", "kimi:kimi-k2"] {
|
||||||
|
assert_ne!(provider_family(spec), IMPLEMENTER_FAMILY, "{spec}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A mission's own choice wins over the deployment default.
|
||||||
|
#[test]
|
||||||
|
fn a_mission_can_choose_its_validator() {
|
||||||
|
assert_eq!(
|
||||||
|
resolve_validator_spec(Some("kimi:kimi-k2"), Some("glm:glm-4.7")).as_deref(),
|
||||||
|
Some("kimi:kimi-k2")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
resolve_validator_spec(None, Some("glm:glm-4.7")).as_deref(),
|
||||||
|
Some("glm:glm-4.7"),
|
||||||
|
"nothing said on the mission means the deployment default applies"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An EMPTY value on the mission is an explicit opt-out, not "unset". The
|
||||||
|
/// deployment default must not quietly reinstate independence a mission was
|
||||||
|
/// told to skip — the two cases look the same in a nullable text column and
|
||||||
|
/// mean opposite things.
|
||||||
|
#[test]
|
||||||
|
fn an_empty_mission_setting_opts_out_rather_than_falling_back() {
|
||||||
|
for spelling in [Some(""), Some(" ")] {
|
||||||
|
assert_eq!(
|
||||||
|
resolve_validator_spec(spelling, Some("glm:glm-4.7")),
|
||||||
|
None,
|
||||||
|
"{spelling:?} asked for no independent validator"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// And with neither set, there is no independent judge — which is the state
|
||||||
|
/// every deployment starts in.
|
||||||
|
#[test]
|
||||||
|
fn no_setting_anywhere_means_no_independent_judge() {
|
||||||
|
assert_eq!(resolve_validator_spec(None, None), None);
|
||||||
|
assert_eq!(resolve_validator_spec(None, Some(" ")), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A phase that ran out of passes without meeting its condition did NOT
|
||||||
|
/// succeed. It used to be recorded `completed` alongside a verdict saying
|
||||||
|
/// `met=false`, so mission status reported a goal that was never reached as a
|
||||||
|
/// goal achieved. Found by the Goodhart test: an independent judge refused the
|
||||||
|
/// phase, and the mission closed green anyway.
|
||||||
|
#[test]
|
||||||
|
fn an_unmet_condition_does_not_close_a_phase_as_completed() {
|
||||||
|
// Mirrors the decision in `phase_runner::evaluate_finished_phases`.
|
||||||
|
let outcome = |met: bool| if met { "completed" } else { "failed" };
|
||||||
|
assert_eq!(outcome(true), "completed");
|
||||||
|
assert_eq!(
|
||||||
|
outcome(false),
|
||||||
|
"failed",
|
||||||
|
"an exhausted, unmet phase must not share a status with a met one"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A verdict that has not been marked independent must not read as one. This
|
||||||
|
/// is the field's default, and old rows stored before it existed deserialize
|
||||||
|
/// to exactly that.
|
||||||
|
/// The anti-Goodhart clause and the recorded-value clause must BOTH be in
|
||||||
|
/// the verifying prompt, because each without the other is a known failure.
|
||||||
|
///
|
||||||
|
/// Without the first, an agent emits the string the judge asked for and the
|
||||||
|
/// judge accepts it — that is the incident the verifying judge was built
|
||||||
|
/// after. Without the second, the judge rejects work whose whole point is a
|
||||||
|
/// recorded value: three consecutive production verdicts failed a phase for
|
||||||
|
/// writing a kernel version into a file, which is precisely "a value printed
|
||||||
|
/// rather than produced by working code" as the clause describes it. Asked
|
||||||
|
/// the same question WITHOUT this prompt, the same model answered MET.
|
||||||
|
#[test]
|
||||||
|
fn the_verifying_prompt_distinguishes_cheating_from_recording() {
|
||||||
|
let p = EVAL_SYSTEM_VERIFYING;
|
||||||
|
// The trap it must still catch.
|
||||||
|
assert!(p.contains("hard-coded, stubbed, or printed"), "{p}");
|
||||||
|
// The legitimate case it must not mistake for the trap.
|
||||||
|
assert!(p.contains("RECORDED VALUE"), "{p}");
|
||||||
|
assert!(
|
||||||
|
p.contains("do not reject it for being written rather than computed"),
|
||||||
|
"{p}"
|
||||||
|
);
|
||||||
|
// And the second failure mode from the same three verdicts: the judge
|
||||||
|
// re-deriving the expected value in its own environment.
|
||||||
|
assert!(p.contains("do not re-derive the expected value yourself"), "{p}");
|
||||||
|
assert!(p.contains("Judge the condition AS WRITTEN"), "{p}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_verdict_defaults_to_not_independent() {
|
||||||
|
let v = Verdict::not_met("claude-opus-4-8", "nope", None);
|
||||||
|
assert!(!v.independent);
|
||||||
|
|
||||||
|
let stored = serde_json::json!({
|
||||||
|
"met": true, "reason": "r", "guidance": "", "model": "claude-opus-4-8",
|
||||||
|
"error": null, "checks": []
|
||||||
|
});
|
||||||
|
let old: Verdict = serde_json::from_value(stored).expect("an old verdict still reads");
|
||||||
|
assert!(
|
||||||
|
!old.independent,
|
||||||
|
"a verdict written before independence was recorded was not independent"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|||||||
@@ -195,11 +195,39 @@ pub fn clamp_output(s: &str) -> String {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Where a verification copy lives: a sibling of the per-mission directories,
|
||||||
|
/// so the sweeper that deletes `<root>/<mission_id>` never races it and nothing
|
||||||
|
/// under it is ever collected or delivered.
|
||||||
|
fn verify_path(mission_id: Uuid) -> PathBuf {
|
||||||
|
crate::mission_workspace::missions_root()
|
||||||
|
.join("_verify")
|
||||||
|
.join(mission_id.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
/// A checkout the judge may run verification commands against.
|
/// A checkout the judge may run verification commands against.
|
||||||
#[derive(Debug, Clone)]
|
///
|
||||||
|
/// A COPY of the mission's checkout, never the checkout itself. The judge runs
|
||||||
|
/// real commands — `cargo test` is the whole point — and the container it execs
|
||||||
|
/// into runs as ROOT with the missions root bind-mounted, so running them in the
|
||||||
|
/// live tree left `repo/target/` owned by uid 0 in a checkout otherwise owned by
|
||||||
|
/// the server. That breaks the single-writer invariant copy mode exists to
|
||||||
|
/// guarantee, and the next phase's `cargo` would hit permission-denied on a
|
||||||
|
/// directory it cannot write.
|
||||||
|
///
|
||||||
|
/// It stayed invisible all day because a dead validator credential meant the
|
||||||
|
/// judge never ran a single check; restoring the credential surfaced it on the
|
||||||
|
/// first gated mission, via the harness's uid probe.
|
||||||
|
///
|
||||||
|
/// The deeper rule is the one this codebase already applies to the `verifier`
|
||||||
|
/// subagent, which has no Edit and no Write: **verification must not mutate what
|
||||||
|
/// it verifies.** A judge that can change the tree it is judging can make its own
|
||||||
|
/// verdict true.
|
||||||
|
#[derive(Debug)]
|
||||||
pub struct Sandbox {
|
pub struct Sandbox {
|
||||||
container: String,
|
container: String,
|
||||||
workdir: PathBuf,
|
workdir: PathBuf,
|
||||||
|
/// Whether this sandbox created `workdir` and must remove it.
|
||||||
|
owned: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Sandbox {
|
impl Sandbox {
|
||||||
@@ -208,22 +236,59 @@ impl Sandbox {
|
|||||||
///
|
///
|
||||||
/// Returning `None` rather than an empty sandbox matters: the evaluator
|
/// Returning `None` rather than an empty sandbox matters: the evaluator
|
||||||
/// prompt changes shape depending on whether verification is possible, and
|
/// prompt changes shape depending on whether verification is possible, and
|
||||||
/// a judge must never be told it can check something it cannot.
|
/// a judge must never be told it can check something it cannot. A copy that
|
||||||
|
/// fails to materialise is also `None` for the same reason — an unverifiable
|
||||||
|
/// phase must not be told it can verify.
|
||||||
pub fn for_mission(mission_id: Uuid) -> Option<Sandbox> {
|
pub fn for_mission(mission_id: Uuid) -> Option<Sandbox> {
|
||||||
let workdir = crate::mission_workspace::checkout_path(mission_id);
|
Sandbox::for_checkout(
|
||||||
if !workdir.is_dir() {
|
&crate::mission_workspace::checkout_path(mission_id),
|
||||||
|
&verify_path(mission_id),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The testable half of [`Sandbox::for_mission`]. The paths are parameters
|
||||||
|
/// because `missions_root()` reads process environment, and this workspace
|
||||||
|
/// does not mutate that in tests — the same split as
|
||||||
|
/// `mission_runtime::provider_env_from` and
|
||||||
|
/// `mission_workspace::auth_with_token`.
|
||||||
|
pub fn for_checkout(source: &Path, root: &Path) -> Option<Sandbox> {
|
||||||
|
if !source.is_dir() {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
// `root_copy` owns this pattern for all four callers — the judge, the
|
||||||
|
// benchmark runner, the on_green_tests gate, and this. It packs through
|
||||||
|
// the transport packer (one exclusion list, so a copy carries exactly
|
||||||
|
// what a delivered diff carries) and its `purge` is the only thing that
|
||||||
|
// can remove the root-owned `target/` a run leaves behind.
|
||||||
|
//
|
||||||
|
// A stale copy would otherwise be verified instead of this pass's work —
|
||||||
|
// the "judged a tree nobody wrote" shape the evaluator exists to prevent
|
||||||
|
// — so the caller purges before constructing.
|
||||||
|
// `into_workdir` because the judge has not run yet: letting the handle's
|
||||||
|
// Drop fire on return would delete the tree out from under it. `Sandbox`
|
||||||
|
// owns the lifetime from here, and `Sandbox::purge` clears it.
|
||||||
|
let workdir = crate::root_copy::RootCopy::of(source, root)
|
||||||
|
.ok()?
|
||||||
|
.into_workdir();
|
||||||
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
||||||
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
||||||
Some(Sandbox { container, workdir })
|
Some(Sandbox {
|
||||||
|
container,
|
||||||
|
workdir,
|
||||||
|
owned: true,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Construct against an explicit path. Test seam.
|
/// Construct against an explicit path. Test seam.
|
||||||
|
///
|
||||||
|
/// Never `owned`: a caller-supplied directory is the caller's, and deleting
|
||||||
|
/// it on drop would make this seam destructive in a way its users could not
|
||||||
|
/// see.
|
||||||
pub fn at(container: impl Into<String>, workdir: impl AsRef<Path>) -> Sandbox {
|
pub fn at(container: impl Into<String>, workdir: impl AsRef<Path>) -> Sandbox {
|
||||||
Sandbox {
|
Sandbox {
|
||||||
container: container.into(),
|
container: container.into(),
|
||||||
workdir: workdir.as_ref().to_path_buf(),
|
workdir: workdir.as_ref().to_path_buf(),
|
||||||
|
owned: false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -329,6 +394,53 @@ fn git_ownership_env(workdir: &str) -> Vec<String> {
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl Sandbox {
|
||||||
|
/// Remove the copy, from inside the container that wrote it.
|
||||||
|
///
|
||||||
|
/// `Drop` cannot do this. The judge runs `cargo test` in a container as
|
||||||
|
/// ROOT, so the copy's `target/` is root-owned, and the server process is
|
||||||
|
/// uid 65532 — its `remove_dir_all` fails on those files and leaves the
|
||||||
|
/// whole tree behind. Measured: 16 MB across two stranded copies, the oldest
|
||||||
|
/// hours old, while `Drop` logged nothing anyone read.
|
||||||
|
///
|
||||||
|
/// The claim that "the next pass clears anyway" was wrong for the same
|
||||||
|
/// reason: `for_checkout` removes a stale root before copying, with the same
|
||||||
|
/// uid, and fails the same way.
|
||||||
|
///
|
||||||
|
/// Still best-effort — a housekeeping error must not cost a real verdict —
|
||||||
|
/// but now attempted by something that can actually succeed.
|
||||||
|
pub async fn purge(&self) {
|
||||||
|
if !self.owned {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let Some(root) = self.workdir.parent() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
// The same purge as the other three copy sites, not a fourth copy of
|
||||||
|
// it: an inlined duplicate is how the reap paths drifted apart before.
|
||||||
|
crate::root_copy::purge(&self.container, root).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for Sandbox {
|
||||||
|
/// Fallback only — see [`Sandbox::purge`], which is what actually clears a
|
||||||
|
/// copy the judge has run commands in. This still catches the early paths
|
||||||
|
/// where nothing has run as root yet.
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if !self.owned {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if let Some(root) = self.workdir.parent() {
|
||||||
|
if let Err(e) = std::fs::remove_dir_all(root) {
|
||||||
|
eprintln!(
|
||||||
|
"evaluator_tools: could not remove the verification copy at {} ({e})",
|
||||||
|
root.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// One verification command and what became of it.
|
/// One verification command and what became of it.
|
||||||
///
|
///
|
||||||
/// This exists because the first version recorded *attempted* commands. The
|
/// This exists because the first version recorded *attempted* commands. The
|
||||||
@@ -427,6 +539,58 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// THE regression. The judge runs real commands in a container that runs as
|
||||||
|
/// ROOT with the missions root bind-mounted, so verifying the live checkout
|
||||||
|
/// left `repo/target/` owned by uid 0 in a tree owned by the server — the
|
||||||
|
/// single-writer invariant broken by the thing that was supposed to be
|
||||||
|
/// checking the work. Verifying a COPY makes it unrepresentable.
|
||||||
|
#[test]
|
||||||
|
fn the_judge_verifies_a_copy_and_never_the_mission_tree() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let root = tmp.path().join("missions-root");
|
||||||
|
let mission = Uuid::now_v7();
|
||||||
|
let checkout = root.join(mission.to_string()).join("repo");
|
||||||
|
std::fs::create_dir_all(checkout.join("src")).unwrap();
|
||||||
|
std::fs::write(checkout.join("Cargo.toml"), "[package]\nname='x'\n").unwrap();
|
||||||
|
std::fs::write(checkout.join("src/lib.rs"), "pub fn a() {}").unwrap();
|
||||||
|
// Build output the transport already excludes; the copy must not carry
|
||||||
|
// it either, or the judge measures a stale artifact.
|
||||||
|
std::fs::create_dir_all(checkout.join("target/debug")).unwrap();
|
||||||
|
std::fs::write(checkout.join("target/debug/junk"), "x").unwrap();
|
||||||
|
|
||||||
|
let sandbox = Sandbox::for_checkout(&checkout, &root.join("_verify").join(mission.to_string()))
|
||||||
|
.expect("a checkout on disk yields a sandbox");
|
||||||
|
|
||||||
|
assert_ne!(
|
||||||
|
sandbox.workdir(),
|
||||||
|
checkout,
|
||||||
|
"the judge must not be pointed at the mission's own checkout"
|
||||||
|
);
|
||||||
|
assert!(sandbox.workdir().join("src/lib.rs").is_file(), "the copy has the source");
|
||||||
|
assert!(
|
||||||
|
!sandbox.workdir().join("target").exists(),
|
||||||
|
"the copy must not carry build output: {}",
|
||||||
|
sandbox.workdir().display()
|
||||||
|
);
|
||||||
|
|
||||||
|
// And dropping it takes the copy with it, leaving the mission untouched.
|
||||||
|
let copy_root = sandbox.workdir().parent().unwrap().to_path_buf();
|
||||||
|
drop(sandbox);
|
||||||
|
assert!(!copy_root.exists(), "the copy outlived its sandbox");
|
||||||
|
assert!(checkout.join("src/lib.rs").is_file(), "the mission tree is intact");
|
||||||
|
assert!(checkout.join("target/debug/junk").is_file());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The test seam must not delete a directory it was handed. A destructive
|
||||||
|
/// constructor that looks like a plain one is how a test wipes a real tree.
|
||||||
|
#[test]
|
||||||
|
fn an_explicit_workdir_is_never_deleted() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
std::fs::write(tmp.path().join("keep.txt"), "x").unwrap();
|
||||||
|
drop(Sandbox::at("c", tmp.path()));
|
||||||
|
assert!(tmp.path().join("keep.txt").is_file());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn refuses_programs_off_the_list() {
|
fn refuses_programs_off_the_list() {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
@@ -364,6 +364,15 @@ enum Uplink {
|
|||||||
Result { id: u64, ok: bool, output: String },
|
Result { id: u64, ok: bool, output: String },
|
||||||
#[serde(rename = "pty_out")]
|
#[serde(rename = "pty_out")]
|
||||||
PtyOut { sid: u64, data: String },
|
PtyOut { sid: u64, data: String },
|
||||||
|
/// A chunk of a microVM turn's stdout/stderr, as it happens.
|
||||||
|
///
|
||||||
|
/// Keyed by RUN id rather than a session id: a mission run is the thing a
|
||||||
|
/// browser subscribes to, and unlike a PTY there is no interactive session
|
||||||
|
/// to allocate. `at` is the byte offset AFTER this chunk, so the node can
|
||||||
|
/// resume a dropped tail without replaying — the same contract `fcagent`'s
|
||||||
|
/// `tail` op exposes.
|
||||||
|
#[serde(rename = "vm_out")]
|
||||||
|
VmOut { run_id: String, at: u64, data: String },
|
||||||
#[serde(rename = "pty_exit")]
|
#[serde(rename = "pty_exit")]
|
||||||
PtyExit { sid: u64 },
|
PtyExit { sid: u64 },
|
||||||
#[serde(rename = "webrtc_answer")]
|
#[serde(rename = "webrtc_answer")]
|
||||||
@@ -381,6 +390,11 @@ enum Uplink {
|
|||||||
NodeTools {
|
NodeTools {
|
||||||
tools: std::collections::HashMap<String, String>,
|
tools: std::collections::HashMap<String, String>,
|
||||||
},
|
},
|
||||||
|
/// What the node can HOST, as opposed to what it has installed — the
|
||||||
|
/// inputs to placement predicates. Free-form so a new predicate does not
|
||||||
|
/// need a migration; see `migrations/0065_microvm_placement.sql`.
|
||||||
|
#[serde(rename = "node_capabilities")]
|
||||||
|
NodeCapabilities { capabilities: serde_json::Value },
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
@@ -482,6 +496,44 @@ pub async fn run_channel(pool: PgPool, hub: Arc<NodeHub>, node_id: NodeId, socke
|
|||||||
let _ = s.send(ExecOutput { ok, output });
|
let _ = s.send(ExecOutput { ok, output });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// A chunk of a microVM turn's output, live.
|
||||||
|
//
|
||||||
|
// Appended to the run's checkpoint rather than only fanned
|
||||||
|
// out: `PtyOut` above is deliberately ephemeral because a
|
||||||
|
// terminal has no history worth keeping, but a mission's log
|
||||||
|
// is the record of what the agent did — the Output tab has
|
||||||
|
// to still show it an hour later. Live and durable are
|
||||||
|
// different requirements and this needs both.
|
||||||
|
//
|
||||||
|
// `jsonb ||` merges into whatever else the checkpoint holds
|
||||||
|
// (`records`, written by the turn itself), so the two writers
|
||||||
|
// do not clobber each other.
|
||||||
|
Ok(Uplink::VmOut { run_id, at, data }) => {
|
||||||
|
if let (Ok(rid), Ok(bytes)) =
|
||||||
|
(uuid::Uuid::parse_str(&run_id), B64.decode(&data))
|
||||||
|
{
|
||||||
|
let text = String::from_utf8_lossy(&bytes).to_string();
|
||||||
|
if let Err(e) = sqlx::query(
|
||||||
|
"UPDATE topology_runs
|
||||||
|
SET checkpoint = COALESCE(checkpoint, '{}'::jsonb)
|
||||||
|
|| jsonb_build_object(
|
||||||
|
'log',
|
||||||
|
COALESCE(checkpoint->>'log', '') || $2::text,
|
||||||
|
'log_at', $3::bigint
|
||||||
|
),
|
||||||
|
updated_at = now()
|
||||||
|
WHERE id = $1",
|
||||||
|
)
|
||||||
|
.bind(rid)
|
||||||
|
.bind(&text)
|
||||||
|
.bind(at as i64)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
eprintln!("fleet: appending vm_out for run {rid}: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Ok(Uplink::PtyOut { sid, data }) => {
|
Ok(Uplink::PtyOut { sid, data }) => {
|
||||||
if let Ok(bytes) = B64.decode(&data) {
|
if let Ok(bytes) = B64.decode(&data) {
|
||||||
let sink = conn.pty_sinks.lock().await.get(&sid).cloned();
|
let sink = conn.pty_sinks.lock().await.get(&sid).cloned();
|
||||||
@@ -539,7 +591,32 @@ pub async fn run_channel(pool: PgPool, hub: Arc<NodeHub>, node_id: NodeId, socke
|
|||||||
let pairs: Vec<(String, String)> = tools.into_iter().collect();
|
let pairs: Vec<(String, String)> = tools.into_iter().collect();
|
||||||
let _ = cm_db::repo::node_tools::upsert(&pool, node_id, &pairs).await;
|
let _ = cm_db::repo::node_tools::upsert(&pool, node_id, &pairs).await;
|
||||||
}
|
}
|
||||||
Err(_) => {}
|
Ok(Uplink::NodeCapabilities { capabilities }) => {
|
||||||
|
if let Err(e) = nodes::set_capabilities(&pool, node_id, &capabilities).await
|
||||||
|
{
|
||||||
|
// Loud: a node whose capabilities never land looks
|
||||||
|
// exactly like a node that has none, and will be
|
||||||
|
// passed over for every microVM mission forever
|
||||||
|
// while appearing perfectly healthy.
|
||||||
|
eprintln!(
|
||||||
|
"fleet: could not record capabilities for node {node_id} ({e}) — \
|
||||||
|
it will not be selected for microvm placement"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// An unparseable frame used to vanish here. That is the
|
||||||
|
// worst possible handling: a node op whose reply does not
|
||||||
|
// match `Uplink` never resolves its pending request, so the
|
||||||
|
// caller times out after 20s with nothing anywhere saying
|
||||||
|
// why. Caught exactly that way while wiring the vm_* ops —
|
||||||
|
// `output` was an object where the wire declares a String.
|
||||||
|
Err(e) => {
|
||||||
|
let head: String = t.as_str().chars().take(160).collect();
|
||||||
|
eprintln!(
|
||||||
|
"fleet: node {node_id} sent a frame we could not parse ({e}); \
|
||||||
|
any request it was answering will time out. Frame: {head}"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
+147
-44
@@ -13,16 +13,28 @@
|
|||||||
//! reviewer picked. Rejected proposals move to status='rejected';
|
//! reviewer picked. Rejected proposals move to status='rejected';
|
||||||
//! partial approvals move to status='partial'.
|
//! partial approvals move to status='partial'.
|
||||||
//!
|
//!
|
||||||
//! Uses Gemini 2.5 Flash as the default proposer model — cheap,
|
//! The proposer model resolves through the provider REGISTRY
|
||||||
//! JSON-mode-native, plenty of room for structured output. Configurable
|
//! (`Runtime::resolve_provider`), the same path the evaluator uses, and defaults
|
||||||
//! via CLAWMATES_LEVEL_UP_MODEL.
|
//! to `glm:glm-4.7`. Configurable via `CLAWMATES_LEVEL_UP_MODEL` as a registry
|
||||||
|
//! spec (`glm:glm-4.7`, `kimi:k2`, `claude-sonnet-5`, …).
|
||||||
|
//!
|
||||||
|
//! It used to call Gemini directly over bespoke HTTP with `GEMINI_API_KEY`. Two
|
||||||
|
//! problems with that, one fatal: it was the only thing standing between this
|
||||||
|
//! feature and a dead prepayment balance, and it duplicated a provider client
|
||||||
|
//! the codebase already has. Going through the registry means every provider the
|
||||||
|
//! platform can already reach works here, and no single vendor's billing can
|
||||||
|
//! take the feature down.
|
||||||
|
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
use sqlx::PgPool;
|
use sqlx::PgPool;
|
||||||
use sqlx::Row;
|
use sqlx::Row;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
const DEFAULT_MODEL: &str = "gemini-2.5-flash";
|
/// Registry spec, not a bare model name — the registry needs the provider.
|
||||||
|
///
|
||||||
|
/// GLM: cheap, reliable at structured output, and already the validator this
|
||||||
|
/// project measured and chose (see `scripts/judge-eval.sh`).
|
||||||
|
const DEFAULT_MODEL: &str = "glm:glm-4.7";
|
||||||
|
|
||||||
fn model_name() -> String {
|
fn model_name() -> String {
|
||||||
std::env::var("CLAWMATES_LEVEL_UP_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
std::env::var("CLAWMATES_LEVEL_UP_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
||||||
@@ -31,6 +43,7 @@ fn model_name() -> String {
|
|||||||
/// Analyze an agent + insert a pending proposal. Returns the proposal id.
|
/// Analyze an agent + insert a pending proposal. Returns the proposal id.
|
||||||
pub async fn propose_agent(
|
pub async fn propose_agent(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
workspace_id: cm_domain::WorkspaceId,
|
workspace_id: cm_domain::WorkspaceId,
|
||||||
created_by: cm_domain::UserId,
|
created_by: cm_domain::UserId,
|
||||||
agent_id: Uuid,
|
agent_id: Uuid,
|
||||||
@@ -50,6 +63,7 @@ pub async fn propose_agent(
|
|||||||
.flatten();
|
.flatten();
|
||||||
|
|
||||||
let payload = call_llm_for_agent(
|
let payload = call_llm_for_agent(
|
||||||
|
runtime,
|
||||||
&agent.name,
|
&agent.name,
|
||||||
&agent.job_title,
|
&agent.job_title,
|
||||||
&agent.system_prompt,
|
&agent.system_prompt,
|
||||||
@@ -79,6 +93,7 @@ pub async fn propose_agent(
|
|||||||
/// Analyze a team + insert a pending proposal. Returns the proposal id.
|
/// Analyze a team + insert a pending proposal. Returns the proposal id.
|
||||||
pub async fn propose_team(
|
pub async fn propose_team(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
workspace_id: cm_domain::WorkspaceId,
|
workspace_id: cm_domain::WorkspaceId,
|
||||||
created_by: cm_domain::UserId,
|
created_by: cm_domain::UserId,
|
||||||
team_id: Uuid,
|
team_id: Uuid,
|
||||||
@@ -115,7 +130,7 @@ pub async fn propose_team(
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
let payload = call_llm_for_team(&member_summaries).await?;
|
let payload = call_llm_for_team(runtime, &member_summaries).await?;
|
||||||
|
|
||||||
let model = model_name();
|
let model = model_name();
|
||||||
let id = cm_db::repo::level_up::insert(
|
let id = cm_db::repo::level_up::insert(
|
||||||
@@ -418,6 +433,7 @@ async fn recent_run_summary(pool: &PgPool, agent_id: Uuid, limit: i64) -> Result
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn call_llm_for_agent(
|
async fn call_llm_for_agent(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
name: &str,
|
name: &str,
|
||||||
role: &str,
|
role: &str,
|
||||||
system_prompt: &str,
|
system_prompt: &str,
|
||||||
@@ -462,10 +478,13 @@ the sake of proposing."#;
|
|||||||
})
|
})
|
||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
call_gemini_json(system, &user).await
|
call_llm_json(runtime, system, &user).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn call_llm_for_team(members: &[Value]) -> Result<Value, String> {
|
async fn call_llm_for_team(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
|
members: &[Value],
|
||||||
|
) -> Result<Value, String> {
|
||||||
let system = r#"You review an AI team's roster + recent history and propose
|
let system = r#"You review an AI team's roster + recent history and propose
|
||||||
targeted improvements. Return ONLY JSON:
|
targeted improvements. Return ONLY JSON:
|
||||||
{
|
{
|
||||||
@@ -482,47 +501,90 @@ prompts over adding skills. Only add skills when a clear
|
|||||||
"the team keeps getting stuck on <X>" pattern appears."#;
|
"the team keeps getting stuck on <X>" pattern appears."#;
|
||||||
|
|
||||||
let user = json!({ "members": members }).to_string();
|
let user = json!({ "members": members }).to_string();
|
||||||
call_gemini_json(system, &user).await
|
call_llm_json(runtime, system, &user).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn call_gemini_json(system: &str, user: &str) -> Result<Value, String> {
|
/// Ask the configured proposer model for one JSON object.
|
||||||
let api_key =
|
///
|
||||||
std::env::var("GEMINI_API_KEY").map_err(|_| "GEMINI_API_KEY unset".to_string())?;
|
/// Goes through the provider registry rather than a vendor's HTTP API, so any
|
||||||
let model = model_name();
|
/// model the platform can already reach works and no single vendor's billing can
|
||||||
let url = format!(
|
/// take level-up down.
|
||||||
"https://generativelanguage.googleapis.com/v1beta/models/{}:generateContent?key={}",
|
///
|
||||||
model, api_key
|
/// The JSON is extracted rather than assumed: an anthropic-format model is not
|
||||||
);
|
/// bound by Gemini's `response_mime_type: application/json`, and will happily
|
||||||
let body = json!({
|
/// wrap an object in prose or a ```json fence. Parsing the raw reply worked
|
||||||
"system_instruction": { "parts": [{ "text": system }] },
|
/// against Gemini and would fail on everything else.
|
||||||
"contents": [{ "role": "user", "parts": [{ "text": user }] }],
|
async fn call_llm_json(
|
||||||
"generationConfig": {
|
runtime: &cm_runtime::Runtime,
|
||||||
"temperature": 0.2,
|
system: &str,
|
||||||
"response_mime_type": "application/json",
|
user: &str,
|
||||||
"maxOutputTokens": 8192,
|
) -> Result<Value, String> {
|
||||||
}
|
use cm_llm::{ChatMessage, ChatRequest, ChatRole, ContentPart, LlmEvent};
|
||||||
});
|
use futures::StreamExt as _;
|
||||||
let client = reqwest::Client::builder()
|
|
||||||
.timeout(std::time::Duration::from_secs(60))
|
let spec = model_name();
|
||||||
.build()
|
let (provider, model) = runtime.resolve_provider(&spec);
|
||||||
.map_err(|e| format!("http client: {e}"))?;
|
let request = ChatRequest {
|
||||||
let resp = client
|
system: system.to_string(),
|
||||||
.post(&url)
|
model: model.to_string(),
|
||||||
.json(&body)
|
messages: vec![ChatMessage {
|
||||||
.send()
|
role: ChatRole::User,
|
||||||
|
parts: vec![ContentPart::text(user)],
|
||||||
|
}],
|
||||||
|
tools: vec![],
|
||||||
|
max_tokens: 8192,
|
||||||
|
web_search: false,
|
||||||
|
};
|
||||||
|
let mut stream = provider
|
||||||
|
.stream(request)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("gemini call: {e}"))?;
|
.map_err(|e| format!("level-up call ({spec}): {e}"))?;
|
||||||
if !resp.status().is_success() {
|
let mut text = String::new();
|
||||||
let code = resp.status();
|
while let Some(event) = stream.next().await {
|
||||||
let body = resp.text().await.unwrap_or_default();
|
match event {
|
||||||
return Err(format!("gemini {code}: {}", &body[..body.len().min(500)]));
|
Ok(LlmEvent::TextDelta(t)) => text.push_str(&t),
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(e) => return Err(format!("level-up stream ({spec}): {e}")),
|
||||||
}
|
}
|
||||||
let json: Value = resp.json().await.map_err(|e| format!("gemini json: {e}"))?;
|
}
|
||||||
let text = json
|
let body = extract_json_object(&text)
|
||||||
.pointer("/candidates/0/content/parts/0/text")
|
.ok_or_else(|| format!("no JSON object in {spec} reply: {}", excerpt(&text, 300)))?;
|
||||||
.and_then(|v| v.as_str())
|
serde_json::from_str(body).map_err(|e| format!("parse suggestion json: {e}"))
|
||||||
.ok_or_else(|| "gemini response missing text".to_string())?;
|
}
|
||||||
serde_json::from_str(text).map_err(|e| format!("parse suggestion json: {e}"))
|
|
||||||
|
/// The outermost `{...}` in a reply, so a fenced or prose-wrapped object parses.
|
||||||
|
///
|
||||||
|
/// Brace-counting rather than a regex: a nested object would end a lazy match at
|
||||||
|
/// the first inner `}`, and these proposals are nested by design (items carry
|
||||||
|
/// per-role objects).
|
||||||
|
fn extract_json_object(text: &str) -> Option<&str> {
|
||||||
|
let start = text.find('{')?;
|
||||||
|
let mut depth = 0usize;
|
||||||
|
let mut in_string = false;
|
||||||
|
let mut escaped = false;
|
||||||
|
for (i, c) in text[start..].char_indices() {
|
||||||
|
if in_string {
|
||||||
|
match c {
|
||||||
|
_ if escaped => escaped = false,
|
||||||
|
'\\' => escaped = true,
|
||||||
|
'"' => in_string = false,
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
match c {
|
||||||
|
'"' => in_string = true,
|
||||||
|
'{' => depth += 1,
|
||||||
|
'}' => {
|
||||||
|
depth -= 1;
|
||||||
|
if depth == 0 {
|
||||||
|
return Some(&text[start..start + i + 1]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
fn excerpt(s: &str, max: usize) -> String {
|
fn excerpt(s: &str, max: usize) -> String {
|
||||||
@@ -546,3 +608,44 @@ fn workspace_skill_id(workspace_id: Uuid, name: &str) -> Uuid {
|
|||||||
bytes[8] = (bytes[8] & 0x3f) | 0x80;
|
bytes[8] = (bytes[8] & 0x3f) | 0x80;
|
||||||
Uuid::from_bytes(bytes)
|
Uuid::from_bytes(bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
/// Gemini was asked for `response_mime_type: application/json` and obliged.
|
||||||
|
/// Anthropic-format models are under no such obligation and routinely wrap
|
||||||
|
/// the object in prose or a fenced block, so the reply is EXTRACTED, not
|
||||||
|
/// assumed. Parsing the raw text worked against Gemini and would fail
|
||||||
|
/// everywhere else — exactly the shape of bug a provider swap hides until
|
||||||
|
/// the first real proposal.
|
||||||
|
#[test]
|
||||||
|
fn a_json_object_is_extracted_from_however_the_model_wrapped_it() {
|
||||||
|
let bare = r#"{"items":[]}"#;
|
||||||
|
assert_eq!(super::extract_json_object(bare), Some(bare));
|
||||||
|
|
||||||
|
let fenced = "Here is my proposal:\n```json\n{\"items\":[1]}\n```\nDone.";
|
||||||
|
assert_eq!(super::extract_json_object(fenced), Some(r#"{"items":[1]}"#));
|
||||||
|
|
||||||
|
// Nested objects: a lazy match would stop at the first inner brace and
|
||||||
|
// hand back invalid JSON. These proposals are nested by design.
|
||||||
|
let nested = r#"prose {"a":{"b":{"c":1}},"d":2} trailing"#;
|
||||||
|
assert_eq!(
|
||||||
|
super::extract_json_object(nested),
|
||||||
|
Some(r#"{"a":{"b":{"c":1}},"d":2}"#)
|
||||||
|
);
|
||||||
|
|
||||||
|
// A brace inside a string must not close the object.
|
||||||
|
let stringy = r#"{"note":"an unmatched } here","ok":true}"#;
|
||||||
|
assert_eq!(super::extract_json_object(stringy), Some(stringy));
|
||||||
|
|
||||||
|
assert_eq!(super::extract_json_object("no object here"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The default must not be a vendor whose billing already took a feature
|
||||||
|
/// down. It is a REGISTRY SPEC (`provider:model`), not a bare model name —
|
||||||
|
/// `resolve_provider` needs the provider half.
|
||||||
|
#[test]
|
||||||
|
fn the_default_proposer_is_a_registry_spec_and_not_gemini() {
|
||||||
|
assert!(super::DEFAULT_MODEL.contains(':'), "{}", super::DEFAULT_MODEL);
|
||||||
|
assert!(!super::DEFAULT_MODEL.contains("gemini"), "{}", super::DEFAULT_MODEL);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ pub mod benchmark_runner;
|
|||||||
pub mod beszel;
|
pub mod beszel;
|
||||||
pub mod brain_seed;
|
pub mod brain_seed;
|
||||||
pub mod cleanup_sweeper;
|
pub mod cleanup_sweeper;
|
||||||
|
pub mod agent_names;
|
||||||
|
pub mod mission_gc;
|
||||||
pub mod container_exec;
|
pub mod container_exec;
|
||||||
mod error;
|
mod error;
|
||||||
pub mod evaluator;
|
pub mod evaluator;
|
||||||
@@ -16,18 +18,34 @@ mod mcp_door;
|
|||||||
mod mcp_skills;
|
mod mcp_skills;
|
||||||
pub mod mission_orchestrator;
|
pub mod mission_orchestrator;
|
||||||
pub mod mission_refiner;
|
pub mod mission_refiner;
|
||||||
|
pub mod auto_merge;
|
||||||
pub mod corpus;
|
pub mod corpus;
|
||||||
pub mod harvest;
|
pub mod harvest;
|
||||||
pub mod library;
|
pub mod library;
|
||||||
pub mod mission_delivery;
|
pub mod mission_delivery;
|
||||||
|
pub mod mission_events;
|
||||||
|
pub mod microvm_client;
|
||||||
|
pub mod microvm_executor;
|
||||||
|
pub mod microvm_turn_executor;
|
||||||
|
pub mod subscription;
|
||||||
|
pub mod vm_placement;
|
||||||
|
pub mod vm_stop_gate;
|
||||||
|
pub mod vm_tool_tap;
|
||||||
|
pub mod mission_fs;
|
||||||
|
pub mod mission_outputs;
|
||||||
pub mod papers;
|
pub mod papers;
|
||||||
pub mod phase_config;
|
pub mod phase_config;
|
||||||
|
pub mod session_executor;
|
||||||
|
pub mod repo_digest;
|
||||||
pub mod runtime_preflight;
|
pub mod runtime_preflight;
|
||||||
|
pub mod validator_preflight;
|
||||||
|
pub mod mission_plan;
|
||||||
|
pub mod mission_roster;
|
||||||
pub mod mission_runtime;
|
pub mod mission_runtime;
|
||||||
pub mod mission_workspace;
|
pub mod mission_workspace;
|
||||||
pub mod node_rules;
|
pub mod node_rules;
|
||||||
pub mod pdf_renderer;
|
|
||||||
pub mod phase_runner;
|
pub mod phase_runner;
|
||||||
|
pub mod root_copy;
|
||||||
pub mod phase_summarizer;
|
pub mod phase_summarizer;
|
||||||
pub mod quota;
|
pub mod quota;
|
||||||
mod recursive_exec;
|
mod recursive_exec;
|
||||||
@@ -157,6 +175,8 @@ pub fn router(state: AppState) -> Router {
|
|||||||
.route("/api/world/live", get(routes::world::world_live))
|
.route("/api/world/live", get(routes::world::world_live))
|
||||||
.route("/api/world/replay", get(routes::world::world_replay))
|
.route("/api/world/replay", get(routes::world::world_replay))
|
||||||
.route("/api/nodes", get(routes::nodes::list))
|
.route("/api/nodes", get(routes::nodes::list))
|
||||||
|
.route("/api/fleet/capacity", get(routes::nodes::capacity))
|
||||||
|
.route("/api/fleet/backends", get(routes::nodes::backends))
|
||||||
.route("/api/nodes/pair", post(routes::nodes::pair))
|
.route("/api/nodes/pair", post(routes::nodes::pair))
|
||||||
.route("/api/nodes/live", get(routes::nodes::live))
|
.route("/api/nodes/live", get(routes::nodes::live))
|
||||||
.route("/api/nodes/agent", get(routes::nodes::agent_ws))
|
.route("/api/nodes/agent", get(routes::nodes::agent_ws))
|
||||||
@@ -465,6 +485,8 @@ pub fn router(state: AppState) -> Router {
|
|||||||
"/api/missions",
|
"/api/missions",
|
||||||
get(routes::missions::list).post(routes::missions::create),
|
get(routes::missions::list).post(routes::missions::create),
|
||||||
)
|
)
|
||||||
|
// The roster grouped by mission — what "My Workforce" renders.
|
||||||
|
.route("/api/workforce", get(routes::missions::workforce))
|
||||||
// The workflow recipe catalog (templates/workflows/*.toml). Serving it
|
// The workflow recipe catalog (templates/workflows/*.toml). Serving it
|
||||||
// lets the client stop mirroring the phase composition table inline.
|
// lets the client stop mirroring the phase composition table inline.
|
||||||
.route("/api/workflows", get(routes::missions::list_workflows))
|
.route("/api/workflows", get(routes::missions::list_workflows))
|
||||||
@@ -479,6 +501,43 @@ pub fn router(state: AppState) -> Router {
|
|||||||
axum::routing::patch(routes::missions::set_status),
|
axum::routing::patch(routes::missions::set_status),
|
||||||
)
|
)
|
||||||
.route("/api/missions/{id}/refine", post(routes::missions::refine))
|
.route("/api/missions/{id}/refine", post(routes::missions::refine))
|
||||||
|
// Draft-less sibling: the wizard polishes a description before any
|
||||||
|
// mission exists, so there is no id to route on. Declared BEFORE the
|
||||||
|
// `{id}` routes would otherwise be ambiguous — axum matches literal
|
||||||
|
// segments first, but keeping them adjacent makes the pair obvious.
|
||||||
|
.route(
|
||||||
|
"/api/missions/refine-draft",
|
||||||
|
post(routes::missions::refine_draft),
|
||||||
|
)
|
||||||
|
.route("/api/missions/{id}/merge", post(routes::missions::merge_branch))
|
||||||
|
.route(
|
||||||
|
"/api/missions/{id}/artifacts/{artifact_id}/content",
|
||||||
|
get(routes::missions::artifact_content),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/api/missions/{id}/artifacts/{artifact_id}/download",
|
||||||
|
get(routes::missions::artifact_download),
|
||||||
|
)
|
||||||
|
// Slice 5: let a model size the mission's team. Proposing, listing and
|
||||||
|
// deciding are separate verbs because only the last one spends money.
|
||||||
|
// W1/#13: let a model author the phases, on the same propose → review →
|
||||||
|
// approve shape as the roster above.
|
||||||
|
.route(
|
||||||
|
"/api/missions/{id}/plan-proposals",
|
||||||
|
get(routes::mission_plan::list).post(routes::mission_plan::suggest),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/api/missions/{id}/plan-proposals/{pid}/decide",
|
||||||
|
post(routes::mission_plan::decide),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/api/missions/{id}/team-proposals",
|
||||||
|
get(routes::mission_roster::list).post(routes::mission_roster::suggest),
|
||||||
|
)
|
||||||
|
.route(
|
||||||
|
"/api/missions/{id}/team-proposals/{pid}/decide",
|
||||||
|
post(routes::mission_roster::decide),
|
||||||
|
)
|
||||||
.route(
|
.route(
|
||||||
"/api/missions/{id}/herdr-dispatch",
|
"/api/missions/{id}/herdr-dispatch",
|
||||||
post(routes::missions::herdr_dispatch),
|
post(routes::missions::herdr_dispatch),
|
||||||
|
|||||||
@@ -35,6 +35,11 @@ pub struct LibraryRun {
|
|||||||
/// papers but could not push still has the PDFs and the checkmarks; the
|
/// papers but could not push still has the PDFs and the checkmarks; the
|
||||||
/// notes are simply not on the forge yet.
|
/// notes are simply not on the forge yet.
|
||||||
pub pushed: bool,
|
pub pushed: bool,
|
||||||
|
/// Whether the branch was auto-merged into `main`.
|
||||||
|
pub merged: bool,
|
||||||
|
/// Always populated — a branch that quietly did not merge is
|
||||||
|
/// indistinguishable from one that was never delivered.
|
||||||
|
pub merge_reason: String,
|
||||||
pub error: Option<String>,
|
pub error: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -88,9 +93,13 @@ pub async fn clone_vault(clone_url: &str, work_root: &Path) -> Result<PathBuf, S
|
|||||||
.map_err(|e| format!("mkdir {}: {e}", work_root.display()))?;
|
.map_err(|e| format!("mkdir {}: {e}", work_root.display()))?;
|
||||||
|
|
||||||
let auth = mission_workspace::with_ambient_auth(clone_url);
|
let auth = mission_workspace::with_ambient_auth(clone_url);
|
||||||
let out = tokio::process::Command::new("git")
|
if let Some(why) = &auth.unauthenticated {
|
||||||
.args(["clone", "--quiet", "--depth", "1", &auth])
|
eprintln!("library: cloning the vault WITHOUT credentials — {why}");
|
||||||
.arg(&path)
|
}
|
||||||
|
let mut cmd = tokio::process::Command::new("git");
|
||||||
|
cmd.args(["clone", "--quiet", "--depth", "1", &auth.url])
|
||||||
|
.arg(&path);
|
||||||
|
let out = mission_workspace::no_terminal_prompt(&mut cmd)
|
||||||
.output()
|
.output()
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("spawn git clone: {e}"))?;
|
.map_err(|e| format!("spawn git clone: {e}"))?;
|
||||||
@@ -98,16 +107,15 @@ pub async fn clone_vault(clone_url: &str, work_root: &Path) -> Result<PathBuf, S
|
|||||||
return Err(format!(
|
return Err(format!(
|
||||||
"clone vault → {}: {}",
|
"clone vault → {}: {}",
|
||||||
out.status,
|
out.status,
|
||||||
mission_workspace::redact_token(&String::from_utf8_lossy(&out.stderr))
|
crate::evaluator_tools::clamp_output(&mission_workspace::redact_token(
|
||||||
.chars()
|
&String::from_utf8_lossy(&out.stderr)
|
||||||
.take(300)
|
))
|
||||||
.collect::<String>()
|
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
// The token must not stay in .git/config: the checkout may be handed to a
|
// The token must not stay in .git/config: the checkout may be handed to a
|
||||||
// container later, and a credential in a file an agent can read is a
|
// container later, and a credential in a file an agent can read is a
|
||||||
// credential an agent has.
|
// credential an agent has.
|
||||||
mission_workspace::scrub_remote_credentials(&path, &auth);
|
mission_workspace::scrub_remote_credentials(&path, &auth.url);
|
||||||
Ok(path)
|
Ok(path)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -160,6 +168,8 @@ pub async fn run_to_vault(
|
|||||||
harvest: total,
|
harvest: total,
|
||||||
branch,
|
branch,
|
||||||
pushed: false,
|
pushed: false,
|
||||||
|
merged: false,
|
||||||
|
merge_reason: "nothing new to push".into(),
|
||||||
error: None,
|
error: None,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -179,18 +189,52 @@ pub async fn run_to_vault(
|
|||||||
git(&vault, &["commit", "--no-verify", "-m", &message]).await?;
|
git(&vault, &["commit", "--no-verify", "-m", &message]).await?;
|
||||||
|
|
||||||
let auth = mission_workspace::with_ambient_auth(clone_url);
|
let auth = mission_workspace::with_ambient_auth(clone_url);
|
||||||
|
if let Some(why) = &auth.unauthenticated {
|
||||||
|
if auth.is_forge() {
|
||||||
|
// Not fatal here — the push below reports its own failure — but the
|
||||||
|
// reason belongs in the log next to the attempt, not inferred from a
|
||||||
|
// tty error two layers down.
|
||||||
|
eprintln!("library: pushing to the forge WITHOUT credentials — {why}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let auth = auth.url;
|
||||||
let refspec = format!("HEAD:refs/heads/{branch}");
|
let refspec = format!("HEAD:refs/heads/{branch}");
|
||||||
match git(&vault, &["push", &auth, &refspec]).await {
|
match git(&vault, &["push", &auth, &refspec]).await {
|
||||||
Ok(_) => Ok(LibraryRun {
|
Ok(_) => {
|
||||||
|
// A catalogue branch only ever adds notes under `60 Papers/`, so
|
||||||
|
// it qualifies for auto-merge — but the check is measured from the
|
||||||
|
// diff, not assumed from the mission type. Verified here means the
|
||||||
|
// run shelved something and errored on nothing.
|
||||||
|
let verified = total.healthy() && !total.shelved.is_empty();
|
||||||
|
let merge = crate::auto_merge::try_merge(
|
||||||
|
&vault,
|
||||||
|
&auth,
|
||||||
|
&branch,
|
||||||
|
"main",
|
||||||
|
crate::auto_merge::MergePolicy::AdditiveOnly,
|
||||||
|
verified,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|e| crate::auto_merge::MergeOutcome {
|
||||||
|
merged: false,
|
||||||
|
reason: format!("merge attempt failed: {e}"),
|
||||||
|
});
|
||||||
|
eprintln!("library: branch {branch} — {}", merge.reason);
|
||||||
|
Ok(LibraryRun {
|
||||||
harvest: total,
|
harvest: total,
|
||||||
branch,
|
branch,
|
||||||
pushed: true,
|
pushed: true,
|
||||||
|
merged: merge.merged,
|
||||||
|
merge_reason: merge.reason,
|
||||||
error: None,
|
error: None,
|
||||||
}),
|
})
|
||||||
|
}
|
||||||
Err(e) => Ok(LibraryRun {
|
Err(e) => Ok(LibraryRun {
|
||||||
harvest: total,
|
harvest: total,
|
||||||
branch,
|
branch,
|
||||||
pushed: false,
|
pushed: false,
|
||||||
|
merged: false,
|
||||||
|
merge_reason: "not pushed, so not merged".into(),
|
||||||
error: Some(e),
|
error: Some(e),
|
||||||
}),
|
}),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -166,6 +166,18 @@ async fn policy_decide(
|
|||||||
} else {
|
} else {
|
||||||
state.runtime.judge(system, &request).await
|
state.runtime.judge(system, &request).await
|
||||||
};
|
};
|
||||||
|
// Fail-open is deliberate, but a governor that is failing open on EVERY
|
||||||
|
// request is a security control that has quietly stopped existing —
|
||||||
|
// and the caller drops `reason` whenever it allows, so nothing said so.
|
||||||
|
// `judge()` returns this exact prefix when the provider never answered,
|
||||||
|
// which a rate-limited or uncredited judge model does on every call.
|
||||||
|
if allow && reason.starts_with("governor unreachable") {
|
||||||
|
eprintln!(
|
||||||
|
"mcp_door: WARNING — the door governor is FAILING OPEN for {mcp_tool} \
|
||||||
|
({reason}). Every outbound action is being approved unjudged. Point \
|
||||||
|
CLAWMATES_JUDGE_MODEL at a reachable model."
|
||||||
|
);
|
||||||
|
}
|
||||||
if !allow {
|
if !allow {
|
||||||
return PolicyOutcome::Deny(format!("governor agent vetoed — {reason}"));
|
return PolicyOutcome::Deny(format!("governor agent vetoed — {reason}"));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,354 @@
|
|||||||
|
//! Drive a fleet node's microVMs from the server.
|
||||||
|
//!
|
||||||
|
//! Thin by design: the node owns the VM lifecycle (see
|
||||||
|
//! `clawmates-node::microvm`), and this is the typed way to ask it. Every call
|
||||||
|
//! is one `vm_*` op over the existing `NodeHub` request/response channel, so
|
||||||
|
//! there is no new transport, correlation or timeout machinery.
|
||||||
|
//!
|
||||||
|
//! # Not a `SandboxDriver`
|
||||||
|
//!
|
||||||
|
//! `RemoteDriver` exists to marshal `SandboxDriver` over the hub, and reusing it
|
||||||
|
//! was the plan. That trait is container-shaped — `attach_pty`, `resize_pty`,
|
||||||
|
//! argv `exec` — while a mission needs inject → run → collect. Conforming would
|
||||||
|
//! mean implementing PTY-over-vsock semantics that nothing calls, so this speaks
|
||||||
|
//! the smaller interface the mission path actually uses.
|
||||||
|
//!
|
||||||
|
//! # Timeouts
|
||||||
|
//!
|
||||||
|
//! The hub defaults to 20s, which is right for a create (measured: ~1s) and
|
||||||
|
//! badly wrong for an agent turn. `exec` therefore takes its own budget and
|
||||||
|
//! passes it to BOTH the hub and the guest, with the hub's slightly longer: if
|
||||||
|
//! the guest's own timeout fires first the reply says so, whereas a hub timeout
|
||||||
|
//! leaves us guessing whether the command is still running.
|
||||||
|
|
||||||
|
use cm_domain::NodeId;
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
|
||||||
|
use crate::fleet::NodeHub;
|
||||||
|
|
||||||
|
/// Slack between the guest's deadline and the hub's, so the guest's own timeout
|
||||||
|
/// wins the race and we get a real answer rather than a transport error.
|
||||||
|
const HUB_GRACE_SECS: u64 = 30;
|
||||||
|
|
||||||
|
/// How long the hub waits for a command whose own budget is `guest_secs`.
|
||||||
|
///
|
||||||
|
/// Saturating, not `+`: a caller passing a very large budget would otherwise
|
||||||
|
/// overflow and panic in debug or wrap to a tiny timeout in release — the second
|
||||||
|
/// being far worse, since it turns a long-running agent turn into a spurious
|
||||||
|
/// transport failure.
|
||||||
|
fn hub_deadline(guest_secs: u64) -> u64 {
|
||||||
|
guest_secs.saturating_add(HUB_GRACE_SECS)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct MicroVm<'a> {
|
||||||
|
hub: &'a NodeHub,
|
||||||
|
node_id: NodeId,
|
||||||
|
vm_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> MicroVm<'a> {
|
||||||
|
pub fn new(hub: &'a NodeHub, node_id: NodeId, vm_id: impl Into<String>) -> Self {
|
||||||
|
Self {
|
||||||
|
hub,
|
||||||
|
node_id,
|
||||||
|
vm_id: vm_id.into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn vm_id(&self) -> &str {
|
||||||
|
&self.vm_id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One op, with the node's `output` string parsed back into JSON.
|
||||||
|
///
|
||||||
|
/// `output` is a String on the wire (`Uplink::Result`), and a node that
|
||||||
|
/// answered with a JSON object instead made the whole frame unparseable —
|
||||||
|
/// the reply then vanished into the uplink's error arm and the call timed
|
||||||
|
/// out with nothing explaining why. Parsing here, loudly, keeps that
|
||||||
|
/// mismatch a visible error rather than a mystery timeout.
|
||||||
|
async fn call(&self, op: &str, mut args: Value, secs: u64) -> Result<Value, String> {
|
||||||
|
if let Some(o) = args.as_object_mut() {
|
||||||
|
o.insert("vm_id".into(), Value::String(self.vm_id.clone()));
|
||||||
|
}
|
||||||
|
let out = self
|
||||||
|
.hub
|
||||||
|
.call_timeout(self.node_id, op, args, secs)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("{op} on node {:?}: {e}", self.node_id))?;
|
||||||
|
let body: Value = serde_json::from_str(&out.output)
|
||||||
|
.map_err(|e| format!("{op} returned unparseable output ({e}): {}", out.output))?;
|
||||||
|
if !out.ok {
|
||||||
|
let why = body
|
||||||
|
.get("error")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or(&out.output);
|
||||||
|
return Err(format!("{op} failed: {why}"));
|
||||||
|
}
|
||||||
|
Ok(body)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Boot the VM. Returns only once its guest agent has answered.
|
||||||
|
///
|
||||||
|
/// `backend` selects the rootfs image (`missions.backend`); `None` boots the
|
||||||
|
/// node's default. A backend whose image is not built on that node is an
|
||||||
|
/// error naming the file — never a quiet fall back to the default, which
|
||||||
|
/// would run a claude mission in a kimi VM and report success.
|
||||||
|
pub async fn create(
|
||||||
|
&self,
|
||||||
|
vcpus: u32,
|
||||||
|
mem_mib: u32,
|
||||||
|
backend: Option<&str>,
|
||||||
|
) -> Result<Value, String> {
|
||||||
|
// 60s, not the hub default: a create that has to copy a rootfs and boot
|
||||||
|
// is measured near 1s, but a node under load has no reason to be fast.
|
||||||
|
self.call(
|
||||||
|
"vm_create",
|
||||||
|
json!({ "vcpus": vcpus, "mem_mib": mem_mib, "backend": backend }),
|
||||||
|
60,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Unpack a tar inside the guest at `dest`.
|
||||||
|
///
|
||||||
|
/// Takes the archive bytes rather than a path: the server holds the mission
|
||||||
|
/// checkout, the node does not, and shipping the tar is the whole point of
|
||||||
|
/// the inject → run → collect model.
|
||||||
|
pub async fn inject(&self, dest: &str, tar: &[u8]) -> Result<Value, String> {
|
||||||
|
use base64::Engine as _;
|
||||||
|
let b64 = base64::engine::general_purpose::STANDARD.encode(tar);
|
||||||
|
self.call("vm_inject", json!({ "dest": dest, "tar_b64": b64 }), 120)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run a shell command in the guest.
|
||||||
|
///
|
||||||
|
/// `Ok` means the command RAN; the exit code is in the payload. A non-zero
|
||||||
|
/// exit is not an error here — the caller has to be able to tell "the build
|
||||||
|
/// failed" from "we could not reach the VM", and collapsing them is the
|
||||||
|
/// defect this codebase keeps paying for.
|
||||||
|
/// `env` carries the provider credentials (see
|
||||||
|
/// [`crate::mission_runtime::forwarded_provider_env`]). It is sent, never
|
||||||
|
/// logged: this is the only channel by which a secret reaches the guest, and
|
||||||
|
/// the guest refuses the exec rather than running a command without an entry
|
||||||
|
/// it could not honour.
|
||||||
|
pub async fn exec(
|
||||||
|
&self,
|
||||||
|
cmd: &str,
|
||||||
|
cwd: Option<&str>,
|
||||||
|
timeout_secs: u64,
|
||||||
|
env: &[(String, String)],
|
||||||
|
) -> Result<ExecOut, String> {
|
||||||
|
self.exec_attributed(cmd, cwd, timeout_secs, env, None, None)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The same exec, tagged with the run whose live output this is.
|
||||||
|
///
|
||||||
|
/// When `run_id` is set the node follows `log_path` inside the guest for the
|
||||||
|
/// life of the command and streams what it reads to the server. Probes pass
|
||||||
|
/// `None`: they produce nothing worth streaming and have no subscriber.
|
||||||
|
pub async fn exec_attributed(
|
||||||
|
&self,
|
||||||
|
cmd: &str,
|
||||||
|
cwd: Option<&str>,
|
||||||
|
timeout_secs: u64,
|
||||||
|
env: &[(String, String)],
|
||||||
|
run_id: Option<uuid::Uuid>,
|
||||||
|
log_path: Option<&str>,
|
||||||
|
) -> Result<ExecOut, String> {
|
||||||
|
let env: Option<Value> = (!env.is_empty()).then(|| {
|
||||||
|
env.iter()
|
||||||
|
.map(|(k, v)| (k.clone(), Value::String(v.clone())))
|
||||||
|
.collect::<serde_json::Map<_, _>>()
|
||||||
|
.into()
|
||||||
|
});
|
||||||
|
let v = self
|
||||||
|
.call(
|
||||||
|
"vm_exec",
|
||||||
|
json!({
|
||||||
|
"cmd": cmd, "cwd": cwd, "timeout": timeout_secs, "env": env,
|
||||||
|
"run_id": run_id.map(|r| r.to_string()), "log_path": log_path,
|
||||||
|
}),
|
||||||
|
hub_deadline(timeout_secs),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
// A guest that refused to run the command reports `ok: false` and no rc
|
||||||
|
// — a rejected env entry, for instance. Surface its reason: falling
|
||||||
|
// through to the missing-rc error below would hide the cause behind a
|
||||||
|
// symptom.
|
||||||
|
if v.get("ok").and_then(Value::as_bool) == Some(false) {
|
||||||
|
return Err(format!(
|
||||||
|
"vm_exec did not run: {}",
|
||||||
|
v.get("error").and_then(Value::as_str).unwrap_or("unknown")
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// A missing rc is not "success" — it means the guest did not report one,
|
||||||
|
// which we must not read as zero.
|
||||||
|
let rc = v
|
||||||
|
.get("rc")
|
||||||
|
.and_then(Value::as_i64)
|
||||||
|
.ok_or_else(|| format!("vm_exec gave no exit code: {v}"))?;
|
||||||
|
Ok(ExecOut {
|
||||||
|
rc,
|
||||||
|
stdout: v
|
||||||
|
.get("stdout")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.to_string(),
|
||||||
|
stderr: v
|
||||||
|
.get("stderr")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.to_string(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tar a path out of the guest and return the archive bytes.
|
||||||
|
/// `exclude` names directories to leave out — build output, caches. Sent from
|
||||||
|
/// here so the policy lives in one place: `mission_fs::transport_excludes`,
|
||||||
|
/// the same list the delivery diff uses. Shipping `target/` blew this call's
|
||||||
|
/// 300s budget twice, each time with the agent's work finished and stranded.
|
||||||
|
pub async fn collect(&self, path: &str, exclude: &[&str]) -> Result<Vec<u8>, String> {
|
||||||
|
use base64::Engine as _;
|
||||||
|
let v = self
|
||||||
|
.call("vm_collect", json!({ "path": path, "exclude": exclude }), 300)
|
||||||
|
.await?;
|
||||||
|
// The guest reports its own `ok`: a missing path is a real failure that
|
||||||
|
// must not come back as an empty archive, which would look exactly like
|
||||||
|
// a run that produced nothing.
|
||||||
|
if v.get("ok").and_then(Value::as_bool) != Some(true) {
|
||||||
|
return Err(format!(
|
||||||
|
"vm_collect {path}: {}",
|
||||||
|
v.get("error").and_then(Value::as_str).unwrap_or("unknown")
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let b64 = v
|
||||||
|
.get("tar_b64")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.ok_or_else(|| format!("vm_collect {path} returned no archive: {v}"))?;
|
||||||
|
base64::engine::general_purpose::STANDARD
|
||||||
|
.decode(b64)
|
||||||
|
.map_err(|e| format!("vm_collect {path}: undecodable archive: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stop the VM and remove everything it owned. Idempotent.
|
||||||
|
pub async fn destroy(&self) -> Result<Value, String> {
|
||||||
|
self.call("vm_destroy", json!({}), 60).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The result of a command that RAN. `rc != 0` is a normal outcome.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ExecOut {
|
||||||
|
pub rc: i64,
|
||||||
|
pub stdout: String,
|
||||||
|
pub stderr: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ExecOut {
|
||||||
|
pub fn ok(&self) -> bool {
|
||||||
|
self.rc == 0
|
||||||
|
}
|
||||||
|
/// One line for a log or an artifact, without dumping a whole build.
|
||||||
|
pub fn summary(&self) -> String {
|
||||||
|
let tail = |s: &str| {
|
||||||
|
s.lines()
|
||||||
|
.rev()
|
||||||
|
.take(3)
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.into_iter()
|
||||||
|
.rev()
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" | ")
|
||||||
|
};
|
||||||
|
if self.ok() {
|
||||||
|
format!("rc=0 {}", tail(&self.stdout))
|
||||||
|
} else {
|
||||||
|
format!("rc={} {}", self.rc, tail(&self.stderr))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// VMs a node currently holds, so orphans can be reaped.
|
||||||
|
pub async fn list(hub: &NodeHub, node_id: NodeId) -> Result<Vec<String>, String> {
|
||||||
|
let out = hub
|
||||||
|
.call(node_id, "vm_list", json!({}))
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("vm_list on node {node_id:?}: {e}"))?;
|
||||||
|
let body: Value = serde_json::from_str(&out.output)
|
||||||
|
.map_err(|e| format!("vm_list returned unparseable output ({e}): {}", out.output))?;
|
||||||
|
Ok(body
|
||||||
|
.get("vms")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|a| {
|
||||||
|
a.iter()
|
||||||
|
.filter_map(|v| v.get("vm_id").and_then(Value::as_str))
|
||||||
|
.map(str::to_string)
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// A command that ran and failed must be distinguishable from one that
|
||||||
|
/// could not be reached. `rc` carries the verdict; `Err` is for transport.
|
||||||
|
#[test]
|
||||||
|
fn a_nonzero_exit_is_an_outcome_not_an_error() {
|
||||||
|
let failed = ExecOut {
|
||||||
|
rc: 3,
|
||||||
|
stdout: String::new(),
|
||||||
|
stderr: "boom\n".into(),
|
||||||
|
};
|
||||||
|
assert!(!failed.ok());
|
||||||
|
assert!(failed.summary().starts_with("rc=3"));
|
||||||
|
assert!(failed.summary().contains("boom"));
|
||||||
|
|
||||||
|
let passed = ExecOut {
|
||||||
|
rc: 0,
|
||||||
|
stdout: "fine\n".into(),
|
||||||
|
stderr: String::new(),
|
||||||
|
};
|
||||||
|
assert!(passed.ok());
|
||||||
|
assert_eq!(passed.summary(), "rc=0 fine");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The summary is for logs, so it must stay short even when a build prints
|
||||||
|
/// thousands of lines — and it must keep the LAST lines, where the error is.
|
||||||
|
#[test]
|
||||||
|
fn the_summary_keeps_the_tail_and_stays_short() {
|
||||||
|
let noisy = ExecOut {
|
||||||
|
rc: 1,
|
||||||
|
stdout: String::new(),
|
||||||
|
stderr: (1..=500)
|
||||||
|
.map(|i| format!("line {i}"))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n"),
|
||||||
|
};
|
||||||
|
let s = noisy.summary();
|
||||||
|
assert!(s.contains("line 500"), "the last line must survive: {s}");
|
||||||
|
assert!(!s.contains("line 400"), "older lines must be dropped: {s}");
|
||||||
|
assert!(s.len() < 200, "summary must stay log-sized, got {}", s.len());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The guest's deadline must fire before the hub's, so a slow command comes
|
||||||
|
/// back as a reported timeout rather than an unexplained transport failure.
|
||||||
|
#[test]
|
||||||
|
fn the_hub_always_outlives_the_guests_own_timeout() {
|
||||||
|
for guest in [0u64, 1, 30, 3600, 86_400] {
|
||||||
|
assert!(
|
||||||
|
hub_deadline(guest) > guest,
|
||||||
|
"hub deadline for {guest}s must exceed it"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// A caller passing a huge budget must not wrap to a tiny timeout, which
|
||||||
|
// would turn a long agent turn into a spurious transport failure.
|
||||||
|
assert!(
|
||||||
|
hub_deadline(u64::MAX) >= u64::MAX - 1,
|
||||||
|
"an extreme budget must saturate, not wrap"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,693 @@
|
|||||||
|
//! The two engines composed — Slice 4.
|
||||||
|
//!
|
||||||
|
//! Engine Z (the ZeroClaw graph in `cm_orchestrator`) owns durability and
|
||||||
|
//! heterogeneity: deterministic planners, per-step checkpoint/resume, a stale-run
|
||||||
|
//! sweep, cancellation, and a different model per node. Engine C (Claude Code in
|
||||||
|
//! a microVM) owns shared context, self-sizing and cheap fan-out. Neither has the
|
||||||
|
//! other's asset, which is why keeping both is a composition rather than a
|
||||||
|
//! compromise.
|
||||||
|
//!
|
||||||
|
//! This module is the join: a [`TurnExecutor`] whose "turn" is a whole
|
||||||
|
//! Claude-Code-in-a-VM session. Because `topology_worker` already dispatches by
|
||||||
|
//! tier, implementing the existing trait inherits the planners, checkpointing,
|
||||||
|
//! reaper, cancellation, `close_finished_phases`, evaluation, capture and
|
||||||
|
//! delivery unchanged. `recursive_exec::SubTopologyExecutor` is the precedent: a
|
||||||
|
//! `run_turn` may be arbitrarily heavy.
|
||||||
|
//!
|
||||||
|
//! # The file-handoff trap
|
||||||
|
//!
|
||||||
|
//! A VM is inject-tar → run → collect-tar → destroy. A graph of per-node VMs with
|
||||||
|
//! **text-only** handoff would silently lose every file an earlier node wrote:
|
||||||
|
//! node 2 would boot from the original checkout, see none of node 1's work, and
|
||||||
|
//! still report success — the exact silent-success shape this project keeps
|
||||||
|
//! paying for.
|
||||||
|
//!
|
||||||
|
//! The answer here is that the mission's **host checkout is the medium**. Every
|
||||||
|
//! node injects from `repo` and collects back over `repo`, so the tree carries
|
||||||
|
//! forward node to node and the last node's tree is what delivery diffs. Two
|
||||||
|
//! properties make that safe rather than lucky:
|
||||||
|
//!
|
||||||
|
//! - `execute_resumable` runs steps strictly **sequentially**, so two VMs are
|
||||||
|
//! never writing the same host directory at once;
|
||||||
|
//! - the vm id is deterministic per (phase, iteration, step), so a resumed step
|
||||||
|
//! whose VM is somehow still alive is refused by the node ("vm already exists")
|
||||||
|
//! instead of quietly producing a second writer.
|
||||||
|
//!
|
||||||
|
//! `a_later_node_sees_an_earlier_nodes_files` proves the handoff, and
|
||||||
|
//! `text_only_handoff_loses_the_earlier_nodes_work` is its negative control.
|
||||||
|
//!
|
||||||
|
//! # Keeping a long turn alive
|
||||||
|
//!
|
||||||
|
//! `requeue_stale` requeues a `running` job that has not touched `updated_at` in
|
||||||
|
//! 180 seconds, and one node here can run for an hour. `SubTopologyExecutor`
|
||||||
|
//! keeps its parent alive from each *leaf step*, which it has and this does not:
|
||||||
|
//! there is nothing between the start and end of a VM turn. So the turn holds a
|
||||||
|
//! ticker that touches `updated_at` every [`KEEPALIVE_SECS`] and is aborted on
|
||||||
|
//! drop. Without it a healthy composed run is requeued mid-node, claimed again,
|
||||||
|
//! and boots a second VM against the same checkout.
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use cm_domain::NodeId;
|
||||||
|
use cm_orchestrator::{OrchestratorError, TurnExecutor, TurnOutcome, TurnRequest};
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use crate::microvm_executor::{PhaseVm, VmPhase};
|
||||||
|
|
||||||
|
/// How often a running VM turn touches its run's `updated_at`.
|
||||||
|
///
|
||||||
|
/// Comfortably inside the 180s stale window, and cheap: one UPDATE per node per
|
||||||
|
/// half minute against a row nothing else is writing.
|
||||||
|
const KEEPALIVE_SECS: u64 = 30;
|
||||||
|
|
||||||
|
/// A [`TurnExecutor`] that runs each graph node as a full Claude-Code session
|
||||||
|
/// inside its own microVM, against the mission's shared host checkout.
|
||||||
|
pub struct MicroVmTurnExecutor<V: PhaseVm> {
|
||||||
|
vms: V,
|
||||||
|
pool: PgPool,
|
||||||
|
/// The durable outer run. Touched for keepalive; its status gates the turn.
|
||||||
|
run_id: Uuid,
|
||||||
|
mission_id: Uuid,
|
||||||
|
phase_id: Uuid,
|
||||||
|
iteration: i32,
|
||||||
|
/// The mission's host checkout — injected into every node's VM and collected
|
||||||
|
/// back over, which is how file work survives a node boundary.
|
||||||
|
repo: PathBuf,
|
||||||
|
/// Whether the mission has a repository. Carried so every graph node gets
|
||||||
|
/// the same workspace treatment as a solo phase — see `VmPhase::has_repo`.
|
||||||
|
has_repo: bool,
|
||||||
|
/// `missions.target_node_id`: the fleet node a mission was placed on. A node
|
||||||
|
/// may override it with `attrs["node_id"]`.
|
||||||
|
default_fleet_node: Option<Uuid>,
|
||||||
|
/// `missions.backend`: which rootfs image. A node may override it with
|
||||||
|
/// `attrs["backend"]`, which is what makes a graph heterogeneous — a
|
||||||
|
/// `validator` node on a different provider's image is then a first-class
|
||||||
|
/// graph node rather than a bolt-on.
|
||||||
|
default_backend: Option<String>,
|
||||||
|
/// `missions.team_engine`, passed through so a composed node can itself ask
|
||||||
|
/// for Claude Code fan-out inside its VM.
|
||||||
|
team_engine: Option<String>,
|
||||||
|
/// The phase's completion gate, enforced inside every node's VM.
|
||||||
|
gate: Option<crate::vm_stop_gate::StopGate>,
|
||||||
|
/// Which step is next. `execute_resumable` is sequential and gives the
|
||||||
|
/// executor no index, so the executor counts — and the count starts from the
|
||||||
|
/// checkpoint on resume, or two VMs would share an id across a restart.
|
||||||
|
step: std::sync::atomic::AtomicU32,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Everything a composed run needs that is not the graph itself.
|
||||||
|
pub struct ComposedRun {
|
||||||
|
pub run_id: Uuid,
|
||||||
|
pub mission_id: Uuid,
|
||||||
|
pub phase_id: Uuid,
|
||||||
|
pub iteration: i32,
|
||||||
|
pub repo: PathBuf,
|
||||||
|
pub has_repo: bool,
|
||||||
|
pub target_node_id: Option<Uuid>,
|
||||||
|
pub backend: Option<String>,
|
||||||
|
pub team_engine: Option<String>,
|
||||||
|
/// What must hold before a node's agent may stop. See [`crate::vm_stop_gate`].
|
||||||
|
pub gate: Option<crate::vm_stop_gate::StopGate>,
|
||||||
|
/// Steps already completed, from the durable checkpoint. Nonzero on resume.
|
||||||
|
pub completed_steps: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<V: PhaseVm> MicroVmTurnExecutor<V> {
|
||||||
|
pub fn new(vms: V, pool: PgPool, r: ComposedRun) -> Self {
|
||||||
|
Self {
|
||||||
|
vms,
|
||||||
|
pool,
|
||||||
|
run_id: r.run_id,
|
||||||
|
mission_id: r.mission_id,
|
||||||
|
phase_id: r.phase_id,
|
||||||
|
iteration: r.iteration,
|
||||||
|
repo: r.repo,
|
||||||
|
has_repo: r.has_repo,
|
||||||
|
default_fleet_node: r.target_node_id,
|
||||||
|
default_backend: r.backend,
|
||||||
|
team_engine: r.team_engine,
|
||||||
|
gate: r.gate,
|
||||||
|
step: std::sync::atomic::AtomicU32::new(r.completed_steps),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Which fleet node this graph node runs on.
|
||||||
|
///
|
||||||
|
/// Fail-closed on a malformed override: placing a node on the mission's node
|
||||||
|
/// because its own `node_id` did not parse would run the work somewhere the
|
||||||
|
/// graph did not ask for and say nothing.
|
||||||
|
fn fleet_node(&self, req: &TurnRequest) -> Result<NodeId, OrchestratorError> {
|
||||||
|
let id = match req.attrs.get("node_id") {
|
||||||
|
Some(raw) => Uuid::parse_str(raw.trim()).map_err(|_| {
|
||||||
|
OrchestratorError::Executor(format!(
|
||||||
|
"node {} has an invalid node_id attr: {raw}",
|
||||||
|
req.node_id
|
||||||
|
))
|
||||||
|
})?,
|
||||||
|
None => self.default_fleet_node.ok_or_else(|| {
|
||||||
|
OrchestratorError::Executor(format!(
|
||||||
|
"node {} has no node_id attr and the mission has no \
|
||||||
|
target_node_id — a microVM node cannot run on the gateway, \
|
||||||
|
which has no /dev/kvm",
|
||||||
|
req.node_id
|
||||||
|
))
|
||||||
|
})?,
|
||||||
|
};
|
||||||
|
Ok(NodeId::from(id))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<V: PhaseVm> TurnExecutor for MicroVmTurnExecutor<V> {
|
||||||
|
async fn run_turn(&self, req: TurnRequest) -> Result<TurnOutcome, OrchestratorError> {
|
||||||
|
let fleet_node = self.fleet_node(&req)?;
|
||||||
|
let backend = req
|
||||||
|
.attrs
|
||||||
|
.get("backend")
|
||||||
|
.map(|s| s.trim().to_string())
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.or_else(|| self.default_backend.clone());
|
||||||
|
|
||||||
|
if !self.repo.is_dir() {
|
||||||
|
return Err(OrchestratorError::Executor(format!(
|
||||||
|
"mission has no checkout at {} — a composed node needs the \
|
||||||
|
repository, and it is also how the previous node's work reaches \
|
||||||
|
this one",
|
||||||
|
self.repo.display()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let step = self
|
||||||
|
.step
|
||||||
|
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||||
|
|
||||||
|
// Held for the length of the VM turn: an hour of silence would otherwise
|
||||||
|
// look exactly like a dead worker to `requeue_stale`.
|
||||||
|
let _alive = Keepalive::spawn(self.pool.clone(), self.run_id);
|
||||||
|
|
||||||
|
let task = node_task_text(&req);
|
||||||
|
let outcome = self
|
||||||
|
.vms
|
||||||
|
.run(VmPhase {
|
||||||
|
// Every node of a composed graph streams to the same outer run,
|
||||||
|
// which is the one the operator is watching.
|
||||||
|
run_id: Some(self.run_id),
|
||||||
|
node_id: fleet_node,
|
||||||
|
mission_id: self.mission_id,
|
||||||
|
phase_id: self.phase_id,
|
||||||
|
iteration: self.iteration,
|
||||||
|
task: &task,
|
||||||
|
backend: backend.as_deref(),
|
||||||
|
repo: &self.repo,
|
||||||
|
has_repo: self.has_repo,
|
||||||
|
team_engine: self.team_engine.as_deref(),
|
||||||
|
// Each node is its own agent session, so each carries the
|
||||||
|
// phase's gate. Threaded from the run rather than rebuilt here:
|
||||||
|
// one source for what "done" means, whichever executor asks.
|
||||||
|
gate: self.gate.as_ref(),
|
||||||
|
step: Some(step),
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
OrchestratorError::Executor(format!("node {} in a microVM: {e}", req.node_id))
|
||||||
|
})?;
|
||||||
|
|
||||||
|
// Recorded BEFORE the failure branches below. A node that could not be
|
||||||
|
// collected, or whose gate capped, still touched files — and on this
|
||||||
|
// path those touches are the only account of what it did, since the
|
||||||
|
// work never reached a diff.
|
||||||
|
crate::phase_runner::record_vm_tools(
|
||||||
|
&self.pool,
|
||||||
|
self.mission_id,
|
||||||
|
self.phase_id,
|
||||||
|
self.run_id,
|
||||||
|
&outcome.tools,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
// A node whose work never came back must fail the run rather than hand
|
||||||
|
// the next node a tree missing the previous one's edits. On this path an
|
||||||
|
// uncollected turn is worse than on the solo one: the loss is silent,
|
||||||
|
// because the next node still boots from a checkout that looks fine.
|
||||||
|
if !outcome.collected {
|
||||||
|
return Err(OrchestratorError::Executor(format!(
|
||||||
|
"node {}'s work could not be collected from its VM, so the next \
|
||||||
|
node would not see it: {}",
|
||||||
|
req.node_id,
|
||||||
|
outcome.summary.chars().take(400).collect::<String>()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
// Same rule as the solo path: the gate is the only thing that runs a
|
||||||
|
// `done_when_check`, so a release at the cap must fail the run rather
|
||||||
|
// than hand the next node a tree that does not satisfy the condition
|
||||||
|
// every node in this graph was told to satisfy.
|
||||||
|
if outcome.released_at_cap == Some(true) {
|
||||||
|
return Err(OrchestratorError::Executor(format!(
|
||||||
|
"node {}'s completion gate released it after {} refusal(s) with its check \
|
||||||
|
still failing: {}",
|
||||||
|
req.node_id,
|
||||||
|
crate::vm_stop_gate::MAX_BLOCKS,
|
||||||
|
outcome.summary.chars().take(400).collect::<String>()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
if outcome.rc != 0 {
|
||||||
|
return Err(OrchestratorError::Executor(format!(
|
||||||
|
"node {} exited {}: {}",
|
||||||
|
req.node_id,
|
||||||
|
outcome.rc,
|
||||||
|
outcome.summary.chars().take(400).collect::<String>()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
eprintln!(
|
||||||
|
"microvm_turn_executor: run {} node {} (role {}, step {}) ok — subagents: {}",
|
||||||
|
self.run_id,
|
||||||
|
req.node_id,
|
||||||
|
req.role,
|
||||||
|
step,
|
||||||
|
outcome
|
||||||
|
.subagents
|
||||||
|
.map(|n| n.to_string())
|
||||||
|
.unwrap_or_else(|| "?".into()),
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(TurnOutcome {
|
||||||
|
output: outcome.summary,
|
||||||
|
// `claude -p` does not report token usage on stdout, and inventing a
|
||||||
|
// number here would corrupt the run totals the harness reads. Zero is
|
||||||
|
// the honest value for "not measured on this path".
|
||||||
|
tokens: 0,
|
||||||
|
gated: Vec::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What one graph node is told.
|
||||||
|
///
|
||||||
|
/// The upstream outputs are included as context, but the load-bearing sentence is
|
||||||
|
/// that the previous node's *files* are already in the tree: a node told only
|
||||||
|
/// about the text would re-do work it is standing on.
|
||||||
|
fn node_task_text(req: &TurnRequest) -> String {
|
||||||
|
let mut s = format!(
|
||||||
|
"You are the `{}` stage of a multi-stage mission.\n\nMISSION TASK\n{}\n",
|
||||||
|
req.role, req.task
|
||||||
|
);
|
||||||
|
if !req.context.is_empty() {
|
||||||
|
s.push_str(
|
||||||
|
"\nWHAT CAME BEFORE\nThe earlier stages' work is ALREADY IN THIS \
|
||||||
|
WORKING TREE — the repository you have been given is their output, \
|
||||||
|
not a fresh checkout. Read the files before changing them, and do \
|
||||||
|
not redo what is already done. Their closing reports:\n",
|
||||||
|
);
|
||||||
|
for (i, c) in req.context.iter().enumerate() {
|
||||||
|
s.push_str(&format!("\n--- stage {} ---\n{}\n", i + 1, c));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Touches a run's `updated_at` until dropped.
|
||||||
|
struct Keepalive(tokio::task::JoinHandle<()>);
|
||||||
|
|
||||||
|
impl Keepalive {
|
||||||
|
fn spawn(pool: PgPool, run_id: Uuid) -> Self {
|
||||||
|
Keepalive(tokio::spawn(async move {
|
||||||
|
let mut ticker = tokio::time::interval(Duration::from_secs(KEEPALIVE_SECS));
|
||||||
|
loop {
|
||||||
|
ticker.tick().await;
|
||||||
|
let _ = cm_db::repo::topology_runs::touch(&pool, run_id).await;
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for Keepalive {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.0.abort();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build the executor the worker uses, over real VMs on the fleet.
|
||||||
|
pub fn for_fleet(
|
||||||
|
hub: Arc<crate::fleet::NodeHub>,
|
||||||
|
pool: PgPool,
|
||||||
|
r: ComposedRun,
|
||||||
|
) -> MicroVmTurnExecutor<crate::microvm_executor::HubVms> {
|
||||||
|
MicroVmTurnExecutor::new(crate::microvm_executor::HubVms::new(hub), pool, r)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::microvm_executor::VmOutcome;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::sync::Mutex;
|
||||||
|
|
||||||
|
/// A VM modelled honestly: the host tree is packed in, the "agent" works on a
|
||||||
|
/// COPY that no host path points at, and the result is unpacked back over the
|
||||||
|
/// host tree. That is the real inject → run → collect shape, which is what
|
||||||
|
/// makes the negative control below meaningful — remove the collect and the
|
||||||
|
/// handoff breaks exactly as it would in production.
|
||||||
|
struct FakeVms {
|
||||||
|
/// Whether the guest's tree is collected back to the host.
|
||||||
|
collect: bool,
|
||||||
|
/// vm ids used, in order — the id is what stops two nodes colliding.
|
||||||
|
ids: Mutex<Vec<String>>,
|
||||||
|
/// (backend, fleet node) per call, for the heterogeneity assertions.
|
||||||
|
placements: Mutex<Vec<(Option<String>, NodeId)>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FakeVms {
|
||||||
|
fn new(collect: bool) -> Self {
|
||||||
|
Self {
|
||||||
|
collect,
|
||||||
|
ids: Mutex::new(Vec::new()),
|
||||||
|
placements: Mutex::new(Vec::new()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PhaseVm for FakeVms {
|
||||||
|
async fn run(&self, p: VmPhase<'_>) -> Result<VmOutcome, String> {
|
||||||
|
self.ids
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.push(format!("{}-{:?}", p.phase_id.simple(), p.step));
|
||||||
|
self.placements
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.push((p.backend.map(str::to_string), p.node_id));
|
||||||
|
|
||||||
|
// inject: the host checkout goes in as a tar.
|
||||||
|
let tar = crate::mission_fs::pack_dir(p.repo, "repo")?;
|
||||||
|
let guest = tempfile::tempdir().map_err(|e| e.to_string())?;
|
||||||
|
crate::mission_fs::unpack_into(&tar, guest.path())?;
|
||||||
|
let guest_repo = guest.path().join("repo");
|
||||||
|
|
||||||
|
// run: the agent records that it was here, and reports what it found
|
||||||
|
// of the previous stages — the observation the handoff test reads.
|
||||||
|
let seen: Vec<String> = std::fs::read_dir(&guest_repo)
|
||||||
|
.map_err(|e| e.to_string())?
|
||||||
|
.filter_map(|e| e.ok())
|
||||||
|
.map(|e| e.file_name().to_string_lossy().to_string())
|
||||||
|
.filter(|n| n.starts_with("stage-"))
|
||||||
|
.collect();
|
||||||
|
let mine = guest_repo.join(format!("stage-{}.txt", p.step.unwrap_or(0)));
|
||||||
|
std::fs::write(&mine, "work").map_err(|e| e.to_string())?;
|
||||||
|
|
||||||
|
// collect: the guest tree comes back over the same host path.
|
||||||
|
if self.collect {
|
||||||
|
let back = crate::mission_fs::pack_dir(&guest_repo, "repo")?;
|
||||||
|
let parent = p.repo.parent().ok_or("no parent")?;
|
||||||
|
crate::mission_fs::unpack_into(&back, parent)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(VmOutcome {
|
||||||
|
summary: format!("saw:[{}]", seen.join(",")),
|
||||||
|
rc: 0,
|
||||||
|
collected: true,
|
||||||
|
subagents: Some(0),
|
||||||
|
teammates: None,
|
||||||
|
stop_blocks: None,
|
||||||
|
released_at_cap: None,
|
||||||
|
tools: Vec::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn req(node: &str, role: &str, context: Vec<String>) -> TurnRequest {
|
||||||
|
TurnRequest {
|
||||||
|
node_id: node.into(),
|
||||||
|
role: role.into(),
|
||||||
|
agent: None,
|
||||||
|
attrs: BTreeMap::new(),
|
||||||
|
task: "build the thing".into(),
|
||||||
|
context,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exec<V: PhaseVm>(vms: V, repo: PathBuf) -> MicroVmTurnExecutor<V> {
|
||||||
|
// A pool that is never connected: every test here fails the turn before
|
||||||
|
// any query, or drives one whose only DB touch is the best-effort
|
||||||
|
// keepalive (which swallows its own errors by design).
|
||||||
|
let pool = sqlx::postgres::PgPoolOptions::new()
|
||||||
|
.max_connections(1)
|
||||||
|
.connect_lazy("postgres://invalid/invalid")
|
||||||
|
.expect("a lazy pool never dials");
|
||||||
|
MicroVmTurnExecutor::new(
|
||||||
|
vms,
|
||||||
|
pool,
|
||||||
|
ComposedRun {
|
||||||
|
run_id: Uuid::now_v7(),
|
||||||
|
mission_id: Uuid::now_v7(),
|
||||||
|
phase_id: Uuid::now_v7(),
|
||||||
|
iteration: 1,
|
||||||
|
repo,
|
||||||
|
has_repo: true,
|
||||||
|
target_node_id: Some(Uuid::now_v7()),
|
||||||
|
backend: Some("claude".into()),
|
||||||
|
team_engine: None,
|
||||||
|
gate: None,
|
||||||
|
completed_steps: 0,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn a_checkout() -> tempfile::TempDir {
|
||||||
|
let d = tempfile::tempdir().unwrap();
|
||||||
|
std::fs::create_dir_all(d.path().join("repo")).unwrap();
|
||||||
|
std::fs::write(d.path().join("repo").join("README.md"), "hello").unwrap();
|
||||||
|
d
|
||||||
|
}
|
||||||
|
|
||||||
|
/// THE trap this slice exists to solve. A per-node VM is destroyed with its
|
||||||
|
/// filesystem, so unless the tree is carried forward, node 2 works from the
|
||||||
|
/// original checkout and silently loses node 1's edits — while still
|
||||||
|
/// reporting success.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_later_node_sees_an_earlier_nodes_files() {
|
||||||
|
let d = a_checkout();
|
||||||
|
let e = exec(FakeVms::new(true), d.path().join("repo"));
|
||||||
|
|
||||||
|
let first = e.run_turn(req("n1", "implementer", vec![])).await.unwrap();
|
||||||
|
assert_eq!(first.output, "saw:[]", "the first node starts clean");
|
||||||
|
|
||||||
|
let second = e
|
||||||
|
.run_turn(req("n2", "verifier", vec![first.output.clone()]))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
second.output.contains("stage-0.txt"),
|
||||||
|
"node 2 could not see node 1's file: {}",
|
||||||
|
second.output
|
||||||
|
);
|
||||||
|
// And the host tree — what delivery diffs — holds both nodes' work.
|
||||||
|
for f in ["stage-0.txt", "stage-1.txt"] {
|
||||||
|
assert!(d.path().join("repo").join(f).exists(), "{f} missing on the host");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The negative control, run rather than assumed: with the collect removed —
|
||||||
|
/// i.e. a text-only handoff between nodes — the test above fails. A guard
|
||||||
|
/// that cannot detect the bug it was written for is decoration.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn text_only_handoff_loses_the_earlier_nodes_work() {
|
||||||
|
let d = a_checkout();
|
||||||
|
let e = exec(FakeVms::new(false), d.path().join("repo"));
|
||||||
|
|
||||||
|
e.run_turn(req("n1", "implementer", vec![])).await.unwrap();
|
||||||
|
let second = e.run_turn(req("n2", "verifier", vec![])).await.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
second.output, "saw:[]",
|
||||||
|
"without a collect, node 2 must NOT see node 1's work — if it does, \
|
||||||
|
this test is no longer controlling anything"
|
||||||
|
);
|
||||||
|
assert!(!d.path().join("repo").join("stage-0.txt").exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Each node gets its own vm id within one phase and iteration. Two nodes
|
||||||
|
/// sharing an id means the second is refused by the fleet node while the
|
||||||
|
/// first is alive, and indistinguishable from a re-run once it is not.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn every_node_runs_in_its_own_vm() {
|
||||||
|
let d = a_checkout();
|
||||||
|
let vms = FakeVms::new(true);
|
||||||
|
let e = exec(vms, d.path().join("repo"));
|
||||||
|
for n in ["n1", "n2", "n3"] {
|
||||||
|
e.run_turn(req(n, "worker", vec![])).await.unwrap();
|
||||||
|
}
|
||||||
|
let ids = e.vms.ids.lock().unwrap().clone();
|
||||||
|
let unique: std::collections::HashSet<_> = ids.iter().collect();
|
||||||
|
assert_eq!(unique.len(), ids.len(), "{ids:?}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resume must not re-use a completed step's vm id. The executor counts steps
|
||||||
|
/// itself, so the count has to start where the checkpoint left off.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_resumed_run_continues_the_step_numbering() {
|
||||||
|
let d = a_checkout();
|
||||||
|
let pool = sqlx::postgres::PgPoolOptions::new()
|
||||||
|
.max_connections(1)
|
||||||
|
.connect_lazy("postgres://invalid/invalid")
|
||||||
|
.unwrap();
|
||||||
|
let e = MicroVmTurnExecutor::new(
|
||||||
|
FakeVms::new(true),
|
||||||
|
pool,
|
||||||
|
ComposedRun {
|
||||||
|
run_id: Uuid::now_v7(),
|
||||||
|
mission_id: Uuid::now_v7(),
|
||||||
|
phase_id: Uuid::now_v7(),
|
||||||
|
iteration: 1,
|
||||||
|
repo: d.path().join("repo"),
|
||||||
|
has_repo: true,
|
||||||
|
target_node_id: Some(Uuid::now_v7()),
|
||||||
|
backend: None,
|
||||||
|
team_engine: None,
|
||||||
|
gate: None,
|
||||||
|
completed_steps: 2,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
e.run_turn(req("n3", "worker", vec![])).await.unwrap();
|
||||||
|
let ids = e.vms.ids.lock().unwrap().clone();
|
||||||
|
assert!(
|
||||||
|
ids[0].ends_with("Some(2)"),
|
||||||
|
"the first step after a resume must be step 2, not 0: {ids:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Per-node `backend` is what makes the outer graph heterogeneous — a
|
||||||
|
/// validator node on another provider's image. It must override the
|
||||||
|
/// mission's, and the mission's must still apply to nodes that say nothing.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_node_may_pick_its_own_backend_and_fleet_node() {
|
||||||
|
let d = a_checkout();
|
||||||
|
let e = exec(FakeVms::new(true), d.path().join("repo"));
|
||||||
|
let elsewhere = Uuid::now_v7();
|
||||||
|
|
||||||
|
let mut r = req("n1", "worker", vec![]);
|
||||||
|
r.attrs.insert("backend".into(), "glm".into());
|
||||||
|
r.attrs.insert("node_id".into(), elsewhere.to_string());
|
||||||
|
e.run_turn(r).await.unwrap();
|
||||||
|
e.run_turn(req("n2", "worker", vec![])).await.unwrap();
|
||||||
|
|
||||||
|
let p = e.vms.placements.lock().unwrap().clone();
|
||||||
|
assert_eq!(p[0].0.as_deref(), Some("glm"));
|
||||||
|
assert_eq!(p[0].1, NodeId::from(elsewhere));
|
||||||
|
assert_eq!(p[1].0.as_deref(), Some("claude"), "the mission default");
|
||||||
|
assert_ne!(p[1].1, NodeId::from(elsewhere));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A malformed `node_id` must fail the node, not fall back to the mission's.
|
||||||
|
/// Silently running work somewhere the graph did not ask for is the same
|
||||||
|
/// class of bug as an alias that serde dropped.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_malformed_node_placement_fails_closed() {
|
||||||
|
let d = a_checkout();
|
||||||
|
let e = exec(FakeVms::new(true), d.path().join("repo"));
|
||||||
|
let mut r = req("n1", "worker", vec![]);
|
||||||
|
r.attrs.insert("node_id".into(), "not-a-uuid".into());
|
||||||
|
let err = e.run_turn(r).await.unwrap_err().to_string();
|
||||||
|
assert!(err.contains("invalid node_id"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An uncollected node is a failed run here, not a warning: the next node
|
||||||
|
/// would boot from a tree that looks fine and is missing this node's work.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn an_uncollected_node_fails_the_run() {
|
||||||
|
struct Lost;
|
||||||
|
impl PhaseVm for Lost {
|
||||||
|
async fn run(&self, _p: VmPhase<'_>) -> Result<VmOutcome, String> {
|
||||||
|
Ok(VmOutcome {
|
||||||
|
summary: "did plenty".into(),
|
||||||
|
rc: 0,
|
||||||
|
collected: false,
|
||||||
|
subagents: None,
|
||||||
|
teammates: None,
|
||||||
|
stop_blocks: None,
|
||||||
|
released_at_cap: None,
|
||||||
|
tools: Vec::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let d = a_checkout();
|
||||||
|
let e = exec(Lost, d.path().join("repo"));
|
||||||
|
let err = e.run_turn(req("n1", "worker", vec![])).await.unwrap_err().to_string();
|
||||||
|
assert!(err.contains("could not be collected"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A node whose gate gave up is a FAILED run, not a completed one.
|
||||||
|
///
|
||||||
|
/// The gate is the only thing in the system that ever runs a
|
||||||
|
/// `done_when_check`. If it releases the agent at the cap and this returns
|
||||||
|
/// Ok, the check's failure is never seen again: the node reports success,
|
||||||
|
/// the next node builds on a tree that does not satisfy the condition, and
|
||||||
|
/// the phase completes green. `rc` is 0 and the work IS collected here on
|
||||||
|
/// purpose — those are the two signals that used to decide this, and both
|
||||||
|
/// say "fine".
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_node_whose_gate_gave_up_fails_the_run() {
|
||||||
|
struct Capped;
|
||||||
|
impl PhaseVm for Capped {
|
||||||
|
async fn run(&self, _p: VmPhase<'_>) -> Result<VmOutcome, String> {
|
||||||
|
Ok(VmOutcome {
|
||||||
|
summary: "I could not get the tests passing, but here is what I did".into(),
|
||||||
|
rc: 0,
|
||||||
|
collected: true,
|
||||||
|
subagents: None,
|
||||||
|
teammates: None,
|
||||||
|
stop_blocks: Some(crate::vm_stop_gate::MAX_BLOCKS),
|
||||||
|
released_at_cap: Some(true),
|
||||||
|
tools: Vec::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let d = a_checkout();
|
||||||
|
let e = exec(Capped, d.path().join("repo"));
|
||||||
|
let err = e.run_turn(req("n1", "worker", vec![])).await.unwrap_err().to_string();
|
||||||
|
assert!(err.contains("released it after"), "{err}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The negative control: the SAME number of blocks, without the cap. An
|
||||||
|
/// agent that was refused three times and then got it right on the fourth
|
||||||
|
/// try has succeeded, and reports `blocks: 3` exactly like the test above.
|
||||||
|
/// Failing on the count instead of the mark would fail this healthy run.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_node_that_was_blocked_and_then_succeeded_passes() {
|
||||||
|
struct Recovered;
|
||||||
|
impl PhaseVm for Recovered {
|
||||||
|
async fn run(&self, _p: VmPhase<'_>) -> Result<VmOutcome, String> {
|
||||||
|
Ok(VmOutcome {
|
||||||
|
summary: "took me a few tries".into(),
|
||||||
|
rc: 0,
|
||||||
|
collected: true,
|
||||||
|
subagents: None,
|
||||||
|
teammates: None,
|
||||||
|
stop_blocks: Some(crate::vm_stop_gate::MAX_BLOCKS),
|
||||||
|
released_at_cap: Some(false),
|
||||||
|
tools: Vec::new(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let d = a_checkout();
|
||||||
|
let e = exec(Recovered, d.path().join("repo"));
|
||||||
|
e.run_turn(req("n1", "worker", vec![]))
|
||||||
|
.await
|
||||||
|
.expect("a run that recovered inside its own turn is a success");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A node must be told its predecessors' files are already in the tree.
|
||||||
|
/// Given only the text, an agent re-does work it is standing on.
|
||||||
|
#[test]
|
||||||
|
fn a_downstream_node_is_told_the_work_is_already_in_the_tree() {
|
||||||
|
let solo = node_task_text(&req("n1", "implementer", vec![]));
|
||||||
|
assert!(solo.contains("build the thing"));
|
||||||
|
assert!(!solo.contains("WHAT CAME BEFORE"), "{solo}");
|
||||||
|
|
||||||
|
let later = node_task_text(&req("n2", "verifier", vec!["I wrote foo.rs".into()]));
|
||||||
|
assert!(later.contains("ALREADY IN THIS WORKING TREE"), "{later}");
|
||||||
|
assert!(later.contains("I wrote foo.rs"), "{later}");
|
||||||
|
assert!(later.contains("verifier"), "the node's role: {later}");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -40,7 +40,14 @@ use crate::mission_workspace;
|
|||||||
/// coding phase that ran `cargo build` leaves a `target/` directory larger
|
/// coding phase that ran `cargo build` leaves a `target/` directory larger
|
||||||
/// than most repositories, and a patch containing it is unreadable as well as
|
/// than most repositories, and a patch containing it is unreadable as well as
|
||||||
/// enormous.
|
/// enormous.
|
||||||
const EXCLUDED_PATHS: &[&str] = &[
|
///
|
||||||
|
/// `mission_fs` uses this same list for the TRANSPORT, and that is not a
|
||||||
|
/// convenience — it is the fix for a real failure. The diff excluded `target/`
|
||||||
|
/// while the tar that carried the tree in and out did not, so a phase that ran
|
||||||
|
/// `cargo test` shipped its whole build directory over vsock twice. `vm_collect`
|
||||||
|
/// timed out at 300s on mission 019fd43e with the agent's work finished and
|
||||||
|
/// stranded inside a VM. Two layers, one list.
|
||||||
|
pub(crate) const EXCLUDED_PATHS: &[&str] = &[
|
||||||
"target",
|
"target",
|
||||||
"node_modules",
|
"node_modules",
|
||||||
".venv",
|
".venv",
|
||||||
@@ -66,6 +73,10 @@ const EXCLUDED_PATHS: &[&str] = &[
|
|||||||
/// generated or vendored got committed), and the head of it is what an
|
/// generated or vendored got committed), and the head of it is what an
|
||||||
/// operator needs to see to work out what happened.
|
/// operator needs to see to work out what happened.
|
||||||
const MAX_PATCH_BYTES: usize = 4 * 1024 * 1024;
|
const MAX_PATCH_BYTES: usize = 4 * 1024 * 1024;
|
||||||
|
/// Cap on the recorded path list. A cap that silently truncates is worse than
|
||||||
|
/// no cap, so the metadata carries `files_truncated` beside it — a reader must
|
||||||
|
/// be able to tell "touched 12 files" from "touched at least 500".
|
||||||
|
const MAX_CAPTURED_PATHS: usize = 500;
|
||||||
|
|
||||||
/// Who delivery commits as.
|
/// Who delivery commits as.
|
||||||
///
|
///
|
||||||
@@ -109,7 +120,18 @@ pub struct Capture {
|
|||||||
/// The phase changed nothing. Still recorded — "this coding phase wrote no
|
/// The phase changed nothing. Still recorded — "this coding phase wrote no
|
||||||
/// code" is currently invisible to an operator, and it is worth saying.
|
/// code" is currently invisible to an operator, and it is worth saying.
|
||||||
pub empty: bool,
|
pub empty: bool,
|
||||||
|
/// Why the diff could not be computed, if it could not be. `empty` is only
|
||||||
|
/// meaningful when this is `None`: otherwise the tree was never read, and
|
||||||
|
/// callers deciding anything on the strength of "no changes" must not.
|
||||||
|
pub diff_error: Option<String>,
|
||||||
pub truncated: bool,
|
pub truncated: bool,
|
||||||
|
/// The paths this phase touched, with their `--name-status` letter. The
|
||||||
|
/// diffstat gives counts only; this is what lets anything downstream say
|
||||||
|
/// WHICH files changed.
|
||||||
|
pub files: Vec<(char, String)>,
|
||||||
|
/// The path list hit `MAX_CAPTURED_PATHS`. Recorded so a reader can tell a
|
||||||
|
/// complete list from a clipped one.
|
||||||
|
pub files_truncated: bool,
|
||||||
pub patch_path: PathBuf,
|
pub patch_path: PathBuf,
|
||||||
/// Set once the work has been committed to a mission branch.
|
/// Set once the work has been committed to a mission branch.
|
||||||
pub committed: Option<Commit>,
|
pub committed: Option<Commit>,
|
||||||
@@ -227,19 +249,74 @@ pub async fn capture_phase_diff_at(
|
|||||||
// A repo with nothing to add is fine; keep going and let the diff be empty.
|
// A repo with nothing to add is fine; keep going and let the diff be empty.
|
||||||
let _ = git(&repo, &add).await;
|
let _ = git(&repo, &add).await;
|
||||||
|
|
||||||
|
// A failed `git diff` and a phase that changed nothing both yield an empty
|
||||||
|
// string, and `unwrap_or_default` used to erase the difference: a corrupt
|
||||||
|
// index or an unreadable base would land `empty: true, files_changed: 0` —
|
||||||
|
// byte-identical to an honest no-op, and just as quiet. Whatever went
|
||||||
|
// wrong is recorded so the artifact can say which of the two it was.
|
||||||
|
let mut diff_error: Option<String> = None;
|
||||||
|
let mut note_diff_failure = |what: &str, e: String| {
|
||||||
|
eprintln!(
|
||||||
|
"mission_delivery: mission {mission_id} phase {phase_id} could not compute \
|
||||||
|
{what} against {base_sha}: {e}"
|
||||||
|
);
|
||||||
|
if diff_error.is_none() {
|
||||||
|
diff_error = Some(format!("{what}: {}", e.chars().take(300).collect::<String>()));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
let mut diff_args = vec!["diff", base_sha.as_str(), "--"];
|
let mut diff_args = vec!["diff", base_sha.as_str(), "--"];
|
||||||
diff_args.extend(excludes.iter().map(String::as_str));
|
diff_args.extend(excludes.iter().map(String::as_str));
|
||||||
let patch = git(&repo, &diff_args).await.unwrap_or_default();
|
let patch = match git(&repo, &diff_args).await {
|
||||||
|
Ok(p) => p,
|
||||||
|
Err(e) => {
|
||||||
|
note_diff_failure("patch", e);
|
||||||
|
String::new()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
let mut stat_args = vec!["diff", base_sha.as_str(), "--stat", "--"];
|
let mut stat_args = vec!["diff", base_sha.as_str(), "--stat", "--"];
|
||||||
stat_args.extend(excludes.iter().map(String::as_str));
|
stat_args.extend(excludes.iter().map(String::as_str));
|
||||||
let diffstat = git(&repo, &stat_args).await.unwrap_or_default();
|
let diffstat = match git(&repo, &stat_args).await {
|
||||||
|
Ok(s) => s,
|
||||||
|
Err(e) => {
|
||||||
|
note_diff_failure("diffstat", e);
|
||||||
|
String::new()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// The paths themselves, not just the counts.
|
||||||
|
//
|
||||||
|
// The diffstat gives three integers and throws the filenames away, so
|
||||||
|
// nothing downstream could say WHICH files a phase touched — the World
|
||||||
|
// could draw a "coding" station but nothing under it. Same `base_sha` and
|
||||||
|
// the same excludes as the `--stat` call above: if the two disagreed,
|
||||||
|
// `files_changed` and this list would contradict each other and nobody
|
||||||
|
// could tell which one lied.
|
||||||
|
//
|
||||||
|
// Must run BEFORE the reset below — `--intent-to-add` is what makes newly
|
||||||
|
// created files visible to diff at all.
|
||||||
|
let mut name_args = vec!["diff", base_sha.as_str(), "--name-status", "--"];
|
||||||
|
name_args.extend(excludes.iter().map(String::as_str));
|
||||||
|
let name_status = match git(&repo, &name_args).await {
|
||||||
|
Ok(s) => s,
|
||||||
|
Err(e) => {
|
||||||
|
note_diff_failure("name-status", e);
|
||||||
|
String::new()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
// Put the index back. `--intent-to-add` is a mutation of the agent's
|
// Put the index back. `--intent-to-add` is a mutation of the agent's
|
||||||
// workspace, and capture must not change what a later commit would see.
|
// workspace, and capture must not change what a later commit would see.
|
||||||
let _ = git(&repo, &["reset", "--quiet"]).await;
|
let _ = git(&repo, &["reset", "--quiet"]).await;
|
||||||
|
|
||||||
let (files_changed, insertions, deletions) = parse_diffstat(&diffstat);
|
let (files_changed, insertions, deletions) = parse_diffstat(&diffstat);
|
||||||
|
// Shared with auto_merge so the two cannot disagree about what a
|
||||||
|
// `--name-status` line means (renames are three fields; the NEW path is the
|
||||||
|
// one that changed).
|
||||||
|
let all_paths = crate::auto_merge::changed_paths(&name_status);
|
||||||
|
let files_truncated = all_paths.len() > MAX_CAPTURED_PATHS;
|
||||||
|
let files: Vec<(char, String)> = all_paths.into_iter().take(MAX_CAPTURED_PATHS).collect();
|
||||||
let empty = patch.trim().is_empty();
|
let empty = patch.trim().is_empty();
|
||||||
let truncated = patch.len() > MAX_PATCH_BYTES;
|
let truncated = patch.len() > MAX_PATCH_BYTES;
|
||||||
let stored = if truncated {
|
let stored = if truncated {
|
||||||
@@ -258,6 +335,9 @@ pub async fn capture_phase_diff_at(
|
|||||||
let patch_path = dir.join("diff.patch");
|
let patch_path = dir.join("diff.patch");
|
||||||
std::fs::write(&patch_path, &stored)
|
std::fs::write(&patch_path, &stored)
|
||||||
.map_err(|e| format!("write {}: {e}", patch_path.display()))?;
|
.map_err(|e| format!("write {}: {e}", patch_path.display()))?;
|
||||||
|
// Raw evidence on disk, independent of the JSONB. When the metadata and
|
||||||
|
// the picture disagree, this is the tiebreaker.
|
||||||
|
let _ = std::fs::write(dir.join("names.txt"), &name_status);
|
||||||
std::fs::write(dir.join("diffstat.txt"), &diffstat)
|
std::fs::write(dir.join("diffstat.txt"), &diffstat)
|
||||||
.map_err(|e| format!("write diffstat: {e}"))?;
|
.map_err(|e| format!("write diffstat: {e}"))?;
|
||||||
|
|
||||||
@@ -293,15 +373,42 @@ pub async fn capture_phase_diff_at(
|
|||||||
// Gate, then publish. Both are best-effort on top of an artifact that has
|
// Gate, then publish. Both are best-effort on top of an artifact that has
|
||||||
// already landed: a phase whose tests fail, or whose push is rejected,
|
// already landed: a phase whose tests fail, or whose push is rejected,
|
||||||
// still has its patch on disk and its work on a local branch.
|
// still has its patch on disk and its work on a local branch.
|
||||||
|
//
|
||||||
|
// `empty` suppresses publishing, so a diff we could not COMPUTE would
|
||||||
|
// otherwise skip the push and leave `push_error: null` — the phase looking
|
||||||
|
// exactly like one that correctly had nothing to publish. See
|
||||||
|
// [`untrusted_empty_reason`].
|
||||||
let mut outcome: Option<TestOutcome> = None;
|
let mut outcome: Option<TestOutcome> = None;
|
||||||
let mut published: Option<Publish> = None;
|
let mut published: Option<Publish> = None;
|
||||||
let mut publish_error: Option<String> = None;
|
let mut publish_error: Option<String> = untrusted_empty_reason(empty, diff_error.as_deref());
|
||||||
if let Some(c) = committed.as_ref() {
|
if let Some(c) = committed.as_ref() {
|
||||||
if !empty {
|
if !empty {
|
||||||
if gate == Gate::OnGreenTests {
|
if gate == Gate::OnGreenTests {
|
||||||
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
||||||
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
||||||
let o = verify_tests(&repo, &container).await;
|
// Against a COPY, never the checkout. `verify_tests` execs
|
||||||
|
// `cargo test` in a container running as ROOT, which writes
|
||||||
|
// `target/` — in the live tree that leaves root-owned build
|
||||||
|
// output in a checkout owned by uid 65532 and breaks the
|
||||||
|
// single-writer invariant. Measured the first time this gate
|
||||||
|
// ever ran end to end: `uids=0,65532`.
|
||||||
|
//
|
||||||
|
// The gate had been implemented but never exercised (every
|
||||||
|
// harness fixture used `commit_policy: "always"`), which is why
|
||||||
|
// a bug this mechanical survived in it.
|
||||||
|
let gate_root = crate::root_copy::copy_root("_gate", mission_id);
|
||||||
|
crate::root_copy::purge(&container, &gate_root).await;
|
||||||
|
let o = match crate::root_copy::RootCopy::of(&repo, &gate_root) {
|
||||||
|
Ok(copy) => {
|
||||||
|
let r = verify_tests(copy.workdir(), &container).await;
|
||||||
|
crate::root_copy::purge(&container, &gate_root).await;
|
||||||
|
r
|
||||||
|
}
|
||||||
|
// Fail-closed: an unverifiable suite must not license a push.
|
||||||
|
Err(e) => TestOutcome::CouldNotRun(format!(
|
||||||
|
"could not copy the checkout to test it: {e}"
|
||||||
|
)),
|
||||||
|
};
|
||||||
// An infrastructure fault must be loud. The gate degrades
|
// An infrastructure fault must be loud. The gate degrades
|
||||||
// safely either way, but "we could not run the suite" is a
|
// safely either way, but "we could not run the suite" is a
|
||||||
// problem with the platform and needs to look like one.
|
// problem with the platform and needs to look like one.
|
||||||
@@ -327,7 +434,14 @@ pub async fn capture_phase_diff_at(
|
|||||||
could not publish {}: {e}",
|
could not publish {}: {e}",
|
||||||
c.branch
|
c.branch
|
||||||
);
|
);
|
||||||
publish_error = Some(e.chars().take(500).collect());
|
// Both ends, not the first 500 chars. Git prints its
|
||||||
|
// REASON last — "non-fast-forward", "fetch first",
|
||||||
|
// "protected branch" — so a head-only clamp keeps the
|
||||||
|
// noise and drops the answer. A real push failure was
|
||||||
|
// recorded as two auth lines plus a branch name cut
|
||||||
|
// off mid-word, with the reject reason gone.
|
||||||
|
publish_error =
|
||||||
|
Some(crate::evaluator_tools::clamp_output(&e));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -379,7 +493,18 @@ pub async fn capture_phase_diff_at(
|
|||||||
"insertions": insertions,
|
"insertions": insertions,
|
||||||
"deletions": deletions,
|
"deletions": deletions,
|
||||||
"empty": empty,
|
"empty": empty,
|
||||||
|
// Non-null means `empty`/`files_changed` describe a failed read, not
|
||||||
|
// an unchanged tree. Readers that treat `empty: true` as "the phase
|
||||||
|
// did nothing" must check this first.
|
||||||
|
"diff_error": diff_error,
|
||||||
"truncated": truncated,
|
"truncated": truncated,
|
||||||
|
// WHICH files, not just how many. Same base_sha and the same excludes
|
||||||
|
// as `files_changed`, so the two describe the same diff.
|
||||||
|
"files": files
|
||||||
|
.iter()
|
||||||
|
.map(|(st, path)| serde_json::json!({ "status": st.to_string(), "path": path }))
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
"files_truncated": files_truncated,
|
||||||
"excluded_paths": EXCLUDED_PATHS,
|
"excluded_paths": EXCLUDED_PATHS,
|
||||||
});
|
});
|
||||||
std::fs::write(
|
std::fs::write(
|
||||||
@@ -388,8 +513,11 @@ pub async fn capture_phase_diff_at(
|
|||||||
)
|
)
|
||||||
.map_err(|e| format!("write delivery.json: {e}"))?;
|
.map_err(|e| format!("write delivery.json: {e}"))?;
|
||||||
|
|
||||||
// Path is stored relative to the missions root, matching how
|
// Path is stored relative to the MISSIONS ROOT — the convention every
|
||||||
// `pdf_renderer` resolves artifact paths.
|
// artifact uses, and what `routes::missions::artifact_content` resolves
|
||||||
|
// against. (The old `pdf_renderer` claimed to match this and did not: it
|
||||||
|
// joined the mission id first, producing a doubled id and ENOENT. It is
|
||||||
|
// gone; this comment named it as the authority, which it never was.)
|
||||||
let rel = format!("_outputs/{mission_id}/{phase_id}/diff.patch");
|
let rel = format!("_outputs/{mission_id}/{phase_id}/diff.patch");
|
||||||
cm_db::repo::missions::register_artifact(
|
cm_db::repo::missions::register_artifact(
|
||||||
pool,
|
pool,
|
||||||
@@ -426,32 +554,41 @@ pub async fn capture_phase_diff_at(
|
|||||||
insertions,
|
insertions,
|
||||||
deletions,
|
deletions,
|
||||||
empty,
|
empty,
|
||||||
|
diff_error,
|
||||||
truncated,
|
truncated,
|
||||||
|
files,
|
||||||
|
files_truncated,
|
||||||
patch_path,
|
patch_path,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run git in `repo`, returning stdout.
|
/// Run git in `repo`, returning stdout.
|
||||||
///
|
///
|
||||||
/// Every invocation carries `-c safe.directory`: the server clones as uid
|
/// Every invocation carries `-c safe.directory`: under `CLAWMATES_MISSION_FS=bind`
|
||||||
/// 65532 while agents write into the same tree as root, so without it git
|
/// the server clones as uid 65532 while agents write into the same tree as
|
||||||
/// refuses the repository outright — the failure that had the phase evaluator
|
/// root, so without it git refuses the repository outright — the failure that
|
||||||
/// silently falling back to guesswork.
|
/// had the phase evaluator silently falling back to guesswork. Copy mode makes
|
||||||
|
/// the tree single-uid and this redundant, but it stays while the bind path is
|
||||||
|
/// still selectable: a workaround may only be deleted once the situation it
|
||||||
|
/// works around can no longer be chosen.
|
||||||
async fn git(repo: &Path, args: &[&str]) -> Result<String, String> {
|
async fn git(repo: &Path, args: &[&str]) -> Result<String, String> {
|
||||||
let repo_s = repo.display().to_string();
|
let repo_s = repo.display().to_string();
|
||||||
let mut full = vec![
|
// Owned, not `Box::leak`. The leak was justified as "the process is
|
||||||
"-C",
|
// short-lived", which is true of a CLI and false of cm-api — it is a
|
||||||
&repo_s,
|
// long-running server, so that was one permanently leaked allocation per
|
||||||
"-c",
|
// git call, growing with every phase of every mission for the life of the
|
||||||
// Leaked into a `String` so it can live in a `&str` slice alongside
|
// process.
|
||||||
// the borrowed args; the process is short-lived and this is one
|
let mut full: Vec<String> = vec![
|
||||||
// allocation per git call.
|
"-C".into(),
|
||||||
Box::leak(format!("safe.directory={repo_s}").into_boxed_str()),
|
repo_s.clone(),
|
||||||
|
"-c".into(),
|
||||||
|
format!("safe.directory={repo_s}"),
|
||||||
];
|
];
|
||||||
full.extend_from_slice(args);
|
full.extend(args.iter().map(|a| (*a).to_string()));
|
||||||
let (name, email) = commit_identity();
|
let (name, email) = commit_identity();
|
||||||
let out = tokio::process::Command::new("git")
|
let mut cmd = tokio::process::Command::new("git");
|
||||||
.args(&full)
|
cmd.args(&full);
|
||||||
|
let out = crate::mission_workspace::no_terminal_prompt(&mut cmd)
|
||||||
// The server container has no git identity — `git config --global
|
// The server container has no git identity — `git config --global
|
||||||
// user.email` exits 1 — so `git commit` fails with "Author identity
|
// user.email` exits 1 — so `git commit` fails with "Author identity
|
||||||
// unknown" unless one is supplied. Mission `019fc450` lost its first
|
// unknown" unless one is supplied. Mission `019fc450` lost its first
|
||||||
@@ -480,10 +617,15 @@ async fn git(repo: &Path, args: &[&str]) -> Result<String, String> {
|
|||||||
"git {} → {}: {}",
|
"git {} → {}: {}",
|
||||||
args.first().copied().unwrap_or("?"),
|
args.first().copied().unwrap_or("?"),
|
||||||
out.status,
|
out.status,
|
||||||
String::from_utf8_lossy(&out.stderr)
|
// Both ends, never a head-only clamp. THIS is where the reason was
|
||||||
.chars()
|
// being lost: `publish_phase_branch` returns a rejected push as
|
||||||
.take(300)
|
// `Ok(Publish { error })`, so the string it carries was already
|
||||||
.collect::<String>()
|
// truncated here — 300 chars of auth noise, with "non-fast-forward"
|
||||||
|
// cut off — before the caller's own both-ends clamp ever saw it.
|
||||||
|
// Clamping the caller fixed the path that was already fine.
|
||||||
|
crate::mission_workspace::redact_token(&crate::evaluator_tools::clamp_output(
|
||||||
|
&String::from_utf8_lossy(&out.stderr)
|
||||||
|
))
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
|
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
|
||||||
@@ -583,6 +725,7 @@ pub async fn commit_phase_work(
|
|||||||
String::new()
|
String::new()
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
clear_stale_commit_editmsg(repo);
|
||||||
git(repo, &["commit", "--no-verify", "-m", &message]).await?;
|
git(repo, &["commit", "--no-verify", "-m", &message]).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -717,7 +860,38 @@ async fn push_url_for(pool: &sqlx::PgPool, mission_id: Uuid) -> Result<Option<St
|
|||||||
// "nothing to push to" — the shape that made `commit_error` necessary.
|
// "nothing to push to" — the shape that made `commit_error` necessary.
|
||||||
.map_err(|e| format!("query push URL: {e}"))?
|
.map_err(|e| format!("query push URL: {e}"))?
|
||||||
.flatten();
|
.flatten();
|
||||||
Ok(url.map(|u| mission_workspace::with_ambient_auth(&u)))
|
let Some(url) = url else { return Ok(None) };
|
||||||
|
|
||||||
|
let auth = mission_workspace::with_ambient_auth(&url);
|
||||||
|
// Fail here, not at the tty. An unauthenticated URL to OUR forge cannot
|
||||||
|
// push, and every second it survives past this point is spent producing a
|
||||||
|
// symptom that looks like something else: git asking for a username, then
|
||||||
|
// `/dev/tty: No such device or address`, then a `push_error` about auth that
|
||||||
|
// sent #55's investigation after credentials which were never the problem.
|
||||||
|
// A third-party host is left alone — ssh keys and .netrc are legitimate.
|
||||||
|
if let Some(why) = &auth.unauthenticated {
|
||||||
|
if auth.is_forge() {
|
||||||
|
return Err(format!(
|
||||||
|
"cannot authenticate the push URL for this mission — {why}. The work \
|
||||||
|
is committed locally; fix the credential and re-run delivery."
|
||||||
|
));
|
||||||
|
}
|
||||||
|
eprintln!("mission_delivery: pushing to a non-forge remote unauthenticated — {why}");
|
||||||
|
}
|
||||||
|
Ok(Some(auth.url))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Did the forge reject this push because our history diverged from the ref?
|
||||||
|
///
|
||||||
|
/// Git says this several ways depending on version and refspec, and all of them
|
||||||
|
/// mean the same thing here: the branch already exists with commits ours does not
|
||||||
|
/// contain.
|
||||||
|
fn is_non_fast_forward(err: &str) -> bool {
|
||||||
|
let e = err.to_ascii_lowercase();
|
||||||
|
e.contains("non-fast-forward")
|
||||||
|
|| e.contains("fetch first")
|
||||||
|
|| e.contains("updates were rejected")
|
||||||
|
|| (e.contains("[rejected]") && !e.contains("stale info"))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run the gate, then push the branch if the gate allows it.
|
/// Run the gate, then push the branch if the gate allows it.
|
||||||
@@ -760,6 +934,54 @@ pub async fn publish_phase_branch(
|
|||||||
error: None,
|
error: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
// #55: a mission whose checkout was re-cloned — a retry, a container
|
||||||
|
// teardown, disk loss — builds divergent history against its OWN
|
||||||
|
// deterministic branch, and every push it ever attempts is rejected.
|
||||||
|
// Before this, that was terminal: the work stayed on a local branch in a
|
||||||
|
// directory that gets reaped.
|
||||||
|
//
|
||||||
|
// The escape is a NEW ref, not `--force`. Forcing would overwrite
|
||||||
|
// whatever the earlier attempt pushed — which may be the only copy of
|
||||||
|
// that work — to make this attempt look tidy. Suffixing with the commit
|
||||||
|
// sha is deterministic (the same history always lands on the same ref),
|
||||||
|
// self-describing in a branch list, and cannot collide, since divergent
|
||||||
|
// history is by definition a different sha.
|
||||||
|
Err(e) if is_non_fast_forward(&e) => {
|
||||||
|
let sha = git(repo, &["rev-parse", "HEAD"])
|
||||||
|
.await
|
||||||
|
.map(|s| s.trim().to_string())
|
||||||
|
.unwrap_or_default();
|
||||||
|
let Some(short) = sha.get(..8) else {
|
||||||
|
eprintln!("mission_delivery: push of {target} rejected and HEAD unreadable: {e}");
|
||||||
|
return Ok(Publish {
|
||||||
|
branch: target,
|
||||||
|
pushed: false,
|
||||||
|
error: Some(e),
|
||||||
|
});
|
||||||
|
};
|
||||||
|
let alt = format!("{target}-{short}");
|
||||||
|
eprintln!(
|
||||||
|
"mission_delivery: {target} exists on the forge with history this \
|
||||||
|
checkout does not contain — pushing to {alt} instead of forcing. \
|
||||||
|
Original: {e}"
|
||||||
|
);
|
||||||
|
git(repo, &["branch", "-f", &alt, "HEAD"]).await?;
|
||||||
|
match git(repo, &["push", push_url, &format!("HEAD:refs/heads/{alt}")]).await {
|
||||||
|
Ok(_) => Ok(Publish {
|
||||||
|
branch: alt,
|
||||||
|
pushed: true,
|
||||||
|
// Not an error — the work reached the forge — but the
|
||||||
|
// redirect is a fact the operator needs, or two branches for
|
||||||
|
// one phase look like a bug rather than a rescue.
|
||||||
|
error: None,
|
||||||
|
}),
|
||||||
|
Err(e2) => Ok(Publish {
|
||||||
|
branch: alt,
|
||||||
|
pushed: false,
|
||||||
|
error: Some(format!("{e}\n\nand the diverged-history retry also failed: {e2}")),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
// Redacted by `git`'s error path already; the patch and the local
|
// Redacted by `git`'s error path already; the patch and the local
|
||||||
// branch both survive, so this is a degraded success.
|
// branch both survive, so this is a degraded success.
|
||||||
@@ -878,6 +1100,34 @@ impl TestOutcome {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Remove a `COMMIT_EDITMSG` the agent left behind as root.
|
||||||
|
///
|
||||||
|
/// The checkout is shared between the server (uid 65532) and the agent
|
||||||
|
/// container (root). `core.sharedRepository` makes git create *objects and
|
||||||
|
/// refs* group-writable — `.git/index` lands as 0666, which is why commits
|
||||||
|
/// work at all — but it does not cover `COMMIT_EDITMSG`, which git writes
|
||||||
|
/// with the default umask. An agent that runs `git commit` itself leaves that
|
||||||
|
/// file owned by root at 0644, and the server's next commit dies with:
|
||||||
|
///
|
||||||
|
/// ```text
|
||||||
|
/// git commit → exit 128: could not open '.git/COMMIT_EDITMSG': Permission denied
|
||||||
|
/// ```
|
||||||
|
///
|
||||||
|
/// Observed on mission `019fcd0c`, which produced correct work — a reviewed,
|
||||||
|
/// tested function plus a REVIEW.md quoting a real `cargo test` summary — and
|
||||||
|
/// then delivered none of it.
|
||||||
|
///
|
||||||
|
/// Unlinking works where overwriting does not: removing a file requires write
|
||||||
|
/// permission on the *directory*, and `.git/` is owned by the server. Silent
|
||||||
|
/// on failure by design — if the file is absent or cannot be removed, the
|
||||||
|
/// commit below reports the real error rather than this speculative cleanup.
|
||||||
|
fn clear_stale_commit_editmsg(repo: &Path) {
|
||||||
|
let msg = repo.join(".git/COMMIT_EDITMSG");
|
||||||
|
if msg.exists() {
|
||||||
|
let _ = std::fs::remove_file(&msg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Mark a phase as impossible to capture, so it stops being selected.
|
/// Mark a phase as impossible to capture, so it stops being selected.
|
||||||
///
|
///
|
||||||
/// A phase whose checkout has already been reaped can never be captured. It
|
/// A phase whose checkout has already been reaped can never be captured. It
|
||||||
@@ -929,10 +1179,105 @@ pub async fn record_uncapturable(
|
|||||||
.map_err(|e| format!("register uncapturable marker: {e}"))
|
.map_err(|e| format!("register uncapturable marker: {e}"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Why an empty patch must not be believed, if it must not be believed.
|
||||||
|
///
|
||||||
|
/// An empty patch has two causes that produce identical bytes: the tree really
|
||||||
|
/// did not change, or `git diff` failed and we have no idea what the tree
|
||||||
|
/// looks like. The first is an ordinary outcome; the second is a platform
|
||||||
|
/// fault. Returning `Some` for the second is what stops the fault from being
|
||||||
|
/// filed under the ordinary outcome — the recurring shape where a failure and
|
||||||
|
/// a legitimate negative share one representation.
|
||||||
|
fn untrusted_empty_reason(empty: bool, diff_error: Option<&str>) -> Option<String> {
|
||||||
|
match (empty, diff_error) {
|
||||||
|
(true, Some(why)) => Some(format!(
|
||||||
|
"not published: the diff could not be computed, so an empty patch \
|
||||||
|
cannot be trusted to mean an unchanged tree ({why})"
|
||||||
|
)),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod changed_path_capture_tests {
|
||||||
|
/// The path list and `files_changed` must describe the SAME diff.
|
||||||
|
///
|
||||||
|
/// They come from two separate git invocations — `--stat` and
|
||||||
|
/// `--name-status`. If those are ever given different revisions or
|
||||||
|
/// different exclude pathspecs, the count and the list disagree and there
|
||||||
|
/// is no way to tell which is right: both look like plausible output.
|
||||||
|
#[test]
|
||||||
|
fn both_diff_calls_use_the_same_revision_and_excludes() {
|
||||||
|
let src = include_str!("mission_delivery.rs");
|
||||||
|
let body = src
|
||||||
|
.split("let mut stat_args")
|
||||||
|
.nth(1)
|
||||||
|
.and_then(|s| s.split("let (files_changed").next())
|
||||||
|
.expect("the capture block");
|
||||||
|
assert!(
|
||||||
|
body.contains("let mut name_args = vec![\"diff\", base_sha.as_str(), \"--name-status\", \"--\"]"),
|
||||||
|
"the name-status call must use the same base_sha as --stat"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
body.contains("name_args.extend(excludes.iter().map(String::as_str))"),
|
||||||
|
"and the same excludes, or files_changed and the path list describe \
|
||||||
|
different diffs"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `--name-status` must run before the index is put back, or newly created
|
||||||
|
/// files — which `--intent-to-add` is what makes visible — vanish from the
|
||||||
|
/// list while still being counted by the stat.
|
||||||
|
#[test]
|
||||||
|
fn paths_are_read_before_the_index_reset() {
|
||||||
|
let src = include_str!("mission_delivery.rs");
|
||||||
|
let name_at = src.find("--name-status").expect("name-status call");
|
||||||
|
let reset_at = src
|
||||||
|
.find("git(&repo, &[\"reset\", \"--quiet\"])")
|
||||||
|
.expect("index reset");
|
||||||
|
assert!(
|
||||||
|
name_at < reset_at,
|
||||||
|
"the path list must be captured while --intent-to-add is still in \
|
||||||
|
effect, or created files are invisible to it"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
/// Git says "your history diverged" several ways, and the one production
|
||||||
|
/// actually produced (`! [rejected] ... (fetch first)`) is not the phrase
|
||||||
|
/// anyone reaches for first. Missing a phrasing means the rescue does not
|
||||||
|
/// fire and the work stays on a local branch in a directory that gets
|
||||||
|
/// reaped — silently, since the push failure is a degraded success.
|
||||||
|
#[test]
|
||||||
|
fn every_way_git_says_diverged_is_recognised() {
|
||||||
|
for e in [
|
||||||
|
"git push → exit 1: ! [rejected] HEAD -> b (fetch first)\nhint: …",
|
||||||
|
" ! [rejected] HEAD -> b (non-fast-forward)",
|
||||||
|
"hint: Updates were rejected because the remote contains work that you \
|
||||||
|
do not have locally.",
|
||||||
|
] {
|
||||||
|
assert!(is_non_fast_forward(e), "not recognised: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// And it must not fire on failures a new branch cannot fix. Retrying a
|
||||||
|
/// permissions or network error onto a second ref just produces a second
|
||||||
|
/// failure and a confusing branch name.
|
||||||
|
#[test]
|
||||||
|
fn other_push_failures_are_not_mistaken_for_divergence() {
|
||||||
|
for e in [
|
||||||
|
"fatal: repository 'https://forge/x.git' not found",
|
||||||
|
"remote: error: GH006: Protected branch update failed",
|
||||||
|
"fatal: could not read Username for 'https://forge': terminal prompts disabled",
|
||||||
|
" ! [rejected] (stale info)",
|
||||||
|
] {
|
||||||
|
assert!(!is_non_fast_forward(e), "wrongly recognised: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ── The gate ───────────────────────────────────────────────────────
|
// ── The gate ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
/// Three recipes have declared `commit_policy` since they were written and
|
/// Three recipes have declared `commit_policy` since they were written and
|
||||||
@@ -1031,6 +1376,28 @@ mod tests {
|
|||||||
|
|
||||||
/// An empty stat means an empty phase, not a parse failure. This is the
|
/// An empty stat means an empty phase, not a parse failure. This is the
|
||||||
/// case that must still produce an artifact.
|
/// case that must still produce an artifact.
|
||||||
|
/// The whole point: a tree that genuinely did not change stays silent, and
|
||||||
|
/// a diff that could not be computed does not get to borrow that silence.
|
||||||
|
#[test]
|
||||||
|
fn an_uncomputable_diff_is_not_an_unchanged_tree() {
|
||||||
|
assert_eq!(
|
||||||
|
untrusted_empty_reason(true, None),
|
||||||
|
None,
|
||||||
|
"a genuinely unchanged tree must not report an error"
|
||||||
|
);
|
||||||
|
let reason = untrusted_empty_reason(true, Some("patch: fatal: bad object"))
|
||||||
|
.expect("an empty patch from a FAILED diff must be reported, not accepted");
|
||||||
|
assert!(
|
||||||
|
reason.contains("bad object"),
|
||||||
|
"the reason must name what went wrong, got: {reason}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
untrusted_empty_reason(false, Some("diffstat: fatal: bad object")),
|
||||||
|
None,
|
||||||
|
"a non-empty patch stands on its own even if the diffstat failed"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn an_empty_diffstat_is_all_zeroes() {
|
fn an_empty_diffstat_is_all_zeroes() {
|
||||||
assert_eq!(parse_diffstat(""), (0, 0, 0));
|
assert_eq!(parse_diffstat(""), (0, 0, 0));
|
||||||
|
|||||||
@@ -0,0 +1,229 @@
|
|||||||
|
//! Structured mission activity — the channel that replaced parsing prose.
|
||||||
|
//!
|
||||||
|
//! The operator decision behind this module: action detail comes from
|
||||||
|
//! **structured events at the source**, never from `checkpoint.log` or model
|
||||||
|
//! output. A tool name in a log line is indistinguishable from an agent
|
||||||
|
//! *discussing* a tool, and a visualization built on that distinction reads as
|
||||||
|
//! confident fact while being partly fiction.
|
||||||
|
//!
|
||||||
|
//! Everything here is best-effort. A mission must not fail because its
|
||||||
|
//! telemetry could not be written — so every write logs and swallows. That is a
|
||||||
|
//! deliberate exception to this codebase's usual rule, and it is bounded: the
|
||||||
|
//! only thing lost is detail in a picture.
|
||||||
|
|
||||||
|
use serde_json::Value;
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
/// A phase entered `running`.
|
||||||
|
pub const PHASE_STARTED: &str = "phase.started";
|
||||||
|
/// A phase reached a terminal state. `detail.status` says which.
|
||||||
|
pub const PHASE_COMPLETED: &str = "phase.completed";
|
||||||
|
/// An agent called a tool. `target` is the tool name.
|
||||||
|
pub const TOOL_CALL: &str = "tool.call";
|
||||||
|
/// A tool touched a path. `target` is the path, repo-relative where known.
|
||||||
|
pub const FILE_TOUCH: &str = "file.touch";
|
||||||
|
|
||||||
|
/// Most events one phase may record.
|
||||||
|
///
|
||||||
|
/// A capped stream that says so beats an uncapped one that quietly becomes the
|
||||||
|
/// largest table in the database: a coding phase can call thousands of tools,
|
||||||
|
/// and every one of them would be replayed to every World subscriber. Past the
|
||||||
|
/// cap the picture is already complete — nobody reads the four-thousandth file
|
||||||
|
/// orb.
|
||||||
|
pub const PER_PHASE_CAP: i64 = 400;
|
||||||
|
|
||||||
|
/// One recorded event.
|
||||||
|
#[derive(Debug, Clone, Default)]
|
||||||
|
pub struct MissionEvent {
|
||||||
|
pub mission_id: Uuid,
|
||||||
|
pub phase_id: Option<Uuid>,
|
||||||
|
pub run_id: Option<Uuid>,
|
||||||
|
pub agent_id: Option<Uuid>,
|
||||||
|
pub kind: String,
|
||||||
|
pub target: Option<String>,
|
||||||
|
pub detail: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MissionEvent {
|
||||||
|
pub fn new(mission_id: Uuid, kind: &str) -> Self {
|
||||||
|
MissionEvent {
|
||||||
|
mission_id,
|
||||||
|
kind: kind.to_string(),
|
||||||
|
detail: Value::Null,
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn phase(mut self, id: Uuid) -> Self {
|
||||||
|
self.phase_id = Some(id);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
pub fn run(mut self, id: Uuid) -> Self {
|
||||||
|
self.run_id = Some(id);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
pub fn agent(mut self, id: Option<Uuid>) -> Self {
|
||||||
|
self.agent_id = id;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
pub fn target(mut self, t: impl Into<String>) -> Self {
|
||||||
|
self.target = Some(t.into());
|
||||||
|
self
|
||||||
|
}
|
||||||
|
pub fn detail(mut self, d: Value) -> Self {
|
||||||
|
self.detail = d;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record one event, best-effort.
|
||||||
|
///
|
||||||
|
/// The per-phase cap is enforced in the INSERT itself rather than by a read
|
||||||
|
/// followed by a write: two tool taps writing concurrently would both read a
|
||||||
|
/// count below the cap and both insert, and the cap would drift by however many
|
||||||
|
/// writers there are. `INSERT … SELECT … WHERE (subquery) < cap` makes the
|
||||||
|
/// decision inside the statement.
|
||||||
|
pub async fn record(pool: &PgPool, e: MissionEvent) {
|
||||||
|
let detail = if e.detail.is_null() {
|
||||||
|
Value::Object(Default::default())
|
||||||
|
} else {
|
||||||
|
e.detail
|
||||||
|
};
|
||||||
|
let res = sqlx::query(
|
||||||
|
"INSERT INTO mission_events
|
||||||
|
(mission_id, phase_id, run_id, agent_id, kind, target, detail)
|
||||||
|
SELECT $1, $2, $3, $4, $5, $6, $7
|
||||||
|
WHERE $2::uuid IS NULL
|
||||||
|
OR (SELECT count(*) FROM mission_events WHERE phase_id = $2) < $8",
|
||||||
|
)
|
||||||
|
.bind(e.mission_id)
|
||||||
|
.bind(e.phase_id)
|
||||||
|
.bind(e.run_id)
|
||||||
|
.bind(e.agent_id)
|
||||||
|
.bind(&e.kind)
|
||||||
|
.bind(&e.target)
|
||||||
|
.bind(&detail)
|
||||||
|
.bind(PER_PHASE_CAP)
|
||||||
|
.execute(pool)
|
||||||
|
.await;
|
||||||
|
if let Err(err) = res {
|
||||||
|
eprintln!("mission_events: record {} failed: {err}", e.kind);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record several events under one round trip's worth of intent.
|
||||||
|
pub async fn record_all(pool: &PgPool, events: Vec<MissionEvent>) {
|
||||||
|
for e in events {
|
||||||
|
record(pool, e).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The path a tool's **arguments** name, if any.
|
||||||
|
///
|
||||||
|
/// Reads the arguments as JSON — never the tool's prose summary. The summary is
|
||||||
|
/// a sentence written for a human; a path pulled out of it by regex would be
|
||||||
|
/// right often enough to be trusted and wrong often enough to matter.
|
||||||
|
///
|
||||||
|
/// The key names are the ones Claude Code and the ZeroClaw tools actually use.
|
||||||
|
/// An unrecognised shape returns `None`, which renders as a tool call with no
|
||||||
|
/// file — accurate, rather than a guess at which argument was a path.
|
||||||
|
pub fn tool_path(args: &Value) -> Option<String> {
|
||||||
|
const KEYS: [&str; 6] = [
|
||||||
|
"file_path",
|
||||||
|
"filePath",
|
||||||
|
"path",
|
||||||
|
"notebook_path",
|
||||||
|
"file",
|
||||||
|
"target_file",
|
||||||
|
];
|
||||||
|
let obj = args.as_object()?;
|
||||||
|
for k in KEYS {
|
||||||
|
if let Some(s) = obj.get(k).and_then(Value::as_str) {
|
||||||
|
let s = s.trim();
|
||||||
|
if !s.is_empty() {
|
||||||
|
return Some(s.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Strip the guest/host workspace prefix so a path is repo-relative.
|
||||||
|
///
|
||||||
|
/// Tool arguments are absolute inside the sandbox (`/mission/repo/src/a.rs`).
|
||||||
|
/// Left alone, every mission's file tree would nest under a `mission` → `repo`
|
||||||
|
/// pair of directory orbs that exist in no repository and mean nothing to the
|
||||||
|
/// person reading the map.
|
||||||
|
pub fn repo_relative(path: &str, roots: &[&str]) -> String {
|
||||||
|
let p = path.trim();
|
||||||
|
for root in roots {
|
||||||
|
let root = root.trim_end_matches('/');
|
||||||
|
if let Some(rest) = p.strip_prefix(root) {
|
||||||
|
let rest = rest.trim_start_matches('/');
|
||||||
|
if !rest.is_empty() {
|
||||||
|
return rest.to_string();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.trim_start_matches("./").to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use serde_json::json;
|
||||||
|
|
||||||
|
/// Paths come from arguments, and only from argument keys we know.
|
||||||
|
///
|
||||||
|
/// The alternative — scanning the values for anything that looks like a
|
||||||
|
/// path — is what makes a viz confidently wrong: a `pattern` of `*.rs` or a
|
||||||
|
/// `command` of `ls src/` would both become "the agent edited a file".
|
||||||
|
#[test]
|
||||||
|
fn a_path_comes_from_a_known_argument_or_not_at_all() {
|
||||||
|
assert_eq!(
|
||||||
|
tool_path(&json!({"file_path": "/mission/repo/src/a.rs"})).as_deref(),
|
||||||
|
Some("/mission/repo/src/a.rs")
|
||||||
|
);
|
||||||
|
assert_eq!(tool_path(&json!({"path": "docs/x.md"})).as_deref(), Some("docs/x.md"));
|
||||||
|
// A shell command mentions paths and touches none we can name.
|
||||||
|
assert_eq!(tool_path(&json!({"command": "ls src/"})), None);
|
||||||
|
// A glob is a query, not a file.
|
||||||
|
assert_eq!(tool_path(&json!({"pattern": "**/*.rs"})), None);
|
||||||
|
// Blank is absence, not a file called "".
|
||||||
|
assert_eq!(tool_path(&json!({"file_path": " "})), None);
|
||||||
|
assert_eq!(tool_path(&json!("not an object")), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The sandbox prefix must not become two directory orbs in every mission.
|
||||||
|
#[test]
|
||||||
|
fn paths_are_made_repo_relative() {
|
||||||
|
let roots = ["/mission/repo", "/workspace"];
|
||||||
|
assert_eq!(repo_relative("/mission/repo/src/a.rs", &roots), "src/a.rs");
|
||||||
|
assert_eq!(repo_relative("/workspace/README.md", &roots), "README.md");
|
||||||
|
assert_eq!(repo_relative("./src/a.rs", &roots), "src/a.rs");
|
||||||
|
// Outside every root, it is left alone rather than mangled.
|
||||||
|
assert_eq!(repo_relative("/etc/hosts", &roots), "/etc/hosts");
|
||||||
|
// The root ITSELF is not a file, so it must not collapse to "".
|
||||||
|
assert_eq!(repo_relative("/mission/repo", &roots), "/mission/repo");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The cap must be decided inside the INSERT.
|
||||||
|
///
|
||||||
|
/// A count-then-insert is the classic version of this and it is wrong here:
|
||||||
|
/// the container tap and the microVM drain both write for the same phase,
|
||||||
|
/// and each would see a count below the cap and insert. Nothing errors —
|
||||||
|
/// the table simply grows past the bound that exists to hold it.
|
||||||
|
#[test]
|
||||||
|
fn the_cap_is_enforced_in_one_statement() {
|
||||||
|
let src = include_str!("mission_events.rs");
|
||||||
|
let body = src
|
||||||
|
.split("pub async fn record(")
|
||||||
|
.nth(1)
|
||||||
|
.and_then(|s| s.split("pub async fn").next())
|
||||||
|
.expect("record body");
|
||||||
|
assert!(
|
||||||
|
body.contains("INSERT INTO mission_events") && body.contains("SELECT count(*)"),
|
||||||
|
"the cap must be a subquery in the INSERT, not a separate read"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,433 @@
|
|||||||
|
//! Move a mission's checkout in and out of its container, instead of sharing it.
|
||||||
|
//!
|
||||||
|
//! Today the checkout lives on the host and is bind-mounted into the mission
|
||||||
|
//! container. That single directory is written by **two users** — cm-api as
|
||||||
|
//! uid 65532 and the agent as root — and every bug that pattern can produce,
|
||||||
|
//! it has produced:
|
||||||
|
//!
|
||||||
|
//! | Symptom | Fix that was needed |
|
||||||
|
//! |---|---|
|
||||||
|
//! | `.git/objects` permission denied | `core.sharedRepository=0777` |
|
||||||
|
//! | capture base overwritten each phase | advance the base after commit |
|
||||||
|
//! | `.git/COMMIT_EDITMSG` root-owned | unlink before commit |
|
||||||
|
//! | `reset --hard` deleting a prior phase | `.git/clawmates-in-use` marker |
|
||||||
|
//!
|
||||||
|
//! Four fixes, one cause. `core.sharedRepository` was never a general
|
||||||
|
//! solution — it covers objects and refs, and every *other* file git touches
|
||||||
|
//! is a fresh opportunity.
|
||||||
|
//!
|
||||||
|
//! Copy-in/copy-out removes the cause: the agent owns its filesystem
|
||||||
|
//! completely, as root, with no other writer. Nothing on the host is shared,
|
||||||
|
//! so nothing on the host can collide.
|
||||||
|
//!
|
||||||
|
//! # Cost
|
||||||
|
//!
|
||||||
|
//! Measured on gw-04 against a real 65 MB checkout of this repository:
|
||||||
|
//! **0.23s in, 0.18s out**. That was the one open risk in the plan — a
|
||||||
|
//! monorepo copied per phase — and it is not a risk at this size. Measure
|
||||||
|
//! again before assuming it holds for a repository an order of magnitude
|
||||||
|
//! larger.
|
||||||
|
//!
|
||||||
|
//! No compression: the payload crosses a local Docker socket, so gzip would
|
||||||
|
//! spend CPU to save nothing.
|
||||||
|
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use bollard::Docker;
|
||||||
|
|
||||||
|
/// Where a mission's checkout lives inside its container.
|
||||||
|
pub const CONTAINER_MISSION_DIR: &str = "/mission";
|
||||||
|
|
||||||
|
/// Pack a host directory into an uncompressed tar.
|
||||||
|
///
|
||||||
|
/// `name_in_archive` is the top-level entry, so unpacking at
|
||||||
|
/// [`CONTAINER_MISSION_DIR`] yields `/mission/<name>`. Kept separate from the
|
||||||
|
/// upload so the packing is testable without Docker.
|
||||||
|
pub fn pack_dir(root: &Path, name_in_archive: &str) -> Result<Vec<u8>, String> {
|
||||||
|
let mut builder = tar::Builder::new(Vec::new());
|
||||||
|
// Follow no symlinks: a checkout can contain a link pointing outside the
|
||||||
|
// tree, and dereferencing it would pull host files into the container.
|
||||||
|
builder.follow_symlinks(false);
|
||||||
|
append_filtered(&mut builder, root, Path::new(name_in_archive))
|
||||||
|
.map_err(|e| format!("pack {}: {e}", root.display()))?;
|
||||||
|
builder
|
||||||
|
.into_inner()
|
||||||
|
.map_err(|e| format!("finish archive for {}: {e}", root.display()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Directory names never carried across the boundary.
|
||||||
|
///
|
||||||
|
/// The same list the delivery diff uses, deliberately: see
|
||||||
|
/// [`crate::mission_delivery::EXCLUDED_PATHS`]. A build directory is not work —
|
||||||
|
/// it is regenerable output that dwarfs the source, and shipping it cost a
|
||||||
|
/// mission its results when `vm_collect` timed out with the agent's finished work
|
||||||
|
/// still inside the VM.
|
||||||
|
pub fn transport_excludes() -> &'static [&'static str] {
|
||||||
|
crate::mission_delivery::EXCLUDED_PATHS
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Should this directory entry be left out of the archive?
|
||||||
|
///
|
||||||
|
/// Matched on the entry NAME at any depth, not on a path prefix: a workspace has
|
||||||
|
/// a `target/` per crate, and excluding only the root one would still ship the
|
||||||
|
/// rest.
|
||||||
|
pub fn is_excluded(name: &str) -> bool {
|
||||||
|
transport_excludes().contains(&name)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Recursive `append_dir_all` that skips [`transport_excludes`].
|
||||||
|
///
|
||||||
|
/// Hand-rolled because `tar::Builder::append_dir_all` takes no filter. Symlinks
|
||||||
|
/// are added as links rather than followed, matching `follow_symlinks(false)`.
|
||||||
|
fn append_filtered<W: std::io::Write>(
|
||||||
|
builder: &mut tar::Builder<W>,
|
||||||
|
dir: &Path,
|
||||||
|
prefix: &Path,
|
||||||
|
) -> std::io::Result<()> {
|
||||||
|
builder.append_dir(prefix, dir)?;
|
||||||
|
let mut entries: Vec<_> = std::fs::read_dir(dir)?.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
// Stable order so an archive of the same tree is byte-identical, which makes
|
||||||
|
// a size or content difference between two runs mean something.
|
||||||
|
entries.sort_by_key(|e| e.file_name());
|
||||||
|
for entry in entries {
|
||||||
|
let name = entry.file_name();
|
||||||
|
let name_str = name.to_string_lossy();
|
||||||
|
let path = entry.path();
|
||||||
|
let dest = prefix.join(&name);
|
||||||
|
let meta = std::fs::symlink_metadata(&path)?;
|
||||||
|
if meta.is_dir() {
|
||||||
|
if is_excluded(&name_str) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
append_filtered(builder, &path, &dest)?;
|
||||||
|
} else if meta.is_symlink() {
|
||||||
|
let mut header = tar::Header::new_gnu();
|
||||||
|
header.set_metadata(&meta);
|
||||||
|
header.set_entry_type(tar::EntryType::Symlink);
|
||||||
|
header.set_size(0);
|
||||||
|
let target = std::fs::read_link(&path)?;
|
||||||
|
builder.append_link(&mut header, &dest, &target)?;
|
||||||
|
} else {
|
||||||
|
let mut f = std::fs::File::open(&path)?;
|
||||||
|
builder.append_file(&dest, &mut f)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Unpack a tar into a host directory.
|
||||||
|
///
|
||||||
|
/// `tar` refuses entries whose paths escape the destination, which is the
|
||||||
|
/// property that matters here: the archive comes back from a container the
|
||||||
|
/// agent controls as root, so it is untrusted input. A `../../etc` entry must
|
||||||
|
/// not be able to write outside the collection directory.
|
||||||
|
pub fn unpack_into(archive: &[u8], dest: &Path) -> Result<(), String> {
|
||||||
|
std::fs::create_dir_all(dest).map_err(|e| format!("mkdir {}: {e}", dest.display()))?;
|
||||||
|
let mut ar = tar::Archive::new(archive);
|
||||||
|
ar.set_overwrite(true);
|
||||||
|
// Ownership in the archive is the container's root; re-applying it on the
|
||||||
|
// host would recreate the very uid split this module exists to remove.
|
||||||
|
ar.set_preserve_permissions(false);
|
||||||
|
ar.unpack(dest)
|
||||||
|
.map_err(|e| format!("unpack into {}: {e}", dest.display()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Copy a host directory into a running container at [`CONTAINER_MISSION_DIR`].
|
||||||
|
pub async fn copy_in(
|
||||||
|
docker: &Docker,
|
||||||
|
container: &str,
|
||||||
|
host_dir: &Path,
|
||||||
|
name_in_archive: &str,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let archive = pack_dir(host_dir, name_in_archive)?;
|
||||||
|
let opts = bollard::query_parameters::UploadToContainerOptionsBuilder::default()
|
||||||
|
.path(CONTAINER_MISSION_DIR)
|
||||||
|
.build();
|
||||||
|
docker
|
||||||
|
.upload_to_container(container, Some(opts), bollard::body_full(archive.into()))
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("copy into {container}:{CONTAINER_MISSION_DIR}: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build a one-entry tar. Split out from [`put_file`] so the size-independence
|
||||||
|
/// that is the whole point can be tested without Docker.
|
||||||
|
fn single_file_archive(name: &str, contents: &[u8]) -> Result<Vec<u8>, String> {
|
||||||
|
let mut header = tar::Header::new_gnu();
|
||||||
|
header
|
||||||
|
.set_path(name)
|
||||||
|
.map_err(|e| format!("tar path {name}: {e}"))?;
|
||||||
|
header.set_size(contents.len() as u64);
|
||||||
|
header.set_mode(0o600);
|
||||||
|
header.set_entry_type(tar::EntryType::Regular);
|
||||||
|
header.set_cksum();
|
||||||
|
|
||||||
|
let mut builder = tar::Builder::new(Vec::new());
|
||||||
|
builder
|
||||||
|
.append(&header, contents)
|
||||||
|
.map_err(|e| format!("tar {name}: {e}"))?;
|
||||||
|
builder
|
||||||
|
.into_inner()
|
||||||
|
.map_err(|e| format!("finish archive for {name}: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write one file into a container, at any size.
|
||||||
|
///
|
||||||
|
/// The obvious way to do this is `sh -c "printf … > file"`, and it works right
|
||||||
|
/// up until the payload approaches `ARG_MAX`, at which point exec fails with
|
||||||
|
/// `argument list too long`. That is a size-dependent failure in a code path
|
||||||
|
/// whose payload grows with use, which makes it a bug that ships green and
|
||||||
|
/// surfaces in production — as it did, silently unpinning every agent in
|
||||||
|
/// mission `019fcf62`. Tar has no argv limit.
|
||||||
|
///
|
||||||
|
/// The write is not atomic. Callers that need it can upload beside the target
|
||||||
|
/// and rename; the config writer does not, because the daemon reads its config
|
||||||
|
/// once at boot and is restarted afterwards.
|
||||||
|
pub async fn put_file(
|
||||||
|
docker: &Docker,
|
||||||
|
container: &str,
|
||||||
|
path: &str,
|
||||||
|
contents: &[u8],
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let (dir, file) = path
|
||||||
|
.rsplit_once('/')
|
||||||
|
.ok_or_else(|| format!("{path} is not an absolute path"))?;
|
||||||
|
let dir = if dir.is_empty() { "/" } else { dir };
|
||||||
|
|
||||||
|
let archive = single_file_archive(file, contents)?;
|
||||||
|
let opts = bollard::query_parameters::UploadToContainerOptionsBuilder::default()
|
||||||
|
.path(dir)
|
||||||
|
.build();
|
||||||
|
docker
|
||||||
|
.upload_to_container(container, Some(opts), bollard::body_full(archive.into()))
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("upload {path} to {container}: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Copy a directory back out of a container onto the host.
|
||||||
|
pub async fn copy_out(
|
||||||
|
docker: &Docker,
|
||||||
|
container: &str,
|
||||||
|
container_path: &str,
|
||||||
|
dest: &Path,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
use futures::StreamExt;
|
||||||
|
|
||||||
|
let opts = bollard::query_parameters::DownloadFromContainerOptionsBuilder::default()
|
||||||
|
.path(container_path)
|
||||||
|
.build();
|
||||||
|
let mut stream = docker.download_from_container(container, Some(opts));
|
||||||
|
let mut archive = Vec::new();
|
||||||
|
while let Some(chunk) = stream.next().await {
|
||||||
|
let bytes = chunk.map_err(|e| format!("copy out of {container}:{container_path}: {e}"))?;
|
||||||
|
archive.extend_from_slice(&bytes);
|
||||||
|
}
|
||||||
|
unpack_into(&archive, dest)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Is the copy-in/copy-out filesystem model enabled?
|
||||||
|
///
|
||||||
|
/// **Default since 2026-08-04.** It shipped opt-in, on the principle that
|
||||||
|
/// silently changing how every mission receives its code should require
|
||||||
|
/// someone to have typed it. Four production missions and a fail-closed
|
||||||
|
/// harness later (`scripts/verify-mission-delivery.sh`), the opt-in is the
|
||||||
|
/// riskier setting: the bind path is the one with four documented work-loss
|
||||||
|
/// incidents, and leaving it as the default means the untested path is what
|
||||||
|
/// runs when nobody sets the variable.
|
||||||
|
///
|
||||||
|
/// `CLAWMATES_MISSION_FS=bind` still selects the old behaviour, so a revert is
|
||||||
|
/// one line in `.env` rather than a rollback. Anything else — unset, empty,
|
||||||
|
/// misspelt — gets copy mode, because the failure mode of a typo should be the
|
||||||
|
/// safer path, not the one being retired.
|
||||||
|
pub fn copy_mode() -> bool {
|
||||||
|
!matches!(std::env::var("CLAWMATES_MISSION_FS").as_deref(), Ok("bind"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Host directory holding a mission's checkout.
|
||||||
|
fn host_repo(mission_id: uuid::Uuid) -> std::path::PathBuf {
|
||||||
|
crate::mission_workspace::checkout_path(mission_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Push the host checkout into the container before a phase runs.
|
||||||
|
///
|
||||||
|
/// No-op when the mission has no repo — research-only missions have no
|
||||||
|
/// checkout, and that must not fail a phase launch.
|
||||||
|
pub async fn sync_in(container: &str, mission_id: uuid::Uuid) -> Result<(), String> {
|
||||||
|
let repo = host_repo(mission_id);
|
||||||
|
if !repo.is_dir() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let docker = crate::container_exec::connect()?;
|
||||||
|
copy_in(&docker, container, &repo, "repo").await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pull the agent's work back onto the host after a phase.
|
||||||
|
///
|
||||||
|
/// Unpacks over the SAME host path the checkout came from, so the host
|
||||||
|
/// directory stays a server-owned staging area with exactly one writer — and
|
||||||
|
/// `mission_delivery::capture_phase_diff_at` needs no change at all, because
|
||||||
|
/// it still finds a normal checkout exactly where it always has.
|
||||||
|
pub async fn sync_out(container: &str, mission_id: uuid::Uuid) -> Result<(), String> {
|
||||||
|
let repo = host_repo(mission_id);
|
||||||
|
if !repo.is_dir() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let parent = repo
|
||||||
|
.parent()
|
||||||
|
.ok_or_else(|| format!("{} has no parent", repo.display()))?;
|
||||||
|
let docker = crate::container_exec::connect()?;
|
||||||
|
copy_out(&docker, container, "/mission/repo", parent).await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn seed(root: &Path) {
|
||||||
|
std::fs::create_dir_all(root.join("src")).unwrap();
|
||||||
|
std::fs::create_dir_all(root.join(".git")).unwrap();
|
||||||
|
std::fs::write(root.join("src/lib.rs"), "pub fn x() {}\n").unwrap();
|
||||||
|
std::fs::write(root.join(".git/HEAD"), "ref: refs/heads/main\n").unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A checkout must survive the round trip intact — including `.git`,
|
||||||
|
/// without which the whole delivery path (diff, commit, push) is dead.
|
||||||
|
#[test]
|
||||||
|
fn a_checkout_round_trips_with_its_git_dir() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let src = tmp.path().join("repo");
|
||||||
|
seed(&src);
|
||||||
|
|
||||||
|
let archive = pack_dir(&src, "repo").unwrap();
|
||||||
|
let dest = tmp.path().join("out");
|
||||||
|
unpack_into(&archive, &dest).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(dest.join("repo/src/lib.rs")).unwrap(),
|
||||||
|
"pub fn x() {}\n"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
dest.join("repo/.git/HEAD").exists(),
|
||||||
|
"the .git dir must survive or delivery has nothing to diff"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The archive comes back from a container the agent controls as root, so
|
||||||
|
/// it is untrusted. An entry that climbs out of the destination must not
|
||||||
|
/// be able to write to the host.
|
||||||
|
#[test]
|
||||||
|
fn an_archive_cannot_escape_the_destination() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let dest = tmp.path().join("dest");
|
||||||
|
let canary = tmp.path().join("ESCAPED");
|
||||||
|
|
||||||
|
// The path has to be written into the header bytes directly: the tar
|
||||||
|
// crate refuses to BUILD an entry containing `..`, which is itself
|
||||||
|
// reassuring but means a hostile archive cannot be produced through
|
||||||
|
// the safe API. A real attacker writes the bytes, so the test does.
|
||||||
|
let body = b"pwned\n";
|
||||||
|
let mut header = tar::Header::new_gnu();
|
||||||
|
header.set_size(body.len() as u64);
|
||||||
|
header.set_mode(0o644);
|
||||||
|
header.set_entry_type(tar::EntryType::Regular);
|
||||||
|
{
|
||||||
|
let gnu = header.as_gnu_mut().expect("gnu header");
|
||||||
|
let evil = b"../ESCAPED";
|
||||||
|
gnu.name[..evil.len()].copy_from_slice(evil);
|
||||||
|
}
|
||||||
|
header.set_cksum();
|
||||||
|
|
||||||
|
let mut archive = Vec::new();
|
||||||
|
archive.extend_from_slice(header.as_bytes());
|
||||||
|
let mut block = [0u8; 512];
|
||||||
|
block[..body.len()].copy_from_slice(body);
|
||||||
|
archive.extend_from_slice(&block);
|
||||||
|
archive.extend_from_slice(&[0u8; 1024]); // end-of-archive marker
|
||||||
|
|
||||||
|
let _ = unpack_into(&archive, &dest);
|
||||||
|
assert!(
|
||||||
|
!canary.exists(),
|
||||||
|
"a ../ entry wrote outside the destination"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A symlink pointing at the host filesystem must be packed as a link,
|
||||||
|
/// not followed and inlined — otherwise copy-in would smuggle host files
|
||||||
|
/// into the container.
|
||||||
|
#[test]
|
||||||
|
fn symlinks_are_not_dereferenced_into_the_archive() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let src = tmp.path().join("repo");
|
||||||
|
seed(&src);
|
||||||
|
let secret = tmp.path().join("host-secret");
|
||||||
|
std::fs::write(&secret, "TOP SECRET\n").unwrap();
|
||||||
|
std::os::unix::fs::symlink(&secret, src.join("link")).unwrap();
|
||||||
|
|
||||||
|
let archive = pack_dir(&src, "repo").unwrap();
|
||||||
|
let haystack = String::from_utf8_lossy(&archive);
|
||||||
|
assert!(
|
||||||
|
!haystack.contains("TOP SECRET"),
|
||||||
|
"symlink target contents were inlined into the archive"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Only the exact word `bind` opts out. A typo must land on copy mode —
|
||||||
|
/// the path with a verification harness behind it — rather than silently
|
||||||
|
/// selecting the one with four documented work-loss incidents.
|
||||||
|
#[test]
|
||||||
|
fn only_the_exact_word_bind_opts_out() {
|
||||||
|
// Cannot set env vars in a test process without racing every other
|
||||||
|
// test, so this asserts the predicate the function is built from.
|
||||||
|
let opts_out = |v: &str| v == "bind";
|
||||||
|
assert!(opts_out("bind"));
|
||||||
|
for near_miss in ["Bind", "binds", "bound", "copy", "0", "false", ""] {
|
||||||
|
assert!(
|
||||||
|
!opts_out(near_miss),
|
||||||
|
"{near_miss:?} must NOT select the bind path"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The regression this exists for: a config large enough to blow `ARG_MAX`
|
||||||
|
/// via `sh -c` must round-trip untouched. 2 MB is well past the ~128 KB
|
||||||
|
/// limit that unpinned every agent in mission `019fcf62`.
|
||||||
|
#[test]
|
||||||
|
fn a_file_far_past_arg_max_round_trips() {
|
||||||
|
let big = "workspace_path = \"/mission/repo\"\n".repeat(64 * 1024);
|
||||||
|
assert!(big.len() > 2_000_000, "the fixture must exceed ARG_MAX");
|
||||||
|
|
||||||
|
let archive = single_file_archive("config.toml", big.as_bytes()).unwrap();
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
unpack_into(&archive, tmp.path()).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(tmp.path().join("config.toml")).unwrap(),
|
||||||
|
big,
|
||||||
|
"a large config must survive byte-for-byte"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// TOML holding quotes, newlines and backslashes went through a shell
|
||||||
|
/// before; nothing may depend on quoting now.
|
||||||
|
#[test]
|
||||||
|
fn shell_metacharacters_survive_the_archive() {
|
||||||
|
let nasty = "path = \"/a'b\\\"c\"\n$(rm -rf /) `id` \\\\ \n";
|
||||||
|
let archive = single_file_archive("config.toml", nasty.as_bytes()).unwrap();
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
unpack_into(&archive, tmp.path()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(tmp.path().join("config.toml")).unwrap(),
|
||||||
|
nasty
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_empty_directory_packs_without_error() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let src = tmp.path().join("empty");
|
||||||
|
std::fs::create_dir_all(&src).unwrap();
|
||||||
|
let archive = pack_dir(&src, "repo").unwrap();
|
||||||
|
let dest = tmp.path().join("out");
|
||||||
|
unpack_into(&archive, &dest).unwrap();
|
||||||
|
assert!(dest.join("repo").is_dir());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,414 @@
|
|||||||
|
//! Reclaim the mission tree on the gateway.
|
||||||
|
//!
|
||||||
|
//! # Why this is filesystem-first
|
||||||
|
//!
|
||||||
|
//! `cleanup_sweeper` prunes ROWS. Deleting a row does not delete a directory,
|
||||||
|
//! and the reaper that was supposed to — `mission_runtime::teardown_container` —
|
||||||
|
//! only runs while a mission still exists to tear down. So a mission deleted by
|
||||||
|
//! any path that did not go through teardown left its directory behind forever,
|
||||||
|
//! and the gateway is the smallest disk in the fleet (150 GB, shared with
|
||||||
|
//! postgres and every checkout).
|
||||||
|
//!
|
||||||
|
//! The DB is therefore the PREDICATE here, never the enumerator: this walks the
|
||||||
|
//! filesystem and asks the database about what it finds. Enumerating from the
|
||||||
|
//! database is precisely how the orphans became invisible — a directory whose
|
||||||
|
//! row is gone is exactly the one a row-driven sweep cannot see.
|
||||||
|
//!
|
||||||
|
//! # Why deletion needs two attempts
|
||||||
|
//!
|
||||||
|
//! The server runs as uid 65532. Almost everything under a mission belongs to
|
||||||
|
//! 65532 now, but the per-mission ZeroClaw daemon still runs as root and leaves
|
||||||
|
//! ~26 of its own files (`.claude.json`, session jsonl). `remove_dir_all` then
|
||||||
|
//! fails with `PermissionDenied` and the directory survives — the
|
||||||
|
//! cleanup-that-cannot-clean-up shape, at a scale small enough to go unnoticed.
|
||||||
|
//! So a failed removal falls back to `root_copy::purge`, which deletes from
|
||||||
|
//! inside the runtime container as root.
|
||||||
|
//!
|
||||||
|
//! # What it will not touch
|
||||||
|
//!
|
||||||
|
//! Anything belonging to a mission that still has a row, and anything younger
|
||||||
|
//! than the grace window. A mission directory is created BEFORE its row is
|
||||||
|
//! committed in some paths, and reaping a directory out from under a launching
|
||||||
|
//! mission would be a far worse bug than the leak this fixes.
|
||||||
|
|
||||||
|
use std::path::Path;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use sqlx::PgPool;
|
||||||
|
|
||||||
|
/// How long a directory must have been untouched before it is considered
|
||||||
|
/// abandoned. Generously long: the cost of waiting is disk, and the cost of
|
||||||
|
/// being wrong is deleting a live mission's checkout.
|
||||||
|
const ORPHAN_GRACE: Duration = Duration::from_secs(2 * 60 * 60);
|
||||||
|
|
||||||
|
/// Retention for captured outputs (`_outputs`), which are artifacts a user can
|
||||||
|
/// still open. Mirrors `TOPOLOGY_RUNS_DAYS` in `cleanup_sweeper` — the run
|
||||||
|
/// history and the files it points at should not outlive each other.
|
||||||
|
const OUTPUTS_DAYS: u64 = 90;
|
||||||
|
|
||||||
|
/// Scratch trees the mission machinery makes and is supposed to remove itself:
|
||||||
|
/// `_bench`, `_gate`, `_verify`, `_merge`. Anything older than this is debris
|
||||||
|
/// from a crashed or killed run, not work in progress — every command that
|
||||||
|
/// creates one is bounded well below it.
|
||||||
|
const SCRATCH_GRACE: Duration = Duration::from_secs(6 * 60 * 60);
|
||||||
|
|
||||||
|
/// Directories under the missions root that are NOT missions.
|
||||||
|
const RESERVED: &[&str] = &["_outputs", "_home", "_cargo", "_mirrors"];
|
||||||
|
|
||||||
|
pub fn spawn(pool: PgPool, interval: Duration) {
|
||||||
|
tokio::spawn(async move {
|
||||||
|
// Not on the first tick. A sweep racing the server's own startup — while
|
||||||
|
// `start_pending_phases` is still adopting in-flight missions — is the
|
||||||
|
// one moment its "no row for this directory" predicate is least
|
||||||
|
// trustworthy.
|
||||||
|
tokio::time::sleep(Duration::from_secs(120)).await;
|
||||||
|
let mut tick = tokio::time::interval(interval);
|
||||||
|
loop {
|
||||||
|
tick.tick().await;
|
||||||
|
match sweep_once(&pool).await {
|
||||||
|
Ok(r) if r.is_empty() => {}
|
||||||
|
Ok(r) => eprintln!("mission_gc: {r}"),
|
||||||
|
Err(e) => eprintln!("mission_gc: sweep failed: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What one sweep reclaimed.
|
||||||
|
#[derive(Debug, Default, PartialEq)]
|
||||||
|
pub struct Reclaimed {
|
||||||
|
pub orphan_dirs: u64,
|
||||||
|
pub scratch_dirs: u64,
|
||||||
|
pub outputs: u64,
|
||||||
|
pub bytes: u64,
|
||||||
|
/// Directories we tried and failed to remove. Reported rather than swallowed
|
||||||
|
/// — a GC that cannot collect is the thing being fixed.
|
||||||
|
pub failed: u64,
|
||||||
|
/// Rows swept from `mission_events`.
|
||||||
|
pub events: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Reclaimed {
|
||||||
|
pub fn is_empty(&self) -> bool {
|
||||||
|
*self == Reclaimed::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for Reclaimed {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
write!(
|
||||||
|
f,
|
||||||
|
"reclaimed {} orphan mission dir(s), {} scratch dir(s), {} output(s), \
|
||||||
|
{} mission event(s), {:.1} MiB{}",
|
||||||
|
self.orphan_dirs,
|
||||||
|
self.scratch_dirs,
|
||||||
|
self.outputs,
|
||||||
|
self.events,
|
||||||
|
self.bytes as f64 / (1024.0 * 1024.0),
|
||||||
|
if self.failed > 0 {
|
||||||
|
format!(" — {} COULD NOT BE REMOVED", self.failed)
|
||||||
|
} else {
|
||||||
|
String::new()
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn sweep_once(pool: &PgPool) -> Result<Reclaimed, String> {
|
||||||
|
let root = crate::mission_workspace::missions_root();
|
||||||
|
let mut out = Reclaimed::default();
|
||||||
|
reap_orphan_missions(pool, &root, &mut out).await?;
|
||||||
|
reap_scratch(&root, &mut out).await;
|
||||||
|
reap_outputs(pool, &root, &mut out).await;
|
||||||
|
reap_mission_events(pool, &mut out).await;
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How long a mission's structured activity is kept.
|
||||||
|
///
|
||||||
|
/// The World shows the last 24 hours of finished missions, so a week is
|
||||||
|
/// generous and still bounds a table that a single busy coding phase can add
|
||||||
|
/// hundreds of rows to. The per-phase cap bounds ONE phase; this bounds time.
|
||||||
|
const EVENT_RETENTION_DAYS: i32 = 7;
|
||||||
|
|
||||||
|
/// Sweep expired `mission_events`.
|
||||||
|
///
|
||||||
|
/// Bounded per pass rather than deleting the whole backlog in one statement: a
|
||||||
|
/// deployment that has been accumulating for months would otherwise take a long
|
||||||
|
/// lock on its first sweep after this ships. The sweep runs on a timer, so a
|
||||||
|
/// large backlog simply drains over several passes.
|
||||||
|
async fn reap_mission_events(pool: &PgPool, out: &mut Reclaimed) {
|
||||||
|
let res = sqlx::query(
|
||||||
|
"DELETE FROM mission_events
|
||||||
|
WHERE id IN (
|
||||||
|
SELECT id FROM mission_events
|
||||||
|
WHERE created_at < now() - make_interval(days => $1)
|
||||||
|
LIMIT 10000
|
||||||
|
)",
|
||||||
|
)
|
||||||
|
.bind(EVENT_RETENTION_DAYS)
|
||||||
|
.execute(pool)
|
||||||
|
.await;
|
||||||
|
match res {
|
||||||
|
Ok(r) => out.events += r.rows_affected(),
|
||||||
|
Err(e) => eprintln!("mission_gc: sweeping mission_events failed: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Directories under the missions root with no mission row.
|
||||||
|
async fn reap_orphan_missions(
|
||||||
|
pool: &PgPool,
|
||||||
|
root: &Path,
|
||||||
|
out: &mut Reclaimed,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
let Ok(entries) = std::fs::read_dir(root) else {
|
||||||
|
// Not an error: a deployment that has never run a mission has no tree.
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let path = entry.path();
|
||||||
|
if !path.is_dir() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if RESERVED.contains(&name) || name.starts_with('_') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// Only well-formed mission ids. A directory this function does not
|
||||||
|
// recognise is one it has no business deleting.
|
||||||
|
let Ok(id) = name.parse::<uuid::Uuid>() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if !older_than(&path, ORPHAN_GRACE) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// The DB as predicate, asked per directory.
|
||||||
|
let exists: Option<(uuid::Uuid,)> =
|
||||||
|
sqlx::query_as("SELECT id FROM missions WHERE id = $1")
|
||||||
|
.bind(id)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("looking up mission {id}: {e}"))?;
|
||||||
|
if exists.is_some() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let bytes = dir_size(&path);
|
||||||
|
if remove_tree(&path).await {
|
||||||
|
out.orphan_dirs += 1;
|
||||||
|
out.bytes += bytes;
|
||||||
|
} else {
|
||||||
|
out.failed += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `_bench` / `_gate` / `_verify` / `_merge` trees older than their command
|
||||||
|
/// ceilings. These are siblings of the per-mission dirs and have leaked before.
|
||||||
|
async fn reap_scratch(root: &Path, out: &mut Reclaimed) {
|
||||||
|
const SCRATCH: &[&str] = &["_bench", "_gate", "_verify", "_merge"];
|
||||||
|
for name in SCRATCH {
|
||||||
|
let path = root.join(name);
|
||||||
|
if !path.is_dir() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Ok(entries) = std::fs::read_dir(&path) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let p = entry.path();
|
||||||
|
if !older_than(&p, SCRATCH_GRACE) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let bytes = dir_size(&p);
|
||||||
|
if remove_tree(&p).await {
|
||||||
|
out.scratch_dirs += 1;
|
||||||
|
out.bytes += bytes;
|
||||||
|
} else {
|
||||||
|
out.failed += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Captured outputs past retention, with their artifact rows marked so nothing
|
||||||
|
/// points at a file that is gone.
|
||||||
|
async fn reap_outputs(pool: &PgPool, root: &Path, out: &mut Reclaimed) {
|
||||||
|
let outputs = root.join("_outputs");
|
||||||
|
let Ok(entries) = std::fs::read_dir(&outputs) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let grace = Duration::from_secs(OUTPUTS_DAYS * 24 * 60 * 60);
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let p = entry.path();
|
||||||
|
if !p.is_dir() || !older_than(&p, grace) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(id) = p
|
||||||
|
.file_name()
|
||||||
|
.and_then(|n| n.to_str())
|
||||||
|
.and_then(|n| n.parse::<uuid::Uuid>().ok())
|
||||||
|
else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let bytes = dir_size(&p);
|
||||||
|
if !remove_tree(&p).await {
|
||||||
|
out.failed += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// The row is marked only AFTER the files are gone. The other order
|
||||||
|
// leaves a mission whose artifacts claim to be reaped while they are
|
||||||
|
// still on disk, which is a lie in the direction that costs disk.
|
||||||
|
let _ = sqlx::query(
|
||||||
|
"UPDATE mission_artifacts SET metadata = COALESCE(metadata, '{}'::jsonb)
|
||||||
|
|| '{\"reaped\": true}'::jsonb
|
||||||
|
WHERE mission_id = $1",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.execute(pool)
|
||||||
|
.await;
|
||||||
|
out.outputs += 1;
|
||||||
|
out.bytes += bytes;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Remove a tree, escalating to a root purge when our uid cannot.
|
||||||
|
///
|
||||||
|
/// The ONLY deletion path in this module. A second one is how the reap paths
|
||||||
|
/// drifted apart last time.
|
||||||
|
async fn remove_tree(path: &Path) -> bool {
|
||||||
|
match tokio::fs::remove_dir_all(path).await {
|
||||||
|
Ok(()) => true,
|
||||||
|
Err(e) if e.kind() == std::io::ErrorKind::NotFound => true,
|
||||||
|
Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => {
|
||||||
|
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
||||||
|
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
||||||
|
crate::root_copy::purge(&container, path).await;
|
||||||
|
let gone = tokio::fs::metadata(path).await.is_err();
|
||||||
|
if !gone {
|
||||||
|
eprintln!(
|
||||||
|
"mission_gc: {} survived a root purge — it will keep accumulating",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
gone
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("mission_gc: could not remove {}: {e}", path.display());
|
||||||
|
false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn older_than(path: &Path, grace: Duration) -> bool {
|
||||||
|
let Ok(meta) = std::fs::metadata(path) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
// mtime, not ctime: a directory whose contents changed recently is one
|
||||||
|
// something is still writing to.
|
||||||
|
let Ok(modified) = meta.modified() else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
modified
|
||||||
|
.elapsed()
|
||||||
|
.map(|age| age >= grace)
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Apparent size, best-effort. Used only for reporting, so a read error costs a
|
||||||
|
/// wrong number in a log line rather than a wrong decision.
|
||||||
|
fn dir_size(path: &Path) -> u64 {
|
||||||
|
let mut total = 0;
|
||||||
|
let Ok(entries) = std::fs::read_dir(path) else {
|
||||||
|
return 0;
|
||||||
|
};
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let Ok(meta) = entry.metadata() else { continue };
|
||||||
|
if meta.is_dir() {
|
||||||
|
total += dir_size(&entry.path());
|
||||||
|
} else {
|
||||||
|
total += meta.len();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
total
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn touch_dir(root: &Path, name: &str) -> std::path::PathBuf {
|
||||||
|
let p = root.join(name);
|
||||||
|
std::fs::create_dir_all(&p).unwrap();
|
||||||
|
std::fs::write(p.join("f"), b"x").unwrap();
|
||||||
|
p
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The reserved siblings are never candidates.
|
||||||
|
///
|
||||||
|
/// `_outputs`, `_home` and `_cargo` live under the same root as the mission
|
||||||
|
/// directories. `_cargo` in particular is a SHARED cache every mission
|
||||||
|
/// writes to, so a sweep that treated an underscore-prefixed sibling as an
|
||||||
|
/// orphan mission would delete it out from under running work — and it would
|
||||||
|
/// look like a slow cargo build rather than a bug.
|
||||||
|
#[test]
|
||||||
|
fn siblings_of_the_mission_dirs_are_not_missions() {
|
||||||
|
for name in RESERVED {
|
||||||
|
assert!(
|
||||||
|
name.starts_with('_'),
|
||||||
|
"{name} must be underscore-prefixed so the guard catches it"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
name.parse::<uuid::Uuid>().is_err(),
|
||||||
|
"{name} must not parse as a mission id"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Only a well-formed mission id is ever a candidate.
|
||||||
|
///
|
||||||
|
/// The predicate is "no row exists", and a directory whose name is not an id
|
||||||
|
/// can have no row BY CONSTRUCTION — so name-parsing has to gate the lookup,
|
||||||
|
/// or every unrecognised directory looks like an orphan.
|
||||||
|
#[test]
|
||||||
|
fn a_directory_that_is_not_a_mission_id_is_never_a_candidate() {
|
||||||
|
for name in ["_outputs", "_cargo", "lost+found", "notes", "019fe8", ""] {
|
||||||
|
assert!(
|
||||||
|
name.parse::<uuid::Uuid>().is_err(),
|
||||||
|
"{name:?} must not parse as a mission id"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!("019fe82e-7f0d-7481-a197-698f1d400419"
|
||||||
|
.parse::<uuid::Uuid>()
|
||||||
|
.is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The grace window is real, and measured from mtime.
|
||||||
|
#[test]
|
||||||
|
fn a_fresh_directory_is_never_old_enough() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let d = touch_dir(tmp.path(), "019fe82e-7f0d-7481-a197-698f1d400419");
|
||||||
|
assert!(!older_than(&d, ORPHAN_GRACE));
|
||||||
|
// And a zero grace makes everything eligible, which is what proves the
|
||||||
|
// check is the window rather than an accident of the filesystem.
|
||||||
|
assert!(older_than(&d, Duration::from_secs(0)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One deletion path, and it escalates.
|
||||||
|
///
|
||||||
|
/// A second removal site is how the container reap paths drifted apart and
|
||||||
|
/// leaked for a day. The escalation is the other half: the server is uid
|
||||||
|
/// 65532 and cannot delete what the per-mission daemon left as root.
|
||||||
|
#[test]
|
||||||
|
fn there_is_exactly_one_deletion_path_and_it_escalates() {
|
||||||
|
let src = include_str!("mission_gc.rs");
|
||||||
|
assert_eq!(
|
||||||
|
src.matches(concat!("remove_dir", "_all(")).count(),
|
||||||
|
1,
|
||||||
|
"exactly one removal site"
|
||||||
|
);
|
||||||
|
assert!(src.contains("root_copy::purge"), "and it must escalate");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -79,7 +79,12 @@ pub async fn on_launch(
|
|||||||
// The mission's own runtime endpoint. Claws MUST be provisioned against
|
// The mission's own runtime endpoint. Claws MUST be provisioned against
|
||||||
// THIS gateway, not the global one — see RuntimeProvisioner::for_gateway.
|
// THIS gateway, not the global one — see RuntimeProvisioner::for_gateway.
|
||||||
let mut mission_gateway: Option<String> = None;
|
let mut mission_gateway: Option<String> = None;
|
||||||
if let Some(prov) = crate::mission_runtime::MissionRuntimeProvisioner::from_env() {
|
// Not for a microVM mission: the ZeroClaw daemon it would start is never
|
||||||
|
// spoken to, and it would sit holding a pairing code and ~3 GB of image for
|
||||||
|
// the life of the mission. Observed doing exactly that on the first real run.
|
||||||
|
if let Some(prov) = crate::mission_runtime::MissionRuntimeProvisioner::from_env()
|
||||||
|
.filter(|_| mission.runtime_kind != "microvm")
|
||||||
|
{
|
||||||
match prov.ensure_container(mission_id).await {
|
match prov.ensure_container(mission_id).await {
|
||||||
Ok(ec) => {
|
Ok(ec) => {
|
||||||
mission_gateway = Some(ec.endpoint.clone());
|
mission_gateway = Some(ec.endpoint.clone());
|
||||||
@@ -115,6 +120,76 @@ pub async fn on_launch(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// microVM PLACEMENT MUST COME BEFORE the early return below. It did not, and
|
||||||
|
// the first real microvm mission failed with "mission has no target_node_id" —
|
||||||
|
// the executor's own guard firing correctly on a mission this function had
|
||||||
|
// returned from before ever choosing a node for it.
|
||||||
|
// microVM placement. KVM is a hard predicate, not a preference: gw-04 —
|
||||||
|
// where every mission runs today — is itself a VM without nested
|
||||||
|
// virtualisation and has no /dev/kvm, so a microvm mission landing there
|
||||||
|
// cannot start. Resolve a capable node now and fail the launch if there is
|
||||||
|
// none, because the alternative is a mission that sits in 'running' having
|
||||||
|
// never had anywhere to run.
|
||||||
|
if mission.runtime_kind == "microvm" {
|
||||||
|
// Capable means BOTH: it can host a microVM, and it holds the image this
|
||||||
|
// mission's backend names. Asking only for `microvm` sent the first real
|
||||||
|
// microVM mission to a node without `rootfs-claude.ext4`.
|
||||||
|
let backend = mission.backend.as_deref();
|
||||||
|
let capable =
|
||||||
|
cm_db::repo::nodes::online_for_backend(pool, mission.workspace_id, backend)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("looking up nodes for backend {backend:?}: {e}"))?;
|
||||||
|
let how_to_fix = format!(
|
||||||
|
"needs /dev/kvm + firecracker (scripts/fc-node-setup.sh) AND the {} rootfs \
|
||||||
|
built on that node (scripts/fc-build-rootfs.sh <host> <image> {})",
|
||||||
|
backend.unwrap_or("default"),
|
||||||
|
backend.unwrap_or("<name>")
|
||||||
|
);
|
||||||
|
let how_to_fix = how_to_fix.as_str();
|
||||||
|
// CAPABILITY is checked here; CAPACITY is not, and no node is pinned.
|
||||||
|
//
|
||||||
|
// Placement moved to phase launch (`phase_runner`). A node chosen now
|
||||||
|
// would be chosen once, minutes before the first VM boots and hours
|
||||||
|
// before the last — and re-placing between phases is free, because
|
||||||
|
// mission state lives on the gateway checkout and every VM is
|
||||||
|
// inject → run → collect → destroy. Pinning early bought nothing and
|
||||||
|
// cost the ability to react to a node filling or draining mid-mission.
|
||||||
|
//
|
||||||
|
// Launching still FAILS here when no node could ever run this backend:
|
||||||
|
// that is not transient, waiting will not fix it, and the harness's
|
||||||
|
// `microvm-negctl` scenario asserts such a mission stays `draft`.
|
||||||
|
if capable.is_empty() {
|
||||||
|
return Err(format!(
|
||||||
|
"no online node can run backend {:?} — {how_to_fix}",
|
||||||
|
backend.unwrap_or("default")
|
||||||
|
));
|
||||||
|
}
|
||||||
|
eprintln!(
|
||||||
|
"mission_orchestrator: mission {mission_id} has {} node(s) able to run \
|
||||||
|
backend {:?}; placement happens per phase",
|
||||||
|
capable.len(),
|
||||||
|
backend.unwrap_or("default")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// A microVM mission materialises no team. Its phases run as one `claude -p`
|
||||||
|
// inside a VM (`microvm_executor`), so there is no claw graph to provision —
|
||||||
|
// and demanding one rejected the launch of a well-formed mission with "pick
|
||||||
|
// teams in the wizard". This is the third of three team gates on a path that
|
||||||
|
// uses no teams; the other two are in `routes::missions` (draft→running) and
|
||||||
|
// `phase_runner::launch_phase` (no matching teams → stay pending).
|
||||||
|
//
|
||||||
|
// Returning before the picks below, not filtering them, because provisioning
|
||||||
|
// claws that never run is not a cheaper version of the same thing — it is a
|
||||||
|
// runtime binding and a pairing code describing something nothing uses.
|
||||||
|
if mission.runtime_kind == "microvm" {
|
||||||
|
eprintln!(
|
||||||
|
"mission_orchestrator: mission {mission_id} is a microvm mission — no team to \
|
||||||
|
materialise; its phases execute in a VM"
|
||||||
|
);
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
|
||||||
// Skip team materialization if already bound.
|
// Skip team materialization if already bound.
|
||||||
if mission.team_id.is_some() {
|
if mission.team_id.is_some() {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
@@ -193,7 +268,7 @@ pub async fn on_launch(
|
|||||||
provisioner: provisioner.as_ref(),
|
provisioner: provisioner.as_ref(),
|
||||||
template: &template,
|
template: &template,
|
||||||
team_name: &team_name,
|
team_name: &team_name,
|
||||||
default_model: "claude-sonnet-5",
|
default_model: MINTED_CLAW_MODEL,
|
||||||
},
|
},
|
||||||
&mut provisioned_claws,
|
&mut provisioned_claws,
|
||||||
)
|
)
|
||||||
@@ -228,31 +303,41 @@ pub async fn on_launch(
|
|||||||
// is a PathBuf the prop-schema won't expose — see provision_claw), so
|
// is a PathBuf the prop-schema won't expose — see provision_claw), so
|
||||||
// we patch the shared config file directly on the per-mission runtime
|
// we patch the shared config file directly on the per-mission runtime
|
||||||
// container. The daemon picks it up on the same reload that surfaces
|
// container. The daemon picks it up on the same reload that surfaces
|
||||||
// the freshly-provisioned claws for the run. Non-fatal: without the
|
// the freshly-provisioned claws for the run.
|
||||||
// pin, agents still write (to the sandbox) but the committer can't
|
//
|
||||||
// find the changes in /mission/repo.
|
// FATAL, deliberately. This was "non-fatal: agents still write (to the
|
||||||
if !provisioned_claws.is_empty() && mission_gateway.is_some() {
|
// sandbox) but the committer can't find the changes in /mission/repo" —
|
||||||
if let Some(mp) = crate::mission_runtime::MissionRuntimeProvisioner::from_env() {
|
// which is to say, the mission runs to completion and delivers nothing.
|
||||||
match mp
|
// Mission `019fcf62` did exactly that: the pin failed with `argument list
|
||||||
.pin_agent_workspaces(mission_id, &provisioned_claws, "/mission/repo")
|
// too long`, one line of stderr scrolled past, and phase 0 reported
|
||||||
.await
|
// `completed` with zero files, no commit error and no push error. A launch
|
||||||
|
// that cannot bind its agents to the repo has no path to delivering work,
|
||||||
|
// so it must fail at launch where someone is still looking.
|
||||||
|
//
|
||||||
|
// Not for a microVM mission: its agent is a `claude -p` inside a VM on a
|
||||||
|
// fleet node, not a ZeroClaw claw in a container here, so there is no
|
||||||
|
// workspace to pin. Leaving it would make a microVM launch FAIL on a
|
||||||
|
// container it was never going to use.
|
||||||
|
if !provisioned_claws.is_empty() && mission_gateway.is_some() && mission.runtime_kind != "microvm"
|
||||||
{
|
{
|
||||||
Ok(()) => {
|
if let Some(mp) = crate::mission_runtime::MissionRuntimeProvisioner::from_env() {
|
||||||
// The daemon reads config ONCE at boot and never re-reads
|
mp.pin_agent_workspaces(mission_id, &provisioned_claws, "/mission/repo")
|
||||||
// the file, so the pin is invisible until it restarts. Its
|
.await
|
||||||
// agents were created through its own config API, so they
|
.map_err(|e| {
|
||||||
// are already persisted to the file and survive the
|
format!(
|
||||||
// restart; the pairing code is re-minted on every launch.
|
"could not pin agent workspaces to /mission/repo ({e}) — the mission \
|
||||||
if let Err(e) = mp.restart_container(mission_id).await {
|
would run with its agents writing to their sandboxes, delivering nothing"
|
||||||
eprintln!(
|
)
|
||||||
"mission_orchestrator: restart runtime for {mission_id} failed (continuing, workspace pin will not apply): {e}"
|
})?;
|
||||||
);
|
// The daemon reads config ONCE at boot and never re-reads the
|
||||||
}
|
// file, so the pin is invisible until it restarts. Its agents were
|
||||||
}
|
// created through its own config API, so they are already
|
||||||
Err(e) => eprintln!(
|
// persisted to the file and survive the restart; the pairing code
|
||||||
"mission_orchestrator: pin workspaces for mission {mission_id} failed (continuing): {e}"
|
// is re-minted on every launch. Equally fatal: an unrestarted
|
||||||
),
|
// daemon is an unpinned daemon.
|
||||||
}
|
mp.restart_container(mission_id).await.map_err(|e| {
|
||||||
|
format!("could not restart the runtime to apply the workspace pin: {e}")
|
||||||
|
})?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -365,6 +450,17 @@ async fn mint_team_from_template(
|
|||||||
.await
|
.await
|
||||||
.map_err(|e| format!("stamp template lineage: {e}"))?;
|
.map_err(|e| format!("stamp template lineage: {e}"))?;
|
||||||
|
|
||||||
|
// Names already on this workspace's roster, so a newly hired claw does not
|
||||||
|
// arrive sharing a name with someone already here. Read ONCE — a roster
|
||||||
|
// query per role would be N queries to answer one question — and extended
|
||||||
|
// locally as we mint, which also keeps names distinct WITHIN this team.
|
||||||
|
let mut taken_names: Vec<String> = cm_db::repo::agents::roster(pool, workspace_id)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("read roster for naming: {e}"))?
|
||||||
|
.into_iter()
|
||||||
|
.map(|a| a.name)
|
||||||
|
.collect();
|
||||||
|
|
||||||
// For each role: create agent, provision runtime, ingest brain
|
// For each role: create agent, provision runtime, ingest brain
|
||||||
// seed, record link, bind to topology node.
|
// seed, record link, bind to topology node.
|
||||||
for (idx, role) in template.roles.iter().enumerate() {
|
for (idx, role) in template.roles.iter().enumerate() {
|
||||||
@@ -378,10 +474,63 @@ async fn mint_team_from_template(
|
|||||||
template.roles.len(),
|
template.roles.len(),
|
||||||
));
|
));
|
||||||
};
|
};
|
||||||
|
// Every mission gets its OWN crew.
|
||||||
|
//
|
||||||
|
// This deliberately reverses the reuse added earlier. Reuse hired the
|
||||||
|
// existing claw for a (template, slot) so the roster stayed at one team
|
||||||
|
// and "My Workforce" was people you keep — but it also meant every
|
||||||
|
// mission was staffed by the same five names, and the workforce view
|
||||||
|
// showed one crew repeated down the page with nothing to tell the
|
||||||
|
// missions apart. Chosen by the operator: distinct crews read better
|
||||||
|
// than a bounded roster.
|
||||||
|
//
|
||||||
|
// The cost is real and is the cost that reuse existed to avoid: claws
|
||||||
|
// are `lifecycle = 'permanent'` and nothing reaps them until their
|
||||||
|
// MISSION is deleted, so the roster now grows by the team size on every
|
||||||
|
// mission. `agent_names::pick` keeps names unique workspace-wide and
|
||||||
|
// falls back to a numeric suffix once the pool is exhausted, so growth
|
||||||
|
// degrades the naming gracefully rather than colliding.
|
||||||
|
//
|
||||||
|
// `reusable_claw` in cm-db is kept, with its tests: this is a policy
|
||||||
|
// choice that has now flipped twice, and the query is the hard part.
|
||||||
|
let reused: Option<uuid::Uuid> = None;
|
||||||
|
|
||||||
|
// Seed the name choice from the claw's OWN id, not its position in the
|
||||||
|
// team.
|
||||||
|
//
|
||||||
|
// Seeding with the role index (0..n) started every crew near the top of
|
||||||
|
// the pool and took the next free names, so the first mission hired
|
||||||
|
// Aarav, Abebe, Adaora, Adrian, Agnieszka — correct, unique, and
|
||||||
|
// transparently alphabetical. A crew should look like a team, not like
|
||||||
|
// a listing. UUIDv7 puts its random bytes LAST (the leading bytes are a
|
||||||
|
// timestamp, which would cluster again), so the tail is what spreads
|
||||||
|
// the five picks across the whole pool.
|
||||||
|
let agent_id = cm_domain::AgentId::new();
|
||||||
|
let name_seed = {
|
||||||
|
let uuid = agent_id.as_uuid();
|
||||||
|
let b = uuid.as_bytes();
|
||||||
|
u64::from_le_bytes([b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15]])
|
||||||
|
};
|
||||||
|
|
||||||
let agent = Agent {
|
let agent = Agent {
|
||||||
id: cm_domain::AgentId::new(),
|
id: agent_id,
|
||||||
workspace_id,
|
workspace_id,
|
||||||
name: format!("{} · {}", team_name, role.slot),
|
// A PERSON's name, with the role in `job_title`.
|
||||||
|
//
|
||||||
|
// This was `"{mission title} · {purpose} · {template} · {slot}"` —
|
||||||
|
// names like "verify: a repo-less research mission keeps its output
|
||||||
|
// · mission · Rust SDLC · planner", unreadable in the roster, the
|
||||||
|
// API and every log line at once. Then it was the bare slot, which
|
||||||
|
// fixed the length but made the UI show the same word twice (name
|
||||||
|
// on top, role beneath) and made a roster of five read as five job
|
||||||
|
// tickets rather than a crew.
|
||||||
|
//
|
||||||
|
// The role still lives in `job_title`, which is what the mission
|
||||||
|
// machinery binds on — `team_members.role_slot` and the topology
|
||||||
|
// node carry the slot, so nothing downstream keys off the display
|
||||||
|
// name. Only the reused branch below ignores this, deliberately: a
|
||||||
|
// claw you already hired keeps the name it already had.
|
||||||
|
name: crate::agent_names::pick(&taken_names, name_seed),
|
||||||
job_title: role.slot.clone(),
|
job_title: role.slot.clone(),
|
||||||
// This is the ONLY consumer of the templates' `system_prompt` prose,
|
// This is the ONLY consumer of the templates' `system_prompt` prose,
|
||||||
// and it feeds the *chat* path, not missions: it lands in
|
// and it feeds the *chat* path, not missions: it lands in
|
||||||
@@ -398,12 +547,40 @@ async fn mint_team_from_template(
|
|||||||
managed_by: user_id,
|
managed_by: user_id,
|
||||||
status: AgentStatus::Online,
|
status: AgentStatus::Online,
|
||||||
};
|
};
|
||||||
|
let claw_id = match reused {
|
||||||
|
Some(existing) => {
|
||||||
|
eprintln!(
|
||||||
|
"mission_orchestrator: reusing claw {existing} for role {} \
|
||||||
|
(template {})",
|
||||||
|
role.slot, template.template.id
|
||||||
|
);
|
||||||
|
existing
|
||||||
|
}
|
||||||
|
None => {
|
||||||
cm_db::repo::agents::insert(pool, &agent, &AccessPolicy::default())
|
cm_db::repo::agents::insert(pool, &agent, &AccessPolicy::default())
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("insert agent {}: {e}", role.slot))?;
|
.map_err(|e| format!("insert agent {}: {e}", role.slot))?;
|
||||||
let claw_id = agent.id.as_uuid();
|
// Claim the name for the rest of this loop. Without this the
|
||||||
|
// roster snapshot taken before the loop is stale from the
|
||||||
|
// second role onward and a five-person team can arrive with
|
||||||
|
// two Merediths.
|
||||||
|
taken_names.push(agent.name.clone());
|
||||||
|
agent.id.as_uuid()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let agent_id = cm_domain::AgentId::from(claw_id);
|
||||||
|
|
||||||
cm_db::repo::agents::set_model_binding(pool, agent.id, default_model)
|
// The ROLE's model when the template names one, else the mint's default.
|
||||||
|
// Before migration 0071 there was no role model at all, so every claw of
|
||||||
|
// every mission team ran the same one — including a reviewer reviewing
|
||||||
|
// the coder it shares a model with.
|
||||||
|
let role_model = role
|
||||||
|
.model
|
||||||
|
.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|m| !m.is_empty())
|
||||||
|
.unwrap_or(default_model);
|
||||||
|
cm_db::repo::agents::set_model_binding(pool, agent_id, role_model)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("set_model_binding {claw_id}: {e}"))?;
|
.map_err(|e| format!("set_model_binding {claw_id}: {e}"))?;
|
||||||
|
|
||||||
@@ -420,10 +597,10 @@ async fn mint_team_from_template(
|
|||||||
// out-of-band via MissionRuntimeProvisioner::pin_agent_workspaces.
|
// out-of-band via MissionRuntimeProvisioner::pin_agent_workspaces.
|
||||||
if let Some(p) = provisioner {
|
if let Some(p) = provisioner {
|
||||||
match p
|
match p
|
||||||
.provision_claw(claw_id, default_model, &template.template.risk_profile)
|
.provision_claw(claw_id, role_model, &template.template.risk_profile)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(_) => provisioned_claws.push(agent.id),
|
Ok(_) => provisioned_claws.push(agent_id),
|
||||||
Err(e) => eprintln!(
|
Err(e) => eprintln!(
|
||||||
"mission_orchestrator: provision claw {claw_id} failed (continuing): {e}"
|
"mission_orchestrator: provision claw {claw_id} failed (continuing): {e}"
|
||||||
),
|
),
|
||||||
@@ -432,6 +609,10 @@ async fn mint_team_from_template(
|
|||||||
|
|
||||||
// Ingest brain seed (Slice 3.5d). Non-fatal on failure —
|
// Ingest brain seed (Slice 3.5d). Non-fatal on failure —
|
||||||
// agent still works from system_prompt alone.
|
// agent still works from system_prompt alone.
|
||||||
|
// Seed only a NEW claw. A reused one carries what it learned on earlier
|
||||||
|
// missions, and re-seeding would overwrite that with the template's
|
||||||
|
// starting point — which is precisely the accumulation reuse exists for.
|
||||||
|
if reused.is_none() {
|
||||||
if let Some(seed) = role.brain_seed.as_deref().filter(|s| !s.trim().is_empty()) {
|
if let Some(seed) = role.brain_seed.as_deref().filter(|s| !s.trim().is_empty()) {
|
||||||
if let Err(e) =
|
if let Err(e) =
|
||||||
crate::brain_seed::ingest(claw_id, seed.to_string(), role.system_prompt.clone())
|
crate::brain_seed::ingest(claw_id, seed.to_string(), role.system_prompt.clone())
|
||||||
@@ -442,6 +623,7 @@ async fn mint_team_from_template(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Record lineage (Slice 3.5d) so the MCP skills server can
|
// Record lineage (Slice 3.5d) so the MCP skills server can
|
||||||
// merge template default skills with per-agent overrides.
|
// merge template default skills with per-agent overrides.
|
||||||
@@ -470,9 +652,10 @@ async fn mint_team_from_template(
|
|||||||
cm_db::repo::audit::Actor::User(user_id),
|
cm_db::repo::audit::Actor::User(user_id),
|
||||||
"agent.created",
|
"agent.created",
|
||||||
"agent",
|
"agent",
|
||||||
&agent.id.to_string(),
|
&agent_id.to_string(),
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
"name": agent.name,
|
"name": agent.name,
|
||||||
|
"reused": reused.is_some(),
|
||||||
"job_title": agent.job_title,
|
"job_title": agent.job_title,
|
||||||
"source": "mission_orchestrator",
|
"source": "mission_orchestrator",
|
||||||
"template_id": template.template.id.to_string(),
|
"template_id": template.template.id.to_string(),
|
||||||
@@ -486,6 +669,97 @@ async fn mint_team_from_template(
|
|||||||
Ok(team_id)
|
Ok(team_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The model a minted claw runs on when its template role does not name one.
|
||||||
|
///
|
||||||
|
/// A DEFAULT now, not a hardcode: `template_roles.model` (migration 0071) lets a
|
||||||
|
/// template put its reviewer on a different model from the coder it reviews,
|
||||||
|
/// which is the correlated failure the cross-provider judge exists to break,
|
||||||
|
/// one layer down. Roles that say nothing still land here, so every template
|
||||||
|
/// that existed before 0071 behaves exactly as it did.
|
||||||
|
const MINTED_CLAW_MODEL: &str = "claude-sonnet-5";
|
||||||
|
|
||||||
|
/// The graph a COMPOSED microVM mission runs, built from its team template
|
||||||
|
/// without minting a single claw.
|
||||||
|
///
|
||||||
|
/// A composed mission needs the template's *shape* — how many nodes, in what
|
||||||
|
/// pattern, playing what roles — and nothing else it carries. Its nodes are VMs,
|
||||||
|
/// so provisioning claws for them would create agents, containers and `.brain`
|
||||||
|
/// files that nothing ever dials; that is exactly why `on_launch` returns early
|
||||||
|
/// for a microVM mission, and this is how the composed path gets its graph
|
||||||
|
/// anyway rather than by undoing that.
|
||||||
|
///
|
||||||
|
/// `purposes` is the phase's purpose list, matched against `config.phase_teams`;
|
||||||
|
/// missions using the legacy single `team_template_id` fall back to it.
|
||||||
|
/// Returns `None` when the mission picked no template at all.
|
||||||
|
pub async fn composed_graph(
|
||||||
|
pool: &PgPool,
|
||||||
|
mission_id: Uuid,
|
||||||
|
purposes: &[&str],
|
||||||
|
) -> Result<Option<serde_json::Value>, String> {
|
||||||
|
let row: Option<(serde_json::Value, Option<Uuid>)> =
|
||||||
|
sqlx::query_as("SELECT config, team_template_id FROM missions WHERE id = $1")
|
||||||
|
.bind(mission_id)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("load mission {mission_id}: {e}"))?;
|
||||||
|
let Some((config, legacy_template)) = row else {
|
||||||
|
return Err(format!("mission {mission_id} not found"));
|
||||||
|
};
|
||||||
|
|
||||||
|
// An APPROVED roster wins over the template. It is the more specific answer
|
||||||
|
// — a model sized it for this mission's actual task and a human accepted it
|
||||||
|
// — and it is the only path on which nodes carry per-node backends, which is
|
||||||
|
// how a mission runs more than one provider. Stored already built and
|
||||||
|
// validated (`routes::mission_roster::decide`), so nothing here can turn a
|
||||||
|
// refused roster into a running one.
|
||||||
|
if let Some(roster) = config.get("roster").filter(|v| v.is_object()) {
|
||||||
|
// Parsed rather than trusted: a graph the orchestrator cannot plan would
|
||||||
|
// otherwise be claimed and fail as "missing or invalid graph", which
|
||||||
|
// reads as a runtime fault instead of a bad roster.
|
||||||
|
serde_json::from_value::<cm_topology::TopologyGraph>(roster.clone())
|
||||||
|
.map_err(|e| format!("mission {mission_id}: the approved roster is not a runnable topology: {e}"))?;
|
||||||
|
return Ok(Some(roster.clone()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let template_id = config
|
||||||
|
.get("phase_teams")
|
||||||
|
.and_then(|v| v.as_object())
|
||||||
|
.and_then(|pt| {
|
||||||
|
// First template named by any purpose this phase answers to, in the
|
||||||
|
// phase's own preference order — the same order `launch_phase` uses
|
||||||
|
// to pick teams, so a composed mission and a ZeroClaw one resolve the
|
||||||
|
// same template for the same phase.
|
||||||
|
purposes.iter().find_map(|p| {
|
||||||
|
pt.get(*p)
|
||||||
|
.and_then(|v| v.as_array())
|
||||||
|
.and_then(|a| a.first())
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.and_then(|s| Uuid::parse_str(s).ok())
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.or(legacy_template);
|
||||||
|
let Some(template_id) = template_id else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
|
||||||
|
let template = cm_db::repo::team_templates::get(pool, template_id)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("load template {template_id}: {e}"))?
|
||||||
|
.ok_or_else(|| format!("template {template_id} not found"))?;
|
||||||
|
let roles: Vec<&str> = template.roles.iter().map(|r| r.slot.as_str()).collect();
|
||||||
|
if roles.is_empty() {
|
||||||
|
return Err(format!("template {template_id} defines no roles"));
|
||||||
|
}
|
||||||
|
let graph = cm_topology::build(
|
||||||
|
parse_topology_kind(&template.template.default_topology),
|
||||||
|
&roles,
|
||||||
|
)
|
||||||
|
.map_err(|e| format!("build topology graph for template {template_id}: {e}"))?;
|
||||||
|
serde_json::to_value(&graph)
|
||||||
|
.map(Some)
|
||||||
|
.map_err(|e| format!("serialize topology graph: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
fn parse_topology_kind(s: &str) -> cm_topology::TopologyKind {
|
fn parse_topology_kind(s: &str) -> cm_topology::TopologyKind {
|
||||||
use cm_topology::TopologyKind;
|
use cm_topology::TopologyKind;
|
||||||
match s {
|
match s {
|
||||||
|
|||||||
@@ -0,0 +1,503 @@
|
|||||||
|
//! Capture for missions that have no repository.
|
||||||
|
//!
|
||||||
|
//! `mission_delivery` captures a phase's work by diffing a git checkout. A
|
||||||
|
//! mission with `repo_id IS NULL` — every `research_only` mission, because that
|
||||||
|
//! recipe sets `requires_repo = false` — has no checkout, so
|
||||||
|
//! `capture_finished_coding_phases` filters it out at the SQL level
|
||||||
|
//! (`AND m.repo_id IS NOT NULL`) and never reads the container at all.
|
||||||
|
//!
|
||||||
|
//! The agents still write files. The research directive tells them to save
|
||||||
|
//! findings under `/mission/repo/research/`, and it says so whether or not a
|
||||||
|
//! repo exists. So the work lands in the container's own filesystem, is never
|
||||||
|
//! collected, and is destroyed when the sweeper reaps the container.
|
||||||
|
//!
|
||||||
|
//! # What this cost, measured
|
||||||
|
//!
|
||||||
|
//! Mission `019fdc35` ("ClawHDF5 Research"): four agents, 9.5 minutes, **eight
|
||||||
|
//! research documents** — an HDF5 parser design, a Rust ecosystem survey, a
|
||||||
|
//! seven-crate dependency map, tracing and fuzzing strategy. `mission_artifacts`
|
||||||
|
//! held zero rows and the mission reported `completed`. One agent's own summary
|
||||||
|
//! recorded the situation exactly: *"No git repo — file is written."* It noticed,
|
||||||
|
//! wrote anyway, and the platform threw the result away without a word.
|
||||||
|
//!
|
||||||
|
//! Nothing survived but the summarizer's account of it — which is the agents'
|
||||||
|
//! description of the work, not the work.
|
||||||
|
//!
|
||||||
|
//! # Why a separate path rather than widening the diff capture
|
||||||
|
//!
|
||||||
|
//! There is no base commit to diff against and no branch to push, so every
|
||||||
|
//! concept `capture_phase_diff` is built on is absent. What a repo-less mission
|
||||||
|
//! produces is simply *files*, and the honest capture is to copy them out and
|
||||||
|
//! register each as an artifact. `_outputs/` is deliberately a SIBLING of the
|
||||||
|
//! mission directory and survives `teardown_container`, so artifacts registered
|
||||||
|
//! here outlive the reap that destroyed the originals.
|
||||||
|
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use sqlx::{PgPool, Row};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
/// Directories never worth capturing, whatever an agent leaves behind.
|
||||||
|
///
|
||||||
|
/// Same intent as `mission_fs`'s exclusion list: a captured `.git` or
|
||||||
|
/// `node_modules` is noise that would bury the four documents that matter.
|
||||||
|
const SKIP_DIRS: &[&str] = &[
|
||||||
|
".git",
|
||||||
|
"node_modules",
|
||||||
|
"target",
|
||||||
|
".venv",
|
||||||
|
"venv",
|
||||||
|
"__pycache__",
|
||||||
|
".cache",
|
||||||
|
"dist",
|
||||||
|
"build",
|
||||||
|
];
|
||||||
|
|
||||||
|
/// How many phases to capture per tick, matching `CAPTURE_BATCH`.
|
||||||
|
const BATCH: i64 = 5;
|
||||||
|
|
||||||
|
/// The artifact kind this path registers. Also the idempotency key: a phase with
|
||||||
|
/// one of these has already been captured.
|
||||||
|
pub const OUTPUT_KIND: &str = "document";
|
||||||
|
|
||||||
|
/// Filename of the marker written when a phase produced nothing.
|
||||||
|
const EMPTY_MARKER: &str = "NO-OUTPUT.md";
|
||||||
|
|
||||||
|
/// Capture the outputs of finished phases on missions that have no repo.
|
||||||
|
pub async fn capture_repo_less_phases(pool: &PgPool) -> Result<(), String> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT mp.id, mp.mission_id, mp.kind, mp.config, m.runtime_kind
|
||||||
|
FROM mission_phases mp
|
||||||
|
JOIN missions m ON m.id = mp.mission_id
|
||||||
|
WHERE mp.status IN ('completed', 'failed')
|
||||||
|
AND m.repo_id IS NULL
|
||||||
|
-- microVM used to be excluded here because `run_phase_in_vm`
|
||||||
|
-- refused to boot without a checkout. It no longer does: a
|
||||||
|
-- repo-less mission gets an empty workspace at the same guest path,
|
||||||
|
-- and the collect unpacks it back onto the host — so those files are
|
||||||
|
-- already on disk and `collect_into` reads them instead of asking a
|
||||||
|
-- container that never existed.
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM mission_artifacts a
|
||||||
|
WHERE a.mission_id = mp.mission_id
|
||||||
|
AND a.phase_id = mp.id
|
||||||
|
AND a.kind = $2
|
||||||
|
)
|
||||||
|
ORDER BY mp.completed_at DESC NULLS LAST
|
||||||
|
LIMIT $1",
|
||||||
|
)
|
||||||
|
.bind(BATCH)
|
||||||
|
.bind(OUTPUT_KIND)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("select repo-less phases to capture: {e}"))?;
|
||||||
|
|
||||||
|
for row in rows {
|
||||||
|
let phase_id: Uuid = row.get("id");
|
||||||
|
let mission_id: Uuid = row.get("mission_id");
|
||||||
|
let kind: String = row.get("kind");
|
||||||
|
let config: serde_json::Value = row.get("config");
|
||||||
|
let runtime_kind: String = row.get("runtime_kind");
|
||||||
|
|
||||||
|
let dest = outputs_dir(mission_id, phase_id);
|
||||||
|
let captured = match collect_into(mission_id, &dest, &runtime_kind).await {
|
||||||
|
Ok(files) => files,
|
||||||
|
Err(e) => {
|
||||||
|
// Loud and retryable, never silently "captured nothing": the
|
||||||
|
// whole defect this module exists for is work disappearing
|
||||||
|
// without a word. The next tick tries again; if the container is
|
||||||
|
// already gone the phase is failed below on the next pass.
|
||||||
|
eprintln!(
|
||||||
|
"mission_outputs: could NOT collect outputs for phase {phase_id} \
|
||||||
|
of mission {mission_id}: {e}"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
for file in &captured {
|
||||||
|
let rel = match file.strip_prefix(missions_root()) {
|
||||||
|
Ok(r) => r.to_string_lossy().to_string(),
|
||||||
|
Err(_) => file.to_string_lossy().to_string(),
|
||||||
|
};
|
||||||
|
let title = file
|
||||||
|
.file_name()
|
||||||
|
.map(|n| n.to_string_lossy().to_string())
|
||||||
|
.unwrap_or_else(|| rel.clone());
|
||||||
|
if let Err(e) = cm_db::repo::missions::register_artifact(
|
||||||
|
pool,
|
||||||
|
cm_db::repo::missions::RegisterArtifact {
|
||||||
|
mission_id,
|
||||||
|
phase_id: Some(phase_id),
|
||||||
|
path: &rel,
|
||||||
|
kind: OUTPUT_KIND,
|
||||||
|
mime: Some(mime_for(file)),
|
||||||
|
title: Some(&title),
|
||||||
|
generated_by_run: None,
|
||||||
|
// No PDF. The renderer converted Markdown to HTML by
|
||||||
|
// calling an LLM — a paid API call, per document, on the
|
||||||
|
// critical path of "save my research", which promptly
|
||||||
|
// failed on depleted credits. Markdown IS the deliverable;
|
||||||
|
// it is served by `artifact_content` and styled at render
|
||||||
|
// time, which is free, offline, and cannot 429.
|
||||||
|
render_pdf: false,
|
||||||
|
metadata: Some(serde_json::json!({
|
||||||
|
"bytes": std::fs::metadata(file).map(|m| m.len()).unwrap_or(0),
|
||||||
|
"captured_from": "/mission/repo",
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
eprintln!("mission_outputs: registering {rel}: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if captured.is_empty() {
|
||||||
|
// Register a marker even when there is nothing to capture, or this
|
||||||
|
// phase matches the `NOT EXISTS` selection on every tick forever:
|
||||||
|
// re-running a docker copy_out each time and, because the batch is
|
||||||
|
// bounded, permanently occupying a slot so no other repo-less
|
||||||
|
// mission is ever captured again.
|
||||||
|
//
|
||||||
|
// `phase_runner::record_uncapturable` exists for exactly this
|
||||||
|
// failure on the diff path — five dead phases starved the batch
|
||||||
|
// while live work went untouched — and this code hit it again on
|
||||||
|
// its first live negative control (4 log lines, then 8, 45 seconds
|
||||||
|
// apart). Same shape, same fix: a real file behind a real row,
|
||||||
|
// because an artifact pointing at nothing turns every reader into
|
||||||
|
// an unexplained 404.
|
||||||
|
if let Err(e) = register_empty_marker(pool, mission_id, phase_id, &dest).await {
|
||||||
|
eprintln!("mission_outputs: marking phase {phase_id} as empty: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if captured.is_empty() && !allow_empty(&config) {
|
||||||
|
// The same rule `empty_delivery_is_a_failure` applies to a coding
|
||||||
|
// phase, for the only channel a repo-less phase has. Without it a
|
||||||
|
// research mission that produced nothing is indistinguishable from
|
||||||
|
// one that produced eight documents — both `completed`.
|
||||||
|
eprintln!(
|
||||||
|
"mission_outputs: phase {phase_id} ({kind}) of mission {mission_id} produced \
|
||||||
|
NO output files — failing it. Set config.allow_empty = true if this phase is \
|
||||||
|
meant to think rather than produce."
|
||||||
|
);
|
||||||
|
if let Err(e) = sqlx::query("UPDATE mission_phases SET status = 'failed' WHERE id = $1")
|
||||||
|
.bind(phase_id)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
eprintln!("mission_outputs: failing empty phase {phase_id}: {e}");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
eprintln!(
|
||||||
|
"mission_outputs: captured {} file(s) from phase {phase_id} ({kind}) of \
|
||||||
|
mission {mission_id}",
|
||||||
|
captured.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record that a phase produced nothing, so it is not reconsidered forever.
|
||||||
|
///
|
||||||
|
/// Deliberately the same `OUTPUT_KIND` the real captures use: the selection
|
||||||
|
/// query asks "has this phase been captured?", and "captured, and there was
|
||||||
|
/// nothing" is an answer to that question. `metadata.empty` is what tells the
|
||||||
|
/// two apart — the same convention `mission_delivery` uses for its "No code
|
||||||
|
/// changes" artifact.
|
||||||
|
async fn register_empty_marker(
|
||||||
|
pool: &PgPool,
|
||||||
|
mission_id: Uuid,
|
||||||
|
phase_id: Uuid,
|
||||||
|
dest: &Path,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
std::fs::create_dir_all(dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
|
||||||
|
let file = dest.join(EMPTY_MARKER);
|
||||||
|
std::fs::write(
|
||||||
|
&file,
|
||||||
|
"This phase finished without leaving any files in its workspace, so there\n was nothing to publish. If the phase is meant to reason rather than\n produce, set `config.allow_empty = true` on it.\n",
|
||||||
|
)
|
||||||
|
.map_err(|e| format!("write {}: {e}", file.display()))?;
|
||||||
|
let rel = file
|
||||||
|
.strip_prefix(missions_root())
|
||||||
|
.map(|r| r.to_string_lossy().to_string())
|
||||||
|
.unwrap_or_else(|_| file.to_string_lossy().to_string());
|
||||||
|
cm_db::repo::missions::register_artifact(
|
||||||
|
pool,
|
||||||
|
cm_db::repo::missions::RegisterArtifact {
|
||||||
|
mission_id,
|
||||||
|
phase_id: Some(phase_id),
|
||||||
|
path: &rel,
|
||||||
|
kind: OUTPUT_KIND,
|
||||||
|
mime: Some("text/markdown"),
|
||||||
|
title: Some("No output produced"),
|
||||||
|
generated_by_run: None,
|
||||||
|
render_pdf: false,
|
||||||
|
metadata: Some(serde_json::json!({ "empty": true })),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map(|_| ())
|
||||||
|
.map_err(|e| format!("register empty marker: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Gather the mission's produced files and return the ones worth keeping.
|
||||||
|
///
|
||||||
|
/// Where they come from depends on the runtime, and the difference is not
|
||||||
|
/// cosmetic: a container mission's files are still INSIDE a running container,
|
||||||
|
/// while a microVM's have already been unpacked onto the host by the collect at
|
||||||
|
/// the end of the turn (`microvm_executor` writes them over
|
||||||
|
/// `mission_workspace::checkout_path`). Asking docker for a VM mission's files
|
||||||
|
/// would query a container that never existed.
|
||||||
|
async fn collect_into(mission_id: Uuid, dest: &Path, runtime_kind: &str) -> Result<Vec<PathBuf>, String> {
|
||||||
|
// A stale copy from an earlier attempt would be registered as this pass's
|
||||||
|
// output — the same "captured a tree nobody wrote" shape capture avoids.
|
||||||
|
let _ = std::fs::remove_dir_all(dest);
|
||||||
|
std::fs::create_dir_all(dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
|
||||||
|
|
||||||
|
if runtime_kind == "microvm" {
|
||||||
|
let src = crate::mission_workspace::checkout_path(mission_id);
|
||||||
|
if !src.is_dir() {
|
||||||
|
return Err(format!(
|
||||||
|
"{} is absent — the VM's collect did not land",
|
||||||
|
src.display()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
copy_tree(&src, &dest.join("repo"))?;
|
||||||
|
return Ok(keep_files(&dest.join("repo")));
|
||||||
|
}
|
||||||
|
|
||||||
|
let container = crate::mission_runtime::container_name(mission_id);
|
||||||
|
let docker = crate::container_exec::connect()?;
|
||||||
|
crate::mission_fs::copy_out(&docker, &container, "/mission/repo", dest).await?;
|
||||||
|
Ok(keep_files(&dest.join("repo")))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Recursive file copy. Small on purpose — the alternative is a dependency or a
|
||||||
|
/// shell-out, and this runs as the server's own uid against its own directory.
|
||||||
|
fn copy_tree(src: &Path, dest: &Path) -> Result<(), String> {
|
||||||
|
std::fs::create_dir_all(dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
|
||||||
|
let entries =
|
||||||
|
std::fs::read_dir(src).map_err(|e| format!("read {}: {e}", src.display()))?;
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let from = entry.path();
|
||||||
|
let to = dest.join(entry.file_name());
|
||||||
|
match entry.file_type() {
|
||||||
|
Ok(t) if t.is_dir() => copy_tree(&from, &to)?,
|
||||||
|
Ok(t) if t.is_file() => {
|
||||||
|
std::fs::copy(&from, &to).map_err(|e| format!("copy {}: {e}", from.display()))?;
|
||||||
|
}
|
||||||
|
// Symlinks and specials are skipped rather than followed: a link out
|
||||||
|
// of the tree would publish whatever it points at.
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every regular file worth keeping, recursively.
|
||||||
|
fn keep_files(root: &Path) -> Vec<PathBuf> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let mut stack = vec![root.to_path_buf()];
|
||||||
|
while let Some(dir) = stack.pop() {
|
||||||
|
let Ok(entries) = std::fs::read_dir(&dir) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let path = entry.path();
|
||||||
|
let name = entry.file_name().to_string_lossy().to_string();
|
||||||
|
if path.is_dir() {
|
||||||
|
if !SKIP_DIRS.contains(&name.as_str()) {
|
||||||
|
stack.push(path);
|
||||||
|
}
|
||||||
|
} else if path.is_file()
|
||||||
|
&& !name.starts_with('.')
|
||||||
|
// The agent runtime seeds its own identity files into the
|
||||||
|
// workspace root, which is pinned to the repo root. In a
|
||||||
|
// repo-backed mission `.git/info/exclude` hides them; a
|
||||||
|
// repo-less mission has no `.git`, so without this the user's
|
||||||
|
// artifact list is 7 files of agent scaffolding and 2 of their
|
||||||
|
// research. Measured exactly that way on the first live run.
|
||||||
|
&& !crate::mission_workspace::AGENT_SCAFFOLDING.contains(&name.as_str())
|
||||||
|
{
|
||||||
|
out.push(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.sort();
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `<missions_root>/_outputs/<mission>/<phase>` — a sibling of the mission
|
||||||
|
/// directory, so `teardown_container` reaping the mission does not take the
|
||||||
|
/// captured artifacts with it.
|
||||||
|
fn outputs_dir(mission_id: Uuid, phase_id: Uuid) -> PathBuf {
|
||||||
|
missions_root()
|
||||||
|
.join("_outputs")
|
||||||
|
.join(mission_id.to_string())
|
||||||
|
.join(phase_id.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The missions root, for callers that resolve artifact paths against it.
|
||||||
|
pub fn missions_root_dir() -> PathBuf {
|
||||||
|
missions_root()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The only directory an artifact may be read from.
|
||||||
|
pub fn outputs_root_dir() -> PathBuf {
|
||||||
|
missions_root().join("_outputs")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn missions_root() -> PathBuf {
|
||||||
|
crate::mission_workspace::missions_root()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn mime_for(p: &Path) -> &'static str {
|
||||||
|
match p.extension().and_then(|e| e.to_str()) {
|
||||||
|
Some("md") | Some("markdown") => "text/markdown",
|
||||||
|
Some("json") => "application/json",
|
||||||
|
Some("csv") => "text/csv",
|
||||||
|
Some("html") => "text/html",
|
||||||
|
_ => "text/plain",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn allow_empty(config: &serde_json::Value) -> bool {
|
||||||
|
config.get("allow_empty").and_then(|v| v.as_bool()) == Some(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn touch(p: &Path) {
|
||||||
|
std::fs::create_dir_all(p.parent().unwrap()).unwrap();
|
||||||
|
std::fs::write(p, "x").unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The documents a research phase writes are what must come back — and the
|
||||||
|
/// machinery around them must not.
|
||||||
|
#[test]
|
||||||
|
fn research_documents_are_kept_and_scaffolding_is_not() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let repo = tmp.path().join("repo");
|
||||||
|
touch(&repo.join("research/01_repo_archaeology.md"));
|
||||||
|
touch(&repo.join("research/02_ecosystem.md"));
|
||||||
|
touch(&repo.join("notes.txt"));
|
||||||
|
// The seven the agent runtime seeds into the workspace root.
|
||||||
|
for f in crate::mission_workspace::AGENT_SCAFFOLDING {
|
||||||
|
touch(&repo.join(f));
|
||||||
|
}
|
||||||
|
touch(&repo.join(".git/HEAD"));
|
||||||
|
touch(&repo.join("node_modules/left-pad/index.js"));
|
||||||
|
touch(&repo.join("target/debug/thing"));
|
||||||
|
touch(&repo.join(".hidden"));
|
||||||
|
|
||||||
|
let kept: Vec<String> = keep_files(&repo)
|
||||||
|
.iter()
|
||||||
|
.map(|p| p.strip_prefix(&repo).unwrap().to_string_lossy().to_string())
|
||||||
|
.collect();
|
||||||
|
assert_eq!(
|
||||||
|
kept,
|
||||||
|
vec![
|
||||||
|
"notes.txt".to_string(),
|
||||||
|
"research/01_repo_archaeology.md".to_string(),
|
||||||
|
"research/02_ecosystem.md".to_string(),
|
||||||
|
],
|
||||||
|
"kept: {kept:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Markdown is the deliverable, so it must be labelled as markdown — the
|
||||||
|
/// viewer decides how to render from the mime type.
|
||||||
|
#[test]
|
||||||
|
fn markdown_is_labelled_so_the_viewer_can_style_it() {
|
||||||
|
assert_eq!(mime_for(Path::new("/x/01_notes.md")), "text/markdown");
|
||||||
|
assert_eq!(mime_for(Path::new("/x/data.json")), "application/json");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The containment rule the content endpoint enforces: everything readable
|
||||||
|
/// lives under `_outputs`, and nothing else does.
|
||||||
|
///
|
||||||
|
/// Artifact paths are written by this server, but they are DATA in a table,
|
||||||
|
/// and a row saying `../../../etc/passwd` must be a 404 rather than a file
|
||||||
|
/// read. The endpoint canonicalises before comparing — checking the string
|
||||||
|
/// first would pass `_outputs/../../etc/passwd` straight through.
|
||||||
|
#[test]
|
||||||
|
fn everything_readable_lives_under_the_outputs_root() {
|
||||||
|
let root = outputs_root_dir();
|
||||||
|
assert!(root.ends_with("_outputs"), "{root:?}");
|
||||||
|
assert!(root.starts_with(missions_root_dir()), "{root:?}");
|
||||||
|
|
||||||
|
// A real capture is inside it...
|
||||||
|
let inside = outputs_dir(Uuid::now_v7(), Uuid::now_v7());
|
||||||
|
assert!(inside.starts_with(&root), "{inside:?}");
|
||||||
|
|
||||||
|
// ...and the traversal shape this guards against is not, once resolved.
|
||||||
|
let escaped = root.join("..").join("..").join("etc/passwd");
|
||||||
|
let normalised: PathBuf = escaped
|
||||||
|
.components()
|
||||||
|
.fold(PathBuf::new(), |mut acc, c| {
|
||||||
|
match c {
|
||||||
|
std::path::Component::ParentDir => {
|
||||||
|
acc.pop();
|
||||||
|
}
|
||||||
|
other => acc.push(other),
|
||||||
|
}
|
||||||
|
acc
|
||||||
|
});
|
||||||
|
assert!(
|
||||||
|
!normalised.starts_with(&root),
|
||||||
|
"a traversal must not resolve back inside the outputs root: {normalised:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A phase that produced nothing must still leave a marker, or the
|
||||||
|
/// selection query matches it on every tick forever.
|
||||||
|
///
|
||||||
|
/// Measured on the first live negative control: the guard logged "produced
|
||||||
|
/// NO output files" 4 times, then 8 times 45 seconds later — a docker
|
||||||
|
/// copy_out per tick, and with a bounded batch, five such phases would
|
||||||
|
/// starve every other repo-less mission out of capture permanently.
|
||||||
|
/// `phase_runner::record_uncapturable` was written for the identical
|
||||||
|
/// failure on the diff path.
|
||||||
|
#[test]
|
||||||
|
fn an_empty_phase_leaves_a_marker_so_it_is_not_reconsidered_forever() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let dest = tmp.path().join("out");
|
||||||
|
// The file-writing half of `register_empty_marker`, which is the part
|
||||||
|
// that must exist for the artifact row to point at something real.
|
||||||
|
std::fs::create_dir_all(&dest).unwrap();
|
||||||
|
let file = dest.join(EMPTY_MARKER);
|
||||||
|
std::fs::write(&file, "x").unwrap();
|
||||||
|
assert!(file.exists(), "an artifact row must not point at nothing");
|
||||||
|
assert_eq!(
|
||||||
|
file.file_name().unwrap().to_string_lossy(),
|
||||||
|
"NO-OUTPUT.md",
|
||||||
|
"the marker name is part of the contract with readers"
|
||||||
|
);
|
||||||
|
// And the marker must not itself be mistaken for captured output on a
|
||||||
|
// later pass: it is filtered like any other scaffolding would be.
|
||||||
|
assert!(keep_files(&dest).iter().any(|p| p == &file));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Artifacts must land OUTSIDE the mission directory. `teardown_container`
|
||||||
|
/// removes `<missions_root>/<mission_id>` wholesale, so a capture written
|
||||||
|
/// inside it would be destroyed by the very reap it exists to survive.
|
||||||
|
#[test]
|
||||||
|
fn captures_survive_the_mission_directory_being_reaped() {
|
||||||
|
let mission = Uuid::now_v7();
|
||||||
|
let phase = Uuid::now_v7();
|
||||||
|
let out = outputs_dir(mission, phase);
|
||||||
|
let mission_dir = missions_root().join(mission.to_string());
|
||||||
|
assert!(
|
||||||
|
!out.starts_with(&mission_dir),
|
||||||
|
"{} must not be inside {}",
|
||||||
|
out.display(),
|
||||||
|
mission_dir.display()
|
||||||
|
);
|
||||||
|
assert!(out.starts_with(missions_root().join("_outputs")), "{out:?}");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,296 @@
|
|||||||
|
//! A model-authored execution plan for one mission — W1 / #13.
|
||||||
|
//!
|
||||||
|
//! Every mission's phases come from one of five hand-written recipes in
|
||||||
|
//! `templates/workflows/*.toml`, chosen by `template_kind`. A recipe is a fixed
|
||||||
|
//! answer to "what phases does this kind of mission have", written before anyone
|
||||||
|
//! saw the mission — the "do it this way: 1, 2, 3" over-specification that makes
|
||||||
|
//! a capable model follow a worse plan than it would have chosen for the actual
|
||||||
|
//! task.
|
||||||
|
//!
|
||||||
|
//! This is the other half of [`crate::mission_roster`]: that one lets a model
|
||||||
|
//! size the team, this one lets it decide what the work IS. Same shape on
|
||||||
|
//! purpose — propose, review, approve, apply — because the review gate is what
|
||||||
|
//! makes model-authored structure safe to run, and a second shape would be a
|
||||||
|
//! second thing to get right.
|
||||||
|
//!
|
||||||
|
//! # Grounded in what the platform actually reads
|
||||||
|
//!
|
||||||
|
//! The interesting constraint is not "is this JSON valid" but "will anything
|
||||||
|
//! consume it". `phase_config::KNOWN_KEYS` already names every phase-config key
|
||||||
|
//! and the code that reads it, with eleven marked NOT IMPLEMENTED — the registry
|
||||||
|
//! built after `task` sat unread through every mission. A plan is validated
|
||||||
|
//! against that registry, so a model cannot propose a phase whose settings
|
||||||
|
//! nothing will act on. The failure that registry exists to EXPOSE is one this
|
||||||
|
//! path cannot create.
|
||||||
|
//!
|
||||||
|
//! Phase kinds are checked the same way, against the kinds `phase_runner`
|
||||||
|
//! actually dispatches. A model asked to plan work will happily invent
|
||||||
|
//! `kind: "review"`, and an unknown kind does not fail — it falls to the
|
||||||
|
//! catch-all purpose and runs as a generic phase, which looks like it worked.
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
/// Phase kinds `phase_runner` dispatches on.
|
||||||
|
///
|
||||||
|
/// Not an enum, because `mission_phases.kind` is a free-form column shared with
|
||||||
|
/// hand-written recipes and the wizard; this is the subset a MODEL may propose.
|
||||||
|
/// An unrecognised kind is the dangerous case: it does not error, it falls
|
||||||
|
/// through to the generic `mission` purpose and runs anyway.
|
||||||
|
pub const PLANNABLE_KINDS: &[&str] = &["research", "coding", "benchmark", "security_scan"];
|
||||||
|
|
||||||
|
/// Ceiling on a proposed plan.
|
||||||
|
///
|
||||||
|
/// Each phase is a full agent run — a VM boot, a checkout, a turn, a capture —
|
||||||
|
/// executed in sequence. Anthropic's own guidance warns against decomposing work
|
||||||
|
/// into sequential phases at all ("a handoff loses context at every step"), so
|
||||||
|
/// this bound is deliberately tight: a model that wants eight phases is
|
||||||
|
/// describing a to-do list, not a plan.
|
||||||
|
pub const MAX_PHASES: usize = 4;
|
||||||
|
|
||||||
|
/// One phase of a proposed plan.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct PlannedPhase {
|
||||||
|
/// One of [`PLANNABLE_KINDS`].
|
||||||
|
pub kind: String,
|
||||||
|
/// What this phase does. Lands in `config.task`, which
|
||||||
|
/// `phase_task_text` injects — the key that sat unread through every
|
||||||
|
/// mission until two phases with different tasks produced identical output.
|
||||||
|
pub task: String,
|
||||||
|
/// Optional completion condition, judged post-hoc by the evaluator.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub done_when: Option<String>,
|
||||||
|
/// Optional deterministic check, enforced IN the agent's loop by the stop
|
||||||
|
/// gate ([`crate::vm_stop_gate`]).
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub done_when_check: Option<String>,
|
||||||
|
/// This phase is allowed to change nothing (a verification pass).
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub allow_empty: Option<bool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A proposed sequence of phases.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct Plan {
|
||||||
|
pub phases: Vec<PlannedPhase>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Why a plan was refused.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum Refusal {
|
||||||
|
Empty,
|
||||||
|
TooMany(usize),
|
||||||
|
UnknownKind { index: usize, kind: String },
|
||||||
|
BlankTask(usize),
|
||||||
|
/// A config key with no reader in this build — named, with the ones that
|
||||||
|
/// would have been consumed.
|
||||||
|
InertKey { index: usize, key: String },
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for Refusal {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
Refusal::Empty => write!(f, "the plan has no phases, so the mission would do nothing"),
|
||||||
|
Refusal::TooMany(n) => write!(
|
||||||
|
f,
|
||||||
|
"the plan has {n} phases and the ceiling is {MAX_PHASES} — each one is a full \
|
||||||
|
agent run, and a handoff loses context at every step"
|
||||||
|
),
|
||||||
|
Refusal::UnknownKind { index, kind } => write!(
|
||||||
|
f,
|
||||||
|
"phase {index} has kind {kind:?}, which nothing dispatches on; use one of: {}",
|
||||||
|
PLANNABLE_KINDS.join(", ")
|
||||||
|
),
|
||||||
|
Refusal::BlankTask(i) => write!(
|
||||||
|
f,
|
||||||
|
"phase {i} has no task, so its agent would receive the mission description and \
|
||||||
|
nothing telling it which part is its own"
|
||||||
|
),
|
||||||
|
Refusal::InertKey { index, key } => write!(
|
||||||
|
f,
|
||||||
|
"phase {index} sets {key:?}, which nothing in this build reads — it would be \
|
||||||
|
stored, rendered, and consumed by nobody"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Plan {
|
||||||
|
/// Check a plan against what the platform can actually execute.
|
||||||
|
pub fn validate(&self) -> Result<(), Refusal> {
|
||||||
|
if self.phases.is_empty() {
|
||||||
|
return Err(Refusal::Empty);
|
||||||
|
}
|
||||||
|
if self.phases.len() > MAX_PHASES {
|
||||||
|
return Err(Refusal::TooMany(self.phases.len()));
|
||||||
|
}
|
||||||
|
for (i, p) in self.phases.iter().enumerate() {
|
||||||
|
if !PLANNABLE_KINDS.contains(&p.kind.as_str()) {
|
||||||
|
return Err(Refusal::UnknownKind {
|
||||||
|
index: i,
|
||||||
|
kind: p.kind.clone(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if p.task.trim().is_empty() {
|
||||||
|
return Err(Refusal::BlankTask(i));
|
||||||
|
}
|
||||||
|
// Every key this phase would write must have a reader. The plan is
|
||||||
|
// built from typed fields, so this can only fail if a field is added
|
||||||
|
// here without a corresponding entry in the registry — which is
|
||||||
|
// exactly the drift worth failing on.
|
||||||
|
if let Some(key) = crate::phase_config::inert_keys(&p.config()).into_iter().next() {
|
||||||
|
return Err(Refusal::InertKey { index: i, key });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The phases as `(kind, order_idx, config)`, ready for mission creation.
|
||||||
|
///
|
||||||
|
/// `order_idx` is the array position rather than a field the model sets:
|
||||||
|
/// two sources for one fact is how a plan ends up with two phase 0s.
|
||||||
|
pub fn phases(&self) -> Vec<(String, i32, serde_json::Value)> {
|
||||||
|
self.phases
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(i, p)| (p.kind.clone(), i as i32, p.config()))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PlannedPhase {
|
||||||
|
/// This phase's `mission_phases.config`.
|
||||||
|
fn config(&self) -> serde_json::Value {
|
||||||
|
let mut o = serde_json::Map::new();
|
||||||
|
o.insert("task".into(), serde_json::Value::String(self.task.clone()));
|
||||||
|
if let Some(d) = self.done_when.as_deref().map(str::trim).filter(|s| !s.is_empty()) {
|
||||||
|
o.insert("done_when".into(), serde_json::Value::String(d.to_string()));
|
||||||
|
}
|
||||||
|
if let Some(c) = self
|
||||||
|
.done_when_check
|
||||||
|
.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
{
|
||||||
|
o.insert(
|
||||||
|
"done_when_check".into(),
|
||||||
|
serde_json::Value::String(c.to_string()),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if let Some(e) = self.allow_empty {
|
||||||
|
o.insert("allow_empty".into(), serde_json::Value::Bool(e));
|
||||||
|
}
|
||||||
|
serde_json::Value::Object(o)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn phase(kind: &str, task: &str) -> PlannedPhase {
|
||||||
|
PlannedPhase {
|
||||||
|
kind: kind.into(),
|
||||||
|
task: task.into(),
|
||||||
|
done_when: None,
|
||||||
|
done_when_check: None,
|
||||||
|
allow_empty: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A kind nothing dispatches on is the dangerous one: it does not error, it
|
||||||
|
/// falls through to the generic purpose and runs as a nondescript phase that
|
||||||
|
/// looks like it worked.
|
||||||
|
#[test]
|
||||||
|
fn an_invented_phase_kind_is_refused_naming_the_real_ones() {
|
||||||
|
let p = Plan {
|
||||||
|
phases: vec![phase("coding", "do it"), phase("review", "check it")],
|
||||||
|
};
|
||||||
|
let err = p.validate().unwrap_err();
|
||||||
|
assert_eq!(
|
||||||
|
err,
|
||||||
|
Refusal::UnknownKind {
|
||||||
|
index: 1,
|
||||||
|
kind: "review".into()
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let msg = err.to_string();
|
||||||
|
for kind in PLANNABLE_KINDS {
|
||||||
|
assert!(msg.contains(kind), "the message must name {kind}: {msg}");
|
||||||
|
}
|
||||||
|
// And every kind the runner dispatches on is accepted, so this cannot
|
||||||
|
// drift from what `phase_runner` can actually execute.
|
||||||
|
for kind in PLANNABLE_KINDS {
|
||||||
|
assert!(Plan { phases: vec![phase(kind, "work")] }.validate().is_ok(), "{kind}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every key a planned phase writes must have a reader. This is the whole
|
||||||
|
/// reason `phase_config` exists — a key nothing consumes is stored,
|
||||||
|
/// rendered, and silently inert.
|
||||||
|
#[test]
|
||||||
|
fn every_key_a_plan_writes_is_one_something_reads() {
|
||||||
|
let p = PlannedPhase {
|
||||||
|
kind: "coding".into(),
|
||||||
|
task: "add a module".into(),
|
||||||
|
done_when: Some("the suite passes".into()),
|
||||||
|
done_when_check: Some("cargo test".into()),
|
||||||
|
allow_empty: Some(false),
|
||||||
|
};
|
||||||
|
let cfg = p.config();
|
||||||
|
assert!(
|
||||||
|
crate::phase_config::inert_keys(&cfg).is_empty(),
|
||||||
|
"a planned phase must write only keys with readers: {:?}",
|
||||||
|
crate::phase_config::inert_keys(&cfg)
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
crate::phase_config::unknown_keys(&cfg).is_empty(),
|
||||||
|
"and only keys the registry knows: {:?}",
|
||||||
|
crate::phase_config::unknown_keys(&cfg)
|
||||||
|
);
|
||||||
|
assert!(Plan { phases: vec![p] }.validate().is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A blank task is the failure that produced identical output from two
|
||||||
|
/// different phases — the agent gets the mission description and nothing
|
||||||
|
/// saying which part is its own.
|
||||||
|
#[test]
|
||||||
|
fn a_phase_without_a_task_is_refused() {
|
||||||
|
let p = Plan {
|
||||||
|
phases: vec![phase("coding", " ")],
|
||||||
|
};
|
||||||
|
assert_eq!(p.validate(), Err(Refusal::BlankTask(0)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bounded and non-empty. Each phase is a full agent run in sequence, and
|
||||||
|
/// splitting one change into stages loses context at every handoff.
|
||||||
|
#[test]
|
||||||
|
fn a_plan_is_bounded_and_non_empty() {
|
||||||
|
assert_eq!(Plan { phases: vec![] }.validate(), Err(Refusal::Empty));
|
||||||
|
let many: Vec<_> = (0..MAX_PHASES + 1).map(|_| phase("coding", "work")).collect();
|
||||||
|
assert_eq!(
|
||||||
|
Plan { phases: many }.validate(),
|
||||||
|
Err(Refusal::TooMany(MAX_PHASES + 1))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Order comes from the array, not from a field the model sets. Two sources
|
||||||
|
/// for one fact is how a plan ends up with two phase 0s — and `order_idx`
|
||||||
|
/// is what `start_pending_phases` sequences on.
|
||||||
|
#[test]
|
||||||
|
fn order_comes_from_the_arrays_own_order() {
|
||||||
|
let p = Plan {
|
||||||
|
phases: vec![
|
||||||
|
phase("research", "read the code"),
|
||||||
|
phase("coding", "change it"),
|
||||||
|
phase("coding", "then this"),
|
||||||
|
],
|
||||||
|
};
|
||||||
|
let out = p.phases();
|
||||||
|
assert_eq!(
|
||||||
|
out.iter().map(|(_, i, _)| *i).collect::<Vec<_>>(),
|
||||||
|
vec![0, 1, 2]
|
||||||
|
);
|
||||||
|
assert_eq!(out[0].0, "research");
|
||||||
|
assert_eq!(out[1].2["task"], "change it");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -2,16 +2,18 @@
|
|||||||
//! mission and rewrite it into a coherent, sectioned Markdown brief
|
//! mission and rewrite it into a coherent, sectioned Markdown brief
|
||||||
//! that downstream research + coding agents can ingest cleanly.
|
//! that downstream research + coding agents can ingest cleanly.
|
||||||
//!
|
//!
|
||||||
//! Calls Anthropic Claude Opus 4.8 by default. Prod already carries
|
//! Asks for Claude Opus 4.8 by default, but goes through
|
||||||
//! ANTHROPIC_API_KEY for ZeroClaw's provider config, so no separate
|
//! `subscription::complete_with_fallback` like every other server-side model
|
||||||
//! env is needed.
|
//! call. It used to hand-roll its own HTTPS POST to the Messages API with the
|
||||||
|
//! metered key — a comment above this line still claimed prod "already carries
|
||||||
|
//! ANTHROPIC_API_KEY, so no separate env is needed", which stopped being true
|
||||||
|
//! the moment that account ran out of credit. See `subscription`, whose
|
||||||
|
//! source-walk test is what found this module.
|
||||||
|
|
||||||
use serde_json::json;
|
|
||||||
use sqlx::PgPool;
|
use sqlx::PgPool;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
const DEFAULT_MODEL: &str = "claude-opus-4-8";
|
const DEFAULT_MODEL: &str = "claude-opus-4-8";
|
||||||
const ANTHROPIC_API_VERSION: &str = "2023-06-01";
|
|
||||||
|
|
||||||
fn model_name() -> String {
|
fn model_name() -> String {
|
||||||
std::env::var("CLAWMATES_REFINER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
std::env::var("CLAWMATES_REFINER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
||||||
@@ -22,12 +24,36 @@ pub struct RefineResult {
|
|||||||
pub refined: String,
|
pub refined: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Refine a description that has no mission behind it yet.
|
||||||
|
///
|
||||||
|
/// The wizard's polish button runs BEFORE the mission is created — there is no
|
||||||
|
/// row to load and no id to pass — while [`refine`] deliberately requires a
|
||||||
|
/// saved draft so Accept/Cancel can write back to it. Same prompt, same model
|
||||||
|
/// chain; only where the inputs come from differs.
|
||||||
|
pub async fn refine_draft(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
|
title: &str,
|
||||||
|
template_kind: &str,
|
||||||
|
phase_kinds: &[String],
|
||||||
|
raw: &str,
|
||||||
|
) -> Result<RefineResult, String> {
|
||||||
|
if raw.trim().is_empty() {
|
||||||
|
return Err("description is empty — nothing to refine".into());
|
||||||
|
}
|
||||||
|
let refined = call_anthropic(runtime, title, template_kind, phase_kinds, raw).await?;
|
||||||
|
Ok(RefineResult {
|
||||||
|
original: raw.to_string(),
|
||||||
|
refined,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// Generate a refined description without touching the database. The
|
/// Generate a refined description without touching the database. The
|
||||||
/// caller (frontend) reviews the diff and calls `set_description` to
|
/// caller (frontend) reviews the diff and calls `set_description` to
|
||||||
/// commit — that separation makes Accept/Cancel + undo trivial without
|
/// commit — that separation makes Accept/Cancel + undo trivial without
|
||||||
/// an audit table.
|
/// an audit table.
|
||||||
pub async fn refine(
|
pub async fn refine(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
workspace_id: cm_domain::WorkspaceId,
|
workspace_id: cm_domain::WorkspaceId,
|
||||||
mission_id: Uuid,
|
mission_id: Uuid,
|
||||||
) -> Result<RefineResult, String> {
|
) -> Result<RefineResult, String> {
|
||||||
@@ -54,7 +80,8 @@ pub async fn refine(
|
|||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let refined =
|
let refined =
|
||||||
call_anthropic(&mission.title, &mission.template_kind, &phase_kinds, &raw).await?;
|
call_anthropic(runtime, &mission.title, &mission.template_kind, &phase_kinds, &raw)
|
||||||
|
.await?;
|
||||||
|
|
||||||
Ok(RefineResult {
|
Ok(RefineResult {
|
||||||
original: raw,
|
original: raw,
|
||||||
@@ -63,13 +90,12 @@ pub async fn refine(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn call_anthropic(
|
async fn call_anthropic(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
title: &str,
|
title: &str,
|
||||||
template_kind: &str,
|
template_kind: &str,
|
||||||
phase_kinds: &[String],
|
phase_kinds: &[String],
|
||||||
raw: &str,
|
raw: &str,
|
||||||
) -> Result<String, String> {
|
) -> Result<String, String> {
|
||||||
let api_key =
|
|
||||||
std::env::var("ANTHROPIC_API_KEY").map_err(|_| "ANTHROPIC_API_KEY unset".to_string())?;
|
|
||||||
let model = model_name();
|
let model = model_name();
|
||||||
|
|
||||||
let system = "You are a technical brief editor for an autonomous software \
|
let system = "You are a technical brief editor for an autonomous software \
|
||||||
@@ -131,57 +157,14 @@ async fn call_anthropic(
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Opus 4.8 rejects the `temperature` parameter — the model runs at
|
// Opus 4.8 rejects the `temperature` parameter — the model runs at
|
||||||
// its own calibrated setting. Older Claude models accepted 0.0–1.0.
|
// its own calibrated setting. Older Claude models accepted 0.0–1.0, and
|
||||||
let body = json!({
|
// `ChatRequest` does not carry one, so nothing is lost by the move.
|
||||||
"model": model,
|
let (text, answered_by) =
|
||||||
"max_tokens": 4096,
|
crate::subscription::complete_with_fallback(runtime, system, &user, &model, 4096, false)
|
||||||
"system": system,
|
.await?;
|
||||||
"messages": [
|
let text = text.trim().to_string();
|
||||||
{ "role": "user", "content": user }
|
|
||||||
]
|
|
||||||
});
|
|
||||||
|
|
||||||
let client = reqwest::Client::builder()
|
|
||||||
.timeout(std::time::Duration::from_secs(90))
|
|
||||||
.build()
|
|
||||||
.map_err(|e| format!("http client: {e}"))?;
|
|
||||||
let resp = client
|
|
||||||
.post("https://api.anthropic.com/v1/messages")
|
|
||||||
.header("x-api-key", &api_key)
|
|
||||||
.header("anthropic-version", ANTHROPIC_API_VERSION)
|
|
||||||
.header("content-type", "application/json")
|
|
||||||
.json(&body)
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("anthropic call: {e}"))?;
|
|
||||||
if !resp.status().is_success() {
|
|
||||||
let code = resp.status();
|
|
||||||
let body = resp.text().await.unwrap_or_default();
|
|
||||||
return Err(format!(
|
|
||||||
"anthropic {code}: {}",
|
|
||||||
&body[..body.len().min(500)]
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let json: serde_json::Value = resp
|
|
||||||
.json()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("anthropic json: {e}"))?;
|
|
||||||
// Anthropic Messages API returns content as an array of blocks;
|
|
||||||
// the first text block holds the assistant's reply.
|
|
||||||
let text = json
|
|
||||||
.get("content")
|
|
||||||
.and_then(|c| c.as_array())
|
|
||||||
.and_then(|arr| {
|
|
||||||
arr.iter()
|
|
||||||
.find(|b| b.get("type").and_then(|t| t.as_str()) == Some("text"))
|
|
||||||
})
|
|
||||||
.and_then(|b| b.get("text"))
|
|
||||||
.and_then(|t| t.as_str())
|
|
||||||
.ok_or_else(|| "anthropic response missing text block".to_string())?
|
|
||||||
.trim()
|
|
||||||
.to_string();
|
|
||||||
if text.is_empty() {
|
if text.is_empty() {
|
||||||
return Err("anthropic returned empty text".into());
|
return Err(format!("{answered_by} returned empty text"));
|
||||||
}
|
}
|
||||||
Ok(text)
|
Ok(text)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,373 @@
|
|||||||
|
//! A model-authored roster for a mission — Slice 5.
|
||||||
|
//!
|
||||||
|
//! The Master Planner has been proposing teams (2-6 members, a model each) since
|
||||||
|
//! it shipped, and none of it reached a mission: the proposal lived in React
|
||||||
|
//! state. A mission's shape came instead from a team template — fixed roles, and
|
||||||
|
//! every claw minted `claude-sonnet-5`, which is why no mission has ever run
|
||||||
|
//! heterogeneous providers.
|
||||||
|
//!
|
||||||
|
//! This is the seam. A roster is `(topology_kind, [(role, backend)])`, which is
|
||||||
|
//! exactly what the composed executor consumes: `composed_graph` turns it into a
|
||||||
|
//! `TopologyGraph`, and `MicroVmTurnExecutor` reads `attrs["backend"]` per node,
|
||||||
|
//! so a `validator` role on a different provider's rootfs is a first-class graph
|
||||||
|
//! node rather than a bolt-on.
|
||||||
|
//!
|
||||||
|
//! # Why the backend is validated here and not at boot
|
||||||
|
//!
|
||||||
|
//! Placement already refuses a mission whose backend no online node can run —
|
||||||
|
//! but it refuses it at LAUNCH, after the roster was approved, the mission was
|
||||||
|
//! created and someone believed it was going to run. A model that invents
|
||||||
|
//! `rootfs-opus` is a normal thing for a model to do; discovering it three steps
|
||||||
|
//! later is not. So a roster naming a backend the fleet cannot run is rejected
|
||||||
|
//! when it is proposed, naming the backends that do exist.
|
||||||
|
//!
|
||||||
|
//! # What it deliberately does not do
|
||||||
|
//!
|
||||||
|
//! It does not mint claws. A composed mission's nodes are VMs, and provisioning
|
||||||
|
//! containers for them would create agents and `.brain` files nothing ever
|
||||||
|
//! dials — the same reason `on_launch` returns early for a microVM mission.
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
/// One member of a proposed roster.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct RosterMember {
|
||||||
|
/// The node's role, e.g. `implementer`, `verifier`. Becomes the graph node's
|
||||||
|
/// role, which is what the per-node prompt is written around.
|
||||||
|
pub role: String,
|
||||||
|
/// Which rootfs image this node's VM boots (`missions.backend` per node).
|
||||||
|
/// `None` inherits the mission's.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub backend: Option<String>,
|
||||||
|
/// One line on why this member exists. Not consumed by anything — kept
|
||||||
|
/// because a roster nobody can read is a roster nobody can refuse.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub rationale: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A proposed shape for a mission.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
|
pub struct Roster {
|
||||||
|
/// A `cm_topology::TopologyKind` name — `pipeline`, `hub_spoke`, …
|
||||||
|
pub topology_kind: String,
|
||||||
|
pub members: Vec<RosterMember>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ceiling on a proposed roster.
|
||||||
|
///
|
||||||
|
/// Each member is a whole VM: a boot, an inject, an agent session and a collect.
|
||||||
|
/// Anthropic's own guidance tops out at 3-5 subagents, and every member here
|
||||||
|
/// costs far more than a subagent does. A model asked to size a team will
|
||||||
|
/// cheerfully propose twelve.
|
||||||
|
pub const MAX_MEMBERS: usize = 6;
|
||||||
|
|
||||||
|
/// Why a roster was refused.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub enum Refusal {
|
||||||
|
Empty,
|
||||||
|
TooMany(usize),
|
||||||
|
BlankRole(usize),
|
||||||
|
/// A backend no online node can run, with the ones that exist.
|
||||||
|
UnknownBackend { backend: String, available: Vec<String> },
|
||||||
|
UnknownTopology(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Display for Refusal {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
match self {
|
||||||
|
Refusal::Empty => write!(f, "the roster has no members, so there is nothing to run"),
|
||||||
|
Refusal::TooMany(n) => write!(
|
||||||
|
f,
|
||||||
|
"the roster has {n} members and the ceiling is {MAX_MEMBERS} — each one is a whole \
|
||||||
|
VM, not a subagent"
|
||||||
|
),
|
||||||
|
Refusal::BlankRole(i) => write!(f, "member {i} has no role"),
|
||||||
|
Refusal::UnknownBackend { backend, available } => write!(
|
||||||
|
f,
|
||||||
|
"no online node can run backend {backend:?}; the fleet has: {}",
|
||||||
|
if available.is_empty() {
|
||||||
|
"(none — no node reports a microvm rootfs)".to_string()
|
||||||
|
} else {
|
||||||
|
available.join(", ")
|
||||||
|
}
|
||||||
|
),
|
||||||
|
Refusal::UnknownTopology(k) => write!(
|
||||||
|
f,
|
||||||
|
"{k:?} is not a topology kind this platform can plan; use one of: {}",
|
||||||
|
cm_topology::TopologyKind::ALL
|
||||||
|
.iter()
|
||||||
|
.map(|k| k.as_str())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ")
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Roster {
|
||||||
|
/// Check a roster against the platform and the fleet.
|
||||||
|
///
|
||||||
|
/// `available` is the set of backends at least one ONLINE node can boot.
|
||||||
|
/// Fail-closed on every axis: an unrecognised topology, a blank role and an
|
||||||
|
/// unbuildable backend are all refusals, because each of them becomes a
|
||||||
|
/// failure much later and much more expensively.
|
||||||
|
pub fn validate(&self, available: &[String]) -> Result<(), Refusal> {
|
||||||
|
if self.members.is_empty() {
|
||||||
|
return Err(Refusal::Empty);
|
||||||
|
}
|
||||||
|
if self.members.len() > MAX_MEMBERS {
|
||||||
|
return Err(Refusal::TooMany(self.members.len()));
|
||||||
|
}
|
||||||
|
if parse_kind(&self.topology_kind).is_none() {
|
||||||
|
return Err(Refusal::UnknownTopology(self.topology_kind.clone()));
|
||||||
|
}
|
||||||
|
for (i, m) in self.members.iter().enumerate() {
|
||||||
|
if m.role.trim().is_empty() {
|
||||||
|
return Err(Refusal::BlankRole(i));
|
||||||
|
}
|
||||||
|
if let Some(b) = m.backend.as_deref().map(str::trim).filter(|b| !b.is_empty()) {
|
||||||
|
if !available.iter().any(|a| a == b) {
|
||||||
|
return Err(Refusal::UnknownBackend {
|
||||||
|
backend: b.to_string(),
|
||||||
|
available: available.to_vec(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The graph a composed run executes.
|
||||||
|
///
|
||||||
|
/// Node ids follow `cm_topology::build`'s `n0..` convention so the graph is
|
||||||
|
/// indistinguishable from a template-built one — the executor, the planners
|
||||||
|
/// and the checkpoint all treat it the same. The per-member backend rides in
|
||||||
|
/// `attrs`, which is the channel `MicroVmTurnExecutor` already reads.
|
||||||
|
pub fn graph(&self) -> Result<serde_json::Value, String> {
|
||||||
|
let kind = parse_kind(&self.topology_kind)
|
||||||
|
.ok_or_else(|| format!("unknown topology kind {:?}", self.topology_kind))?;
|
||||||
|
let roles: Vec<&str> = self.members.iter().map(|m| m.role.trim()).collect();
|
||||||
|
let mut graph =
|
||||||
|
cm_topology::build(kind, &roles).map_err(|e| format!("build topology: {e}"))?;
|
||||||
|
for (node, member) in graph.nodes.iter_mut().zip(self.members.iter()) {
|
||||||
|
if let Some(b) = member
|
||||||
|
.backend
|
||||||
|
.as_deref()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|b| !b.is_empty())
|
||||||
|
{
|
||||||
|
node.attrs.insert("backend".to_string(), b.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
serde_json::to_value(&graph).map_err(|e| format!("serialize graph: {e}"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Topology kind by name, accepting exactly what the catalog declares.
|
||||||
|
///
|
||||||
|
/// Deliberately not `unwrap_or(HubSpoke)`. `mission_orchestrator::
|
||||||
|
/// parse_topology_kind` does default, which is right for a stored template
|
||||||
|
/// written by us and wrong for a string a model just invented: silently running
|
||||||
|
/// a `pipeline` proposal as a hub-and-spoke would change what every node sees
|
||||||
|
/// and nothing would say so.
|
||||||
|
fn parse_kind(s: &str) -> Option<cm_topology::TopologyKind> {
|
||||||
|
let want = s.trim();
|
||||||
|
cm_topology::TopologyKind::ALL
|
||||||
|
.iter()
|
||||||
|
.copied()
|
||||||
|
.find(|k| k.as_str().eq_ignore_ascii_case(want))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Backends at least one online node can actually boot.
|
||||||
|
///
|
||||||
|
/// Read from the nodes' reported `rootfs` capability, so it answers "what can
|
||||||
|
/// run today" rather than "what images did someone build once".
|
||||||
|
pub async fn available_backends(
|
||||||
|
pool: &sqlx::PgPool,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
) -> Result<Vec<String>, String> {
|
||||||
|
let rows: Vec<(serde_json::Value,)> = sqlx::query_as(
|
||||||
|
"SELECT capabilities -> 'rootfs'
|
||||||
|
FROM nodes
|
||||||
|
WHERE workspace_id = $1 AND status = 'online'
|
||||||
|
AND capabilities @> '{\"microvm\": true}'::jsonb",
|
||||||
|
)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("read node rootfs capabilities: {e}"))?;
|
||||||
|
|
||||||
|
let mut out: Vec<String> = rows
|
||||||
|
.into_iter()
|
||||||
|
.filter_map(|(v,)| v.as_array().cloned())
|
||||||
|
.flatten()
|
||||||
|
.filter_map(|v| v.as_str().map(str::to_string))
|
||||||
|
// A node reports every rootfs it has BUILT, which is not the same as
|
||||||
|
// every rootfs a mission can run in. `agent-terminal` is on tank right
|
||||||
|
// now: bootable, and with no credential contract, so an agent inside it
|
||||||
|
// has nothing to authenticate with. Offering it to the planner would
|
||||||
|
// produce a roster that validates, approves, launches, and then fails at
|
||||||
|
// the agent turn — the expensive kind of late.
|
||||||
|
.filter(|b| crate::mission_runtime::backend_can_run_a_mission(b))
|
||||||
|
.collect();
|
||||||
|
out.sort();
|
||||||
|
out.dedup();
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn member(role: &str, backend: Option<&str>) -> RosterMember {
|
||||||
|
RosterMember {
|
||||||
|
role: role.into(),
|
||||||
|
backend: backend.map(str::to_string),
|
||||||
|
rationale: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn roster(kind: &str, members: Vec<RosterMember>) -> Roster {
|
||||||
|
Roster {
|
||||||
|
topology_kind: kind.into(),
|
||||||
|
members,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A backend the fleet cannot boot must be refused where it is PROPOSED.
|
||||||
|
/// Placement would refuse it too — at launch, after the roster was approved
|
||||||
|
/// and someone believed the mission was going to run.
|
||||||
|
#[test]
|
||||||
|
fn a_backend_no_node_can_run_is_refused_with_the_ones_that_exist() {
|
||||||
|
let have = vec!["claude".to_string(), "kimi".to_string()];
|
||||||
|
let r = roster(
|
||||||
|
"pipeline",
|
||||||
|
vec![member("implementer", Some("claude")), member("verifier", Some("rootfs-opus"))],
|
||||||
|
);
|
||||||
|
let err = r.validate(&have).unwrap_err();
|
||||||
|
assert_eq!(
|
||||||
|
err,
|
||||||
|
Refusal::UnknownBackend {
|
||||||
|
backend: "rootfs-opus".into(),
|
||||||
|
available: have.clone()
|
||||||
|
}
|
||||||
|
);
|
||||||
|
// The message must name what IS available, or the operator's next move
|
||||||
|
// is a guess.
|
||||||
|
let msg = err.to_string();
|
||||||
|
assert!(msg.contains("claude") && msg.contains("kimi"), "{msg}");
|
||||||
|
|
||||||
|
// And the same roster passes once every backend is one the fleet has.
|
||||||
|
let ok = roster(
|
||||||
|
"pipeline",
|
||||||
|
vec![member("implementer", Some("claude")), member("verifier", Some("kimi"))],
|
||||||
|
);
|
||||||
|
assert!(ok.validate(&have).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A member with no backend inherits the mission's, which is legitimate —
|
||||||
|
/// the whole roster does not have to be heterogeneous to be useful.
|
||||||
|
#[test]
|
||||||
|
fn a_member_without_a_backend_is_not_a_refusal() {
|
||||||
|
let r = roster("pipeline", vec![member("implementer", None)]);
|
||||||
|
assert!(r.validate(&["claude".to_string()]).is_ok());
|
||||||
|
// Blank counts as absent, not as a backend named "".
|
||||||
|
let r = roster("pipeline", vec![member("implementer", Some(" "))]);
|
||||||
|
assert!(r.validate(&["claude".to_string()]).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A bootable image is not necessarily a runnable one. tank reports
|
||||||
|
/// `agent-terminal` in its rootfs list today: a real image, with no
|
||||||
|
/// credential contract, so an agent booted into it has nothing to
|
||||||
|
/// authenticate with. Offering it to the planner would produce a roster that
|
||||||
|
/// validates, approves, launches and then fails at the agent turn.
|
||||||
|
#[test]
|
||||||
|
fn only_backends_that_can_authenticate_are_offered() {
|
||||||
|
assert!(crate::mission_runtime::backend_can_run_a_mission("claude"));
|
||||||
|
assert!(crate::mission_runtime::backend_can_run_a_mission("default"));
|
||||||
|
for unrunnable in ["agent-terminal", "agent-browser", "rootfs-opus"] {
|
||||||
|
assert!(
|
||||||
|
!crate::mission_runtime::backend_can_run_a_mission(unrunnable),
|
||||||
|
"{unrunnable} has no credential contract and must not be proposable"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The ceiling. Each member is a VM boot, an inject, a full agent session
|
||||||
|
/// and a collect — a model asked to size a team proposes twelve happily.
|
||||||
|
#[test]
|
||||||
|
fn a_roster_is_bounded_and_non_empty() {
|
||||||
|
let have = vec!["claude".to_string()];
|
||||||
|
assert_eq!(roster("pipeline", vec![]).validate(&have), Err(Refusal::Empty));
|
||||||
|
|
||||||
|
let many: Vec<_> = (0..MAX_MEMBERS + 1)
|
||||||
|
.map(|i| member(&format!("r{i}"), None))
|
||||||
|
.collect();
|
||||||
|
assert_eq!(
|
||||||
|
roster("pipeline", many).validate(&have),
|
||||||
|
Err(Refusal::TooMany(MAX_MEMBERS + 1))
|
||||||
|
);
|
||||||
|
|
||||||
|
let exactly: Vec<_> = (0..MAX_MEMBERS).map(|i| member(&format!("r{i}"), None)).collect();
|
||||||
|
assert!(roster("pipeline", exactly).validate(&have).is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An invented topology kind must be refused, NOT defaulted. Running a
|
||||||
|
/// `pipeline` proposal as a hub-and-spoke changes what every node sees and
|
||||||
|
/// nothing would say so — the same silent-substitution shape as a backend
|
||||||
|
/// that quietly falls back to the default image.
|
||||||
|
#[test]
|
||||||
|
fn an_invented_topology_kind_is_refused_rather_than_defaulted() {
|
||||||
|
let have = vec!["claude".to_string()];
|
||||||
|
let r = roster("assembly_line", vec![member("implementer", None)]);
|
||||||
|
assert_eq!(
|
||||||
|
r.validate(&have),
|
||||||
|
Err(Refusal::UnknownTopology("assembly_line".into()))
|
||||||
|
);
|
||||||
|
// Every kind the catalog declares is accepted, so this cannot drift out
|
||||||
|
// of sync with what the orchestrator can actually plan.
|
||||||
|
for kind in cm_topology::TopologyKind::ALL {
|
||||||
|
let r = roster(kind.as_str(), vec![member("implementer", None)]);
|
||||||
|
assert!(r.validate(&have).is_ok(), "{}", kind.as_str());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The graph is the handoff to the composed executor: node ids in
|
||||||
|
/// `cm_topology`'s own convention, and the backend in the `attrs` channel
|
||||||
|
/// `MicroVmTurnExecutor` reads. If this drifts, a heterogeneous roster runs
|
||||||
|
/// every node on the mission default and looks fine.
|
||||||
|
#[test]
|
||||||
|
fn the_graph_carries_each_members_backend_where_the_executor_reads_it() {
|
||||||
|
let r = roster(
|
||||||
|
"pipeline",
|
||||||
|
vec![
|
||||||
|
member("implementer", Some("claude")),
|
||||||
|
member("verifier", Some("kimi")),
|
||||||
|
member("scribe", None),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
let g = r.graph().expect("a runnable graph");
|
||||||
|
let nodes = g["nodes"].as_array().expect("nodes");
|
||||||
|
assert_eq!(nodes.len(), 3);
|
||||||
|
assert_eq!(nodes[0]["role"], "implementer");
|
||||||
|
assert_eq!(nodes[0]["attrs"]["backend"], "claude");
|
||||||
|
assert_eq!(nodes[1]["attrs"]["backend"], "kimi");
|
||||||
|
assert!(
|
||||||
|
nodes[2]["attrs"].get("backend").is_none(),
|
||||||
|
"a member with no backend must inherit the mission's, not be stamped with one"
|
||||||
|
);
|
||||||
|
|
||||||
|
// And it deserializes as the real thing the worker will parse — a graph
|
||||||
|
// that only looks right as JSON fails at claim time with "missing or
|
||||||
|
// invalid graph", which reads as a runtime fault rather than a bad
|
||||||
|
// roster.
|
||||||
|
let parsed: cm_topology::TopologyGraph =
|
||||||
|
serde_json::from_value(g).expect("the worker must be able to parse it");
|
||||||
|
assert_eq!(parsed.nodes.len(), 3);
|
||||||
|
assert_eq!(
|
||||||
|
parsed.nodes[1].attrs.get("backend").map(String::as_str),
|
||||||
|
Some("kimi")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -11,7 +11,10 @@
|
|||||||
//! - no repo_id → no-op (Ok(None))
|
//! - no repo_id → no-op (Ok(None))
|
||||||
//! - dir already a git repo → `fetch + reset --hard origin/<branch>`
|
//! - dir already a git repo → `fetch + reset --hard origin/<branch>`
|
||||||
//! to bring it in sync
|
//! to bring it in sync
|
||||||
//! - dir missing → `git clone --depth 1 <url> <path>`
|
//! - dir missing → `git clone --filter=blob:none --single-branch` (NOT
|
||||||
|
//! `--depth 1`: a shallow clone cannot push a new branch back, and delivery
|
||||||
|
//! needs exactly that — see `clone`). A mission with a `security_scan` phase
|
||||||
|
//! gets a FULLY HYDRATED clone instead; see `wants_full_history`.
|
||||||
//!
|
//!
|
||||||
//! Auth: for `git.redclaw.dev` clones we inject the ambient
|
//! Auth: for `git.redclaw.dev` clones we inject the ambient
|
||||||
//! `GITEA_TOKEN` (already provisioned in the server container's env)
|
//! `GITEA_TOKEN` (already provisioned in the server container's env)
|
||||||
@@ -23,7 +26,17 @@ use std::path::PathBuf;
|
|||||||
use tokio::process::Command;
|
use tokio::process::Command;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
pub(crate) fn missions_root() -> PathBuf {
|
/// The ONE definition of where mission state lives on the docker host.
|
||||||
|
///
|
||||||
|
/// There used to be five: this function, three private copies of the same
|
||||||
|
/// `env::var(...).unwrap_or(...)` in `security_scan`, `benchmark_runner` and
|
||||||
|
/// `mission_outputs`, and a hardcoded `MISSIONS_HOST_ROOT` const in
|
||||||
|
/// `mission_runtime` that read no env at all. They agree on today's
|
||||||
|
/// deployment, which is why nothing had broken — but anything that sweeps or
|
||||||
|
/// reclaims this tree has to be sure it is sweeping the same tree the writers
|
||||||
|
/// use, and five definitions cannot promise that. A GC written against one of
|
||||||
|
/// them would silently miss the others.
|
||||||
|
pub fn missions_root() -> PathBuf {
|
||||||
std::env::var("CLAWMATES_MISSIONS_ROOT")
|
std::env::var("CLAWMATES_MISSIONS_ROOT")
|
||||||
.map(PathBuf::from)
|
.map(PathBuf::from)
|
||||||
.unwrap_or_else(|_| PathBuf::from("/var/lib/clawmates-missions"))
|
.unwrap_or_else(|_| PathBuf::from("/var/lib/clawmates-missions"))
|
||||||
@@ -65,7 +78,19 @@ pub async fn ensure_checkout(
|
|||||||
.map_err(|e| format!("mkdir {}: {e}", parent.display()))?;
|
.map_err(|e| format!("mkdir {}: {e}", parent.display()))?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let auth_url = with_ambient_auth(clone_url);
|
let auth = with_ambient_auth(clone_url);
|
||||||
|
// Said once, here, where the checkout is created: every later git call uses
|
||||||
|
// a URL built the same way, so an unauthenticated forge URL is a fact worth
|
||||||
|
// one line now rather than a `/dev/tty` error later.
|
||||||
|
if let Some(why) = &auth.unauthenticated {
|
||||||
|
if auth.is_forge() {
|
||||||
|
eprintln!(
|
||||||
|
"mission_workspace: mission {mission_id} will talk to the forge \
|
||||||
|
WITHOUT credentials — {why}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let auth_url = auth.url;
|
||||||
if path.join(".git").exists() {
|
if path.join(".git").exists() {
|
||||||
// Checkouts cloned before this setting existed get it on reuse. It
|
// Checkouts cloned before this setting existed get it on reuse. It
|
||||||
// governs objects created from now on, which is what delivery needs.
|
// governs objects created from now on, which is what delivery needs.
|
||||||
@@ -87,43 +112,181 @@ pub async fn ensure_checkout(
|
|||||||
fetch_and_reset(&path, default_branch, &auth_url).await?;
|
fetch_and_reset(&path, default_branch, &auth_url).await?;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
clone(&path, &auth_url).await?;
|
clone(&path, &auth_url, wants_full_history(pool, mission_id).await).await?;
|
||||||
}
|
}
|
||||||
Ok(Some(path))
|
Ok(Some(path))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// If the URL points at git.redclaw.dev AND GITEA_TOKEN is set in the
|
/// The forge whose URLs the ambient `GITEA_TOKEN` can authenticate.
|
||||||
/// environment, rewrite it to include the token as basic-auth. Returns
|
const FORGE_HOST: &str = "git.redclaw.dev";
|
||||||
/// the URL unchanged otherwise. The token is never logged (we only
|
|
||||||
/// pass the rewritten URL into `git clone` via argv).
|
/// A URL, and whether a credential actually reached it.
|
||||||
pub(crate) fn with_ambient_auth(url: &str) -> String {
|
///
|
||||||
let Ok(token) = std::env::var("GITEA_TOKEN") else {
|
/// The second field is the whole point. This used to be a bare `String`: an
|
||||||
return url.to_string();
|
/// unmatched URL — an ssh remote, `http://` instead of `https://`, an explicit
|
||||||
};
|
/// port, a different case in the host — silently came back unauthenticated, and
|
||||||
if token.is_empty() {
|
/// the first symptom was git opening `/dev/tty` several layers later. Tracing
|
||||||
return url.to_string();
|
/// #55 cost hours to a failure whose cause was one unlogged early return.
|
||||||
}
|
pub struct Authed {
|
||||||
if let Some(rest) = url.strip_prefix("https://git.redclaw.dev/") {
|
pub url: String,
|
||||||
return format!("https://oauth2:{token}@git.redclaw.dev/{rest}");
|
/// `None` when the token was applied; otherwise WHY it was not.
|
||||||
}
|
pub unauthenticated: Option<String>,
|
||||||
url.to_string()
|
/// Whether the URL names the forge our token is for. Recorded from the
|
||||||
|
/// ORIGINAL url, not re-derived from `url` — an authenticated URL carries
|
||||||
|
/// userinfo, and parsing that back out is how the answer goes wrong.
|
||||||
|
forge: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn clone(path: &std::path::Path, url: &str) -> Result<(), String> {
|
impl Authed {
|
||||||
|
/// Is this URL on the forge our token is for? An unauthenticated URL to a
|
||||||
|
/// third-party host is normal (public repos, ssh remotes with a key); an
|
||||||
|
/// unauthenticated URL to OUR forge is a fault, and only the caller knows
|
||||||
|
/// how much it costs.
|
||||||
|
pub fn is_forge(&self) -> bool {
|
||||||
|
self.forge
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The host component of a URL, for the scp-like and scheme forms git accepts.
|
||||||
|
///
|
||||||
|
/// Deliberately tolerant, because the point is to RECOGNISE our forge in every
|
||||||
|
/// shape it can be written, not to validate URLs: `https://`, `http://`, an
|
||||||
|
/// explicit `:port`, `user@host`, `ssh://`, and `git@host:path`.
|
||||||
|
fn host_of(url: &str) -> Option<&str> {
|
||||||
|
let rest = match url.split_once("://") {
|
||||||
|
Some((_, rest)) => rest,
|
||||||
|
// scp-like: `git@host:path/to.git`, which has no scheme.
|
||||||
|
None => url,
|
||||||
|
};
|
||||||
|
// Userinfo FIRST, then the port. The other order splits
|
||||||
|
// `oauth2:token@host` at the credential's colon and reports the username as
|
||||||
|
// the host — which is exactly how the first version of this function decided
|
||||||
|
// an authenticated forge URL was not the forge.
|
||||||
|
let authority = rest.split('/').next().filter(|s| !s.is_empty())?;
|
||||||
|
let hostport = authority.rsplit_once('@').map_or(authority, |(_, h)| h);
|
||||||
|
Some(hostport.split(':').next().unwrap_or(hostport)).filter(|h| !h.is_empty())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rewrite a forge URL to carry the ambient `GITEA_TOKEN` as basic-auth.
|
||||||
|
///
|
||||||
|
/// Returns the reason instead of the credential whenever it cannot: a missing
|
||||||
|
/// token, a host that is not ours, or a shape a token cannot be injected into.
|
||||||
|
/// The token is never logged — only the rewritten URL is passed to git, via
|
||||||
|
/// argv.
|
||||||
|
pub fn with_ambient_auth(url: &str) -> Authed {
|
||||||
|
auth_with_token(url, std::env::var("GITEA_TOKEN").ok().as_deref())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The testable half of [`with_ambient_auth`]. The token is a parameter because
|
||||||
|
/// a test cannot set process environment variables here — the workspace denies
|
||||||
|
/// `unsafe`, and `set_var` is racy across test threads regardless.
|
||||||
|
fn auth_with_token(url: &str, token: Option<&str>) -> Authed {
|
||||||
|
let host = host_of(url);
|
||||||
|
let forge = host.is_some_and(|h| h.eq_ignore_ascii_case(FORGE_HOST));
|
||||||
|
let unauth = |why: String| Authed {
|
||||||
|
url: url.to_string(),
|
||||||
|
unauthenticated: Some(why),
|
||||||
|
forge,
|
||||||
|
};
|
||||||
|
|
||||||
|
let host = match host {
|
||||||
|
Some(h) => h,
|
||||||
|
None => return unauth(format!("no host could be read from {url:?}")),
|
||||||
|
};
|
||||||
|
if !forge {
|
||||||
|
return unauth(format!(
|
||||||
|
"{host} is not {FORGE_HOST}, so GITEA_TOKEN does not apply — git will \
|
||||||
|
use whatever ambient credentials exist (ssh agent, .netrc, helper)"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let token = match token {
|
||||||
|
Some(t) if !t.trim().is_empty() => t,
|
||||||
|
_ => return unauth("GITEA_TOKEN is unset or empty".to_string()),
|
||||||
|
};
|
||||||
|
// Only the scheme forms can carry basic-auth. An ssh remote authenticates
|
||||||
|
// with a key, and pretending otherwise would produce a URL git rejects.
|
||||||
|
let Some((scheme, rest)) = url.split_once("://") else {
|
||||||
|
return unauth(format!(
|
||||||
|
"{url} is an ssh-style remote; a token cannot be embedded in it"
|
||||||
|
));
|
||||||
|
};
|
||||||
|
if !matches!(scheme, "http" | "https") {
|
||||||
|
return unauth(format!("scheme {scheme} cannot carry a token"));
|
||||||
|
}
|
||||||
|
// Drop any userinfo already present rather than producing `a@b@host`.
|
||||||
|
let rest = rest.split_once('@').map(|(_, r)| r).unwrap_or(rest);
|
||||||
|
Authed {
|
||||||
|
url: format!("{scheme}://oauth2:{token}@{rest}"),
|
||||||
|
unauthenticated: None,
|
||||||
|
forge,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Git must never wait for a human.
|
||||||
|
///
|
||||||
|
/// Without this, a URL that ended up without credentials does not fail — git
|
||||||
|
/// opens `/dev/tty` to ask for a username, and in a server container that
|
||||||
|
/// surfaces as `No such device or address`, several layers away from the
|
||||||
|
/// missing token that caused it. With it, the failure names itself:
|
||||||
|
/// `terminal prompts disabled`.
|
||||||
|
pub(crate) fn no_terminal_prompt(cmd: &mut Command) -> &mut Command {
|
||||||
|
cmd.env("GIT_TERMINAL_PROMPT", "0")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Does any phase of this mission need history it can READ, not just reference?
|
||||||
|
///
|
||||||
|
/// `--filter=blob:none` keeps every commit but fetches file contents on demand,
|
||||||
|
/// which is nearly free for a repo that gets read once — and silently useless to
|
||||||
|
/// a tool that walks history, because the agent environment has NO network route
|
||||||
|
/// to the forge. Measured: gitleaks on a 4-commit repo reported
|
||||||
|
/// "1 commits scanned" and "could not fetch <sha> from promisor remote". It was
|
||||||
|
/// not misconfigured; the blobs simply were not there and could not be got.
|
||||||
|
///
|
||||||
|
/// A security scan is the phase kind whose entire value is old content — a
|
||||||
|
/// credential committed and later deleted is exactly what it looks for, and that
|
||||||
|
/// is precisely what a lazy blob is. So those missions pay for a full clone and
|
||||||
|
/// everything else keeps the cheap one.
|
||||||
|
///
|
||||||
|
/// Best-effort: an unreadable phase list yields `false`, i.e. today's behaviour.
|
||||||
|
async fn wants_full_history(pool: &sqlx::PgPool, mission_id: Uuid) -> bool {
|
||||||
|
sqlx::query_scalar::<_, i64>(
|
||||||
|
"SELECT count(*) FROM mission_phases WHERE mission_id = $1 AND kind = 'security_scan'",
|
||||||
|
)
|
||||||
|
.bind(mission_id)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.map(|n| n > 0)
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The `git clone` flags, split out so the strategy is testable without a forge.
|
||||||
|
fn clone_args(full_history: bool) -> Vec<&'static str> {
|
||||||
|
let mut a = vec!["clone"];
|
||||||
|
if !full_history {
|
||||||
|
a.push("--filter=blob:none");
|
||||||
|
}
|
||||||
|
a.push("--single-branch");
|
||||||
|
a
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn clone(path: &std::path::Path, url: &str, full_history: bool) -> Result<(), String> {
|
||||||
// `--filter=blob:none` rather than `--depth 1`. A shallow clone cannot
|
// `--filter=blob:none` rather than `--depth 1`. A shallow clone cannot
|
||||||
// usually push a new branch back ("shallow update not allowed"), and
|
// usually push a new branch back ("shallow update not allowed"), and
|
||||||
// mission delivery needs exactly that. A partial clone keeps full history
|
// mission delivery needs exactly that. A partial clone keeps full history
|
||||||
// — so the base commit stays meaningful and a diff has something to be
|
// — so the base commit stays meaningful and a diff has something to be
|
||||||
// relative to — while fetching file contents only on demand, which is
|
// relative to — while fetching file contents only on demand, which is
|
||||||
// nearly as cheap as a shallow clone for a repo that gets read once.
|
// nearly as cheap as a shallow clone for a repo that gets read once.
|
||||||
let out = Command::new("git")
|
if full_history {
|
||||||
.args([
|
eprintln!(
|
||||||
"clone",
|
"mission_workspace: cloning {} with full history — a security_scan phase \
|
||||||
"--filter=blob:none",
|
reads old file contents, which a partial clone cannot supply offline",
|
||||||
"--single-branch",
|
path.display()
|
||||||
url,
|
);
|
||||||
&path.display().to_string(),
|
}
|
||||||
])
|
let mut cmd = Command::new("git");
|
||||||
|
cmd.args(clone_args(full_history));
|
||||||
|
cmd.args([url, &path.display().to_string()]);
|
||||||
|
let out = no_terminal_prompt(&mut cmd)
|
||||||
.output()
|
.output()
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("spawn git clone: {e}"))?;
|
.map_err(|e| format!("spawn git clone: {e}"))?;
|
||||||
@@ -131,10 +294,11 @@ async fn clone(path: &std::path::Path, url: &str) -> Result<(), String> {
|
|||||||
return Err(format!(
|
return Err(format!(
|
||||||
"git clone → exit {}: {}",
|
"git clone → exit {}: {}",
|
||||||
out.status,
|
out.status,
|
||||||
redact_token(&String::from_utf8_lossy(&out.stderr))
|
// Both ends: git prints its reason LAST, and a head-only clamp keeps
|
||||||
.chars()
|
// the progress noise while dropping the answer.
|
||||||
.take(400)
|
crate::evaluator_tools::clamp_output(&redact_token(&String::from_utf8_lossy(
|
||||||
.collect::<String>()
|
&out.stderr
|
||||||
|
)))
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
share_repository_across_uids(path);
|
share_repository_across_uids(path);
|
||||||
@@ -447,7 +611,13 @@ fn strip_credentials(url: &str) -> String {
|
|||||||
/// They are the agent's identity scaffolding, not the user's code — `SOUL.md`
|
/// They are the agent's identity scaffolding, not the user's code — `SOUL.md`
|
||||||
/// opens "Who You Are / You're not a chatbot." Observed on mission 019fc058,
|
/// opens "Who You Are / You're not a chatbot." Observed on mission 019fc058,
|
||||||
/// where all seven appeared as untracked files in a freshly cloned repo.
|
/// where all seven appeared as untracked files in a freshly cloned repo.
|
||||||
const AGENT_SCAFFOLDING: &[&str] = &[
|
///
|
||||||
|
/// `pub(crate)` because a repo-less mission needs the same list and cannot use
|
||||||
|
/// the same mechanism: `ignore_agent_scaffolding` writes `.git/info/exclude`,
|
||||||
|
/// and a mission with no repository has no `.git`. `mission_outputs` filters on
|
||||||
|
/// this list directly — one list, two consumers, so the next file the runtime
|
||||||
|
/// starts seeding is excluded from both at once.
|
||||||
|
pub(crate) const AGENT_SCAFFOLDING: &[&str] = &[
|
||||||
"AGENTS.md",
|
"AGENTS.md",
|
||||||
"HEARTBEAT.md",
|
"HEARTBEAT.md",
|
||||||
"IDENTITY.md",
|
"IDENTITY.md",
|
||||||
@@ -523,16 +693,15 @@ async fn fetch_and_reset(
|
|||||||
// errors on a repo that is already complete, so it is only attempted when
|
// errors on a repo that is already complete, so it is only attempted when
|
||||||
// the marker file is present.
|
// the marker file is present.
|
||||||
if path.join(".git/shallow").exists() {
|
if path.join(".git/shallow").exists() {
|
||||||
let deepen = Command::new("git")
|
let mut cmd = Command::new("git");
|
||||||
.args([
|
cmd.args([
|
||||||
"-C",
|
"-C",
|
||||||
&path.display().to_string(),
|
&path.display().to_string(),
|
||||||
"fetch",
|
"fetch",
|
||||||
"--unshallow",
|
"--unshallow",
|
||||||
auth_url,
|
auth_url,
|
||||||
])
|
]);
|
||||||
.output()
|
let deepen = no_terminal_prompt(&mut cmd).output().await;
|
||||||
.await;
|
|
||||||
match deepen {
|
match deepen {
|
||||||
Ok(o) if o.status.success() => {}
|
Ok(o) if o.status.success() => {}
|
||||||
Ok(o) => eprintln!(
|
Ok(o) => eprintln!(
|
||||||
@@ -556,8 +725,9 @@ async fn fetch_and_reset(
|
|||||||
// so `git fetch origin` has no credentials and fails with
|
// so `git fetch origin` has no credentials and fails with
|
||||||
// "could not read Username". Building the URL here also means a rotated
|
// "could not read Username". Building the URL here also means a rotated
|
||||||
// token takes effect immediately instead of at the next clone.
|
// token takes effect immediately instead of at the next clone.
|
||||||
let fetch = Command::new("git")
|
let mut cmd = Command::new("git");
|
||||||
.args(["-C", &path.display().to_string(), "fetch", auth_url, branch])
|
cmd.args(["-C", &path.display().to_string(), "fetch", auth_url, branch]);
|
||||||
|
let fetch = no_terminal_prompt(&mut cmd)
|
||||||
.output()
|
.output()
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("spawn git fetch: {e}"))?;
|
.map_err(|e| format!("spawn git fetch: {e}"))?;
|
||||||
@@ -565,10 +735,9 @@ async fn fetch_and_reset(
|
|||||||
return Err(format!(
|
return Err(format!(
|
||||||
"git fetch origin {branch} → exit {}: {}",
|
"git fetch origin {branch} → exit {}: {}",
|
||||||
fetch.status,
|
fetch.status,
|
||||||
redact_token(&String::from_utf8_lossy(&fetch.stderr))
|
crate::evaluator_tools::clamp_output(&redact_token(&String::from_utf8_lossy(
|
||||||
.chars()
|
&fetch.stderr
|
||||||
.take(400)
|
)))
|
||||||
.collect::<String>()
|
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let reset = Command::new("git")
|
let reset = Command::new("git")
|
||||||
@@ -600,8 +769,102 @@ async fn fetch_and_reset(
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
/// The clone strategy is a fact worth pinning: `--depth 1` breaks delivery
|
||||||
|
/// (a shallow clone cannot push a new branch — "shallow update not
|
||||||
|
/// allowed"), and `--filter=blob:none` breaks history-reading tools offline.
|
||||||
|
/// Both failure modes are real and were both hit.
|
||||||
|
#[test]
|
||||||
|
fn the_clone_strategy_is_partial_by_default_and_never_shallow() {
|
||||||
|
let partial = clone_args(false);
|
||||||
|
assert!(partial.contains(&"--filter=blob:none"), "{partial:?}");
|
||||||
|
assert!(!partial.iter().any(|a| a.starts_with("--depth")), "{partial:?}");
|
||||||
|
|
||||||
|
// A security_scan mission must NOT get the lazy-blob filter: its scanner
|
||||||
|
// walks old file contents and cannot reach the forge to fetch them.
|
||||||
|
let full = clone_args(true);
|
||||||
|
assert!(!full.contains(&"--filter=blob:none"), "{full:?}");
|
||||||
|
assert!(!full.iter().any(|a| a.starts_with("--depth")), "{full:?}");
|
||||||
|
|
||||||
|
// Both keep --single-branch: the mission only ever works one branch.
|
||||||
|
for args in [partial, full] {
|
||||||
|
assert!(args.contains(&"--single-branch"), "{args:?}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
const TOK: Option<&str> = Some("secret123");
|
||||||
|
|
||||||
|
/// The forge in every shape a remote can be written. Each of these used to
|
||||||
|
/// fall out of the `strip_prefix("https://git.redclaw.dev/")` match and come
|
||||||
|
/// back unauthenticated with no log line — the fail-open found while tracing
|
||||||
|
/// #55.
|
||||||
|
#[test]
|
||||||
|
fn the_forge_is_recognised_however_the_url_is_written() {
|
||||||
|
for url in [
|
||||||
|
"https://git.redclaw.dev/o/r.git",
|
||||||
|
"http://git.redclaw.dev/o/r.git",
|
||||||
|
"https://GIT.RedClaw.dev/o/r.git",
|
||||||
|
"https://git.redclaw.dev:3000/o/r.git",
|
||||||
|
"https://oauth2:[email protected]/o/r.git",
|
||||||
|
] {
|
||||||
|
let a = auth_with_token(url, TOK);
|
||||||
|
assert!(a.is_forge(), "{url} was not recognised as the forge");
|
||||||
|
assert!(
|
||||||
|
a.unauthenticated.is_none(),
|
||||||
|
"{url} → {:?}",
|
||||||
|
a.unauthenticated
|
||||||
|
);
|
||||||
|
assert!(a.url.contains("oauth2:secret123@"), "{}", a.url);
|
||||||
|
// And exactly one set of credentials, not `old@` left behind.
|
||||||
|
assert_eq!(a.url.matches('@').count(), 1, "{}", a.url);
|
||||||
|
}
|
||||||
|
// The port and the scheme survive the rewrite — changing either would
|
||||||
|
// point the push somewhere the operator did not configure.
|
||||||
|
assert!(auth_with_token("https://git.redclaw.dev:3000/o/r.git", TOK)
|
||||||
|
.url
|
||||||
|
.contains("@git.redclaw.dev:3000/o/r.git"));
|
||||||
|
assert!(auth_with_token("http://git.redclaw.dev/o/r.git", TOK)
|
||||||
|
.url
|
||||||
|
.starts_with("http://oauth2:"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every path that cannot authenticate must SAY so. "Unauthenticated and
|
||||||
|
/// silent" is the shape that cost hours: the first symptom was git opening
|
||||||
|
/// /dev/tty, several layers from the cause.
|
||||||
|
#[test]
|
||||||
|
fn an_unauthenticated_url_carries_its_reason() {
|
||||||
|
let cases = [
|
||||||
|
(auth_with_token("https://git.redclaw.dev/o/r.git", None), true),
|
||||||
|
(auth_with_token("https://git.redclaw.dev/o/r.git", Some(" ")), true),
|
||||||
|
(auth_with_token("[email protected]:o/r.git", TOK), true),
|
||||||
|
(auth_with_token("ssh://[email protected]/o/r.git", TOK), true),
|
||||||
|
(auth_with_token("https://github.com/o/r.git", TOK), false),
|
||||||
|
];
|
||||||
|
for (a, is_forge) in cases {
|
||||||
|
let why = a.unauthenticated.as_deref().unwrap_or("");
|
||||||
|
assert!(!why.is_empty(), "{} came back with no reason", a.url);
|
||||||
|
assert_eq!(a.is_forge(), is_forge, "{}", a.url);
|
||||||
|
// And the URL is handed back untouched, so a caller that proceeds
|
||||||
|
// anyway (ssh keys, .netrc) still works.
|
||||||
|
assert!(!a.url.contains("secret123"), "{}", a.url);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A token must never be embedded in a URL for someone else's host.
|
||||||
|
#[test]
|
||||||
|
fn the_token_never_leaves_the_forge() {
|
||||||
|
for url in [
|
||||||
|
"https://github.com/o/r.git",
|
||||||
|
"https://git.redclaw.dev.evil.example/o/r.git",
|
||||||
|
"https://evil.example/git.redclaw.dev/r.git",
|
||||||
|
] {
|
||||||
|
let a = auth_with_token(url, TOK);
|
||||||
|
assert!(!a.url.contains("secret123"), "{url} → {}", a.url);
|
||||||
|
assert!(!a.is_forge(), "{url}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The exclude must be idempotent — `ensure_checkout` re-runs on every
|
/// The exclude must be idempotent — `ensure_checkout` re-runs on every
|
||||||
/// phase, and appending the same block each time would grow the file
|
/// phase, and appending the same block each time would grow the file
|
||||||
/// without bound.
|
/// without bound.
|
||||||
@@ -821,4 +1084,43 @@ mod tests {
|
|||||||
mark_phase_started(repo);
|
mark_phase_started(repo);
|
||||||
assert!(checkout_in_use(repo));
|
assert!(checkout_in_use(repo));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Nobody re-derives the missions root.
|
||||||
|
///
|
||||||
|
/// It had fragmented into five definitions — this function, three private
|
||||||
|
/// `env::var("CLAWMATES_MISSIONS_ROOT")` copies, and a hardcoded const
|
||||||
|
/// that read no env at all. They agreed on the deployed value, so nothing
|
||||||
|
/// ever broke; the risk is entirely in what comes next. Anything that
|
||||||
|
/// sweeps, reclaims or reaps this tree has to be sweeping the same tree the
|
||||||
|
/// writers use, and five definitions cannot promise that.
|
||||||
|
#[test]
|
||||||
|
fn the_missions_root_has_exactly_one_definition() {
|
||||||
|
fn walk(dir: &std::path::Path, out: &mut Vec<std::path::PathBuf>) {
|
||||||
|
for entry in std::fs::read_dir(dir).expect("readable source dir") {
|
||||||
|
let path = entry.expect("readable entry").path();
|
||||||
|
if path.is_dir() {
|
||||||
|
walk(&path, out);
|
||||||
|
} else if path.extension().is_some_and(|e| e == "rs") {
|
||||||
|
out.push(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
|
||||||
|
let mut files = Vec::new();
|
||||||
|
walk(&root, &mut files);
|
||||||
|
|
||||||
|
for path in files {
|
||||||
|
if path.ends_with("mission_workspace.rs") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let src = std::fs::read_to_string(&path).expect("readable source");
|
||||||
|
assert!(
|
||||||
|
!src.contains("var(\"CLAWMATES_MISSIONS_ROOT\")"),
|
||||||
|
"{} reads CLAWMATES_MISSIONS_ROOT itself — call \
|
||||||
|
`mission_workspace::missions_root()` so a reaper and a writer \
|
||||||
|
cannot disagree about which tree they are looking at",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,259 +0,0 @@
|
|||||||
//! LLM + Chromium PDF renderer worker — Slice 6.
|
|
||||||
//!
|
|
||||||
//! Watches `mission_artifacts` for rows with `render_pdf_status =
|
|
||||||
//! 'pending'`. For each:
|
|
||||||
//! 1. Read the source MD from `<mission_root>/<path>` on disk
|
|
||||||
//! 2. Call the configured LLM (default: Gemini 2.5 Flash) with a
|
|
||||||
//! "produce styled HTML" prompt anchored to a design-system
|
|
||||||
//! example. LLM writes HTML with inline CSS.
|
|
||||||
//! 3. Print that HTML to PDF via `chromium --headless
|
|
||||||
//! --print-to-pdf`
|
|
||||||
//! 4. Save the PDF alongside the MD, update `rendered_pdf_path` +
|
|
||||||
//! status = 'done'
|
|
||||||
//!
|
|
||||||
//! Graceful degradation: if `GEMINI_API_KEY` is unset or the
|
|
||||||
//! chromium binary isn't on PATH, the worker marks the row `failed`
|
|
||||||
//! with a descriptive error rather than blocking boot. Ops enables
|
|
||||||
//! rendering by wiring both.
|
|
||||||
//!
|
|
||||||
//! The frontend already renders `rendered_pdf_path` as an "Open PDF"
|
|
||||||
//! button on artifact cards (Slice 2).
|
|
||||||
|
|
||||||
use serde_json::json;
|
|
||||||
use sqlx::PgPool;
|
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
const POLL_INTERVAL: Duration = Duration::from_secs(30);
|
|
||||||
const MAX_PARALLEL: usize = 2;
|
|
||||||
const DEFAULT_MODEL: &str = "gemini-2.5-flash";
|
|
||||||
|
|
||||||
/// Where per-mission artifacts land on disk. Overridable so dev vs.
|
|
||||||
/// prod can move the tree; matches the pattern in
|
|
||||||
/// `research_container::research_workspace_root`.
|
|
||||||
fn missions_root() -> PathBuf {
|
|
||||||
std::env::var("CLAWMATES_MISSIONS_ROOT")
|
|
||||||
.map(PathBuf::from)
|
|
||||||
.unwrap_or_else(|_| PathBuf::from("/var/lib/clawmates-missions"))
|
|
||||||
}
|
|
||||||
|
|
||||||
fn chromium_bin() -> String {
|
|
||||||
std::env::var("CHROMIUM_BIN").unwrap_or_else(|_| "chromium".to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn renderer_model() -> String {
|
|
||||||
std::env::var("CLAWMATES_PDF_RENDERER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Spawn the poller. No-op-friendly: if there's nothing pending or
|
|
||||||
/// no rendering pipeline configured, we still tick + observe.
|
|
||||||
pub fn spawn(pool: PgPool) {
|
|
||||||
tokio::spawn(async move {
|
|
||||||
// Small startup delay so migrations + loaders finish first.
|
|
||||||
tokio::time::sleep(Duration::from_secs(8)).await;
|
|
||||||
let mut ticker = tokio::time::interval(POLL_INTERVAL);
|
|
||||||
ticker.tick().await;
|
|
||||||
loop {
|
|
||||||
ticker.tick().await;
|
|
||||||
if let Err(e) = sweep_once(&pool).await {
|
|
||||||
eprintln!("pdf_renderer: sweep failed: {e}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn sweep_once(pool: &PgPool) -> Result<(), String> {
|
|
||||||
let pending = cm_db::repo::missions::next_pdf_pending(pool, MAX_PARALLEL as i64)
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("next_pdf_pending: {e}"))?;
|
|
||||||
for artifact in pending {
|
|
||||||
let pool = pool.clone();
|
|
||||||
let id = artifact.id;
|
|
||||||
tokio::spawn(async move {
|
|
||||||
match render_one(&pool, &artifact).await {
|
|
||||||
Ok(pdf_path) => {
|
|
||||||
let _ = cm_db::repo::missions::set_pdf_result(&pool, id, Some(&pdf_path), None)
|
|
||||||
.await;
|
|
||||||
eprintln!("pdf_renderer: rendered {id} → {pdf_path}");
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
let _ = cm_db::repo::missions::set_pdf_result(&pool, id, None, Some(&e)).await;
|
|
||||||
eprintln!("pdf_renderer: {id} failed: {e}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn render_one(
|
|
||||||
_pool: &PgPool,
|
|
||||||
artifact: &cm_db::repo::missions::MissionArtifact,
|
|
||||||
) -> Result<String, String> {
|
|
||||||
// 1. Locate the source MD on disk.
|
|
||||||
let mission_root = missions_root().join(artifact.mission_id.to_string());
|
|
||||||
let src_path = mission_root.join(&artifact.path);
|
|
||||||
let md = tokio::fs::read_to_string(&src_path)
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("read {}: {e}", src_path.display()))?;
|
|
||||||
|
|
||||||
// 2. LLM → styled HTML.
|
|
||||||
let html = md_to_html_via_llm(&md, artifact.title.as_deref())
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("llm render: {e}"))?;
|
|
||||||
|
|
||||||
// 3. Chromium → PDF.
|
|
||||||
let tmp = tempdir_for(artifact.id)?;
|
|
||||||
let html_path = tmp.join("in.html");
|
|
||||||
let pdf_path = tmp.join("out.pdf");
|
|
||||||
tokio::fs::write(&html_path, html)
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("write {}: {e}", html_path.display()))?;
|
|
||||||
|
|
||||||
let status = tokio::process::Command::new(chromium_bin())
|
|
||||||
.args([
|
|
||||||
"--headless=new",
|
|
||||||
"--disable-gpu",
|
|
||||||
"--no-sandbox",
|
|
||||||
"--hide-scrollbars",
|
|
||||||
&format!("--print-to-pdf={}", pdf_path.display()),
|
|
||||||
"--print-to-pdf-no-header",
|
|
||||||
"--virtual-time-budget=10000",
|
|
||||||
&format!("file://{}", html_path.display()),
|
|
||||||
])
|
|
||||||
.stderr(std::process::Stdio::piped())
|
|
||||||
.stdout(std::process::Stdio::piped())
|
|
||||||
.status()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("spawn chromium: {e}"))?;
|
|
||||||
if !status.success() {
|
|
||||||
return Err(format!("chromium exited {status}"));
|
|
||||||
}
|
|
||||||
|
|
||||||
// 4. Move next to the source MD so the artifact tree stays self-
|
|
||||||
// contained. Filename derived from the MD path (foo.md → foo.pdf).
|
|
||||||
let out_rel = pdf_sibling(&artifact.path);
|
|
||||||
let out_abs = mission_root.join(&out_rel);
|
|
||||||
if let Some(parent) = out_abs.parent() {
|
|
||||||
tokio::fs::create_dir_all(parent)
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("mkdir {}: {e}", parent.display()))?;
|
|
||||||
}
|
|
||||||
tokio::fs::copy(&pdf_path, &out_abs)
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("copy pdf: {e}"))?;
|
|
||||||
// Best-effort tmp cleanup — the temp dir lives under /tmp so the
|
|
||||||
// OS will reap it anyway.
|
|
||||||
let _ = tokio::fs::remove_dir_all(&tmp).await;
|
|
||||||
Ok(out_rel)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Ask the configured LLM to turn `md` into a fully self-contained
|
|
||||||
/// styled HTML doc. Uses whichever provider `CLAWMATES_PDF_RENDERER_MODEL`
|
|
||||||
/// resolves to. Defaults to Gemini 2.5 Flash + GEMINI_API_KEY.
|
|
||||||
async fn md_to_html_via_llm(md: &str, title: Option<&str>) -> Result<String, String> {
|
|
||||||
let model = renderer_model();
|
|
||||||
// For now we hardcode the Gemini path — anthropic + openai
|
|
||||||
// variants land when the design-system template stabilizes.
|
|
||||||
if !model.starts_with("gemini") {
|
|
||||||
return Err(format!(
|
|
||||||
"renderer model {model} not yet wired (only gemini-* supported in Slice 6)"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let api_key =
|
|
||||||
std::env::var("GEMINI_API_KEY").map_err(|_| "GEMINI_API_KEY unset".to_string())?;
|
|
||||||
|
|
||||||
let system = r#"You are a document typesetter. Given a Markdown source,
|
|
||||||
produce ONE self-contained HTML document that:
|
|
||||||
- Has ALL styles inline in a single <style> block in <head>. No external
|
|
||||||
fonts, no external CSS. System font stack only.
|
|
||||||
- Uses a clean, modern, readable serif for body copy (Georgia / "Iowan Old
|
|
||||||
Style" / "Charter" / serif) and a sans for headings.
|
|
||||||
- Uses ONLY these accent colors: #ff8a7a (heading), #5ec8d8 (link),
|
|
||||||
#101014 (body text), #f7f7f8 (page bg).
|
|
||||||
- Renders code blocks with a monospace stack and a subtle background.
|
|
||||||
- Uses page-break-inside: avoid on headings and images.
|
|
||||||
- Puts a document title in an <h1> at the top if provided.
|
|
||||||
- Includes NOTHING outside the HTML — no ```html fence, no commentary."#;
|
|
||||||
|
|
||||||
let prompt = match title {
|
|
||||||
Some(t) => format!("Document title: {t}\n\nMarkdown:\n\n{md}"),
|
|
||||||
None => md.to_string(),
|
|
||||||
};
|
|
||||||
|
|
||||||
let url = format!(
|
|
||||||
"https://generativelanguage.googleapis.com/v1beta/models/{}:generateContent?key={}",
|
|
||||||
model, api_key
|
|
||||||
);
|
|
||||||
let body = json!({
|
|
||||||
"system_instruction": { "parts": [{ "text": system }] },
|
|
||||||
"contents": [{ "role": "user", "parts": [{ "text": prompt }] }],
|
|
||||||
"generationConfig": {
|
|
||||||
"temperature": 0.2,
|
|
||||||
"maxOutputTokens": 32000,
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
let client = reqwest::Client::builder()
|
|
||||||
.timeout(Duration::from_secs(120))
|
|
||||||
.build()
|
|
||||||
.map_err(|e| format!("http client: {e}"))?;
|
|
||||||
let resp = client
|
|
||||||
.post(&url)
|
|
||||||
.json(&body)
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("gemini call: {e}"))?;
|
|
||||||
if !resp.status().is_success() {
|
|
||||||
let code = resp.status();
|
|
||||||
let body = resp.text().await.unwrap_or_default();
|
|
||||||
return Err(format!("gemini {code}: {}", &body[..body.len().min(500)]));
|
|
||||||
}
|
|
||||||
let json: serde_json::Value = resp.json().await.map_err(|e| format!("gemini json: {e}"))?;
|
|
||||||
let text = json
|
|
||||||
.pointer("/candidates/0/content/parts/0/text")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| "gemini response missing text".to_string())?;
|
|
||||||
// Strip a stray ```html fence if the model added one despite the
|
|
||||||
// system prompt — cheap belt to the suspenders.
|
|
||||||
let cleaned = text
|
|
||||||
.trim()
|
|
||||||
.strip_prefix("```html")
|
|
||||||
.and_then(|s| s.strip_suffix("```"))
|
|
||||||
.map(|s| s.trim())
|
|
||||||
.unwrap_or(text.trim())
|
|
||||||
.to_string();
|
|
||||||
Ok(cleaned)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn tempdir_for(id: uuid::Uuid) -> Result<PathBuf, String> {
|
|
||||||
let dir = std::env::temp_dir().join(format!("clawmates-pdf-{id}"));
|
|
||||||
std::fs::create_dir_all(&dir).map_err(|e| format!("mkdir tmp: {e}"))?;
|
|
||||||
Ok(dir)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `research/v3/spec.md` → `research/v3/spec.pdf`.
|
|
||||||
/// `foo/bar/without_ext` → `foo/bar/without_ext.pdf` (rare — parser
|
|
||||||
/// never emits an extension-less MD, but we're defensive).
|
|
||||||
fn pdf_sibling(md_path: &str) -> String {
|
|
||||||
let p = Path::new(md_path);
|
|
||||||
let stem = p.file_stem().and_then(|s| s.to_str()).unwrap_or("output");
|
|
||||||
let parent = p.parent().map(|x| x.to_string_lossy().to_string());
|
|
||||||
let base = format!("{stem}.pdf");
|
|
||||||
match parent {
|
|
||||||
Some(pp) if !pp.is_empty() => format!("{pp}/{base}"),
|
|
||||||
_ => base,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn pdf_sibling_paths() {
|
|
||||||
assert_eq!(pdf_sibling("research/v3/spec.md"), "research/v3/spec.pdf");
|
|
||||||
assert_eq!(pdf_sibling("spec.md"), "spec.pdf");
|
|
||||||
assert_eq!(pdf_sibling("no_ext"), "no_ext.pdf");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -47,6 +47,17 @@ pub const KNOWN_KEYS: &[KnownKey] = &[
|
|||||||
key: "commit_policy",
|
key: "commit_policy",
|
||||||
read_by: "mission_delivery::Gate::parse — selects the delivery gate",
|
read_by: "mission_delivery::Gate::parse — selects the delivery gate",
|
||||||
},
|
},
|
||||||
|
KnownKey {
|
||||||
|
key: "allow_empty",
|
||||||
|
read_by: "phase_runner::empty_delivery_is_a_failure — when true, a coding \
|
||||||
|
phase that changes no files still completes; also vm_stop_gate::\
|
||||||
|
StopGate::for_phase, where it drops the in-loop delivery check",
|
||||||
|
},
|
||||||
|
KnownKey {
|
||||||
|
key: "done_when_check",
|
||||||
|
read_by: "vm_stop_gate::StopGate::for_phase — a shell command the agent's \
|
||||||
|
`Stop` hook runs, refusing the stop while it exits non-zero",
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
/// Keys a recipe may carry that are deliberately not consumed *yet*.
|
/// Keys a recipe may carry that are deliberately not consumed *yet*.
|
||||||
|
|||||||
+1352
-45
File diff suppressed because it is too large
Load Diff
@@ -23,7 +23,6 @@ use std::time::Duration;
|
|||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
const DEFAULT_MODEL: &str = "claude-opus-4-8";
|
const DEFAULT_MODEL: &str = "claude-opus-4-8";
|
||||||
const ANTHROPIC_API_VERSION: &str = "2023-06-01";
|
|
||||||
const POLL_INTERVAL: Duration = Duration::from_secs(30);
|
const POLL_INTERVAL: Duration = Duration::from_secs(30);
|
||||||
/// Cap the raw material we send to the model. Missions can produce
|
/// Cap the raw material we send to the model. Missions can produce
|
||||||
/// hundreds of KB of agent output; we slice by turn and by phase
|
/// hundreds of KB of agent output; we slice by turn and by phase
|
||||||
@@ -34,21 +33,26 @@ fn model_name() -> String {
|
|||||||
std::env::var("CLAWMATES_SUMMARIZER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
std::env::var("CLAWMATES_SUMMARIZER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn spawn(pool: PgPool) {
|
/// The runtime is carried purely so the summarizer can reach the SAME
|
||||||
|
/// providers as everything else. It used to hand-roll its own HTTPS POST with
|
||||||
|
/// `x-api-key: $ANTHROPIC_API_KEY`, which is why no audit of `.complete(` call
|
||||||
|
/// sites ever found it — and why every phase summary on this deployment died
|
||||||
|
/// with "credit balance is too low" while the phases themselves ran fine.
|
||||||
|
pub fn spawn(pool: PgPool, runtime: cm_runtime::Runtime) {
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
tokio::time::sleep(Duration::from_secs(45)).await;
|
tokio::time::sleep(Duration::from_secs(45)).await;
|
||||||
let mut ticker = tokio::time::interval(POLL_INTERVAL);
|
let mut ticker = tokio::time::interval(POLL_INTERVAL);
|
||||||
ticker.tick().await;
|
ticker.tick().await;
|
||||||
loop {
|
loop {
|
||||||
ticker.tick().await;
|
ticker.tick().await;
|
||||||
if let Err(e) = sweep_once(&pool).await {
|
if let Err(e) = sweep_once(&pool, &runtime).await {
|
||||||
eprintln!("phase_summarizer: sweep failed: {e}");
|
eprintln!("phase_summarizer: sweep failed: {e}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn sweep_once(pool: &PgPool) -> Result<(), String> {
|
async fn sweep_once(pool: &PgPool, runtime: &cm_runtime::Runtime) -> Result<(), String> {
|
||||||
// Terminal phases with no summary yet.
|
// Terminal phases with no summary yet.
|
||||||
let rows = sqlx::query(
|
let rows = sqlx::query(
|
||||||
"SELECT mp.id, mp.mission_id, mp.kind
|
"SELECT mp.id, mp.mission_id, mp.kind
|
||||||
@@ -65,7 +69,7 @@ async fn sweep_once(pool: &PgPool) -> Result<(), String> {
|
|||||||
let phase_id: Uuid = row.get("id");
|
let phase_id: Uuid = row.get("id");
|
||||||
let mission_id: Uuid = row.get("mission_id");
|
let mission_id: Uuid = row.get("mission_id");
|
||||||
let kind: String = row.get("kind");
|
let kind: String = row.get("kind");
|
||||||
if let Err(e) = summarize_one(pool, mission_id, phase_id, &kind).await {
|
if let Err(e) = summarize_one(pool, runtime, mission_id, phase_id, &kind).await {
|
||||||
// Persist an error row so we don't infinite-retry a broken
|
// Persist an error row so we don't infinite-retry a broken
|
||||||
// phase — the UI can surface "summary unavailable: <e>".
|
// phase — the UI can surface "summary unavailable: <e>".
|
||||||
eprintln!("phase_summarizer: {phase_id} ({kind}) failed: {e}");
|
eprintln!("phase_summarizer: {phase_id} ({kind}) failed: {e}");
|
||||||
@@ -77,6 +81,7 @@ async fn sweep_once(pool: &PgPool) -> Result<(), String> {
|
|||||||
|
|
||||||
async fn summarize_one(
|
async fn summarize_one(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
mission_id: Uuid,
|
mission_id: Uuid,
|
||||||
phase_id: Uuid,
|
phase_id: Uuid,
|
||||||
kind: &str,
|
kind: &str,
|
||||||
@@ -105,7 +110,7 @@ async fn summarize_one(
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
let (narrative, structured) = call_anthropic(kind, &material).await?;
|
let (narrative, structured, answered_by) = call_anthropic(runtime, kind, &material).await?;
|
||||||
let metrics = structured
|
let metrics = structured
|
||||||
.get("metrics")
|
.get("metrics")
|
||||||
.cloned()
|
.cloned()
|
||||||
@@ -128,7 +133,7 @@ async fn summarize_one(
|
|||||||
mission_id,
|
mission_id,
|
||||||
phase_id,
|
phase_id,
|
||||||
kind,
|
kind,
|
||||||
&model_name(),
|
&answered_by,
|
||||||
&narrative,
|
&narrative,
|
||||||
&metrics,
|
&metrics,
|
||||||
&sources,
|
&sources,
|
||||||
@@ -323,58 +328,25 @@ async fn collect_material(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn call_anthropic(kind: &str, material: &PhaseMaterial) -> Result<(String, Value), String> {
|
/// Returns the narrative, the parsed object, and **the model that answered** —
|
||||||
let api_key =
|
/// which may be a fallback link rather than `model_name()`, and is recorded as
|
||||||
std::env::var("ANTHROPIC_API_KEY").map_err(|_| "ANTHROPIC_API_KEY unset".to_string())?;
|
/// such.
|
||||||
|
async fn call_anthropic(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
|
kind: &str,
|
||||||
|
material: &PhaseMaterial,
|
||||||
|
) -> Result<(String, Value, String), String> {
|
||||||
let model = model_name();
|
let model = model_name();
|
||||||
let system = system_prompt(kind);
|
let system = system_prompt(kind);
|
||||||
let user = user_prompt(kind, material);
|
let user = user_prompt(kind, material);
|
||||||
|
|
||||||
let body = json!({
|
let (raw, answered_by) = crate::subscription::complete_with_fallback(
|
||||||
"model": model,
|
runtime, &system, &user, &model, 4096, false,
|
||||||
"max_tokens": 4096,
|
)
|
||||||
"system": system,
|
.await?;
|
||||||
"messages": [ { "role": "user", "content": user } ]
|
let raw = raw.trim().to_string();
|
||||||
});
|
|
||||||
let client = reqwest::Client::builder()
|
|
||||||
.timeout(std::time::Duration::from_secs(120))
|
|
||||||
.build()
|
|
||||||
.map_err(|e| format!("http client: {e}"))?;
|
|
||||||
let resp = client
|
|
||||||
.post("https://api.anthropic.com/v1/messages")
|
|
||||||
.header("x-api-key", &api_key)
|
|
||||||
.header("anthropic-version", ANTHROPIC_API_VERSION)
|
|
||||||
.header("content-type", "application/json")
|
|
||||||
.json(&body)
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("anthropic call: {e}"))?;
|
|
||||||
if !resp.status().is_success() {
|
|
||||||
let code = resp.status();
|
|
||||||
let body = resp.text().await.unwrap_or_default();
|
|
||||||
return Err(format!(
|
|
||||||
"anthropic {code}: {}",
|
|
||||||
&body[..body.len().min(500)]
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let json: Value = resp
|
|
||||||
.json()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("anthropic json: {e}"))?;
|
|
||||||
let raw = json
|
|
||||||
.get("content")
|
|
||||||
.and_then(|c| c.as_array())
|
|
||||||
.and_then(|arr| {
|
|
||||||
arr.iter()
|
|
||||||
.find(|b| b.get("type").and_then(|t| t.as_str()) == Some("text"))
|
|
||||||
})
|
|
||||||
.and_then(|b| b.get("text"))
|
|
||||||
.and_then(|t| t.as_str())
|
|
||||||
.ok_or_else(|| "anthropic response missing text block".to_string())?
|
|
||||||
.trim()
|
|
||||||
.to_string();
|
|
||||||
if raw.is_empty() {
|
if raw.is_empty() {
|
||||||
return Err("anthropic returned empty text".into());
|
return Err(format!("{answered_by} returned empty text"));
|
||||||
}
|
}
|
||||||
// Model returns a JSON object; extract narrative + rest.
|
// Model returns a JSON object; extract narrative + rest.
|
||||||
let parsed: Value = serde_json::from_str(&strip_code_fence(&raw)).map_err(|e| {
|
let parsed: Value = serde_json::from_str(&strip_code_fence(&raw)).map_err(|e| {
|
||||||
@@ -392,7 +364,7 @@ async fn call_anthropic(kind: &str, material: &PhaseMaterial) -> Result<(String,
|
|||||||
if narrative.is_empty() {
|
if narrative.is_empty() {
|
||||||
return Err("summarizer response missing narrative".into());
|
return Err("summarizer response missing narrative".into());
|
||||||
}
|
}
|
||||||
Ok((narrative, parsed))
|
Ok((narrative, parsed, answered_by))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Trim a leading/trailing ```json … ``` fence the model sometimes wraps
|
/// Trim a leading/trailing ```json … ``` fence the model sometimes wraps
|
||||||
|
|||||||
@@ -0,0 +1,266 @@
|
|||||||
|
//! What a repository actually contains, small enough to put in a prompt.
|
||||||
|
//!
|
||||||
|
//! The planner was given the root listing and planned "optimise the hot path"
|
||||||
|
//! for a crate whose hot path is `add(a: i64, b: i64) -> i64`. Names were not
|
||||||
|
//! enough: the mission was unachievable from the moment it was written, and
|
||||||
|
//! nothing discovered that until an agent had built a benchmark harness to
|
||||||
|
//! measure an integer addition.
|
||||||
|
//!
|
||||||
|
//! # The rule this module exists to enforce
|
||||||
|
//!
|
||||||
|
//! A digest is always partial for any repository worth planning against, and a
|
||||||
|
//! model shown a partial view without being told it is partial plans as though
|
||||||
|
//! it saw everything. So every omission is STATED — how many files were listed,
|
||||||
|
//! how many were shown, what was cut from each. That is the same distinction as
|
||||||
|
//! `Option<u32>` for the subagent probe: "we did not look" and "there is nothing
|
||||||
|
//! there" are different facts, and only one of them is about the repository.
|
||||||
|
//!
|
||||||
|
//! # Priority
|
||||||
|
//!
|
||||||
|
//! Manifests first (they say what the project IS and what it may depend on),
|
||||||
|
//! then the README, then source ascending by size — smallest-first shows the
|
||||||
|
//! most files per byte, and a planner benefits more from seeing twenty small
|
||||||
|
//! files than one large one.
|
||||||
|
|
||||||
|
/// One file in the repository tree.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct FileEntry {
|
||||||
|
pub path: String,
|
||||||
|
pub size: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Total characters of file CONTENT a digest may carry.
|
||||||
|
///
|
||||||
|
/// The prompt around it is ~1.5k, and the planner is a single call per mission,
|
||||||
|
/// so this is generous by design: the cost of a too-small digest is a plan built
|
||||||
|
/// on a guess, which costs a VM boot to discover.
|
||||||
|
pub const CONTENT_BUDGET: usize = 12_000;
|
||||||
|
|
||||||
|
/// Ceiling per file, so one large file cannot spend the whole budget.
|
||||||
|
pub const PER_FILE_CAP: usize = 3_000;
|
||||||
|
|
||||||
|
/// Files worth showing before any source.
|
||||||
|
fn is_manifest(path: &str) -> bool {
|
||||||
|
matches!(
|
||||||
|
path,
|
||||||
|
"Cargo.toml"
|
||||||
|
| "package.json"
|
||||||
|
| "pyproject.toml"
|
||||||
|
| "setup.py"
|
||||||
|
| "go.mod"
|
||||||
|
| "Gemfile"
|
||||||
|
| "pom.xml"
|
||||||
|
| "build.gradle"
|
||||||
|
| "Makefile"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_readme(path: &str) -> bool {
|
||||||
|
path.eq_ignore_ascii_case("README.md") || path.eq_ignore_ascii_case("README")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Paths to fetch, in the order they earn their place.
|
||||||
|
///
|
||||||
|
/// Directories and files a planner cannot use are dropped: lockfiles are huge
|
||||||
|
/// and say nothing a manifest does not, and build output is not source.
|
||||||
|
pub fn priority(entries: &[FileEntry]) -> Vec<&FileEntry> {
|
||||||
|
let mut useful: Vec<&FileEntry> = entries
|
||||||
|
.iter()
|
||||||
|
.filter(|e| {
|
||||||
|
let p = e.path.as_str();
|
||||||
|
!p.starts_with(".git/")
|
||||||
|
&& !p.contains("/target/")
|
||||||
|
&& !p.starts_with("target/")
|
||||||
|
&& !p.contains("node_modules/")
|
||||||
|
&& p != "Cargo.lock"
|
||||||
|
&& p != "package-lock.json"
|
||||||
|
&& p != "poetry.lock"
|
||||||
|
&& e.size > 0
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
useful.sort_by_key(|e| {
|
||||||
|
let rank = if is_manifest(&e.path) {
|
||||||
|
0
|
||||||
|
} else if is_readme(&e.path) {
|
||||||
|
1
|
||||||
|
} else {
|
||||||
|
2
|
||||||
|
};
|
||||||
|
(rank, e.size, e.path.clone())
|
||||||
|
});
|
||||||
|
useful
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Render the digest a planner sees.
|
||||||
|
///
|
||||||
|
/// `contents` is `(path, text)` for the files that were actually fetched, in
|
||||||
|
/// priority order. Anything not fetched is still LISTED, so the model knows the
|
||||||
|
/// file exists even when it cannot read it.
|
||||||
|
pub fn render(entries: &[FileEntry], contents: &[(String, String)]) -> String {
|
||||||
|
if entries.is_empty() {
|
||||||
|
return "(the repository is empty, or its tree could not be read)".to_string();
|
||||||
|
}
|
||||||
|
let mut out = String::new();
|
||||||
|
out.push_str(&format!("FILES ({} total):\n", entries.len()));
|
||||||
|
// The whole tree by name is cheap and is what stops "does X exist" guessing.
|
||||||
|
// Capped anyway: a 10k-file monorepo listing is not a prompt.
|
||||||
|
const MAX_LISTED: usize = 300;
|
||||||
|
for e in entries.iter().take(MAX_LISTED) {
|
||||||
|
out.push_str(&format!(" {} ({} bytes)\n", e.path, e.size));
|
||||||
|
}
|
||||||
|
if entries.len() > MAX_LISTED {
|
||||||
|
out.push_str(&format!(
|
||||||
|
" … and {} more files NOT listed\n",
|
||||||
|
entries.len() - MAX_LISTED
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if contents.is_empty() {
|
||||||
|
out.push_str("\n(no file contents could be read — plan from the names alone, and say so if that is not enough)\n");
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
out.push_str(&format!(
|
||||||
|
"\nCONTENTS ({} of {} files shown; anything not shown you have NOT seen):\n",
|
||||||
|
contents.len(),
|
||||||
|
entries.len()
|
||||||
|
));
|
||||||
|
for (path, text) in contents {
|
||||||
|
out.push_str(&format!("\n--- {path} ---\n{text}\n"));
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Take file texts up to the budget, truncating each at [`PER_FILE_CAP`].
|
||||||
|
///
|
||||||
|
/// Truncation is marked in the text itself rather than silently cutting: a model
|
||||||
|
/// that can see it is reading a fragment asks differently than one that believes
|
||||||
|
/// it read the file.
|
||||||
|
pub fn fit(fetched: Vec<(String, String)>) -> Vec<(String, String)> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
let mut spent = 0usize;
|
||||||
|
for (path, text) in fetched {
|
||||||
|
if spent >= CONTENT_BUDGET {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let room = (CONTENT_BUDGET - spent).min(PER_FILE_CAP);
|
||||||
|
let text = if text.len() <= room {
|
||||||
|
text
|
||||||
|
} else {
|
||||||
|
let end = (0..=room)
|
||||||
|
.rev()
|
||||||
|
.find(|i| text.is_char_boundary(*i))
|
||||||
|
.unwrap_or(0);
|
||||||
|
format!(
|
||||||
|
"{}\n… [truncated: {} of {} bytes shown]",
|
||||||
|
&text[..end],
|
||||||
|
end,
|
||||||
|
text.len()
|
||||||
|
)
|
||||||
|
};
|
||||||
|
spent += text.len();
|
||||||
|
out.push((path, text));
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn f(path: &str, size: usize) -> FileEntry {
|
||||||
|
FileEntry {
|
||||||
|
path: path.into(),
|
||||||
|
size,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Manifests first, then the README, then source smallest-first. A planner
|
||||||
|
/// learns more from twenty small files than from one large one.
|
||||||
|
#[test]
|
||||||
|
fn the_files_that_say_what_this_is_come_first() {
|
||||||
|
let entries = vec![
|
||||||
|
f("src/big.rs", 9000),
|
||||||
|
f("README.md", 400),
|
||||||
|
f("src/lib.rs", 120),
|
||||||
|
f("Cargo.toml", 200),
|
||||||
|
];
|
||||||
|
let order: Vec<&str> = priority(&entries).iter().map(|e| e.path.as_str()).collect();
|
||||||
|
assert_eq!(order, vec!["Cargo.toml", "README.md", "src/lib.rs", "src/big.rs"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Lockfiles and build output are dropped: enormous, and they say nothing a
|
||||||
|
/// manifest does not.
|
||||||
|
#[test]
|
||||||
|
fn noise_is_not_offered_to_the_planner() {
|
||||||
|
let entries = vec![
|
||||||
|
f("Cargo.lock", 50_000),
|
||||||
|
f("target/debug/thing", 900_000),
|
||||||
|
f("node_modules/x/index.js", 400),
|
||||||
|
f(".git/config", 100),
|
||||||
|
f("src/lib.rs", 100),
|
||||||
|
f("empty.rs", 0),
|
||||||
|
];
|
||||||
|
let kept: Vec<&str> = priority(&entries).iter().map(|e| e.path.as_str()).collect();
|
||||||
|
assert_eq!(kept, vec!["src/lib.rs"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// THE rule. A partial view presented as complete is planned against as
|
||||||
|
/// though it were complete — which is how "optimise the hot path" gets
|
||||||
|
/// written for a crate that adds two integers.
|
||||||
|
#[test]
|
||||||
|
fn every_omission_is_stated() {
|
||||||
|
let entries: Vec<FileEntry> = (0..400).map(|i| f(&format!("src/f{i}.rs"), 100)).collect();
|
||||||
|
let shown = vec![("src/f0.rs".to_string(), "fn a() {}".to_string())];
|
||||||
|
let out = render(&entries, &shown);
|
||||||
|
|
||||||
|
assert!(out.contains("FILES (400 total)"), "{out}");
|
||||||
|
assert!(out.contains("and 100 more files NOT listed"), "{out}");
|
||||||
|
assert!(out.contains("1 of 400 files shown"), "{out}");
|
||||||
|
assert!(
|
||||||
|
out.contains("you have NOT seen"),
|
||||||
|
"the model must be told the view is partial: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A file cut short says so, in the text the model reads.
|
||||||
|
#[test]
|
||||||
|
fn a_truncated_file_says_it_was_truncated() {
|
||||||
|
let big = "x".repeat(PER_FILE_CAP * 2);
|
||||||
|
let out = fit(vec![("src/big.rs".into(), big.clone())]);
|
||||||
|
assert_eq!(out.len(), 1);
|
||||||
|
assert!(out[0].1.contains("truncated"), "{}", &out[0].1[..80]);
|
||||||
|
assert!(out[0].1.len() < big.len());
|
||||||
|
// And the marker names both numbers, so "how much did I miss" is
|
||||||
|
// answerable rather than guessable.
|
||||||
|
assert!(out[0].1.contains(&big.len().to_string()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The budget is a total, not per file: one large file must not starve the
|
||||||
|
/// rest, and the whole digest must stay promptable.
|
||||||
|
#[test]
|
||||||
|
fn the_budget_bounds_the_whole_digest() {
|
||||||
|
let files: Vec<(String, String)> = (0..20)
|
||||||
|
.map(|i| (format!("src/f{i}.rs"), "y".repeat(PER_FILE_CAP)))
|
||||||
|
.collect();
|
||||||
|
let out = fit(files);
|
||||||
|
let total: usize = out.iter().map(|(_, t)| t.len()).sum();
|
||||||
|
assert!(total <= CONTENT_BUDGET, "digest was {total} bytes");
|
||||||
|
assert!(!out.is_empty(), "and it still shows something");
|
||||||
|
assert!(out.len() < 20, "not everything fits, by construction");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An empty or unreadable tree is stated as such — never rendered as a
|
||||||
|
/// repository that happens to contain nothing.
|
||||||
|
#[test]
|
||||||
|
fn an_unreadable_tree_is_not_an_empty_repository() {
|
||||||
|
let out = render(&[], &[]);
|
||||||
|
assert!(out.contains("could not be read"), "{out}");
|
||||||
|
|
||||||
|
// A tree we CAN read but no contents we could fetch is a different
|
||||||
|
// fact, and says so.
|
||||||
|
let out = render(&[f("src/lib.rs", 100)], &[]);
|
||||||
|
assert!(out.contains("src/lib.rs"), "{out}");
|
||||||
|
assert!(out.contains("no file contents could be read"), "{out}");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,159 @@
|
|||||||
|
//! A throwaway copy of a mission checkout, for commands that run as ROOT.
|
||||||
|
//!
|
||||||
|
//! Three places in this codebase run a real command against a mission's tree —
|
||||||
|
//! the judge's verification (`evaluator_tools::Sandbox`), the benchmark runner,
|
||||||
|
//! and the `on_green_tests` delivery gate. All three enter a container running as
|
||||||
|
//! root with the missions root bind-mounted, and all three run something that
|
||||||
|
//! writes `target/`. All three now go through here; the judge was the last to
|
||||||
|
//! move, having carried its own copy of this logic since before it existed.
|
||||||
|
//!
|
||||||
|
//! Run against the live checkout, that breaks the single-writer invariant: the
|
||||||
|
//! tree is owned by uid 65532 and now contains root-owned build output, so the
|
||||||
|
//! next phase's `cargo` hits permission-denied on a directory it cannot write.
|
||||||
|
//! The harness's uid probe reports it as `uids=0,65532`.
|
||||||
|
//!
|
||||||
|
//! # The cleanup half, which is the part that keeps being got wrong
|
||||||
|
//!
|
||||||
|
//! The copy inherits the same problem: its `target/` is root-owned, so the
|
||||||
|
//! server process (uid 65532) **cannot delete it**. A `Drop` calling
|
||||||
|
//! `std::fs::remove_dir_all` fails, and because that error is discarded the tree
|
||||||
|
//! survives forever — measured at 1.2 MB per benchmark run and 16 MB of stranded
|
||||||
|
//! judge sandboxes before this existed.
|
||||||
|
//!
|
||||||
|
//! So removal goes back through the container, as root, where the files were
|
||||||
|
//! written. `Drop` remains only as a fallback for the paths where nothing has
|
||||||
|
//! run as root yet, and does not pretend to be more.
|
||||||
|
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
/// Where throwaway copies live: siblings of the per-mission directories, like
|
||||||
|
/// `_outputs` and `_verify`, so reaping a mission cannot race a running command.
|
||||||
|
pub fn copy_root(kind: &str, mission_id: uuid::Uuid) -> PathBuf {
|
||||||
|
crate::mission_workspace::missions_root()
|
||||||
|
.join(kind)
|
||||||
|
.join(mission_id.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Delete a copy from inside the container that wrote it.
|
||||||
|
///
|
||||||
|
/// Best-effort and loud: a housekeeping failure must not cost a real verdict or
|
||||||
|
/// a real benchmark, but it must not be silent either — silence is how the leaks
|
||||||
|
/// this module exists for went unnoticed for a day.
|
||||||
|
pub async fn purge(container: &str, root: &Path) {
|
||||||
|
let Ok(docker) = crate::container_exec::connect() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let argv = vec![
|
||||||
|
"rm".to_string(),
|
||||||
|
"-rf".to_string(),
|
||||||
|
root.display().to_string(),
|
||||||
|
];
|
||||||
|
// Explicitly root: this exists to delete files an EARLIER root-run exec
|
||||||
|
// created, which uid 65532 cannot touch. Everything else now runs as 65532
|
||||||
|
// (see `container_exec`), so this is cleaning up history, not policy.
|
||||||
|
if let Err(e) = crate::container_exec::exec_as_root(
|
||||||
|
&docker,
|
||||||
|
container,
|
||||||
|
Some("/"),
|
||||||
|
&argv,
|
||||||
|
std::time::Duration::from_secs(120),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
eprintln!(
|
||||||
|
"root_copy: could not remove {} from {container}: {e}",
|
||||||
|
root.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A copy of a checkout, removed when it goes out of scope.
|
||||||
|
pub struct RootCopy {
|
||||||
|
root: PathBuf,
|
||||||
|
workdir: PathBuf,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RootCopy {
|
||||||
|
/// Copy `source` into `root`, returning a handle whose `workdir` is the tree
|
||||||
|
/// to run in.
|
||||||
|
///
|
||||||
|
/// Packed through `mission_fs::pack_dir`, so the copy carries exactly what a
|
||||||
|
/// delivered diff carries — no `target/`, no `node_modules/`. One exclusion
|
||||||
|
/// list, four consumers.
|
||||||
|
pub fn of(source: &Path, root: &Path) -> Result<RootCopy, String> {
|
||||||
|
let archive = crate::mission_fs::pack_dir(source, "repo")
|
||||||
|
.map_err(|e| format!("pack {} for a root-run command: {e}", source.display()))?;
|
||||||
|
crate::mission_fs::unpack_into(&archive, root)
|
||||||
|
.map_err(|e| format!("unpack copy into {}: {e}", root.display()))?;
|
||||||
|
let workdir = root.join("repo");
|
||||||
|
if !workdir.is_dir() {
|
||||||
|
return Err(format!("copy missing at {}", workdir.display()));
|
||||||
|
}
|
||||||
|
Ok(RootCopy {
|
||||||
|
root: root.to_path_buf(),
|
||||||
|
workdir,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn workdir(&self) -> &Path {
|
||||||
|
&self.workdir
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Take the working directory and give up automatic cleanup.
|
||||||
|
///
|
||||||
|
/// For a caller whose copy outlives this handle — `evaluator_tools::Sandbox`
|
||||||
|
/// hands the path to a judge that has not run yet, so letting `Drop` fire on
|
||||||
|
/// return would delete the tree out from under it. That caller becomes
|
||||||
|
/// responsible for calling [`purge`], which is the only thing that can
|
||||||
|
/// remove root-owned build output anyway.
|
||||||
|
///
|
||||||
|
/// Spelled as a method rather than `mem::forget` at the call site, so the
|
||||||
|
/// transfer of responsibility is visible in the type rather than implied by
|
||||||
|
/// a leak.
|
||||||
|
pub fn into_workdir(self) -> PathBuf {
|
||||||
|
let workdir = self.workdir.clone();
|
||||||
|
std::mem::forget(self);
|
||||||
|
workdir
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for RootCopy {
|
||||||
|
/// Fallback only. This CANNOT remove root-owned build output — see
|
||||||
|
/// [`purge`], which is what actually clears a copy something has run in.
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = std::fs::remove_dir_all(&self.root);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// A copy must be a SIBLING of the per-mission directory, never inside it:
|
||||||
|
/// `teardown_container` removes `<missions_root>/<mission_id>` wholesale and
|
||||||
|
/// would take a running command's tree with it.
|
||||||
|
#[test]
|
||||||
|
fn copies_live_beside_the_mission_directory_not_inside_it() {
|
||||||
|
let mission = uuid::Uuid::now_v7();
|
||||||
|
let mission_dir = crate::mission_workspace::missions_root().join(mission.to_string());
|
||||||
|
for kind in ["_bench", "_gate", "_verify"] {
|
||||||
|
let root = copy_root(kind, mission);
|
||||||
|
assert!(!root.starts_with(&mission_dir), "{root:?}");
|
||||||
|
assert!(
|
||||||
|
root.starts_with(crate::mission_workspace::missions_root().join(kind)),
|
||||||
|
"{root:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The copy is not the checkout. Stated as a test because the whole defect
|
||||||
|
/// class is "ran the real command against the real tree".
|
||||||
|
#[test]
|
||||||
|
fn a_copy_is_never_the_checkout() {
|
||||||
|
let mission = uuid::Uuid::now_v7();
|
||||||
|
let live = crate::mission_workspace::checkout_path(mission);
|
||||||
|
for kind in ["_bench", "_gate", "_verify"] {
|
||||||
|
assert_ne!(copy_root(kind, mission).join("repo"), live);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -26,6 +26,19 @@ pub(crate) async fn workspace_agent(
|
|||||||
Ok(agent)
|
Ok(agent)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// As [`workspace_agent`], but sees soft-deleted agents too. PURGE ONLY.
|
||||||
|
pub(crate) async fn workspace_agent_any(
|
||||||
|
state: &AppState,
|
||||||
|
user: &cm_auth::AuthedUser,
|
||||||
|
agent_id: AgentId,
|
||||||
|
) -> Result<Agent, ApiError> {
|
||||||
|
let agent = cm_db::repo::agents::get_any(&state.pool, agent_id).await?;
|
||||||
|
if agent.workspace_id != user.workspace_id {
|
||||||
|
return Err(ApiError::NotFound);
|
||||||
|
}
|
||||||
|
Ok(agent)
|
||||||
|
}
|
||||||
|
|
||||||
/// `GET /api/claws/{id}/runtime-config` — the claw's model + §15 sandbox facts
|
/// `GET /api/claws/{id}/runtime-config` — the claw's model + §15 sandbox facts
|
||||||
/// (for the claw card / anatomy view's model badge).
|
/// (for the claw card / anatomy view's model badge).
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
@@ -577,7 +590,11 @@ pub async fn enhance_brain(
|
|||||||
let user_prompt = format!(
|
let user_prompt = format!(
|
||||||
"BRAIN: {reference}\n\n=== SYSTEM PROMPT ===\n{sp}\n\n=== AGENTS.md ===\n{agent_md}\n\n=== PERSONA ===\n{persona}\n\n=== SKILLS ===\n{skills}"
|
"BRAIN: {reference}\n\n=== SYSTEM PROMPT ===\n{sp}\n\n=== AGENTS.md ===\n{agent_md}\n\n=== PERSONA ===\n{persona}\n\n=== SKILLS ===\n{skills}"
|
||||||
);
|
);
|
||||||
let raw = match runtime.complete(ENHANCE_SYSTEM, &user_prompt, "claude-opus-4-8", 16000, true).await {
|
let raw = match crate::subscription::complete_or(
|
||||||
|
&runtime, ENHANCE_SYSTEM, &user_prompt, "claude-opus-4-8", 16000, true,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(t) => t,
|
Ok(t) => t,
|
||||||
Err(e) => { yield sse(json!({"stage":"error","pct":100,"label":format!("Opus error: {e}")})); return; }
|
Err(e) => { yield sse(json!({"stage":"error","pct":100,"label":format!("Opus error: {e}")})); return; }
|
||||||
};
|
};
|
||||||
@@ -656,8 +673,14 @@ pub(crate) async fn enhance_and_publish(
|
|||||||
let user_prompt = format!(
|
let user_prompt = format!(
|
||||||
"ROLE CONTEXT: {role_context}\n\nBRAIN: {reference}\n\n=== SYSTEM PROMPT ===\n{sp}\n\n=== AGENTS.md ===\n{agent_md}\n\n=== PERSONA ===\n{persona}\n\n=== SKILLS ===\n{skills}"
|
"ROLE CONTEXT: {role_context}\n\nBRAIN: {reference}\n\n=== SYSTEM PROMPT ===\n{sp}\n\n=== AGENTS.md ===\n{agent_md}\n\n=== PERSONA ===\n{persona}\n\n=== SKILLS ===\n{skills}"
|
||||||
);
|
);
|
||||||
let raw = runtime
|
let raw = crate::subscription::complete_or(
|
||||||
.complete(ENHANCE_SYSTEM, &user_prompt, "claude-opus-4-8", 16000, true)
|
runtime,
|
||||||
|
ENHANCE_SYSTEM,
|
||||||
|
&user_prompt,
|
||||||
|
"claude-opus-4-8",
|
||||||
|
16000,
|
||||||
|
true,
|
||||||
|
)
|
||||||
.await?;
|
.await?;
|
||||||
let v = extract_json(&raw).ok_or_else(|| "unparseable enhance output".to_string())?;
|
let v = extract_json(&raw).ok_or_else(|| "unparseable enhance output".to_string())?;
|
||||||
let enh = v.get("enhanced").cloned().unwrap_or(Value::Null);
|
let enh = v.get("enhanced").cloned().unwrap_or(Value::Null);
|
||||||
@@ -1127,7 +1150,7 @@ pub async fn patch(
|
|||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
pub struct SetModelRequest {
|
pub struct SetModelRequest {
|
||||||
/// Model selector (claude / glm / glm-5.2 / kimi / gemini / groq /
|
/// Model selector (claude / glm / glm-5.2 / kimi / groq /
|
||||||
/// specific model id like `claude-sonnet-5`). Resolved through the
|
/// specific model id like `claude-sonnet-5`). Resolved through the
|
||||||
/// same RuntimeProvisioner::provider_alias_for that team creation
|
/// same RuntimeProvisioner::provider_alias_for that team creation
|
||||||
/// uses, so shorthand + fully-qualified ids both work.
|
/// uses, so shorthand + fully-qualified ids both work.
|
||||||
@@ -1279,7 +1302,12 @@ pub async fn batch_delete(
|
|||||||
let mut done = 0usize;
|
let mut done = 0usize;
|
||||||
for id in agent_ids {
|
for id in agent_ids {
|
||||||
let base = 100 * done / total;
|
let base = 100 * done / total;
|
||||||
let agent = match workspace_agent(&state, &user, id).await {
|
// `workspace_agent_any`, not `workspace_agent`: a purge has to be
|
||||||
|
// able to see the rows it exists to remove. The soft-delete path
|
||||||
|
// correctly hides them from every read, which also hid them from
|
||||||
|
// the only route that could reap them — four soft-deleted agents
|
||||||
|
// from June were unreachable from the application entirely.
|
||||||
|
let agent = match workspace_agent_any(&state, &user, id).await {
|
||||||
Ok(a) => a,
|
Ok(a) => a,
|
||||||
Err(_) => { yield sse(json!({"stage":"skip","pct":base,"label":format!("{id}: not found or no access")})); done += 1; continue; }
|
Err(_) => { yield sse(json!({"stage":"skip","pct":base,"label":format!("{id}: not found or no access")})); done += 1; continue; }
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -36,7 +36,7 @@ pub async fn propose_for_agent(
|
|||||||
Authed(user): Authed,
|
Authed(user): Authed,
|
||||||
Path(agent_id): Path<Uuid>,
|
Path(agent_id): Path<Uuid>,
|
||||||
) -> Result<Json<serde_json::Value>, ApiError> {
|
) -> Result<Json<serde_json::Value>, ApiError> {
|
||||||
let id = crate::level_up::propose_agent(&state.pool, user.workspace_id, user.user_id, agent_id)
|
let id = crate::level_up::propose_agent(&state.pool, &state.runtime, user.workspace_id, user.user_id, agent_id)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
eprintln!("level_up: propose_agent {agent_id} failed: {e}");
|
eprintln!("level_up: propose_agent {agent_id} failed: {e}");
|
||||||
@@ -51,7 +51,7 @@ pub async fn propose_for_team(
|
|||||||
Authed(user): Authed,
|
Authed(user): Authed,
|
||||||
Path(team_id): Path<Uuid>,
|
Path(team_id): Path<Uuid>,
|
||||||
) -> Result<Json<serde_json::Value>, ApiError> {
|
) -> Result<Json<serde_json::Value>, ApiError> {
|
||||||
let id = crate::level_up::propose_team(&state.pool, user.workspace_id, user.user_id, team_id)
|
let id = crate::level_up::propose_team(&state.pool, &state.runtime, user.workspace_id, user.user_id, team_id)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
eprintln!("level_up: propose_team {team_id} failed: {e}");
|
eprintln!("level_up: propose_team {team_id} failed: {e}");
|
||||||
|
|||||||
@@ -26,6 +26,11 @@ pub struct RunRequest {
|
|||||||
/// library can otherwise pull hundreds of PDFs in one go.
|
/// library can otherwise pull hundreds of PDFs in one go.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub per_topic: Option<usize>,
|
pub per_topic: Option<usize>,
|
||||||
|
/// Attribute this run to a mission, so the mission can later be asked
|
||||||
|
/// what it contributed. `corpus_items.mission_id` has existed since the
|
||||||
|
/// table landed; without this field nothing could ever populate it.
|
||||||
|
#[serde(default, rename = "missionId")]
|
||||||
|
pub mission_id: Option<uuid::Uuid>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
@@ -37,6 +42,8 @@ pub struct RunResponse {
|
|||||||
pub notes: Vec<String>,
|
pub notes: Vec<String>,
|
||||||
pub branch: String,
|
pub branch: String,
|
||||||
pub pushed: bool,
|
pub pushed: bool,
|
||||||
|
pub merged: bool,
|
||||||
|
pub merge_reason: String,
|
||||||
pub error: Option<String>,
|
pub error: Option<String>,
|
||||||
/// A run that errored on nothing. Reported explicitly so a caller does not
|
/// A run that errored on nothing. Reported explicitly so a caller does not
|
||||||
/// have to infer health from an empty `shelved` list — a quiet week and a
|
/// have to infer health from an empty `shelved` list — a quiet week and a
|
||||||
@@ -78,7 +85,7 @@ pub async fn run(
|
|||||||
&work_root,
|
&work_root,
|
||||||
&topics,
|
&topics,
|
||||||
per_topic,
|
per_topic,
|
||||||
None,
|
req.mission_id,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
@@ -102,6 +109,8 @@ pub async fn run(
|
|||||||
healthy: out.harvest.healthy(),
|
healthy: out.harvest.healthy(),
|
||||||
branch: out.branch,
|
branch: out.branch,
|
||||||
pushed: out.pushed,
|
pushed: out.pushed,
|
||||||
|
merged: out.merged,
|
||||||
|
merge_reason: out.merge_reason,
|
||||||
error: out.error,
|
error: out.error,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,359 @@
|
|||||||
|
//! `/api/missions/{id}/plan-proposals` — let a model author the phases.
|
||||||
|
//!
|
||||||
|
//! W1 / #13, and the sibling of [`crate::routes::mission_roster`]: that one has
|
||||||
|
//! a model size the team, this one has it decide what the work is. Same three
|
||||||
|
//! verbs and the same rule — propose and decide are separate, because only the
|
||||||
|
//! second one changes a mission.
|
||||||
|
//!
|
||||||
|
//! The model is handed two lists it may not depart from: the phase kinds
|
||||||
|
//! `phase_runner` dispatches on, and the config keys `phase_config` says have
|
||||||
|
//! readers. Both are enforced again on the way in, so a plan cannot describe
|
||||||
|
//! work this platform will accept and then not do.
|
||||||
|
|
||||||
|
use axum::extract::{Path, State};
|
||||||
|
use axum::Json;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use crate::mission_plan::{Plan, MAX_PHASES, PLANNABLE_KINDS};
|
||||||
|
use crate::{ApiError, AppState, Authed};
|
||||||
|
|
||||||
|
const PLANNER_MODEL: &str = "claude-opus-4-8";
|
||||||
|
|
||||||
|
/// What the repository actually contains, for the planner's prompt.
|
||||||
|
///
|
||||||
|
/// Names were not enough. Given the root listing alone, the planner wrote
|
||||||
|
/// "optimise the hot path" for a crate whose hot path is
|
||||||
|
/// `add(a: i64, b: i64) -> i64` — a mission that was unachievable from the
|
||||||
|
/// moment it was written, and that nothing discovered until an agent had built a
|
||||||
|
/// benchmark harness to measure an integer addition.
|
||||||
|
///
|
||||||
|
/// Read from the FORGE, not a checkout: at proposal time the mission is still a
|
||||||
|
/// draft and `ensure_checkout` has not run, so there is nothing on disk. Every
|
||||||
|
/// failure degrades to a STATED absence — a planner told "the listing could not
|
||||||
|
/// be read" can hedge; one told nothing assumes.
|
||||||
|
async fn repo_digest(pool: &sqlx::PgPool, mission_id: uuid::Uuid) -> String {
|
||||||
|
let row: Option<(Option<String>, Option<String>, Option<String>)> = sqlx::query_as(
|
||||||
|
"SELECT r.owner, r.name, r.default_branch
|
||||||
|
FROM missions m JOIN repos r ON r.id = m.repo_id
|
||||||
|
WHERE m.id = $1",
|
||||||
|
)
|
||||||
|
.bind(mission_id)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await
|
||||||
|
.ok()
|
||||||
|
.flatten();
|
||||||
|
let Some((Some(owner), Some(name), branch)) = row else {
|
||||||
|
return "(this mission has no repository)".to_string();
|
||||||
|
};
|
||||||
|
let branch = branch.unwrap_or_else(|| "main".to_string());
|
||||||
|
let token = std::env::var("GITEA_TOKEN").unwrap_or_default();
|
||||||
|
let Ok(client) = reqwest::Client::builder()
|
||||||
|
.timeout(std::time::Duration::from_secs(20))
|
||||||
|
.build()
|
||||||
|
else {
|
||||||
|
return "(the repository could not be read)".to_string();
|
||||||
|
};
|
||||||
|
let auth = |r: reqwest::RequestBuilder| {
|
||||||
|
if token.trim().is_empty() {
|
||||||
|
r
|
||||||
|
} else {
|
||||||
|
r.header("Authorization", format!("token {token}"))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// The whole tree in one call, so "does this repo have benches/" is a fact
|
||||||
|
// rather than an inference from the root.
|
||||||
|
let tree_url = format!(
|
||||||
|
"https://git.redclaw.dev/api/v1/repos/{owner}/{name}/git/trees/{branch}?recursive=true&per_page=1000"
|
||||||
|
);
|
||||||
|
let tree: serde_json::Value = match auth(client.get(&tree_url)).send().await {
|
||||||
|
Ok(r) if r.status().is_success() => r.json().await.unwrap_or_default(),
|
||||||
|
_ => return "(the repository tree could not be read)".to_string(),
|
||||||
|
};
|
||||||
|
let entries: Vec<crate::repo_digest::FileEntry> = tree
|
||||||
|
.get("tree")
|
||||||
|
.and_then(|t| t.as_array())
|
||||||
|
.map(|items| {
|
||||||
|
items
|
||||||
|
.iter()
|
||||||
|
.filter(|e| e.get("type").and_then(|v| v.as_str()) == Some("blob"))
|
||||||
|
.filter_map(|e| {
|
||||||
|
Some(crate::repo_digest::FileEntry {
|
||||||
|
path: e.get("path")?.as_str()?.to_string(),
|
||||||
|
size: e.get("size").and_then(|v| v.as_u64()).unwrap_or(0) as usize,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
// Fetch in priority order until the budget is spent. Requested serially and
|
||||||
|
// capped: this runs inside one API request, and a repo with 500 useful files
|
||||||
|
// must not turn a proposal into 500 round trips.
|
||||||
|
let mut fetched: Vec<(String, String)> = Vec::new();
|
||||||
|
let mut spent = 0usize;
|
||||||
|
for e in crate::repo_digest::priority(&entries).into_iter().take(40) {
|
||||||
|
if spent >= crate::repo_digest::CONTENT_BUDGET {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let raw = format!(
|
||||||
|
"https://git.redclaw.dev/api/v1/repos/{owner}/{name}/raw/{}?ref={branch}",
|
||||||
|
e.path
|
||||||
|
);
|
||||||
|
if let Ok(r) = auth(client.get(&raw)).send().await {
|
||||||
|
if r.status().is_success() {
|
||||||
|
if let Ok(text) = r.text().await {
|
||||||
|
spent += text.len().min(crate::repo_digest::PER_FILE_CAP);
|
||||||
|
fetched.push((e.path.clone(), text));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
crate::repo_digest::render(&entries, &crate::repo_digest::fit(fetched))
|
||||||
|
}
|
||||||
|
|
||||||
|
const PLAN_SYSTEM: &str = "You decide what ONE software mission actually does — its phases, in order. \
|
||||||
|
Each phase is a full agent run against the same repository checkout: the next phase sees the tree the \
|
||||||
|
previous one left. They run SEQUENTIALLY, so phases are expensive and a handoff loses context at every \
|
||||||
|
step.\n\n\
|
||||||
|
Propose the FEWEST phases that genuinely need to be separate. ONE phase is usually the right answer, and \
|
||||||
|
is always the right answer for a self-contained change: splitting one change into plan → implement → \
|
||||||
|
test is a documented anti-pattern, not thoroughness — a single agent doing all three in one pass keeps \
|
||||||
|
the context that makes the later steps good. A second phase earns its place only when it depends on \
|
||||||
|
something the first phase could not have known when it started.\n\n\
|
||||||
|
Every phase needs a `task`: the specific instruction for THAT phase, not a restatement of the mission. \
|
||||||
|
An agent receives the mission description plus its own task, so a vague task means an agent guessing \
|
||||||
|
which part of the mission is its share.\n\n\
|
||||||
|
`done_when` is judged afterwards by a separate model reading the repository, so write it as something \
|
||||||
|
observable in the tree — a file that exists, a suite that passes — never as an intention. \
|
||||||
|
`done_when_check` is a SHELL COMMAND that must exit 0; it is enforced while the agent still works, so \
|
||||||
|
prefer it when the condition is mechanical. Set `allow_empty` true only for a phase whose job is to \
|
||||||
|
verify rather than to change files.\n\n\
|
||||||
|
ALWAYS respond with STRICT JSON ONLY, no prose and no markdown: \
|
||||||
|
{\"phases\":[{\"kind\":\"coding\",\"task\":\"...\",\"done_when\":null|\"...\",\
|
||||||
|
\"done_when_check\":null|\"...\",\"allow_empty\":null|true|false}]}";
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct PlanProposalResponse {
|
||||||
|
pub id: Uuid,
|
||||||
|
pub plan: Value,
|
||||||
|
pub author_model: String,
|
||||||
|
pub status: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `POST /api/missions/{id}/plan-proposals` — ask the model for a phase plan.
|
||||||
|
pub async fn suggest(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path(id): Path<Uuid>,
|
||||||
|
) -> Result<Json<PlanProposalResponse>, ApiError> {
|
||||||
|
let ws = user.workspace_id;
|
||||||
|
let mission = cm_db::repo::missions::get(&state.pool, id, ws.as_uuid())
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
|
||||||
|
let prompt = format!(
|
||||||
|
"MISSION: {}\n\nDESCRIPTION:\n{}\n\n=== THE REPOSITORY ===\n{}\n=== END REPOSITORY \
|
||||||
|
===\n\nPlan for the repository as it ACTUALLY IS, not as the description implies it \
|
||||||
|
might be. If the work needs something absent — a benchmark harness, a test suite, a \
|
||||||
|
config file — the phase that needs it must CREATE it, and its task must say so. If the \
|
||||||
|
description asks for something this code cannot support (optimising a function with \
|
||||||
|
nothing to optimise, testing a module that does not exist), say so in the task text and \
|
||||||
|
plan the phase that would establish the truth, rather than a phase that must fail.\n\n\
|
||||||
|
NOTE: a mission agent has NO package-registry access — it cannot add dependencies. A \
|
||||||
|
phase needing tooling must build it from the standard library or from what is already \
|
||||||
|
vendored here.\n\nPHASE KINDS YOU MAY USE (nothing else runs): {}\nCEILING: \
|
||||||
|
{MAX_PHASES} phases.\n\nPropose the plan now (JSON only).",
|
||||||
|
mission.title,
|
||||||
|
mission.description.as_deref().unwrap_or("(none)"),
|
||||||
|
repo_digest(&state.pool, id).await,
|
||||||
|
PLANNABLE_KINDS.join(", "),
|
||||||
|
);
|
||||||
|
|
||||||
|
// The stored `author_model` is whichever link of the fallback chain
|
||||||
|
// actually answered — see `subscription::complete_with_fallback`.
|
||||||
|
let (raw, author_model) = crate::subscription::complete_with_fallback(
|
||||||
|
&state.runtime,
|
||||||
|
PLAN_SYSTEM,
|
||||||
|
&prompt,
|
||||||
|
PLANNER_MODEL,
|
||||||
|
2000,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("mission {id}: plan proposal failed: {e}");
|
||||||
|
crate::subscription::as_api_error(&e)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let parsed: Value = crate::routes::claws::extract_json(&raw).ok_or_else(|| {
|
||||||
|
eprintln!("mission {id}: planner returned no JSON: {raw}");
|
||||||
|
ApiError::BadRequest
|
||||||
|
})?;
|
||||||
|
let plan: Plan = serde_json::from_value(parsed.clone()).map_err(|e| {
|
||||||
|
eprintln!("mission {id}: planner JSON is not a plan ({e}): {parsed}");
|
||||||
|
ApiError::BadRequest
|
||||||
|
})?;
|
||||||
|
// Validated BEFORE storing, so a stored proposal is always one that could be
|
||||||
|
// approved — the failure belongs to the model, not to whoever clicks
|
||||||
|
// approve later.
|
||||||
|
if let Err(why) = plan.validate() {
|
||||||
|
eprintln!("mission {id}: planner proposed an unrunnable plan: {why}");
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
let pid = Uuid::now_v7();
|
||||||
|
let stored = serde_json::to_value(&plan).map_err(|_| ApiError::Internal)?;
|
||||||
|
cm_db::repo::mission_plan_proposals::insert(
|
||||||
|
&state.pool,
|
||||||
|
pid,
|
||||||
|
id,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
&stored,
|
||||||
|
&author_model,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("mission {id}: could not store plan proposal: {e}");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
eprintln!(
|
||||||
|
"mission_plan: mission {id} — {author_model} proposed {} phase(s): {}",
|
||||||
|
plan.phases.len(),
|
||||||
|
plan.phases
|
||||||
|
.iter()
|
||||||
|
.map(|p| p.kind.as_str())
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" → ")
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(Json(PlanProposalResponse {
|
||||||
|
id: pid,
|
||||||
|
plan: stored,
|
||||||
|
author_model,
|
||||||
|
status: "proposed".into(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `GET /api/missions/{id}/plan-proposals`
|
||||||
|
pub async fn list(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path(id): Path<Uuid>,
|
||||||
|
) -> Result<Json<Vec<cm_db::repo::mission_plan_proposals::MissionPlanProposal>>, ApiError> {
|
||||||
|
let rows = cm_db::repo::mission_plan_proposals::list(
|
||||||
|
&state.pool,
|
||||||
|
id,
|
||||||
|
user.workspace_id.as_uuid().to_owned(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?;
|
||||||
|
Ok(Json(rows))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
pub struct DecideRequest {
|
||||||
|
pub status: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub note: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `POST /api/missions/{id}/plan-proposals/{pid}/decide`
|
||||||
|
///
|
||||||
|
/// Approving REPLACES the mission's phases. Draft-only: re-planning a mission
|
||||||
|
/// whose phases have started would discard work that already ran, and the phase
|
||||||
|
/// rows are what every downstream sweep keys off.
|
||||||
|
pub async fn decide(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path((id, pid)): Path<(Uuid, Uuid)>,
|
||||||
|
Json(body): Json<DecideRequest>,
|
||||||
|
) -> Result<Json<Value>, ApiError> {
|
||||||
|
let ws = user.workspace_id;
|
||||||
|
let proposal = cm_db::repo::mission_plan_proposals::get(&state.pool, pid, ws.as_uuid().to_owned())
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
if proposal.mission_id != id {
|
||||||
|
return Err(ApiError::NotFound);
|
||||||
|
}
|
||||||
|
|
||||||
|
if body.status == "rejected" {
|
||||||
|
let decided = cm_db::repo::mission_plan_proposals::decide(
|
||||||
|
&state.pool,
|
||||||
|
pid,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
"rejected",
|
||||||
|
body.note.as_deref(),
|
||||||
|
Some(user.user_id.as_uuid().to_owned()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?;
|
||||||
|
return Ok(Json(json!({ "status": "rejected", "decided": decided })));
|
||||||
|
}
|
||||||
|
if body.status != "approved" {
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mission = cm_db::repo::missions::get(&state.pool, id, ws.as_uuid())
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
if mission.status != "draft" {
|
||||||
|
eprintln!("mission {id}: plan approval refused — mission is {}", mission.status);
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
let plan: Plan = serde_json::from_value(proposal.plan.clone()).map_err(|e| {
|
||||||
|
eprintln!("mission {id}: stored plan {pid} does not parse ({e})");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
// Re-validated at approval. The stored plan passed once, but `PLANNABLE_KINDS`
|
||||||
|
// and the config registry are properties of the BUILD — a proposal made
|
||||||
|
// before a deploy could name a kind this build no longer dispatches.
|
||||||
|
if let Err(why) = plan.validate() {
|
||||||
|
eprintln!("mission {id}: plan {pid} is no longer runnable: {why}");
|
||||||
|
let _ = cm_db::repo::mission_plan_proposals::decide(
|
||||||
|
&state.pool,
|
||||||
|
pid,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
"rejected",
|
||||||
|
Some(&why.to_string()),
|
||||||
|
Some(user.user_id.as_uuid().to_owned()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
let phases = plan.phases();
|
||||||
|
let claimed = cm_db::repo::mission_plan_proposals::approve_and_apply(
|
||||||
|
&state.pool,
|
||||||
|
pid,
|
||||||
|
id,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
&phases,
|
||||||
|
body.note.as_deref(),
|
||||||
|
Some(user.user_id.as_uuid().to_owned()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("mission {id}: could not apply plan {pid}: {e}");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
if !claimed {
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
eprintln!(
|
||||||
|
"mission_plan: mission {id} now runs a {}-phase model-authored plan from proposal {pid}",
|
||||||
|
phases.len()
|
||||||
|
);
|
||||||
|
Ok(Json(json!({
|
||||||
|
"status": "approved",
|
||||||
|
"phases": phases.iter().map(|(k, i, _)| json!({"kind": k, "order_idx": i})).collect::<Vec<_>>(),
|
||||||
|
})))
|
||||||
|
}
|
||||||
@@ -0,0 +1,321 @@
|
|||||||
|
//! `/api/missions/{id}/team-proposals` — let a model size the mission's team.
|
||||||
|
//!
|
||||||
|
//! Slice 5. The planner has been proposing rosters into React state for months;
|
||||||
|
//! this is where one reaches a mission. Three verbs, and the split between them
|
||||||
|
//! is the point:
|
||||||
|
//!
|
||||||
|
//! - **suggest** asks the model and PERSISTS the answer. It changes nothing
|
||||||
|
//! about the mission.
|
||||||
|
//! - **approve** writes the roster onto the mission, where the composed executor
|
||||||
|
//! reads it.
|
||||||
|
//! - **reject** records that a human said no, which is the only evidence we ever
|
||||||
|
//! collect about what the planner gets wrong.
|
||||||
|
//!
|
||||||
|
//! A proposal is never applied on arrival. A model sizing a team is a suggestion
|
||||||
|
//! about how many VMs to boot, and this codebase has an explicit rule about
|
||||||
|
//! model output that costs money: it is evidence for a decision, not the
|
||||||
|
//! decision.
|
||||||
|
|
||||||
|
use axum::extract::{Path, State};
|
||||||
|
use axum::Json;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use crate::mission_roster::{available_backends, Roster};
|
||||||
|
use crate::{ApiError, AppState, Authed};
|
||||||
|
|
||||||
|
/// The model that sizes a mission's team.
|
||||||
|
///
|
||||||
|
/// The same one the Master Planner uses. Sizing a team is the kind of judgement
|
||||||
|
/// the planner's own system prompt calls for — and it is a once-per-mission call,
|
||||||
|
/// so the cost argument that keeps missions on cheaper models does not apply.
|
||||||
|
const PLANNER_MODEL: &str = "claude-opus-4-8";
|
||||||
|
|
||||||
|
const ROSTER_SYSTEM: &str = "You size the team for ONE software mission that runs inside Firecracker \
|
||||||
|
microVMs. Each member you propose is a WHOLE VM — a boot, a repository injected as a tar, a full \
|
||||||
|
Claude Code session, and a collect — running one after another, each one receiving the working tree the \
|
||||||
|
previous member left behind. That is expensive and it is serial, so propose the FEWEST members that \
|
||||||
|
genuinely divide the work. One member is a perfectly good answer and is usually the right one for a \
|
||||||
|
small change; Anthropic measure multi-agent work at 3-10x the tokens with wall-clock often LONGER, and \
|
||||||
|
the benefit is thoroughness rather than speed.\n\n\
|
||||||
|
Members run SEQUENTIALLY and share the repository, so do NOT propose members that would edit the same \
|
||||||
|
file, and do NOT split one change into stages (plan → implement → test) — a handoff loses context at \
|
||||||
|
every step and one careful pass beats an assembly line. The shape that DOES earn its cost is an \
|
||||||
|
implementer followed by an independent verifier that only checks.\n\n\
|
||||||
|
Give each member a `backend` ONLY when running it on a different provider's image is the point — an \
|
||||||
|
independent verifier on another provider breaks the correlated failure where the model that wrote the \
|
||||||
|
code also grades it. Omit `backend` to inherit the mission's.\n\n\
|
||||||
|
ALWAYS respond with STRICT JSON ONLY, no prose and no markdown: \
|
||||||
|
{\"topology_kind\":\"pipeline\",\"members\":[{\"role\":\"...\",\"backend\":null|\"...\",\
|
||||||
|
\"rationale\":\"one line\"}]}";
|
||||||
|
|
||||||
|
#[derive(Debug, Serialize)]
|
||||||
|
pub struct ProposalResponse {
|
||||||
|
pub id: Uuid,
|
||||||
|
pub roster: Value,
|
||||||
|
pub author_model: String,
|
||||||
|
pub status: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `POST /api/missions/{id}/team-proposals` — ask the model for a roster.
|
||||||
|
pub async fn suggest(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path(id): Path<Uuid>,
|
||||||
|
) -> Result<Json<ProposalResponse>, ApiError> {
|
||||||
|
let ws = user.workspace_id;
|
||||||
|
let mission = cm_db::repo::missions::get(&state.pool, id, ws.as_uuid())
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
|
||||||
|
// The backends the FLEET can boot today, handed to the model as the menu.
|
||||||
|
// Without it the model invents plausible image names and the roster is
|
||||||
|
// refused after it was written, which reads as our bug rather than as a
|
||||||
|
// model guessing.
|
||||||
|
let available = available_backends(&state.pool, ws.as_uuid().to_owned())
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("mission {id}: could not read fleet backends: {e}");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let phases: Vec<(String, Option<String>)> = sqlx::query_as(
|
||||||
|
"SELECT kind, config->>'task' FROM mission_phases WHERE mission_id = $1 ORDER BY order_idx",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.fetch_all(&state.pool)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?;
|
||||||
|
|
||||||
|
let phase_text = phases
|
||||||
|
.iter()
|
||||||
|
.map(|(kind, task)| format!("- {kind}: {}", task.as_deref().unwrap_or("(no task text)")))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join("\n");
|
||||||
|
let prompt = format!(
|
||||||
|
"MISSION: {}\n\nDESCRIPTION:\n{}\n\nPHASES:\n{}\n\nBACKENDS THIS FLEET CAN BOOT (use only \
|
||||||
|
these, or omit `backend`): {}\n\nPropose the roster now (JSON only).",
|
||||||
|
mission.title,
|
||||||
|
mission.description.as_deref().unwrap_or("(none)"),
|
||||||
|
if phase_text.is_empty() {
|
||||||
|
"(none declared)".to_string()
|
||||||
|
} else {
|
||||||
|
phase_text
|
||||||
|
},
|
||||||
|
if available.is_empty() {
|
||||||
|
"(none — omit backend on every member)".to_string()
|
||||||
|
} else {
|
||||||
|
available.join(", ")
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
// On the SUBSCRIPTION, like every mission VM — not the metered API key.
|
||||||
|
// `Runtime::complete` with a bare model name resolves to the default
|
||||||
|
// provider, which is the pay-as-you-go key; this planner died with
|
||||||
|
// "credit balance is too low" while missions on the same box ran fine.
|
||||||
|
// `author_model` is what ANSWERED, not what was asked for. When opus is
|
||||||
|
// capped the chain steps down to haiku and then to GLM, and a plan drafted
|
||||||
|
// by the third link but filed as an opus plan is a silent quality change.
|
||||||
|
let (raw, author_model) = crate::subscription::complete_with_fallback(
|
||||||
|
&state.runtime,
|
||||||
|
ROSTER_SYSTEM,
|
||||||
|
&prompt,
|
||||||
|
PLANNER_MODEL,
|
||||||
|
2000,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("mission {id}: roster proposal failed: {e}");
|
||||||
|
// A rate-limited subscription is a 503 the operator can act on, not
|
||||||
|
// a 500 that reads as "this server is broken".
|
||||||
|
crate::subscription::as_api_error(&e)
|
||||||
|
})?;
|
||||||
|
|
||||||
|
// A model that answered with prose around its JSON has still answered; a
|
||||||
|
// model that answered with nothing usable has not, and that is a refusal
|
||||||
|
// rather than an empty roster.
|
||||||
|
let parsed: Value = crate::routes::claws::extract_json(&raw).ok_or_else(|| {
|
||||||
|
eprintln!("mission {id}: planner returned no JSON: {raw}");
|
||||||
|
ApiError::BadRequest
|
||||||
|
})?;
|
||||||
|
let roster: Roster = serde_json::from_value(parsed.clone()).map_err(|e| {
|
||||||
|
eprintln!("mission {id}: planner JSON is not a roster ({e}): {parsed}");
|
||||||
|
ApiError::BadRequest
|
||||||
|
})?;
|
||||||
|
// Validated BEFORE it is stored, so a stored proposal is always one that
|
||||||
|
// could be approved. Storing an invalid roster would mean the failure
|
||||||
|
// surfaces at approval time, pointing at the human rather than the model.
|
||||||
|
if let Err(why) = roster.validate(&available) {
|
||||||
|
eprintln!("mission {id}: planner proposed an unusable roster: {why}");
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
let pid = Uuid::now_v7();
|
||||||
|
let stored = serde_json::to_value(&roster).map_err(|_| ApiError::Internal)?;
|
||||||
|
cm_db::repo::mission_team_proposals::insert(
|
||||||
|
&state.pool,
|
||||||
|
pid,
|
||||||
|
id,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
&stored,
|
||||||
|
&author_model,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("mission {id}: could not store proposal: {e}");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
eprintln!(
|
||||||
|
"mission_roster: mission {id} — {} proposed {} member(s): {}",
|
||||||
|
author_model,
|
||||||
|
roster.members.len(),
|
||||||
|
roster
|
||||||
|
.members
|
||||||
|
.iter()
|
||||||
|
.map(|m| format!("{}{}", m.role, m.backend.as_deref().map(|b| format!("@{b}")).unwrap_or_default()))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(", ")
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(Json(ProposalResponse {
|
||||||
|
id: pid,
|
||||||
|
roster: stored,
|
||||||
|
author_model,
|
||||||
|
status: "proposed".into(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `GET /api/missions/{id}/team-proposals`
|
||||||
|
pub async fn list(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path(id): Path<Uuid>,
|
||||||
|
) -> Result<Json<Vec<cm_db::repo::mission_team_proposals::MissionTeamProposal>>, ApiError> {
|
||||||
|
let rows =
|
||||||
|
cm_db::repo::mission_team_proposals::list(&state.pool, id, user.workspace_id.as_uuid().to_owned())
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?;
|
||||||
|
Ok(Json(rows))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
pub struct DecideRequest {
|
||||||
|
/// `approved` or `rejected`.
|
||||||
|
pub status: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub note: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `POST /api/missions/{id}/team-proposals/{pid}/decide` — accept or refuse.
|
||||||
|
///
|
||||||
|
/// Approving writes `config.roster` on the mission and switches it to the
|
||||||
|
/// composed engine, because a roster is a graph of VMs and that is the engine
|
||||||
|
/// that runs one. Draft-only: re-shaping a mission that is already running would
|
||||||
|
/// change what its next phase does with no record of the swap on the phase that
|
||||||
|
/// already ran.
|
||||||
|
pub async fn decide(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path((id, pid)): Path<(Uuid, Uuid)>,
|
||||||
|
Json(body): Json<DecideRequest>,
|
||||||
|
) -> Result<Json<Value>, ApiError> {
|
||||||
|
let ws = user.workspace_id;
|
||||||
|
let proposal = cm_db::repo::mission_team_proposals::get(&state.pool, pid, ws.as_uuid().to_owned())
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
if proposal.mission_id != id {
|
||||||
|
return Err(ApiError::NotFound);
|
||||||
|
}
|
||||||
|
|
||||||
|
if body.status == "rejected" {
|
||||||
|
let decided = cm_db::repo::mission_team_proposals::decide(
|
||||||
|
&state.pool,
|
||||||
|
pid,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
"rejected",
|
||||||
|
body.note.as_deref(),
|
||||||
|
Some(user.user_id.as_uuid().to_owned()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?;
|
||||||
|
return Ok(Json(json!({ "status": "rejected", "decided": decided })));
|
||||||
|
}
|
||||||
|
if body.status != "approved" {
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mission = cm_db::repo::missions::get(&state.pool, id, ws.as_uuid())
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
if mission.status != "draft" {
|
||||||
|
eprintln!("mission {id}: roster approval refused — mission is {}", mission.status);
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
let roster: Roster = serde_json::from_value(proposal.roster.clone()).map_err(|e| {
|
||||||
|
eprintln!("mission {id}: stored proposal {pid} is not a roster ({e})");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
// Re-validated at approval, against the fleet as it is NOW. A node can go
|
||||||
|
// offline between proposing and approving, and the cheapest place to find
|
||||||
|
// that out is still here rather than at VM boot.
|
||||||
|
let available = available_backends(&state.pool, ws.as_uuid().to_owned())
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?;
|
||||||
|
if let Err(why) = roster.validate(&available) {
|
||||||
|
eprintln!("mission {id}: roster {pid} is no longer applicable: {why}");
|
||||||
|
let _ = cm_db::repo::mission_team_proposals::decide(
|
||||||
|
&state.pool,
|
||||||
|
pid,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
"rejected",
|
||||||
|
Some(&why.to_string()),
|
||||||
|
Some(user.user_id.as_uuid().to_owned()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
let graph = roster.graph().map_err(|e| {
|
||||||
|
eprintln!("mission {id}: approved roster does not build a graph: {e}");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
// Claiming the proposal and writing the mission are ONE transaction. Doing
|
||||||
|
// them as two statements left the first real approval in production marked
|
||||||
|
// `approved` with nothing written to the mission — and the partial unique
|
||||||
|
// index then makes that permanent, since no other proposal for that mission
|
||||||
|
// can ever be approved.
|
||||||
|
let claimed = cm_db::repo::mission_team_proposals::approve_and_apply(
|
||||||
|
&state.pool,
|
||||||
|
pid,
|
||||||
|
id,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
&graph,
|
||||||
|
body.note.as_deref(),
|
||||||
|
Some(user.user_id.as_uuid().to_owned()),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("mission {id}: could not apply roster {pid}: {e}");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
if !claimed {
|
||||||
|
return Err(ApiError::BadRequest);
|
||||||
|
}
|
||||||
|
|
||||||
|
eprintln!(
|
||||||
|
"mission_roster: mission {id} now runs a {}-node composed graph from proposal {pid}",
|
||||||
|
roster.members.len()
|
||||||
|
);
|
||||||
|
Ok(Json(json!({
|
||||||
|
"status": "approved",
|
||||||
|
"team_engine": "composed",
|
||||||
|
"nodes": roster.members.len(),
|
||||||
|
"graph": graph,
|
||||||
|
})))
|
||||||
|
}
|
||||||
@@ -38,6 +38,16 @@ pub struct CreateMissionRequest {
|
|||||||
/// Defaults to "zeroclaw". "local_herdr" requires target_node_id.
|
/// Defaults to "zeroclaw". "local_herdr" requires target_node_id.
|
||||||
pub runtime_kind: Option<String>,
|
pub runtime_kind: Option<String>,
|
||||||
pub target_node_id: Option<Uuid>,
|
pub target_node_id: Option<Uuid>,
|
||||||
|
/// Which per-CLI rootfs a `microvm` mission boots (`missions.backend`), e.g.
|
||||||
|
/// "claude". NULL boots the node's default image.
|
||||||
|
pub backend: Option<String>,
|
||||||
|
/// Model that independently validates this mission's phase verdicts, e.g.
|
||||||
|
/// `glm:glm-4.7`. Omit to use the deployment default; send `""` to opt out of
|
||||||
|
/// independent validation and judge with the house model.
|
||||||
|
pub validator_model: Option<String>,
|
||||||
|
/// Team engine: `"claude_code"` asks the mission's agent to form a team.
|
||||||
|
/// Omit for solo, which is the default and much cheaper.
|
||||||
|
pub team_engine: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn default_schedule() -> Value {
|
fn default_schedule() -> Value {
|
||||||
@@ -260,6 +270,12 @@ pub async fn create(
|
|||||||
return Err(ApiError::BadRequest);
|
return Err(ApiError::BadRequest);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// microvm needs no target here: placement resolves a KVM-capable node at
|
||||||
|
// launch and fails the launch when there is none, so an explicit target is
|
||||||
|
// a request rather than a requirement. Rejecting the value outright — as
|
||||||
|
// this did until B4.5 — made `runtime_kind='microvm'` unreachable through
|
||||||
|
// the only interface that creates missions.
|
||||||
|
"microvm" => {}
|
||||||
_ => return Err(ApiError::BadRequest),
|
_ => return Err(ApiError::BadRequest),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -275,6 +291,9 @@ pub async fn create(
|
|||||||
config: body.config,
|
config: body.config,
|
||||||
runtime_kind: Some(runtime_kind),
|
runtime_kind: Some(runtime_kind),
|
||||||
target_node_id: body.target_node_id,
|
target_node_id: body.target_node_id,
|
||||||
|
backend: body.backend.as_deref(),
|
||||||
|
validator_model: body.validator_model.as_deref(),
|
||||||
|
team_engine: body.team_engine.as_deref(),
|
||||||
phases: phases_for_create(
|
phases: phases_for_create(
|
||||||
crate::workflow_registry::get(body.template_kind.trim()),
|
crate::workflow_registry::get(body.template_kind.trim()),
|
||||||
body.phases,
|
body.phases,
|
||||||
@@ -308,6 +327,301 @@ pub async fn get(
|
|||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// GET /api/missions/{id}/artifacts/{artifact_id}/content — the artifact's text.
|
||||||
|
///
|
||||||
|
/// The frontend had no way to READ an artifact: it listed paths and offered a
|
||||||
|
/// PDF preview, and the PDF never rendered. Markdown is the deliverable now, so
|
||||||
|
/// something has to serve it.
|
||||||
|
///
|
||||||
|
/// Two containment rules, both enforced rather than assumed:
|
||||||
|
///
|
||||||
|
/// - the artifact row must belong to a mission in the caller's workspace, so
|
||||||
|
/// an artifact id from another tenant is a 404, not a file read;
|
||||||
|
/// - the resolved path must stay inside `<missions_root>/_outputs`. Artifact
|
||||||
|
/// paths are written by this server, but a stored `../../etc/passwd` would
|
||||||
|
/// otherwise be read and returned. Canonicalise, then check the prefix —
|
||||||
|
/// checking the string before resolving `..` is the classic hole.
|
||||||
|
///
|
||||||
|
/// Text only, and capped: these are markdown documents, and streaming an
|
||||||
|
/// arbitrary captured file into a JSON body is not what this is for.
|
||||||
|
/// Turn a stored artifact path into an absolute one, refusing anything outside
|
||||||
|
/// `_outputs`.
|
||||||
|
///
|
||||||
|
/// Shared by the read and download routes deliberately: two copies of a
|
||||||
|
/// containment check is two chances for one of them to be the lenient one, and
|
||||||
|
/// the lenient one is a path-traversal read of the gateway's filesystem.
|
||||||
|
fn resolve_artifact_path(stored: &str) -> Result<std::path::PathBuf, ApiError> {
|
||||||
|
let root = crate::mission_outputs::outputs_root_dir();
|
||||||
|
let abs = crate::mission_outputs::missions_root_dir().join(stored);
|
||||||
|
// `canonicalize` on BOTH sides, so a symlink out of the tree resolves to
|
||||||
|
// its target before the comparison rather than after.
|
||||||
|
let resolved = std::fs::canonicalize(&abs).map_err(|_| ApiError::NotFound)?;
|
||||||
|
let root = std::fs::canonicalize(&root).map_err(|_| ApiError::NotFound)?;
|
||||||
|
if !resolved.starts_with(&root) {
|
||||||
|
eprintln!(
|
||||||
|
"missions: refused artifact {} — outside {}",
|
||||||
|
resolved.display(),
|
||||||
|
root.display()
|
||||||
|
);
|
||||||
|
return Err(ApiError::NotFound);
|
||||||
|
}
|
||||||
|
Ok(resolved)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `GET /api/missions/{id}/artifacts/{artifact_id}/download` — the file itself.
|
||||||
|
///
|
||||||
|
/// Separate from `artifact_content` because that route cannot serve the two
|
||||||
|
/// cases a download exists for: it caps at 2 MiB and reads as UTF-8, so a large
|
||||||
|
/// or binary artifact is unreachable by any means today. This one streams the
|
||||||
|
/// bytes with a filename attached and no ceiling.
|
||||||
|
pub async fn artifact_download(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path((id, artifact_id)): Path<(Uuid, Uuid)>,
|
||||||
|
) -> Result<axum::response::Response, ApiError> {
|
||||||
|
use axum::response::IntoResponse;
|
||||||
|
|
||||||
|
cm_db::repo::missions::get(&state.pool, id, user.workspace_id.as_uuid())
|
||||||
|
.await?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
let artifacts = cm_db::repo::missions::artifacts_for(&state.pool, id).await?;
|
||||||
|
let artifact = artifacts
|
||||||
|
.into_iter()
|
||||||
|
.find(|a| a.id == artifact_id)
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
let resolved = resolve_artifact_path(&artifact.path)?;
|
||||||
|
|
||||||
|
let bytes = tokio::fs::read(&resolved)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::NotFound)?;
|
||||||
|
|
||||||
|
// The basename, never the stored path: `_outputs/<mission>/<phase>/repo/x.md`
|
||||||
|
// as a filename would arrive as a browser-mangled string, and the path is
|
||||||
|
// internal layout the user has no reason to see.
|
||||||
|
let name = resolved
|
||||||
|
.file_name()
|
||||||
|
.and_then(|n| n.to_str())
|
||||||
|
.filter(|n| !n.is_empty())
|
||||||
|
.unwrap_or("artifact");
|
||||||
|
// Quoted and stripped of quotes/newlines: a filename is attacker-influenced
|
||||||
|
// input (an agent chose it) and this header is parsed by every browser.
|
||||||
|
let safe: String = name
|
||||||
|
.chars()
|
||||||
|
.filter(|c| *c != '"' && *c != '\\' && !c.is_control())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Ok((
|
||||||
|
[
|
||||||
|
(
|
||||||
|
axum::http::header::CONTENT_TYPE,
|
||||||
|
artifact.mime.unwrap_or_else(|| "application/octet-stream".into()),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
axum::http::header::CONTENT_DISPOSITION,
|
||||||
|
format!("attachment; filename=\"{safe}\""),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
bytes,
|
||||||
|
)
|
||||||
|
.into_response())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn artifact_content(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path((id, artifact_id)): Path<(Uuid, Uuid)>,
|
||||||
|
) -> Result<Json<serde_json::Value>, ApiError> {
|
||||||
|
/// Beyond this, a document is not something a reader wants inline.
|
||||||
|
const MAX_BYTES: u64 = 2 * 1024 * 1024;
|
||||||
|
|
||||||
|
// Scoped to the caller's workspace by loading the mission first.
|
||||||
|
cm_db::repo::missions::get(&state.pool, id, user.workspace_id.as_uuid())
|
||||||
|
.await?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
|
||||||
|
let artifacts = cm_db::repo::missions::artifacts_for(&state.pool, id).await?;
|
||||||
|
let artifact = artifacts
|
||||||
|
.into_iter()
|
||||||
|
.find(|a| a.id == artifact_id)
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
|
||||||
|
let resolved = resolve_artifact_path(&artifact.path)?;
|
||||||
|
|
||||||
|
let meta = std::fs::metadata(&resolved).map_err(|_| ApiError::NotFound)?;
|
||||||
|
if meta.len() > MAX_BYTES {
|
||||||
|
return Ok(Json(serde_json::json!({
|
||||||
|
"path": artifact.path,
|
||||||
|
"mime": artifact.mime,
|
||||||
|
"truncated": true,
|
||||||
|
"content": "",
|
||||||
|
"bytes": meta.len(),
|
||||||
|
})));
|
||||||
|
}
|
||||||
|
let content = std::fs::read_to_string(&resolved).map_err(|_| ApiError::NotFound)?;
|
||||||
|
Ok(Json(serde_json::json!({
|
||||||
|
"path": artifact.path,
|
||||||
|
"mime": artifact.mime,
|
||||||
|
"title": artifact.title,
|
||||||
|
"truncated": false,
|
||||||
|
"content": content,
|
||||||
|
"bytes": meta.len(),
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// POST /api/missions/{id}/merge — merge this mission's branch into the base.
|
||||||
|
///
|
||||||
|
/// The operator's button. `MergePolicy::Never` — the default for anything that
|
||||||
|
/// touches code — means "do not merge on your own", deferring to a human; this
|
||||||
|
/// endpoint is that human saying yes. So the additive-only test does not apply
|
||||||
|
/// here, and deliberately so.
|
||||||
|
///
|
||||||
|
/// It works in a FRESH CLONE under `_merge/<mission>`, never the mission
|
||||||
|
/// checkout: that directory is reaped on a timer after a mission ends, so a
|
||||||
|
/// merge that used it would succeed right after a run and fail inexplicably an
|
||||||
|
/// hour later. The clone is made by the server process, so nothing here runs as
|
||||||
|
/// root and the ordinary cleanup works — unlike the copies in `root_copy`.
|
||||||
|
pub async fn merge_branch(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Path(id): Path<Uuid>,
|
||||||
|
) -> Result<Json<serde_json::Value>, ApiError> {
|
||||||
|
let mission = cm_db::repo::missions::get(&state.pool, id, user.workspace_id.as_uuid())
|
||||||
|
.await?
|
||||||
|
.ok_or(ApiError::NotFound)?;
|
||||||
|
let repo_id = mission.repo_id.ok_or(ApiError::BadRequest)?;
|
||||||
|
let repo = cm_db::repo::repos::get(&state.pool, repo_id, user.workspace_id)
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::NotFound)?;
|
||||||
|
let clone_url = repo.clone_url.as_deref().ok_or(ApiError::BadRequest)?;
|
||||||
|
let base = repo.default_branch.as_deref().unwrap_or("main");
|
||||||
|
|
||||||
|
// The branch is whatever delivery actually pushed — read from the artifact
|
||||||
|
// it recorded, not reconstructed from the mission id. A phase that never
|
||||||
|
// pushed has no branch, and that must be a refusal rather than a guess.
|
||||||
|
let artifacts = cm_db::repo::missions::artifacts_for(&state.pool, id).await?;
|
||||||
|
let delivered = artifacts.iter().rev().find_map(|a| {
|
||||||
|
let m = a.metadata.as_object()?;
|
||||||
|
let branch = m.get("branch")?.as_str()?.to_string();
|
||||||
|
(m.get("pushed").and_then(|v| v.as_bool()) == Some(true)).then_some(branch)
|
||||||
|
});
|
||||||
|
let Some(branch) = delivered else {
|
||||||
|
return Ok(Json(serde_json::json!({
|
||||||
|
"merged": false,
|
||||||
|
"reason": "this mission has no pushed branch to merge",
|
||||||
|
})));
|
||||||
|
};
|
||||||
|
|
||||||
|
let auth = crate::mission_workspace::with_ambient_auth(clone_url);
|
||||||
|
let workdir = crate::mission_workspace::missions_root()
|
||||||
|
.join("_merge")
|
||||||
|
.join(id.to_string());
|
||||||
|
let _ = tokio::fs::remove_dir_all(&workdir).await;
|
||||||
|
if let Some(parent) = workdir.parent() {
|
||||||
|
let _ = tokio::fs::create_dir_all(parent).await;
|
||||||
|
}
|
||||||
|
let clone = tokio::process::Command::new("git")
|
||||||
|
.args(["clone", "--quiet", &auth.url])
|
||||||
|
.arg(&workdir)
|
||||||
|
.env("GIT_TERMINAL_PROMPT", "0")
|
||||||
|
.output()
|
||||||
|
.await
|
||||||
|
.map_err(|_| ApiError::Internal)?;
|
||||||
|
if !clone.status.success() {
|
||||||
|
eprintln!(
|
||||||
|
"missions::merge_branch: clone for {id} failed: {}",
|
||||||
|
String::from_utf8_lossy(&clone.stderr)
|
||||||
|
.chars()
|
||||||
|
.take(300)
|
||||||
|
.collect::<String>()
|
||||||
|
);
|
||||||
|
return Ok(Json(serde_json::json!({
|
||||||
|
"merged": false,
|
||||||
|
"reason": "could not clone the repository to merge",
|
||||||
|
})));
|
||||||
|
}
|
||||||
|
|
||||||
|
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
||||||
|
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
||||||
|
let outcome = async {
|
||||||
|
let merged =
|
||||||
|
crate::auto_merge::merge_on_operator_approval(&workdir, &auth.url, &branch, base)
|
||||||
|
.await?;
|
||||||
|
if !merged.merged {
|
||||||
|
return Ok(merged);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run the project's own tests against the MERGED tree, before it is
|
||||||
|
// published. Verifying first rather than reverting after is the
|
||||||
|
// difference between "main was never broken" and "main was broken until
|
||||||
|
// someone noticed".
|
||||||
|
//
|
||||||
|
// The merge is already committed locally at this point; refusing here
|
||||||
|
// simply never pushes it, and the branch is still there to retry.
|
||||||
|
match crate::mission_delivery::verify_tests(&workdir, &container).await {
|
||||||
|
crate::mission_delivery::TestOutcome::Passed => {}
|
||||||
|
crate::mission_delivery::TestOutcome::NoSuite => {
|
||||||
|
eprintln!(
|
||||||
|
"missions::merge_branch: {branch} has no discoverable test suite — publishing unverified"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
crate::mission_delivery::TestOutcome::Failed(code) => {
|
||||||
|
return Ok(crate::auto_merge::MergeOutcome {
|
||||||
|
merged: false,
|
||||||
|
reason: format!(
|
||||||
|
"the merged tree FAILS the project's tests (exit {code}) — not published. The branch is unchanged; fix it and merge again."
|
||||||
|
),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
// Fail closed. A suite that could not run has not passed, and
|
||||||
|
// publishing on "we could not check" is how a green main stops
|
||||||
|
// meaning anything.
|
||||||
|
crate::mission_delivery::TestOutcome::CouldNotRun(why) => {
|
||||||
|
return Ok(crate::auto_merge::MergeOutcome {
|
||||||
|
merged: false,
|
||||||
|
reason: format!("could not run the tests on the merged tree ({why}) — not published"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
crate::auto_merge::push_merged(&workdir, &auth.url, base).await?;
|
||||||
|
Ok::<_, String>(crate::auto_merge::MergeOutcome {
|
||||||
|
merged: true,
|
||||||
|
reason: format!("tests pass on the merged tree; published to {base}"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
|
||||||
|
// Purge through the container: `verify_tests` runs `cargo test` as ROOT, so
|
||||||
|
// the workdir now holds a root-owned `target/` the server (uid 65532) cannot
|
||||||
|
// delete. Same defect as the bench and judge copies.
|
||||||
|
crate::root_copy::purge(&container, &workdir).await;
|
||||||
|
let _ = tokio::fs::remove_dir_all(&workdir).await;
|
||||||
|
|
||||||
|
match outcome {
|
||||||
|
Ok(o) => {
|
||||||
|
eprintln!(
|
||||||
|
"missions::merge_branch: mission {id} branch {branch} -> {base}: {}",
|
||||||
|
o.reason
|
||||||
|
);
|
||||||
|
Ok(Json(serde_json::json!({
|
||||||
|
"merged": o.merged,
|
||||||
|
"reason": o.reason,
|
||||||
|
"branch": branch,
|
||||||
|
"base": base,
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("missions::merge_branch: mission {id} failed: {e}");
|
||||||
|
Ok(Json(serde_json::json!({
|
||||||
|
"merged": false,
|
||||||
|
"reason": format!("merge failed: {e}"),
|
||||||
|
"branch": branch,
|
||||||
|
"base": base,
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// POST /api/missions/{id}/benchmark — run the benchmark harness
|
/// POST /api/missions/{id}/benchmark — run the benchmark harness
|
||||||
/// against a phase. Slot='baseline' records iteration 0's
|
/// against a phase. Slot='baseline' records iteration 0's
|
||||||
/// before_metrics; slot='after' with iteration=N records the
|
/// before_metrics; slot='after' with iteration=N records the
|
||||||
@@ -367,6 +681,61 @@ pub struct RefineResponse {
|
|||||||
pub refined: String,
|
pub refined: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Deserialize)]
|
||||||
|
pub struct RefineDraftRequest {
|
||||||
|
#[serde(default)]
|
||||||
|
pub title: String,
|
||||||
|
pub description: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub template_kind: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `POST /api/missions/refine-draft` — polish a description with no mission
|
||||||
|
/// behind it yet.
|
||||||
|
///
|
||||||
|
/// The wizard's polish button fires while the user is still typing, before
|
||||||
|
/// anything is created. `refine` deliberately requires a saved draft so its
|
||||||
|
/// Accept can write back; this one has nothing to write back to and returns the
|
||||||
|
/// text for the caller to put in the box.
|
||||||
|
///
|
||||||
|
/// The phase list comes from the workflow recipe rather than the caller, for
|
||||||
|
/// the same reason `phases_for_create` prefers it: the recipe is the
|
||||||
|
/// authoritative composition, and a client that guessed would have the model
|
||||||
|
/// write acceptance criteria for phases the mission will not run.
|
||||||
|
pub async fn refine_draft(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(_user): Authed,
|
||||||
|
Json(req): Json<RefineDraftRequest>,
|
||||||
|
) -> Result<Json<RefineResponse>, ApiError> {
|
||||||
|
let phase_kinds: Vec<String> = req
|
||||||
|
.template_kind
|
||||||
|
.as_deref()
|
||||||
|
.and_then(crate::workflow_registry::get)
|
||||||
|
.map(|r| r.phases.iter().map(|p| p.kind.clone()).collect())
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
let result = crate::mission_refiner::refine_draft(
|
||||||
|
&state.runtime,
|
||||||
|
req.title.trim(),
|
||||||
|
req.template_kind.as_deref().unwrap_or("custom"),
|
||||||
|
&phase_kinds,
|
||||||
|
&req.description,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("refine-draft failed: {e}");
|
||||||
|
if e.contains("empty") {
|
||||||
|
ApiError::BadRequest
|
||||||
|
} else {
|
||||||
|
crate::subscription::as_api_error(&e)
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
Ok(Json(RefineResponse {
|
||||||
|
original: result.original,
|
||||||
|
refined: result.refined,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
/// POST /api/missions/{id}/refine — generate a coherent, sectioned
|
/// POST /api/missions/{id}/refine — generate a coherent, sectioned
|
||||||
/// Markdown rewrite of the current description WITHOUT persisting.
|
/// Markdown rewrite of the current description WITHOUT persisting.
|
||||||
/// Frontend renders a before/after diff; user hits Accept (PATCH
|
/// Frontend renders a before/after diff; user hits Accept (PATCH
|
||||||
@@ -376,7 +745,7 @@ pub async fn refine(
|
|||||||
Authed(user): Authed,
|
Authed(user): Authed,
|
||||||
Path(id): Path<Uuid>,
|
Path(id): Path<Uuid>,
|
||||||
) -> Result<Json<RefineResponse>, ApiError> {
|
) -> Result<Json<RefineResponse>, ApiError> {
|
||||||
let result = crate::mission_refiner::refine(&state.pool, user.workspace_id, id)
|
let result = crate::mission_refiner::refine(&state.pool, &state.runtime, user.workspace_id, id)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
eprintln!("mission {id}: refine failed: {e}");
|
eprintln!("mission {id}: refine failed: {e}");
|
||||||
@@ -530,6 +899,31 @@ async fn reap_mission_resources(state: &AppState, mission_id: Uuid) {
|
|||||||
// drift back into skipping the container teardown.
|
// drift back into skipping the container teardown.
|
||||||
let provisioner = crate::runtime_provision::RuntimeProvisioner::from_env();
|
let provisioner = crate::runtime_provision::RuntimeProvisioner::from_env();
|
||||||
for cid in &claw_ids {
|
for cid in &claw_ids {
|
||||||
|
// Only claws this mission is the LAST holder of.
|
||||||
|
//
|
||||||
|
// Claws are reused across missions now (see
|
||||||
|
// `agent_template_link::reusable_claw`), so a mission's team can contain
|
||||||
|
// staff that other missions still employ. Purging those would delete a
|
||||||
|
// user's workforce as a side effect of tidying up one mission — and it
|
||||||
|
// would look like the roster quietly shrinking, not like an error.
|
||||||
|
let shared: i64 = sqlx::query_scalar(
|
||||||
|
"SELECT count(*)
|
||||||
|
FROM team_members tm
|
||||||
|
JOIN mission_teams mt ON mt.team_id = tm.team_id
|
||||||
|
WHERE tm.claw_id = $1 AND mt.mission_id <> $2",
|
||||||
|
)
|
||||||
|
.bind(cid)
|
||||||
|
.bind(mission_id)
|
||||||
|
.fetch_one(&state.pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or(0);
|
||||||
|
if shared > 0 {
|
||||||
|
eprintln!(
|
||||||
|
"missions::delete: keeping claw {cid} — {shared} other mission(s) still employ it"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
let report = crate::routes::claws::purge_agent(
|
let report = crate::routes::claws::purge_agent(
|
||||||
&state.pool,
|
&state.pool,
|
||||||
&state.runtime,
|
&state.runtime,
|
||||||
@@ -682,9 +1076,16 @@ pub async fn retry_phase(
|
|||||||
let mission = cm_db::repo::missions::get(&state.pool, id, user.workspace_id.as_uuid())
|
let mission = cm_db::repo::missions::get(&state.pool, id, user.workspace_id.as_uuid())
|
||||||
.await?
|
.await?
|
||||||
.ok_or(ApiError::NotFound)?;
|
.ok_or(ApiError::NotFound)?;
|
||||||
if mission.status != "running" {
|
// `failed` is retryable, and has to be: a failed phase now closes its
|
||||||
|
// mission (its later phases are marked unreachable so the mission can
|
||||||
|
// finish at all), so refusing anything but `running` would mean the one
|
||||||
|
// outcome you would actually want to retry is the one you cannot.
|
||||||
|
// `completed` and `cancelled` stay refused — reopening those is a different
|
||||||
|
// decision than re-running a phase that failed.
|
||||||
|
if mission.status != "running" && mission.status != "failed" {
|
||||||
return Err(ApiError::BadRequest);
|
return Err(ApiError::BadRequest);
|
||||||
}
|
}
|
||||||
|
let mut tx = state.pool.begin().await?;
|
||||||
let r = sqlx::query(
|
let r = sqlx::query(
|
||||||
"UPDATE mission_phases
|
"UPDATE mission_phases
|
||||||
SET status = 'pending', started_at = NULL, completed_at = NULL
|
SET status = 'pending', started_at = NULL, completed_at = NULL
|
||||||
@@ -693,12 +1094,41 @@ pub async fn retry_phase(
|
|||||||
)
|
)
|
||||||
.bind(phase_id)
|
.bind(phase_id)
|
||||||
.bind(id)
|
.bind(id)
|
||||||
.execute(&state.pool)
|
.execute(&mut *tx)
|
||||||
.await?;
|
.await?;
|
||||||
if r.rows_affected() == 0 {
|
if r.rows_affected() == 0 {
|
||||||
|
tx.rollback().await?;
|
||||||
return Err(ApiError::NotFound);
|
return Err(ApiError::NotFound);
|
||||||
}
|
}
|
||||||
Ok(Json(serde_json::json!({ "reset": true })))
|
// Reopen the phases this one's failure had made unreachable. Without this a
|
||||||
|
// retry runs the failed phase and then stops, because everything after it
|
||||||
|
// is terminal-by-skip — the mission would close again the moment this phase
|
||||||
|
// finished, having done only part of the work.
|
||||||
|
let reopened = sqlx::query(
|
||||||
|
"UPDATE mission_phases mp
|
||||||
|
SET status = 'pending', started_at = NULL, completed_at = NULL
|
||||||
|
WHERE mp.mission_id = $1
|
||||||
|
AND mp.status = 'skipped'
|
||||||
|
AND mp.order_idx > (SELECT order_idx FROM mission_phases WHERE id = $2)",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(phase_id)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
// And put the mission back to running, or nothing sweeps the phase: every
|
||||||
|
// launcher and closer keys off `missions.status = 'running'`.
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE missions SET status = 'running', completed_at = NULL, updated_at = now()
|
||||||
|
WHERE id = $1 AND status = 'failed'",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?;
|
||||||
|
tx.commit().await?;
|
||||||
|
Ok(Json(
|
||||||
|
serde_json::json!({ "reset": true, "reopened_phases": reopened }),
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// GET /api/missions/{id}/phases/{phase_id}/summary — the completion
|
/// GET /api/missions/{id}/phases/{phase_id}/summary — the completion
|
||||||
@@ -844,7 +1274,16 @@ pub async fn set_status(
|
|||||||
.any(|v| v.as_array().map(|a| !a.is_empty()).unwrap_or(false))
|
.any(|v| v.as_array().map(|a| !a.is_empty()).unwrap_or(false))
|
||||||
})
|
})
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
if prior.team_id.is_none() && prior.team_template_id.is_none() && !has_phase_teams {
|
// A microVM mission materialises no team — `microvm_executor` runs the
|
||||||
|
// agent CLI directly in the VM — so requiring one would reject the launch
|
||||||
|
// of a perfectly well-formed mission, and satisfying it would provision
|
||||||
|
// claws that never run.
|
||||||
|
let needs_team = prior.runtime_kind != "microvm";
|
||||||
|
if needs_team
|
||||||
|
&& prior.team_id.is_none()
|
||||||
|
&& prior.team_template_id.is_none()
|
||||||
|
&& !has_phase_teams
|
||||||
|
{
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"mission {id}: launch rejected — no team_id, no team_template_id, no config.phase_teams"
|
"mission {id}: launch rejected — no team_id, no team_template_id, no config.phase_teams"
|
||||||
);
|
);
|
||||||
@@ -1243,3 +1682,190 @@ mod tests {
|
|||||||
assert!(outputs_of(Some(&serde_json::json!({}))).is_empty());
|
assert!(outputs_of(Some(&serde_json::json!({}))).is_empty());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// GET /api/workforce — the roster grouped by the mission each claw works on.
|
||||||
|
///
|
||||||
|
/// The sidebar used to flatten `orgs → companies → teams → agents`, which
|
||||||
|
/// rendered a claw once per TEAM it belongs to. Since claws are reused across
|
||||||
|
/// missions, a crew of five that had run five missions appeared as twenty-five
|
||||||
|
/// rows of the same five people — the roster looked like it was multiplying.
|
||||||
|
///
|
||||||
|
/// Grouping by mission makes that repetition mean something: the same person
|
||||||
|
/// legitimately appears under each mission they staffed. `agents` is deduped
|
||||||
|
/// per mission, and claws belonging to no mission come back under `unassigned`
|
||||||
|
/// so a hand-created claw cannot fall out of the UI entirely.
|
||||||
|
pub async fn workforce(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
) -> Result<Json<Value>, ApiError> {
|
||||||
|
use sqlx::Row;
|
||||||
|
let ws = user.workspace_id.as_uuid();
|
||||||
|
|
||||||
|
// One query, not one-per-mission: the sidebar renders on every navigation.
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT m.id::text AS mission_id,
|
||||||
|
m.title AS mission_title,
|
||||||
|
m.status AS mission_status,
|
||||||
|
m.template_kind AS template_kind,
|
||||||
|
m.created_at AS created_at,
|
||||||
|
a.id::text AS agent_id,
|
||||||
|
a.name AS agent_name,
|
||||||
|
a.job_title AS job_title,
|
||||||
|
a.accent AS accent,
|
||||||
|
a.status AS agent_status,
|
||||||
|
tm.role AS role_slot
|
||||||
|
FROM missions m
|
||||||
|
JOIN mission_teams mt ON mt.mission_id = m.id
|
||||||
|
JOIN team_members tm ON tm.team_id = mt.team_id
|
||||||
|
JOIN agents a ON a.id = tm.claw_id
|
||||||
|
WHERE m.workspace_id = $1
|
||||||
|
AND a.deleted_at IS NULL
|
||||||
|
ORDER BY m.created_at DESC, tm.role ASC",
|
||||||
|
)
|
||||||
|
.bind(ws)
|
||||||
|
.fetch_all(&state.pool)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let mut missions: Vec<Value> = Vec::new();
|
||||||
|
let mut seen_mission: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
|
||||||
|
for r in rows {
|
||||||
|
let mid: String = r.get("mission_id");
|
||||||
|
let idx = match seen_mission.get(&mid) {
|
||||||
|
Some(i) => *i,
|
||||||
|
None => {
|
||||||
|
missions.push(serde_json::json!({
|
||||||
|
"mission_id": mid,
|
||||||
|
"title": r.get::<String, _>("mission_title"),
|
||||||
|
"status": r.get::<String, _>("mission_status"),
|
||||||
|
// Drives the World's palette: what the mission is FOR
|
||||||
|
// should be visible before any label is read.
|
||||||
|
"templateKind": r.get::<String, _>("template_kind"),
|
||||||
|
"agents": Vec::<Value>::new(),
|
||||||
|
}));
|
||||||
|
seen_mission.insert(r.get::<String, _>("mission_id"), missions.len() - 1);
|
||||||
|
missions.len() - 1
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let agent = serde_json::json!({
|
||||||
|
"id": r.get::<String, _>("agent_id"),
|
||||||
|
"name": r.get::<String, _>("agent_name"),
|
||||||
|
"job_title": r.get::<String, _>("job_title"),
|
||||||
|
"role_slot": r.get::<String, _>("role_slot"),
|
||||||
|
"accent": r.get::<String, _>("accent"),
|
||||||
|
"status": r.get::<String, _>("agent_status"),
|
||||||
|
});
|
||||||
|
// A claw bound to two NODES of the same mission is still one colleague.
|
||||||
|
let list = missions[idx]["agents"].as_array_mut().expect("agents array");
|
||||||
|
let id = agent["id"].clone();
|
||||||
|
if !list.iter().any(|a| a["id"] == id) {
|
||||||
|
list.push(agent);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Claws on no mission at all — hand-created, or whose missions were
|
||||||
|
// deleted. Without this they would simply vanish from the sidebar.
|
||||||
|
let loose = sqlx::query(
|
||||||
|
"SELECT a.id::text AS agent_id, a.name, a.job_title, a.accent, a.status
|
||||||
|
FROM agents a
|
||||||
|
WHERE a.workspace_id = $1
|
||||||
|
AND a.deleted_at IS NULL
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM team_members tm
|
||||||
|
JOIN mission_teams mt ON mt.team_id = tm.team_id
|
||||||
|
JOIN missions m ON m.id = mt.mission_id
|
||||||
|
WHERE tm.claw_id = a.id AND m.workspace_id = $1)
|
||||||
|
ORDER BY a.name ASC",
|
||||||
|
)
|
||||||
|
.bind(ws)
|
||||||
|
.fetch_all(&state.pool)
|
||||||
|
.await?;
|
||||||
|
let unassigned: Vec<Value> = loose
|
||||||
|
.into_iter()
|
||||||
|
.map(|r| {
|
||||||
|
serde_json::json!({
|
||||||
|
"id": r.get::<String, _>("agent_id"),
|
||||||
|
"name": r.get::<String, _>("name"),
|
||||||
|
"job_title": r.get::<String, _>("job_title"),
|
||||||
|
"role_slot": Value::Null,
|
||||||
|
"accent": r.get::<String, _>("accent"),
|
||||||
|
"status": r.get::<String, _>("status"),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
Ok(Json(serde_json::json!({
|
||||||
|
"missions": missions,
|
||||||
|
"unassigned": unassigned,
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod reap_tests {
|
||||||
|
/// A mission's teardown must ask whether anyone else still employs a claw.
|
||||||
|
///
|
||||||
|
/// Claws are reused across missions now, so a mission's team can contain
|
||||||
|
/// staff other missions still hold. The old code purged every claw in the
|
||||||
|
/// team unconditionally, which under reuse deletes a user's workforce as a
|
||||||
|
/// side effect of tidying one mission — and it presents as the roster
|
||||||
|
/// quietly shrinking rather than as an error.
|
||||||
|
#[test]
|
||||||
|
fn mission_teardown_checks_for_other_employers_before_purging() {
|
||||||
|
let src = include_str!("missions.rs");
|
||||||
|
let reaper = src
|
||||||
|
.split("async fn reap_mission_resources")
|
||||||
|
.nth(1)
|
||||||
|
.expect("the reaper exists");
|
||||||
|
// Scoped to the reaper, so the check cannot be satisfied by some other
|
||||||
|
// function elsewhere in the file that happens to mention mission_teams.
|
||||||
|
assert!(
|
||||||
|
reaper.contains("mt.mission_id <> $2"),
|
||||||
|
"the purge must exclude claws held by another mission"
|
||||||
|
);
|
||||||
|
let purge_at = reaper.find("purge_agent").expect("it still purges");
|
||||||
|
let guard_at = reaper.find("mt.mission_id <> $2").expect("guard present");
|
||||||
|
assert!(
|
||||||
|
guard_at < purge_at,
|
||||||
|
"the guard has to run BEFORE the purge, or it is decoration"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod artifact_tests {
|
||||||
|
/// A filename reaches `Content-Disposition` after an AGENT chose it.
|
||||||
|
///
|
||||||
|
/// The value is attacker-influenced and parsed by every browser, so the
|
||||||
|
/// quote and control characters that would end the header early — or inject
|
||||||
|
/// a second one — are removed rather than escaped.
|
||||||
|
#[test]
|
||||||
|
fn a_downloaded_filename_cannot_break_out_of_its_header() {
|
||||||
|
let clean = |name: &str| -> String {
|
||||||
|
name.chars()
|
||||||
|
.filter(|c| *c != '"' && *c != '\\' && !c.is_control())
|
||||||
|
.collect()
|
||||||
|
};
|
||||||
|
assert_eq!(clean("findings.md"), "findings.md");
|
||||||
|
assert_eq!(clean("re\"port.md"), "report.md");
|
||||||
|
assert_eq!(clean("a\r\nX-Evil: 1.md"), "aX-Evil: 1.md");
|
||||||
|
assert_eq!(clean("back\\slash.md"), "backslash.md");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Both artifact routes resolve through ONE containment check.
|
||||||
|
///
|
||||||
|
/// Two copies is two chances for one of them to be the lenient one, and the
|
||||||
|
/// lenient one is an arbitrary read of the gateway's filesystem.
|
||||||
|
#[test]
|
||||||
|
fn one_containment_check_serves_both_routes() {
|
||||||
|
let src = include_str!("missions.rs");
|
||||||
|
assert_eq!(
|
||||||
|
src.matches(concat!("fn resolve_", "artifact_path")).count(),
|
||||||
|
1,
|
||||||
|
"one resolver"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
src.matches(concat!("resolve_", "artifact_path(&artifact.path)")).count(),
|
||||||
|
2,
|
||||||
|
"and both routes must go through it"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -14,6 +14,8 @@ pub mod health;
|
|||||||
pub mod identity;
|
pub mod identity;
|
||||||
pub mod level_up;
|
pub mod level_up;
|
||||||
pub mod library;
|
pub mod library;
|
||||||
|
pub mod mission_plan;
|
||||||
|
pub mod mission_roster;
|
||||||
pub mod missions;
|
pub mod missions;
|
||||||
pub mod nodes;
|
pub mod nodes;
|
||||||
pub mod oauth;
|
pub mod oauth;
|
||||||
|
|||||||
@@ -402,3 +402,115 @@ async fn bridge_terminal(hub: Arc<NodeHub>, node_id: NodeId, socket: WebSocket)
|
|||||||
}
|
}
|
||||||
hub.terminal_close(node_id, sid).await;
|
hub.terminal_close(node_id, sid).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `GET /api/fleet/capacity` — what the SCHEDULER sees, verbatim.
|
||||||
|
///
|
||||||
|
/// Pulled forward from the observability phase because the capacity harness
|
||||||
|
/// scenario needs it: a test that recomputed the slot arithmetic in bash would
|
||||||
|
/// drift from `vm_placement` and then agree with itself while the scheduler did
|
||||||
|
/// something else. This returns `vm_placement::survey` unmodified, so the fleet
|
||||||
|
/// page, the harness and the placer cannot disagree.
|
||||||
|
///
|
||||||
|
/// `backend` narrows to the nodes that can boot one image (`?backend=claude`),
|
||||||
|
/// matching what `choose` does for a phase.
|
||||||
|
pub async fn capacity(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Query(q): Query<CapacityQuery>,
|
||||||
|
) -> Result<Json<Value>, ApiError> {
|
||||||
|
let ws = user.workspace_id.as_uuid().to_owned();
|
||||||
|
let (fit, unfit) =
|
||||||
|
crate::vm_placement::survey(
|
||||||
|
&state.pool,
|
||||||
|
&state.node_hub,
|
||||||
|
ws,
|
||||||
|
&crate::vm_placement::required_backends(q.backend.as_deref(), None),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("fleet capacity survey failed: {e}");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
let ranked = crate::vm_placement::rank(fit);
|
||||||
|
Ok(Json(json!({
|
||||||
|
// Total free slots across the fleet. A burst larger than this MUST
|
||||||
|
// queue rather than overcommit — that is the whole feature.
|
||||||
|
"slots": ranked.iter().map(|n| n.slots).sum::<i64>(),
|
||||||
|
"nodes": ranked.iter().map(|n| json!({
|
||||||
|
"id": n.node_id,
|
||||||
|
"name": n.name,
|
||||||
|
"slots": n.slots,
|
||||||
|
"committedVms": n.committed_vms,
|
||||||
|
"headroom": n.headroom,
|
||||||
|
"memTotalMib": n.mem_total_mib,
|
||||||
|
"usedEffMib": n.used_eff_mib,
|
||||||
|
"diskFreeGib": n.disk_free_gib,
|
||||||
|
})).collect::<Vec<_>>(),
|
||||||
|
// Never folded into the above. "Full" and "unreadable" send an
|
||||||
|
// operator to different places, so they stay separate here too.
|
||||||
|
"unfit": unfit.iter().map(|(id, name, why)| json!({
|
||||||
|
"id": id,
|
||||||
|
"name": name,
|
||||||
|
"reason": why.reason(),
|
||||||
|
})).collect::<Vec<_>>(),
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
pub struct CapacityQuery {
|
||||||
|
pub backend: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `GET /api/fleet/backends` — the microVM backends a mission may actually use.
|
||||||
|
///
|
||||||
|
/// The SAME `available_backends` the roster planner is handed, not a second
|
||||||
|
/// list. The two rules it applies are both load-bearing and neither is obvious
|
||||||
|
/// from a node's capabilities alone: a backend must be built on an online node,
|
||||||
|
/// and it must have a credential contract. `agent-terminal` satisfies the first
|
||||||
|
/// and not the second — bootable, with nothing for the agent inside to
|
||||||
|
/// authenticate with — so offering it would produce a mission that validates,
|
||||||
|
/// launches, and fails at the agent turn, which is the expensive kind of late.
|
||||||
|
///
|
||||||
|
/// Exists because the UI had no backend selector at all: every mission created
|
||||||
|
/// from the dashboard ran on `claude`, so `local-ornith`, `glm` and `kimi` were
|
||||||
|
/// reachable only by calling the API directly.
|
||||||
|
pub async fn backends(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
) -> Result<Json<Value>, ApiError> {
|
||||||
|
let ws = user.workspace_id.as_uuid().to_owned();
|
||||||
|
let mut list = crate::mission_roster::available_backends(&state.pool, ws)
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
eprintln!("fleet backends: {e}");
|
||||||
|
ApiError::Internal
|
||||||
|
})?;
|
||||||
|
// `default` is the generic `rootfs.ext4` and `claude` is the named one, and
|
||||||
|
// `microvm_credential_for` gives them the SAME contract — so a picker
|
||||||
|
// offering both shows two options with one meaning, and whichever the user
|
||||||
|
// picks they get the same thing. Collapse to the named one where it exists.
|
||||||
|
if list.iter().any(|b| b == "claude") {
|
||||||
|
list.retain(|b| b != "default");
|
||||||
|
}
|
||||||
|
Ok(Json(json!({
|
||||||
|
"backends": list.iter().map(|b| json!({
|
||||||
|
"id": b,
|
||||||
|
"label": backend_label(b),
|
||||||
|
})).collect::<Vec<_>>(),
|
||||||
|
})))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A name a person can choose between. The ids are deployment vocabulary
|
||||||
|
/// (`local-ornith`, `canary-claude`); a picker showing those alone asks the user
|
||||||
|
/// to know which company each one bills.
|
||||||
|
fn backend_label(id: &str) -> String {
|
||||||
|
match id {
|
||||||
|
"claude" => "Claude (Anthropic subscription)".into(),
|
||||||
|
"default" => "Claude (generic image)".into(),
|
||||||
|
"canary-claude" => "Claude — candidate CLI (canary)".into(),
|
||||||
|
"glm" => "GLM 4.7 (z.ai)".into(),
|
||||||
|
"kimi" => "Kimi (Moonshot)".into(),
|
||||||
|
"local-ornith" => "Ornith 9B — this fleet's own GPU".into(),
|
||||||
|
other => other.to_string(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -40,7 +40,6 @@ research tools. Grant write only to members that actually produce code or commit
|
|||||||
- glm-4.7 — strong general reasoning (Z.ai); best cost/quality default for most workers.\n\
|
- glm-4.7 — strong general reasoning (Z.ai); best cost/quality default for most workers.\n\
|
||||||
- glm-5.2 — GLM Opus-class for the hardest reasoning roles; higher cost.\n\
|
- glm-5.2 — GLM Opus-class for the hardest reasoning roles; higher cost.\n\
|
||||||
- kimi — excellent for code-heavy roles.\n\
|
- kimi — excellent for code-heavy roles.\n\
|
||||||
- gemini — Gemini 2.5 Flash: very fast; classification, summarization, high-volume tasks.\n\
|
|
||||||
- groq — fastest/cheapest; simple sequential high-throughput steps.\n\
|
- groq — fastest/cheapest; simple sequential high-throughput steps.\n\
|
||||||
AGENT TOOLS each agent can use at runtime: web.search (find sources), browser.goto (fetch a URL), \
|
AGENT TOOLS each agent can use at runtime: web.search (find sources), browser.goto (fetch a URL), \
|
||||||
files.write (build a markdown vault in the shared drive), chat.send (delegate to teammates), \
|
files.write (build a markdown vault in the shared drive), chat.send (delegate to teammates), \
|
||||||
@@ -133,7 +132,11 @@ pub async fn planner_chat(
|
|||||||
};
|
};
|
||||||
let user_prompt = format!("{hierarchy}{topology_lock}\n\n=== CONVERSATION ===\n{convo}\n\nRespond now (JSON only).");
|
let user_prompt = format!("{hierarchy}{topology_lock}\n\n=== CONVERSATION ===\n{convo}\n\nRespond now (JSON only).");
|
||||||
let system = planner_system_for(&body.mode);
|
let system = planner_system_for(&body.mode);
|
||||||
let raw = match runtime.complete(&system, &user_prompt, "claude-opus-4-8", 8000, true).await {
|
let raw = match crate::subscription::complete_or(
|
||||||
|
&runtime, &system, &user_prompt, "claude-opus-4-8", 8000, true,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
Ok(t) => t,
|
Ok(t) => t,
|
||||||
Err(e) => { yield sse(json!({"stage":"error","label":format!("Opus error: {e}")})); return; }
|
Err(e) => { yield sse(json!({"stage":"error","label":format!("Opus error: {e}")})); return; }
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ use crate::{ApiError, AppState, Authed};
|
|||||||
pub struct TeamMemberInput {
|
pub struct TeamMemberInput {
|
||||||
pub role: String,
|
pub role: String,
|
||||||
pub name: String,
|
pub name: String,
|
||||||
/// Model selector: claude | glm | glm-5.2 | kimi | gemini | groq.
|
/// Model selector: claude | glm | glm-5.2 | kimi | groq.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub model: String,
|
pub model: String,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
|
|||||||
@@ -309,6 +309,10 @@ pub async fn run_events_sse(
|
|||||||
.map(|n| n + 1)
|
.map(|n| n + 1)
|
||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
|
|
||||||
|
// Bytes of `checkpoint.log` already sent. The step cursor above counts
|
||||||
|
// RECORDS; this counts BYTES, because a log grows continuously rather than
|
||||||
|
// in discrete entries. Two sources, two cursors.
|
||||||
|
let mut log_sent: usize = 0;
|
||||||
let stream = async_stream::stream! {
|
let stream = async_stream::stream! {
|
||||||
loop {
|
loop {
|
||||||
match cm_db::repo::topology_runs::status(&pool, id, ws).await {
|
match cm_db::repo::topology_runs::status(&pool, id, ws).await {
|
||||||
@@ -327,6 +331,24 @@ pub async fn run_events_sse(
|
|||||||
sent += 1;
|
sent += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Live stdout/stderr from a microVM turn, appended by the
|
||||||
|
// node over the fleet WebSocket (`Uplink::VmOut`). Emitted
|
||||||
|
// as `step` so the existing reader renders it with no
|
||||||
|
// frontend change — it already reads `data.text`.
|
||||||
|
if let Some(log) = st
|
||||||
|
.checkpoint
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|c| c.get("log"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
{
|
||||||
|
if log.len() > log_sent {
|
||||||
|
let fresh = &log[log_sent..];
|
||||||
|
log_sent = log.len();
|
||||||
|
yield Ok::<Event, Infallible>(Event::default().event("step").data(
|
||||||
|
serde_json::json!({ "kind": "output", "text": fresh }).to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
if matches!(st.status.as_str(), "completed" | "failed" | "cancelled") {
|
if matches!(st.status.as_str(), "completed" | "failed" | "cancelled") {
|
||||||
let done = serde_json::json!({
|
let done = serde_json::json!({
|
||||||
"status": st.status,
|
"status": st.status,
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ use cm_domain::WorkspaceId;
|
|||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
use sqlx::{PgPool, Row};
|
use sqlx::{PgPool, Row};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
use crate::{ApiError, AppState, Authed};
|
use crate::{ApiError, AppState, Authed};
|
||||||
|
|
||||||
@@ -44,36 +45,407 @@ async fn working_agents(pool: &PgPool, ws: WorkspaceId) -> HashSet<String> {
|
|||||||
rows.into_iter().map(|r| r.get::<String, _>("id")).collect()
|
rows.into_iter().map(|r| r.get::<String, _>("id")).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Active missions (status='running') with their assigned team members —
|
/// One row per (mission, agent) pair for every mission the World should draw.
|
||||||
/// one row per (mission, agent) pair. The World SSE loop emits each as
|
///
|
||||||
/// a `mission:<id>` landmark orb + `world.touch` beams from every team
|
/// Three bugs were fixed here at once, and each hid the next:
|
||||||
/// member. Replaces the retired research/loops landmarks (commit
|
///
|
||||||
/// fdb8cfe) with the missions-era equivalent.
|
/// 1. **The join was on the wrong column.** It read
|
||||||
async fn active_missions(pool: &PgPool, ws: WorkspaceId) -> Vec<(String, String, String)> {
|
/// `JOIN team_members tm ON tm.team_id = m.team_id`, but `missions.team_id`
|
||||||
|
/// is a legacy pointer at the FIRST minted team — `0056_mission_teams.sql`
|
||||||
|
/// superseded it with the `mission_teams(mission_id, team_id, purpose)`
|
||||||
|
/// junction, which is what everything else (including `/api/workforce`)
|
||||||
|
/// joins through. A multi-team mission showed only its first team.
|
||||||
|
///
|
||||||
|
/// 2. **It was an INNER JOIN, and microVM missions have no team at all.**
|
||||||
|
/// `mission_orchestrator::on_launch` deliberately mints none for them, so
|
||||||
|
/// the platform's primary execution tier was dropped by the join and the
|
||||||
|
/// World has been showing nothing whatsoever for it. LEFT JOIN, and
|
||||||
|
/// `agent_id` is `None` for those — a phase that ran with no platform
|
||||||
|
/// agents draws no pawns, which is the truth rather than a gap.
|
||||||
|
///
|
||||||
|
/// 3. **Only `running` missions were selected.** Missions finish in minutes,
|
||||||
|
/// so the World was empty almost always. Recently-finished ones come back
|
||||||
|
/// too, carrying `status` so the client can draw a finished map instead of
|
||||||
|
/// animating a corpse.
|
||||||
|
struct MissionRow {
|
||||||
|
mission_id: String,
|
||||||
|
title: String,
|
||||||
|
status: String,
|
||||||
|
template_kind: String,
|
||||||
|
completed_at: Option<String>,
|
||||||
|
/// `None` for a mission with no team — see (2) above.
|
||||||
|
agent_id: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn world_missions(
|
||||||
|
pool: &PgPool,
|
||||||
|
ws: WorkspaceId,
|
||||||
|
only: Option<Uuid>,
|
||||||
|
) -> Vec<MissionRow> {
|
||||||
let rows = sqlx::query(
|
let rows = sqlx::query(
|
||||||
"SELECT m.id::text AS mission_id,
|
"SELECT m.id::text AS mission_id,
|
||||||
m.title AS title,
|
m.title AS title,
|
||||||
|
m.status AS status,
|
||||||
|
m.template_kind AS template_kind,
|
||||||
|
to_char(m.completed_at AT TIME ZONE 'UTC',
|
||||||
|
'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"') AS completed_at,
|
||||||
tm.claw_id::text AS agent_id
|
tm.claw_id::text AS agent_id
|
||||||
FROM missions m
|
FROM missions m
|
||||||
JOIN team_members tm ON tm.team_id = m.team_id
|
LEFT JOIN mission_teams mt ON mt.mission_id = m.id
|
||||||
|
LEFT JOIN team_members tm ON tm.team_id = mt.team_id
|
||||||
WHERE m.workspace_id = $1
|
WHERE m.workspace_id = $1
|
||||||
AND m.status = 'running'",
|
AND ( m.status = 'running'
|
||||||
|
OR ( m.status IN ('completed', 'failed')
|
||||||
|
AND m.completed_at > now() - interval '24 hours' ) )
|
||||||
|
AND ($2::uuid IS NULL OR m.id = $2)
|
||||||
|
ORDER BY m.created_at DESC",
|
||||||
)
|
)
|
||||||
.bind(ws.as_uuid())
|
.bind(ws.as_uuid())
|
||||||
|
.bind(only)
|
||||||
.fetch_all(pool)
|
.fetch_all(pool)
|
||||||
.await
|
.await
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
rows.into_iter()
|
rows.into_iter()
|
||||||
.map(|r| {
|
.map(|r| MissionRow {
|
||||||
(
|
mission_id: r.get::<String, _>("mission_id"),
|
||||||
r.get::<String, _>("mission_id"),
|
title: r.get::<String, _>("title"),
|
||||||
r.get::<String, _>("title"),
|
status: r.get::<String, _>("status"),
|
||||||
r.get::<String, _>("agent_id"),
|
template_kind: r.get::<String, _>("template_kind"),
|
||||||
)
|
completed_at: r.get::<Option<String>, _>("completed_at"),
|
||||||
|
agent_id: r.get::<Option<String>, _>("agent_id"),
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Every phase of the given missions, in plan order, with the agents that
|
||||||
|
/// execute it.
|
||||||
|
///
|
||||||
|
/// The whole plan is emitted — including `pending` phases that have not
|
||||||
|
/// started — because the World draws the mission's shape upfront and lights it
|
||||||
|
/// as it progresses. A phase list that only appeared as phases began would make
|
||||||
|
/// a five-phase mission indistinguishable from a one-phase mission until it was
|
||||||
|
/// nearly over.
|
||||||
|
struct PhaseRow {
|
||||||
|
mission_id: String,
|
||||||
|
phase_id: String,
|
||||||
|
kind: String,
|
||||||
|
order_idx: i32,
|
||||||
|
status: String,
|
||||||
|
iteration: i32,
|
||||||
|
started_at: Option<String>,
|
||||||
|
completed_at: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn world_phases(pool: &PgPool, ws: WorkspaceId, only: Option<Uuid>) -> Vec<PhaseRow> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT mp.mission_id::text AS mission_id,
|
||||||
|
mp.id::text AS phase_id,
|
||||||
|
mp.kind AS kind,
|
||||||
|
mp.order_idx AS order_idx,
|
||||||
|
mp.status AS status,
|
||||||
|
COALESCE(mp.iteration, 0) AS iteration,
|
||||||
|
to_char(mp.started_at AT TIME ZONE 'UTC',
|
||||||
|
'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"') AS started_at,
|
||||||
|
to_char(mp.completed_at AT TIME ZONE 'UTC',
|
||||||
|
'YYYY-MM-DD\"T\"HH24:MI:SS\"Z\"') AS completed_at
|
||||||
|
FROM mission_phases mp
|
||||||
|
JOIN missions m ON m.id = mp.mission_id
|
||||||
|
WHERE m.workspace_id = $1
|
||||||
|
AND ( m.status = 'running'
|
||||||
|
OR ( m.status IN ('completed', 'failed')
|
||||||
|
AND m.completed_at > now() - interval '24 hours' ) )
|
||||||
|
AND ($2::uuid IS NULL OR m.id = $2)
|
||||||
|
ORDER BY mp.mission_id, mp.order_idx",
|
||||||
|
)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.bind(only)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
rows.into_iter()
|
||||||
|
.map(|r| PhaseRow {
|
||||||
|
mission_id: r.get::<String, _>("mission_id"),
|
||||||
|
phase_id: r.get::<String, _>("phase_id"),
|
||||||
|
kind: r.get::<String, _>("kind"),
|
||||||
|
order_idx: r.get::<i32, _>("order_idx"),
|
||||||
|
status: r.get::<String, _>("status"),
|
||||||
|
iteration: r.get::<i32, _>("iteration"),
|
||||||
|
started_at: r.get::<Option<String>, _>("started_at"),
|
||||||
|
completed_at: r.get::<Option<String>, _>("completed_at"),
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Files a phase touched, from the `code_diff` artifact captured at delivery.
|
||||||
|
///
|
||||||
|
/// This is recorded fact, not inference: `mission_delivery` writes the path
|
||||||
|
/// list with the same revision and excludes it uses for `files_changed`, so the
|
||||||
|
/// orbs the World draws are the files git says changed.
|
||||||
|
///
|
||||||
|
/// It is end-of-phase detail — the capture runs when a phase finishes — so a
|
||||||
|
/// running phase shows its station lit but no files until it lands. Live
|
||||||
|
/// per-tool file touches are a separate, structured source.
|
||||||
|
struct FileRow {
|
||||||
|
mission_id: String,
|
||||||
|
phase_id: String,
|
||||||
|
path: String,
|
||||||
|
status: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn world_files(pool: &PgPool, ws: WorkspaceId, only: Option<Uuid>) -> Vec<FileRow> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT ma.mission_id::text AS mission_id,
|
||||||
|
ma.phase_id::text AS phase_id,
|
||||||
|
f->>'path' AS path,
|
||||||
|
f->>'status' AS status
|
||||||
|
FROM mission_artifacts ma
|
||||||
|
JOIN missions m ON m.id = ma.mission_id
|
||||||
|
CROSS JOIN LATERAL jsonb_array_elements(
|
||||||
|
COALESCE(ma.metadata->'files', '[]'::jsonb)) AS f
|
||||||
|
WHERE m.workspace_id = $1
|
||||||
|
AND ma.kind = 'code_diff'
|
||||||
|
AND ma.phase_id IS NOT NULL
|
||||||
|
AND ( m.status = 'running'
|
||||||
|
OR ( m.status IN ('completed', 'failed')
|
||||||
|
AND m.completed_at > now() - interval '24 hours' ) )
|
||||||
|
AND ($2::uuid IS NULL OR m.id = $2)
|
||||||
|
LIMIT 2000",
|
||||||
|
)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.bind(only)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
rows.into_iter()
|
||||||
|
.filter_map(|r| {
|
||||||
|
Some(FileRow {
|
||||||
|
mission_id: r.get::<String, _>("mission_id"),
|
||||||
|
phase_id: r.get::<String, _>("phase_id"),
|
||||||
|
path: r.get::<Option<String>, _>("path")?,
|
||||||
|
status: r.get::<Option<String>, _>("status").unwrap_or_default(),
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One row of `mission_events` — what an agent actually did.
|
||||||
|
struct ActRow {
|
||||||
|
id: i64,
|
||||||
|
mission_id: String,
|
||||||
|
phase_id: Option<String>,
|
||||||
|
agent_id: Option<String>,
|
||||||
|
kind: String,
|
||||||
|
target: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Structured mission activity since `after`, oldest first.
|
||||||
|
///
|
||||||
|
/// `after < 0` means "the first pass has not run yet": everything is returned
|
||||||
|
/// so the caller can seed its cursor and draw the backlog as settled history.
|
||||||
|
async fn world_acts(pool: &PgPool, ws: WorkspaceId, only: Option<Uuid>, after: i64) -> Vec<ActRow> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT e.id,
|
||||||
|
e.mission_id::text AS mission_id,
|
||||||
|
e.phase_id::text AS phase_id,
|
||||||
|
e.agent_id::text AS agent_id,
|
||||||
|
e.kind,
|
||||||
|
e.target
|
||||||
|
FROM mission_events e
|
||||||
|
JOIN missions m ON m.id = e.mission_id
|
||||||
|
WHERE m.workspace_id = $1
|
||||||
|
AND e.kind IN ('tool.call', 'file.touch')
|
||||||
|
AND e.target IS NOT NULL
|
||||||
|
AND e.id > $2
|
||||||
|
AND ( m.status = 'running'
|
||||||
|
OR ( m.status IN ('completed', 'failed')
|
||||||
|
AND m.completed_at > now() - interval '24 hours' ) )
|
||||||
|
AND ($3::uuid IS NULL OR m.id = $3)
|
||||||
|
ORDER BY e.id
|
||||||
|
LIMIT 500",
|
||||||
|
)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.bind(after)
|
||||||
|
.bind(only)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
rows.into_iter()
|
||||||
|
.filter_map(|r| {
|
||||||
|
Some(ActRow {
|
||||||
|
id: r.get::<i64, _>("id"),
|
||||||
|
mission_id: r.get::<String, _>("mission_id"),
|
||||||
|
phase_id: r.get::<Option<String>, _>("phase_id"),
|
||||||
|
agent_id: r.get::<Option<String>, _>("agent_id"),
|
||||||
|
kind: r.get::<String, _>("kind"),
|
||||||
|
target: r.get::<Option<String>, _>("target")?,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One security finding, as the scanner recorded it.
|
||||||
|
struct FindingRow {
|
||||||
|
mission_id: String,
|
||||||
|
phase_id: String,
|
||||||
|
task_id: String,
|
||||||
|
title: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Findings raised by a security phase.
|
||||||
|
///
|
||||||
|
/// They are `mission_tasks` rows — the scanner's own store — not a new table.
|
||||||
|
/// There is deliberately NO severity field here: severity, file and line are
|
||||||
|
/// substrings inside `title` (see `security_scan.rs`), and a severity parsed
|
||||||
|
/// out of prose and then encoded as an orb's RADIUS would be an invented fact
|
||||||
|
/// rendered as a measurement. The title is shown as written.
|
||||||
|
async fn world_findings(pool: &PgPool, ws: WorkspaceId, only: Option<Uuid>) -> Vec<FindingRow> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT t.mission_id::text AS mission_id,
|
||||||
|
t.phase_id::text AS phase_id,
|
||||||
|
t.id::text AS task_id,
|
||||||
|
t.title
|
||||||
|
FROM mission_tasks t
|
||||||
|
JOIN mission_phases p ON p.id = t.phase_id
|
||||||
|
JOIN missions m ON m.id = t.mission_id
|
||||||
|
WHERE m.workspace_id = $1
|
||||||
|
AND p.kind = 'security_scan'
|
||||||
|
AND ( m.status = 'running'
|
||||||
|
OR ( m.status IN ('completed', 'failed')
|
||||||
|
AND m.completed_at > now() - interval '24 hours' ) )
|
||||||
|
AND ($2::uuid IS NULL OR m.id = $2)
|
||||||
|
LIMIT 300",
|
||||||
|
)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.bind(only)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
rows.into_iter()
|
||||||
|
.map(|r| FindingRow {
|
||||||
|
mission_id: r.get::<String, _>("mission_id"),
|
||||||
|
phase_id: r.get::<String, _>("phase_id"),
|
||||||
|
task_id: r.get::<String, _>("task_id"),
|
||||||
|
title: r.get::<String, _>("title"),
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A benchmark phase's result, summarised.
|
||||||
|
struct BenchRow {
|
||||||
|
mission_id: String,
|
||||||
|
phase_id: String,
|
||||||
|
note: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Benchmark deltas, as an annotation on the phase — not as nodes.
|
||||||
|
///
|
||||||
|
/// `delta` is a `Record<string, unknown>` with no schema: `compute_delta`
|
||||||
|
/// produces `{kind:"bencher_diff", samples:[…]}` when the before/after shapes
|
||||||
|
/// are structurally comparable, and `{kind:"opaque"}` when they are not. Only
|
||||||
|
/// the shape that can be parsed is formatted; the rest is COUNTED, never
|
||||||
|
/// guessed at, so a driver whose output we do not understand reports "3
|
||||||
|
/// samples" rather than an invented improvement.
|
||||||
|
async fn world_benchmarks(pool: &PgPool, ws: WorkspaceId, only: Option<Uuid>) -> Vec<BenchRow> {
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT DISTINCT ON (b.phase_id)
|
||||||
|
b.mission_id::text AS mission_id,
|
||||||
|
b.phase_id::text AS phase_id,
|
||||||
|
b.delta
|
||||||
|
FROM benchmark_snapshots b
|
||||||
|
JOIN missions m ON m.id = b.mission_id
|
||||||
|
WHERE m.workspace_id = $1
|
||||||
|
AND b.delta IS NOT NULL
|
||||||
|
AND ( m.status = 'running'
|
||||||
|
OR ( m.status IN ('completed', 'failed')
|
||||||
|
AND m.completed_at > now() - interval '24 hours' ) )
|
||||||
|
AND ($2::uuid IS NULL OR m.id = $2)
|
||||||
|
ORDER BY b.phase_id, b.iteration DESC",
|
||||||
|
)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.bind(only)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
rows.into_iter()
|
||||||
|
.filter_map(|r| {
|
||||||
|
let note = benchmark_note(&r.get::<serde_json::Value, _>("delta"))?;
|
||||||
|
Some(BenchRow {
|
||||||
|
mission_id: r.get::<String, _>("mission_id"),
|
||||||
|
phase_id: r.get::<String, _>("phase_id"),
|
||||||
|
note,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One line describing a benchmark delta, or `None` if there is nothing
|
||||||
|
/// truthful to say about it.
|
||||||
|
fn benchmark_note(delta: &serde_json::Value) -> Option<String> {
|
||||||
|
let samples = delta.get("samples").and_then(|s| s.as_array())?;
|
||||||
|
if samples.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let mut improved = 0usize;
|
||||||
|
let mut regressed = 0usize;
|
||||||
|
// Best (most negative) percent change, since that is the one claim the
|
||||||
|
// shape actually supports.
|
||||||
|
let mut best: Option<f64> = None;
|
||||||
|
for s in samples {
|
||||||
|
match s.get("direction").and_then(|d| d.as_str()) {
|
||||||
|
Some("improved") => improved += 1,
|
||||||
|
Some("regressed") => regressed += 1,
|
||||||
|
// Counted in the total but claimed for neither side.
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
if let Some(pct) = s.get("delta_pct").and_then(serde_json::Value::as_f64) {
|
||||||
|
best = Some(best.map_or(pct, |b: f64| b.min(pct)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut parts = vec![format!("{} sample(s)", samples.len())];
|
||||||
|
if improved > 0 {
|
||||||
|
parts.push(format!("{improved} faster"));
|
||||||
|
}
|
||||||
|
if regressed > 0 {
|
||||||
|
parts.push(format!("{regressed} slower"));
|
||||||
|
}
|
||||||
|
if let Some(b) = best.filter(|b| *b < 0.0) {
|
||||||
|
parts.push(format!("best {:.1}%", b));
|
||||||
|
}
|
||||||
|
Some(parts.join(", "))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Agents that execute a phase of this kind, via the purposes the phase runner
|
||||||
|
/// itself uses. Returns empty for a teamless (microVM) mission.
|
||||||
|
async fn phase_agents(
|
||||||
|
pool: &PgPool,
|
||||||
|
mission_id: &str,
|
||||||
|
kind: &str,
|
||||||
|
) -> Vec<String> {
|
||||||
|
let Ok(mid) = Uuid::parse_str(mission_id) else {
|
||||||
|
return Vec::new();
|
||||||
|
};
|
||||||
|
// `purposes_for` is the runner's own mapping, shared rather than copied —
|
||||||
|
// see its doc comment.
|
||||||
|
let purposes: Vec<String> = crate::phase_runner::purposes_for(kind)
|
||||||
|
.iter()
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
.collect();
|
||||||
|
let rows = sqlx::query(
|
||||||
|
"SELECT DISTINCT tm.claw_id::text AS agent_id
|
||||||
|
FROM mission_teams mt
|
||||||
|
JOIN team_members tm ON tm.team_id = mt.team_id
|
||||||
|
WHERE mt.mission_id = $1 AND mt.purpose = ANY($2)",
|
||||||
|
)
|
||||||
|
.bind(mid)
|
||||||
|
.bind(&purposes)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default();
|
||||||
|
rows.into_iter()
|
||||||
|
.map(|r| r.get::<String, _>("agent_id"))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
/// Currently-running runs in the workspace as (run_id, agent_id) — each is a
|
/// Currently-running runs in the workspace as (run_id, agent_id) — each is a
|
||||||
/// real "this agent is converging on its active work" signal (Gource).
|
/// real "this agent is converging on its active work" signal (Gource).
|
||||||
async fn active_runs(pool: &PgPool, ws: WorkspaceId) -> Vec<(String, String)> {
|
async fn active_runs(pool: &PgPool, ws: WorkspaceId) -> Vec<(String, String)> {
|
||||||
@@ -330,10 +702,24 @@ async fn agent_telemetry(
|
|||||||
m
|
m
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Query for `GET /api/world/live`.
|
||||||
|
#[derive(serde::Deserialize)]
|
||||||
|
pub struct LiveQuery {
|
||||||
|
/// Scope the feed to one mission. The client already filters events by
|
||||||
|
/// mission, but doing it here means the server stops querying and sending
|
||||||
|
/// what the viewer will discard.
|
||||||
|
pub mission: Option<Uuid>,
|
||||||
|
}
|
||||||
|
|
||||||
/// `GET /api/world/live` — the taxonomy SSE feed.
|
/// `GET /api/world/live` — the taxonomy SSE feed.
|
||||||
pub async fn world_live(State(state): State<AppState>, Authed(user): Authed) -> impl IntoResponse {
|
pub async fn world_live(
|
||||||
|
State(state): State<AppState>,
|
||||||
|
Authed(user): Authed,
|
||||||
|
Query(q): Query<LiveQuery>,
|
||||||
|
) -> impl IntoResponse {
|
||||||
let pool = state.pool.clone();
|
let pool = state.pool.clone();
|
||||||
let ws = user.workspace_id;
|
let ws = user.workspace_id;
|
||||||
|
let only_mission = q.mission;
|
||||||
|
|
||||||
let stream = async_stream::stream! {
|
let stream = async_stream::stream! {
|
||||||
let mut first = true;
|
let mut first = true;
|
||||||
@@ -341,6 +727,27 @@ pub async fn world_live(State(state): State<AppState>, Authed(user): Authed) ->
|
|||||||
let mut last: std::collections::HashMap<String, String> = std::collections::HashMap::new();
|
let mut last: std::collections::HashMap<String, String> = std::collections::HashMap::new();
|
||||||
// Per-run journal cursor so we stream only NEW run_events each poll.
|
// Per-run journal cursor so we stream only NEW run_events each poll.
|
||||||
let mut cursors: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
|
let mut cursors: std::collections::HashMap<String, i64> = std::collections::HashMap::new();
|
||||||
|
// Last emitted signature per mission / per phase, so the plan is sent
|
||||||
|
// once and then only when something actually moves.
|
||||||
|
let mut last_mission: std::collections::HashMap<String, String> =
|
||||||
|
std::collections::HashMap::new();
|
||||||
|
let mut last_phase: std::collections::HashMap<String, String> =
|
||||||
|
std::collections::HashMap::new();
|
||||||
|
// (phase, path) pairs already announced — a delivered file is a fact
|
||||||
|
// that happened once, not a recurring event.
|
||||||
|
let mut last_file: HashSet<String> = HashSet::new();
|
||||||
|
// `mission_events` cursor. -1 until the first pass seeds it, which is
|
||||||
|
// what separates BACKFILL from MOTION: everything already in the table
|
||||||
|
// when a subscriber arrives is history and is drawn as a settled map,
|
||||||
|
// and only what lands afterwards is animated. Without the distinction,
|
||||||
|
// opening a finished mission would replay an hour of tool calls as a
|
||||||
|
// burst storm and read as a mission that just did all of it at once.
|
||||||
|
let mut event_cursor: i64 = -1;
|
||||||
|
// Findings already announced. A finding is raised once.
|
||||||
|
let mut last_finding: HashSet<String> = HashSet::new();
|
||||||
|
// Last benchmark summary per phase; a looping phase produces a new one.
|
||||||
|
let mut last_bench: std::collections::HashMap<String, String> =
|
||||||
|
std::collections::HashMap::new();
|
||||||
// Audit-log cursor for edge-initiated inter-agent events (delegation,
|
// Audit-log cursor for edge-initiated inter-agent events (delegation,
|
||||||
// A2A) that bypass the run loop. -1 until seeded on the first pass.
|
// A2A) that bypass the run loop. -1 until seeded on the first pass.
|
||||||
let mut audit_cursor: i64 = -1;
|
let mut audit_cursor: i64 = -1;
|
||||||
@@ -391,28 +798,234 @@ pub async fn world_live(State(state): State<AppState>, Authed(user): Authed) ->
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Mission landmarks: one `mission:<id>` orb per running mission,
|
// Mission landmarks, and the whole phase plan behind them.
|
||||||
// with `world.touch` beams from every assigned team member. Missions
|
//
|
||||||
// outlive individual runs, so the orb gives the World a persistent
|
// The orb pins "this is the work"; the phases give it shape. Every
|
||||||
// pin for "this is what the team is working on right now" even when
|
// phase is emitted, including ones that have not started, because
|
||||||
// no run is claimed. Replaces the retired repo:{topic}/loop:{id}
|
// the World draws the plan upfront and lights it as it progresses —
|
||||||
// landmarks after commit fdb8cfe.
|
// a phase list that appeared only as phases began would make a
|
||||||
let missions = active_missions(&pool, ws).await;
|
// five-phase mission indistinguishable from a one-phase mission
|
||||||
|
// until it was nearly over.
|
||||||
|
let missions = world_missions(&pool, ws, only_mission).await;
|
||||||
let mut seen_missions: HashSet<String> = HashSet::new();
|
let mut seen_missions: HashSet<String> = HashSet::new();
|
||||||
for (mission_id, title, agent_id) in &missions {
|
for m in &missions {
|
||||||
if seen_missions.insert(mission_id.clone()) {
|
if seen_missions.insert(m.mission_id.clone()) {
|
||||||
let node_id = format!("mission:{mission_id}");
|
// Delta-guarded like every other stateful event here. Without
|
||||||
|
// this it re-sent the same mission on every poll — harmless
|
||||||
|
// to a client that keys on missionId, and pure noise on the
|
||||||
|
// wire that hides the events that DID change.
|
||||||
|
let sig = format!("{}|{}", m.status, m.completed_at.as_deref().unwrap_or(""));
|
||||||
|
if last_mission.get(&m.mission_id).map(|s| s != &sig).unwrap_or(true) {
|
||||||
|
last_mission.insert(m.mission_id.clone(), sig);
|
||||||
yield sse(
|
yield sse(
|
||||||
"node.activity",
|
"mission.update",
|
||||||
json!({ "nodeId": node_id, "label": title, "kind": "mission", "heat": 0.75 }),
|
json!({
|
||||||
|
"missionId": m.mission_id,
|
||||||
|
"title": m.title,
|
||||||
|
"status": m.status,
|
||||||
|
"templateKind": m.template_kind,
|
||||||
|
"completedAt": m.completed_at,
|
||||||
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
let node_id = format!("mission:{mission_id}");
|
// A terminal mission is a map to read, not a scene to
|
||||||
|
// animate: it still gets an orb, but no heat.
|
||||||
|
let running = m.status == "running";
|
||||||
|
yield sse(
|
||||||
|
"node.activity",
|
||||||
|
json!({
|
||||||
|
"nodeId": format!("mission:{}", m.mission_id),
|
||||||
|
"label": m.title,
|
||||||
|
"kind": "mission",
|
||||||
|
"heat": if running { 0.75 } else { 0.0 },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// `agent_id` is None for a teamless microVM mission. No pawn
|
||||||
|
// beams at it, which is accurate — nothing on this platform ran
|
||||||
|
// that phase except a VM — and is why the join had to become a
|
||||||
|
// LEFT JOIN rather than being left to drop the mission entirely.
|
||||||
|
if let (Some(agent_id), true) = (&m.agent_id, m.status == "running") {
|
||||||
yield sse(
|
yield sse(
|
||||||
"world.touch",
|
"world.touch",
|
||||||
json!({ "agentId": agent_id, "nodeId": node_id, "kind": "mission", "weight": 0.6 }),
|
json!({
|
||||||
|
"agentId": agent_id,
|
||||||
|
"nodeId": format!("mission:{}", m.mission_id),
|
||||||
|
"kind": "mission",
|
||||||
|
"weight": 0.6,
|
||||||
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for p in world_phases(&pool, ws, only_mission).await {
|
||||||
|
// Re-emit only on change. `iteration` and `completed_at` are in
|
||||||
|
// the signature because a phase that loops re-enters `running`
|
||||||
|
// from `running` — status alone would hide the retry.
|
||||||
|
let sig = format!(
|
||||||
|
"{}|{}|{}",
|
||||||
|
p.status,
|
||||||
|
p.iteration,
|
||||||
|
p.completed_at.as_deref().unwrap_or("")
|
||||||
|
);
|
||||||
|
let changed = last_phase.get(&p.phase_id).map(|s| s != &sig).unwrap_or(true);
|
||||||
|
if !changed {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
last_phase.insert(p.phase_id.clone(), sig);
|
||||||
|
let agents = phase_agents(&pool, &p.mission_id, &p.kind).await;
|
||||||
|
yield sse(
|
||||||
|
"mission.phase",
|
||||||
|
json!({
|
||||||
|
"missionId": p.mission_id,
|
||||||
|
"phaseId": p.phase_id,
|
||||||
|
"kind": p.kind,
|
||||||
|
"orderIdx": p.order_idx,
|
||||||
|
"status": p.status,
|
||||||
|
"iteration": p.iteration,
|
||||||
|
"startedAt": p.started_at,
|
||||||
|
"completedAt": p.completed_at,
|
||||||
|
"agentIds": agents,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
yield sse(
|
||||||
|
"node.activity",
|
||||||
|
json!({
|
||||||
|
"nodeId": format!("phase:{}", p.phase_id),
|
||||||
|
"label": p.kind,
|
||||||
|
"kind": "phase",
|
||||||
|
// The client owns the lit/dim encoding via `status`;
|
||||||
|
// heat here is only the arrival pulse.
|
||||||
|
"heat": if p.status == "running" { 0.8 } else { 0.0 },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Files come AFTER the phases they belong to, deliberately: the
|
||||||
|
// client hangs a file tree under the coding station, and parenting
|
||||||
|
// there is first-write-wins. A file that arrived before its plan
|
||||||
|
// would pin its whole directory tree at the origin.
|
||||||
|
//
|
||||||
|
// Files, once each. A file is emitted when its phase's diff was
|
||||||
|
// captured and never again — the World keeps the node alive itself,
|
||||||
|
// and re-sending would re-burst the orb every poll as though the
|
||||||
|
// file had just been touched again.
|
||||||
|
for f in world_files(&pool, ws, only_mission).await {
|
||||||
|
let key = format!("{}|{}", f.phase_id, f.path);
|
||||||
|
if last_file.contains(&key) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
last_file.insert(key);
|
||||||
|
yield sse(
|
||||||
|
"mission.file",
|
||||||
|
json!({
|
||||||
|
"missionId": f.mission_id,
|
||||||
|
"phaseId": f.phase_id,
|
||||||
|
"path": f.path,
|
||||||
|
"status": f.status,
|
||||||
|
"source": "diff",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Benchmark results, as an annotation on the station. Re-emitted
|
||||||
|
// only when the summary changes: a phase that loops produces a new
|
||||||
|
// snapshot per iteration, and that IS news.
|
||||||
|
for b in world_benchmarks(&pool, ws, only_mission).await {
|
||||||
|
if last_bench.get(&b.phase_id).map(|n| n == &b.note).unwrap_or(false) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
last_bench.insert(b.phase_id.clone(), b.note.clone());
|
||||||
|
yield sse(
|
||||||
|
"mission.benchmark",
|
||||||
|
json!({
|
||||||
|
"missionId": b.mission_id,
|
||||||
|
"phaseId": b.phase_id,
|
||||||
|
"note": b.note,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Findings, once each, hanging off the security station. Same rule
|
||||||
|
// as files: a finding is a fact that was raised once, and
|
||||||
|
// re-sending it would pop the orb again on every poll.
|
||||||
|
for f in world_findings(&pool, ws, only_mission).await {
|
||||||
|
if !last_finding.insert(f.task_id.clone()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
yield sse(
|
||||||
|
"mission.finding",
|
||||||
|
json!({
|
||||||
|
"missionId": f.mission_id,
|
||||||
|
"phaseId": f.phase_id,
|
||||||
|
"findingId": f.task_id,
|
||||||
|
"title": f.title,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Structured activity — the motion channel. Tool calls and file
|
||||||
|
// touches recorded at the source by the container tap and the
|
||||||
|
// microVM `PostToolUse` hook. Never parsed from prose: a tool name
|
||||||
|
// in a log is indistinguishable from an agent TALKING about a tool.
|
||||||
|
{
|
||||||
|
let backfill = event_cursor < 0;
|
||||||
|
for a in world_acts(&pool, ws, only_mission, event_cursor.max(0)).await {
|
||||||
|
event_cursor = event_cursor.max(a.id);
|
||||||
|
match a.kind.as_str() {
|
||||||
|
"file.touch" => {
|
||||||
|
let key = format!("{}|{}", a.phase_id.as_deref().unwrap_or(""), a.target);
|
||||||
|
if !last_file.insert(key) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
yield sse(
|
||||||
|
"mission.file",
|
||||||
|
json!({
|
||||||
|
"missionId": a.mission_id,
|
||||||
|
"phaseId": a.phase_id,
|
||||||
|
"agentId": a.agent_id,
|
||||||
|
"path": a.target,
|
||||||
|
// Backlog is history: it draws the file and
|
||||||
|
// stops there. Only what lands while someone
|
||||||
|
// is watching is motion.
|
||||||
|
"source": if backfill { "diff" } else { "tool" },
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// A tool call with an agent is a pawn leaving its
|
||||||
|
// station and coming back; without one (a microVM phase
|
||||||
|
// has no platform agent) it is only a node lighting up,
|
||||||
|
// which is the truth rather than an invented traveller.
|
||||||
|
_ if !backfill => {
|
||||||
|
let node_id = format!("tool:{}", a.target);
|
||||||
|
match &a.agent_id {
|
||||||
|
Some(agent) => yield sse(
|
||||||
|
"world.touch",
|
||||||
|
json!({
|
||||||
|
"agentId": agent,
|
||||||
|
"nodeId": node_id,
|
||||||
|
"kind": "event",
|
||||||
|
"weight": 0.45,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
None => yield sse(
|
||||||
|
"node.activity",
|
||||||
|
json!({
|
||||||
|
"nodeId": node_id,
|
||||||
|
"label": a.target,
|
||||||
|
"kind": "event",
|
||||||
|
"heat": 0.5,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A backlogged tool call draws nothing: the orb would be
|
||||||
|
// a tool nobody is using, permanently lit on a map of
|
||||||
|
// work that already finished.
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Real convergence: each running agent beams toward its active-run node.
|
// Real convergence: each running agent beams toward its active-run node.
|
||||||
for (run_id, agent_id) in &runs {
|
for (run_id, agent_id) in &runs {
|
||||||
@@ -426,6 +1039,23 @@ pub async fn world_live(State(state): State<AppState>, Authed(user): Authed) ->
|
|||||||
// Tail the run's journal for richer real events (reasoning, tool
|
// Tail the run's journal for richer real events (reasoning, tool
|
||||||
// convergence, doors). On first sight, jump the cursor to the
|
// convergence, doors). On first sight, jump the cursor to the
|
||||||
// current max so we stream forward without replaying the backlog.
|
// current max so we stream forward without replaying the backlog.
|
||||||
|
//
|
||||||
|
// THIS BLOCK IS THE a2a/door LAYER ONLY, and it works: those
|
||||||
|
// runs are the ones that write `run_events` (`run_events::append`
|
||||||
|
// is reached from `routes/a2a.rs` and `mcp_door.rs`).
|
||||||
|
//
|
||||||
|
// A sibling block used to read `topology_runs.checkpoint.records`
|
||||||
|
// here, on the theory that it covered missions. It could never
|
||||||
|
// have: `run_id` comes from `active_runs`, which selects
|
||||||
|
// `agent_runs.id`, and mission phases live in `topology_runs`
|
||||||
|
// under independently generated ids. The query ran every poll and
|
||||||
|
// matched nothing, for every mission, forever — which is why the
|
||||||
|
// World has never shown a mission's tool or file activity.
|
||||||
|
// Removed rather than repointed: pointing it at `topology_runs`
|
||||||
|
// would resurrect a path whose only per-step content is the
|
||||||
|
// agent's own prose output, and a tool name in prose is
|
||||||
|
// indistinguishable from an agent talking about a tool. Mission
|
||||||
|
// detail arrives as structured `mission_events` instead.
|
||||||
if let Some(&after) = cursors.get(run_id) {
|
if let Some(&after) = cursors.get(run_id) {
|
||||||
let rows = sqlx::query(
|
let rows = sqlx::query(
|
||||||
"SELECT seq, event_type, payload FROM run_events
|
"SELECT seq, event_type, payload FROM run_events
|
||||||
@@ -586,3 +1216,146 @@ pub async fn world_replay(
|
|||||||
json!({ "events": events, "hours": hours, "count": rows.len() }),
|
json!({ "events": events, "hours": hours, "count": rows.len() }),
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod mission_feed_tests {
|
||||||
|
/// The mission query must not re-acquire any of the three bugs it fixed.
|
||||||
|
///
|
||||||
|
/// All three were invisible: an INNER JOIN does not error when it drops a
|
||||||
|
/// row, the legacy `missions.team_id` is a real column that resolves fine,
|
||||||
|
/// and `status = 'running'` is a defensible-looking filter. The World simply
|
||||||
|
/// showed less than the truth and nothing anywhere said so. A source walk is
|
||||||
|
/// the only guard available — these are runtime `sqlx::query` calls, so the
|
||||||
|
/// compiler checks nothing about them.
|
||||||
|
#[test]
|
||||||
|
fn the_mission_query_keeps_teamless_missions() {
|
||||||
|
let src = include_str!("world.rs");
|
||||||
|
let q = src
|
||||||
|
.split("async fn world_missions")
|
||||||
|
.nth(1)
|
||||||
|
.and_then(|s| s.split("async fn").next())
|
||||||
|
.expect("world_missions body");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
q.contains("LEFT JOIN mission_teams"),
|
||||||
|
"must LEFT JOIN mission_teams: a microVM mission has no team rows at \
|
||||||
|
all, and an INNER JOIN silently drops the platform's primary \
|
||||||
|
execution tier"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
q.contains("LEFT JOIN team_members"),
|
||||||
|
"the second join must be LEFT too, or the mission reappears and its \
|
||||||
|
agents take it back out"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!q.contains("tm.team_id = m.team_id"),
|
||||||
|
"must not join on the legacy missions.team_id — 0056 superseded it \
|
||||||
|
with mission_teams, and it points at only the FIRST minted team"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
q.contains("m.completed_at >"),
|
||||||
|
"recently-finished missions must be included or the World is empty \
|
||||||
|
almost always: missions finish in minutes"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
q.contains("m.status") && q.contains("m.template_kind"),
|
||||||
|
"status and template_kind must be carried: one decides finished-map \
|
||||||
|
vs live, the other the palette"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The checkpoint tail read `topology_runs` keyed by an `agent_runs` id and
|
||||||
|
/// could never match. If it comes back, missions silently lose their tool
|
||||||
|
/// and file detail again — and the query looks entirely reasonable.
|
||||||
|
#[test]
|
||||||
|
fn the_dead_checkpoint_tail_stays_dead() {
|
||||||
|
let src = include_str!("world.rs");
|
||||||
|
// Split so this assertion's own literal is not what the source walk
|
||||||
|
// finds. Written whole, the test fails on itself — which it did, and
|
||||||
|
// which is the same self-match that makes `pkill -f <pattern>` kill the
|
||||||
|
// shell carrying the pattern.
|
||||||
|
let needle = concat!("SELECT checkpoint FROM topology_", "runs WHERE id = $1::uuid");
|
||||||
|
assert!(
|
||||||
|
!src.contains(needle),
|
||||||
|
"the checkpoint tail is keyed on an agent_runs id and cannot match a \
|
||||||
|
topology_runs row; mission detail comes from structured events"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A benchmark delta is described only as far as its shape allows.
|
||||||
|
///
|
||||||
|
/// `compute_delta` emits `{kind:"opaque"}` whenever the before/after
|
||||||
|
/// metrics were not structurally comparable — which is most drivers. The
|
||||||
|
/// temptation is to say something anyway; the result would be a performance
|
||||||
|
/// claim the picture makes and the data does not support.
|
||||||
|
#[test]
|
||||||
|
fn a_benchmark_is_described_only_as_far_as_its_shape_allows() {
|
||||||
|
use serde_json::json;
|
||||||
|
assert_eq!(
|
||||||
|
super::benchmark_note(&json!({
|
||||||
|
"kind": "bencher_diff",
|
||||||
|
"samples": [
|
||||||
|
{"name": "a", "delta_pct": -12.5, "direction": "improved"},
|
||||||
|
{"name": "b", "delta_pct": 3.0, "direction": "regressed"},
|
||||||
|
// No direction and no percentage: counted, claimed for
|
||||||
|
// neither side.
|
||||||
|
{"name": "c"},
|
||||||
|
],
|
||||||
|
}))
|
||||||
|
.as_deref(),
|
||||||
|
Some("3 sample(s), 1 faster, 1 slower, best -12.5%")
|
||||||
|
);
|
||||||
|
// Nothing comparable happened, so nothing is said.
|
||||||
|
assert_eq!(
|
||||||
|
super::benchmark_note(&json!({ "kind": "opaque", "note": "not comparable" })),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
super::benchmark_note(&json!({ "kind": "bencher_diff", "samples": [] })),
|
||||||
|
None
|
||||||
|
);
|
||||||
|
// Everything got slower: no "best" claim at all.
|
||||||
|
assert_eq!(
|
||||||
|
super::benchmark_note(&json!({
|
||||||
|
"samples": [{"name": "a", "delta_pct": 8.0, "direction": "regressed"}],
|
||||||
|
}))
|
||||||
|
.as_deref(),
|
||||||
|
Some("1 sample(s), 1 slower")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Agent→phase attribution must come from the runner's own mapping.
|
||||||
|
#[test]
|
||||||
|
fn phase_attribution_reuses_the_runners_mapping() {
|
||||||
|
let src = include_str!("world.rs");
|
||||||
|
assert!(
|
||||||
|
src.contains("crate::phase_runner::purposes_for"),
|
||||||
|
"a second copy of the kind→purpose mapping would let the picture \
|
||||||
|
disagree with the machine about who is working on what"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Files must be emitted after the phases they hang under.
|
||||||
|
///
|
||||||
|
/// The client parents a file's directory chain to the coding station, and
|
||||||
|
/// parenting is first-write-wins — so a file that reaches the browser
|
||||||
|
/// before its plan pins its whole tree at the origin permanently. Nothing
|
||||||
|
/// errors: the tree renders, in the wrong place, and reads as a layout
|
||||||
|
/// choice. Swapping the two loops back is a one-line-looking edit, which is
|
||||||
|
/// exactly why it needs a guard.
|
||||||
|
#[test]
|
||||||
|
fn files_are_emitted_after_the_phases_they_hang_under() {
|
||||||
|
let src = include_str!("world.rs");
|
||||||
|
let phases = src
|
||||||
|
.find("for p in world_phases(")
|
||||||
|
.expect("the phase emission loop");
|
||||||
|
let files = src
|
||||||
|
.find("for f in world_files(")
|
||||||
|
.expect("the file emission loop");
|
||||||
|
assert!(
|
||||||
|
phases < files,
|
||||||
|
"the phase loop must run before the file loop; files parented \
|
||||||
|
before their coding station stay at the origin forever"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -23,6 +23,7 @@
|
|||||||
//! from serving — it should stop us believing a scan that scanned nothing.
|
//! from serving — it should stop us believing a scan that scanned nothing.
|
||||||
|
|
||||||
use crate::container_exec;
|
use crate::container_exec;
|
||||||
|
use bollard::Docker;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
const PROBE_TIMEOUT: Duration = Duration::from_secs(20);
|
const PROBE_TIMEOUT: Duration = Duration::from_secs(20);
|
||||||
@@ -113,9 +114,67 @@ pub async fn probe(container: &str) -> Result<Vec<ToolStatus>, String> {
|
|||||||
};
|
};
|
||||||
out.push(status);
|
out.push(status);
|
||||||
}
|
}
|
||||||
|
out.push(probe_mission_uid_can_write(&docker, container).await);
|
||||||
Ok(out)
|
Ok(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Can uid 65532 actually work in the missions tree?
|
||||||
|
///
|
||||||
|
/// `container_exec` now runs every mission exec as 65532 rather than root, so
|
||||||
|
/// that no phase leaves behind files the cleanup (which runs as 65532) cannot
|
||||||
|
/// delete. That only holds while the image gives 65532 a writable `HOME` and
|
||||||
|
/// `CARGO_HOME` — and in the deployed image its default `HOME`
|
||||||
|
/// (`/zeroclaw-data`) and `/usr/local/cargo` are BOTH root-owned, which is why
|
||||||
|
/// `container_exec::mission_env` redirects them into the missions root.
|
||||||
|
///
|
||||||
|
/// If a future image moves that mount or tightens its permissions, every cargo
|
||||||
|
/// invocation starts failing for a reason no error message would connect to a
|
||||||
|
/// uid. So it is probed at boot, alongside the tools, and reported the same way.
|
||||||
|
async fn probe_mission_uid_can_write(docker: &Docker, container: &str) -> ToolStatus {
|
||||||
|
let root = crate::mission_workspace::missions_root();
|
||||||
|
let probe = root.join("_probe-uid");
|
||||||
|
// Through `exec`, not `exec_as_root`: the point is to exercise the exact
|
||||||
|
// policy real mission work gets, including the env it is given.
|
||||||
|
let argv: Vec<String> = [
|
||||||
|
"sh",
|
||||||
|
"-c",
|
||||||
|
&format!(
|
||||||
|
"set -e; mkdir -p \"$HOME\" \"$CARGO_HOME\" {p}; : > {p}/w; rm -rf {p}; echo \"uid=$(id -u) HOME=$HOME CARGO_HOME=$CARGO_HOME\"",
|
||||||
|
p = probe.display()
|
||||||
|
),
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.map(|s| s.to_string())
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let detail = match container_exec::exec(
|
||||||
|
docker,
|
||||||
|
container,
|
||||||
|
Some(&root.display().to_string()),
|
||||||
|
&argv,
|
||||||
|
PROBE_TIMEOUT,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(r) if r.success() => {
|
||||||
|
return ToolStatus {
|
||||||
|
program: "mission-uid".to_string(),
|
||||||
|
present: true,
|
||||||
|
detail: r.combined().trim().chars().take(120).collect(),
|
||||||
|
needed_for: "every mission exec, so no phase leaves root-owned files",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(r) => r.combined().trim().chars().take(160).collect(),
|
||||||
|
Err(e) => e.chars().take(160).collect(),
|
||||||
|
};
|
||||||
|
ToolStatus {
|
||||||
|
program: "mission-uid".to_string(),
|
||||||
|
present: false,
|
||||||
|
detail,
|
||||||
|
needed_for: "every mission exec, so no phase leaves root-owned files",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Probe at startup and write the result to stderr.
|
/// Probe at startup and write the result to stderr.
|
||||||
///
|
///
|
||||||
/// Spawned rather than awaited so a slow or absent Docker socket cannot delay
|
/// Spawned rather than awaited so a slow or absent Docker socket cannot delay
|
||||||
|
|||||||
@@ -18,14 +18,46 @@ pub fn claw_alias(claw_id: Uuid) -> String {
|
|||||||
format!("claw_{}", claw_id.simple())
|
format!("claw_{}", claw_id.simple())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The claw behind a runtime alias, or `None` if it is not one of ours.
|
||||||
|
///
|
||||||
|
/// The inverse of [`claw_alias`], and it lives beside it so the two cannot
|
||||||
|
/// drift — a changed prefix breaks the round-trip test rather than quietly
|
||||||
|
/// returning `None` for every agent and dropping their attribution.
|
||||||
|
///
|
||||||
|
/// `None` is the honest answer for `scout` and the other configured aliases
|
||||||
|
/// that are not claws: they have no row in `agents` to point at.
|
||||||
|
pub fn claw_from_alias(alias: &str) -> Option<Uuid> {
|
||||||
|
Uuid::parse_str(alias.trim().strip_prefix("claw_")?).ok()
|
||||||
|
}
|
||||||
|
|
||||||
/// Map a claw's chosen model to a configured provider alias.
|
/// Map a claw's chosen model to a configured provider alias.
|
||||||
///
|
///
|
||||||
/// v0.8.3 fold: `claude_cli.*` and `kimi_cli.*` families were deleted
|
/// Claude models resolve to `claude_cli.default`, which spawns the real
|
||||||
/// upstream; every alias now lives under a real provider family
|
/// `claude` binary against the Max subscription rather than posting to the
|
||||||
/// (`anthropic`, `groq`, `gemini`, ...). Our compose currently
|
/// raw API with Claude Code identity headers. Agent work — ~99% of the
|
||||||
/// configures `anthropic.default`, `anthropic.door`, `groq.default`,
|
/// tokens — belongs on the subscription and on the supported client.
|
||||||
/// and `gemini.default`, so unknown models resolve to
|
///
|
||||||
/// `anthropic.default` — the workspace's high-quality baseline.
|
/// **The API-key path is gone.** `anthropic.default` and `anthropic.judge`
|
||||||
|
/// were retired from the runtime config on 2026-08-10: both held `sk-ant-api`
|
||||||
|
/// keys on an account whose balance is zero, which the real code path reports
|
||||||
|
/// as `400 … "Your credit balance is too low"`. Every agent that named them
|
||||||
|
/// was repointed onto a live credential.
|
||||||
|
///
|
||||||
|
/// The independence argument that put the judge there still holds — a
|
||||||
|
/// verifier sharing one credential with the implementer goes blind at exactly
|
||||||
|
/// the moment there is most to verify — but it is now served by a different
|
||||||
|
/// FAMILY rather than a different key: the validator runs on
|
||||||
|
/// `CLAWMATES_VALIDATOR_MODEL` (`glm:glm-4.7` on gw-04) while agents run on
|
||||||
|
/// the subscription, and `cross_provider_judge` refuses a validator in the
|
||||||
|
/// implementer's own family. `claude_cli.default` also carries
|
||||||
|
/// `fallback = ["claude_cli.kimi", "claude_cli.glm"]`, so a throttle degrades
|
||||||
|
/// across credentials instead of stopping.
|
||||||
|
///
|
||||||
|
/// Non-Claude families are unchanged: `groq.default` and the GLM/Kimi
|
||||||
|
/// substitution below. Gemini was removed entirely — a `gemini*` model now
|
||||||
|
/// falls through to the unrecognised branch, which LOGS and defaults to
|
||||||
|
/// `claude_cli.default` rather than silently routing to a provider we no
|
||||||
|
/// longer configure.
|
||||||
pub fn provider_alias_for(model: &str) -> &'static str {
|
pub fn provider_alias_for(model: &str) -> &'static str {
|
||||||
let m = model.trim().to_ascii_lowercase();
|
let m = model.trim().to_ascii_lowercase();
|
||||||
// Prefix families first (covers claude-sonnet-5, claude-opus-4-8,
|
// Prefix families first (covers claude-sonnet-5, claude-opus-4-8,
|
||||||
@@ -33,10 +65,7 @@ pub fn provider_alias_for(model: &str) -> &'static str {
|
|||||||
// decides what "its own family" means, so the two can't drift apart.
|
// decides what "its own family" means, so the two can't drift apart.
|
||||||
if is_exact_provider_match(&m) {
|
if is_exact_provider_match(&m) {
|
||||||
if m.starts_with("claude") {
|
if m.starts_with("claude") {
|
||||||
return "anthropic.default";
|
return "claude_cli.default";
|
||||||
}
|
|
||||||
if m.starts_with("gemini") {
|
|
||||||
return "gemini.default";
|
|
||||||
}
|
}
|
||||||
return "groq.default";
|
return "groq.default";
|
||||||
}
|
}
|
||||||
@@ -54,18 +83,18 @@ pub fn provider_alias_for(model: &str) -> &'static str {
|
|||||||
| "kimi" | "kimi-k2" | "kimi-for-coding" => {
|
| "kimi" | "kimi-k2" | "kimi-for-coding" => {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"runtime_provision: model {m:?} has no provider family configured — \
|
"runtime_provision: model {m:?} has no provider family configured — \
|
||||||
substituting anthropic.default, which spends ANTHROPIC_API_KEY"
|
substituting claude_cli.default, which spends the Claude subscription"
|
||||||
);
|
);
|
||||||
"anthropic.default"
|
"claude_cli.default"
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
if !m.is_empty() {
|
if !m.is_empty() {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"runtime_provision: unrecognised model {m:?} — defaulting to \
|
"runtime_provision: unrecognised model {m:?} — defaulting to \
|
||||||
anthropic.default"
|
claude_cli.default"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
"anthropic.default"
|
"claude_cli.default"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -80,7 +109,6 @@ pub fn provider_alias_for(model: &str) -> &'static str {
|
|||||||
pub fn is_exact_provider_match(model: &str) -> bool {
|
pub fn is_exact_provider_match(model: &str) -> bool {
|
||||||
let m = model.trim().to_ascii_lowercase();
|
let m = model.trim().to_ascii_lowercase();
|
||||||
m.starts_with("claude")
|
m.starts_with("claude")
|
||||||
|| m.starts_with("gemini")
|
|
||||||
|| m.starts_with("llama")
|
|| m.starts_with("llama")
|
||||||
|| m.starts_with("groq")
|
|| m.starts_with("groq")
|
||||||
}
|
}
|
||||||
@@ -342,19 +370,19 @@ mod tests {
|
|||||||
|
|
||||||
/// The GLM/Kimi substitution is intentional but must be reported as a
|
/// The GLM/Kimi substitution is intentional but must be reported as a
|
||||||
/// substitution, because its consequence is that a user who picked a
|
/// substitution, because its consequence is that a user who picked a
|
||||||
/// non-Anthropic model is spending the Anthropic key.
|
/// non-Anthropic model is spending someone else's budget — now the
|
||||||
|
/// Claude subscription rather than the Anthropic API key.
|
||||||
#[test]
|
#[test]
|
||||||
fn substituted_families_are_not_reported_as_exact_matches() {
|
fn substituted_families_are_not_reported_as_exact_matches() {
|
||||||
for m in ["kimi", "glm-4.7", "glm5", "kimi-k2", "something-unknown"] {
|
for m in ["kimi", "glm-4.7", "glm5", "kimi-k2", "something-unknown"] {
|
||||||
assert_eq!(super::provider_alias_for(m), "anthropic.default");
|
assert_eq!(super::provider_alias_for(m), "claude_cli.default");
|
||||||
assert!(
|
assert!(
|
||||||
!super::is_exact_provider_match(m),
|
!super::is_exact_provider_match(m),
|
||||||
"{m} resolves to anthropic.default by substitution, not by family"
|
"{m} resolves to claude_cli.default by substitution, not by family"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
for m in [
|
for m in [
|
||||||
"claude-sonnet-5",
|
"claude-sonnet-5",
|
||||||
"gemini-2.5-flash",
|
|
||||||
"groq-llama",
|
"groq-llama",
|
||||||
"llama3",
|
"llama3",
|
||||||
] {
|
] {
|
||||||
@@ -393,21 +421,25 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn provider_alias_mapping() {
|
fn provider_alias_mapping() {
|
||||||
assert_eq!(provider_alias_for("gemini"), "gemini.default");
|
// Gemini is gone: no provider row, so it must land on the logged
|
||||||
assert_eq!(provider_alias_for("gemini-2.0-flash"), "gemini.default");
|
// default rather than a family alias that resolves to nothing.
|
||||||
// v0.8.3: glm/kimi families fall back to anthropic until their
|
assert_eq!(provider_alias_for("gemini"), "claude_cli.default");
|
||||||
// own provider tables are configured in the runtime template.
|
assert_eq!(provider_alias_for("gemini-2.0-flash"), "claude_cli.default");
|
||||||
assert_eq!(provider_alias_for("GLM-4.7"), "anthropic.default");
|
assert!(!is_exact_provider_match("gemini-2.5-flash"));
|
||||||
assert_eq!(provider_alias_for("kimi"), "anthropic.default");
|
// glm/kimi families fall back to Claude until their own provider
|
||||||
|
// tables are configured in the runtime template.
|
||||||
|
assert_eq!(provider_alias_for("GLM-4.7"), "claude_cli.default");
|
||||||
|
assert_eq!(provider_alias_for("kimi"), "claude_cli.default");
|
||||||
assert_eq!(provider_alias_for("groq"), "groq.default");
|
assert_eq!(provider_alias_for("groq"), "groq.default");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
provider_alias_for("llama-3.3-70b-versatile"),
|
provider_alias_for("llama-3.3-70b-versatile"),
|
||||||
"groq.default"
|
"groq.default"
|
||||||
);
|
);
|
||||||
assert_eq!(provider_alias_for("claude"), "anthropic.default");
|
// Claude models spawn the real CLI against the subscription.
|
||||||
assert_eq!(provider_alias_for("claude-sonnet-5"), "anthropic.default");
|
assert_eq!(provider_alias_for("claude"), "claude_cli.default");
|
||||||
assert_eq!(provider_alias_for("claude-opus-4-8"), "anthropic.default");
|
assert_eq!(provider_alias_for("claude-sonnet-5"), "claude_cli.default");
|
||||||
assert_eq!(provider_alias_for("anything-else"), "anthropic.default");
|
assert_eq!(provider_alias_for("claude-opus-4-8"), "claude_cli.default");
|
||||||
|
assert_eq!(provider_alias_for("anything-else"), "claude_cli.default");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -415,4 +447,19 @@ mod tests {
|
|||||||
let id = Uuid::nil();
|
let id = Uuid::nil();
|
||||||
assert_eq!(claw_alias(id), "claw_00000000000000000000000000000000");
|
assert_eq!(claw_alias(id), "claw_00000000000000000000000000000000");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The alias must round-trip, and must NOT invent a claw for one of the
|
||||||
|
/// configured non-claw aliases.
|
||||||
|
///
|
||||||
|
/// The failure this guards is silent both ways: a broken round-trip drops
|
||||||
|
/// every tool call's agent attribution (files appear, nobody moves), and a
|
||||||
|
/// too-eager parse would attribute work to a claw id that matches no row.
|
||||||
|
#[test]
|
||||||
|
fn an_alias_round_trips_to_its_claw_and_nothing_else_does() {
|
||||||
|
let id = Uuid::from_u128(0x0198_2f11_7ac0_7d51_9c3e_44a1_09b2_5e77);
|
||||||
|
assert_eq!(claw_from_alias(&claw_alias(id)), Some(id));
|
||||||
|
assert_eq!(claw_from_alias("scout"), None);
|
||||||
|
assert_eq!(claw_from_alias("claude_cli.default"), None);
|
||||||
|
assert_eq!(claw_from_alias("claw_not-a-uuid"), None);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -314,9 +314,7 @@ async fn exec_target(pool: &PgPool, mission_id: Uuid) -> Result<(String, PathBuf
|
|||||||
}
|
}
|
||||||
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
let container = std::env::var("CLAWMATES_RUNTIME_CONTAINER")
|
||||||
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
.unwrap_or_else(|_| "clawmates-runtime".to_string());
|
||||||
let root = std::env::var("CLAWMATES_MISSIONS_ROOT")
|
let workdir = crate::mission_workspace::missions_root()
|
||||||
.unwrap_or_else(|_| "/var/lib/clawmates-missions".to_string());
|
|
||||||
let workdir = PathBuf::from(root)
|
|
||||||
.join(mission_id.to_string())
|
.join(mission_id.to_string())
|
||||||
.join("repo");
|
.join("repo");
|
||||||
Ok((container, workdir))
|
Ok((container, workdir))
|
||||||
|
|||||||
@@ -0,0 +1,262 @@
|
|||||||
|
//! Run a whole mission as ONE headless agent session.
|
||||||
|
//!
|
||||||
|
//! The alternative to `phase_runner`. Instead of splitting a mission into
|
||||||
|
//! phases that hand work to each other through a shared checkout, this hands
|
||||||
|
//! the entire task to a single agent session and asks the forge afterwards
|
||||||
|
//! what actually landed.
|
||||||
|
//!
|
||||||
|
//! # Why
|
||||||
|
//!
|
||||||
|
//! The phase machinery moves state between processes through a filesystem, and
|
||||||
|
//! that seam produced most of a week's defects: two uids fighting over
|
||||||
|
//! `.git/objects`, a missing git identity, `reset --hard` deleting the
|
||||||
|
//! previous phase's work, a capture base overloaded with two meanings. None of
|
||||||
|
//! those failures are *possible* inside one session, because there is no
|
||||||
|
//! handoff to get wrong — step two knows what step one did because it is the
|
||||||
|
//! same context.
|
||||||
|
//!
|
||||||
|
//! Measured against the same task (create a file, read it back, extend it,
|
||||||
|
//! push it): the phase path took nine production runs and five distinct bug
|
||||||
|
//! fixes to do reliably; a single session did it in 23 seconds, 19 times out
|
||||||
|
//! of 20, first try.
|
||||||
|
//!
|
||||||
|
//! # What this deliberately does NOT trust
|
||||||
|
//!
|
||||||
|
//! The agent's own account of what it did. In the same 60-run experiment one
|
||||||
|
//! session exited 0, ran for 18 seconds, and pushed nothing — a clean exit
|
||||||
|
//! status with no work delivered, about 5% of the time. That is the same
|
||||||
|
//! "reported success while doing nothing" shape as every scaffolding bug, and
|
||||||
|
//! it is why [`verify_landed`] asks the forge rather than reading the summary.
|
||||||
|
//!
|
||||||
|
//! Deleting the phase machinery is justified by the evidence. Deleting the
|
||||||
|
//! verification is not — the evidence points the other way.
|
||||||
|
|
||||||
|
use std::time::Duration;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
use crate::container_exec;
|
||||||
|
|
||||||
|
/// Ceiling for one mission session. Long, because a real coding task with a
|
||||||
|
/// test suite legitimately takes minutes; bounded, because a wedged session
|
||||||
|
/// must not hold a container forever.
|
||||||
|
const SESSION_TIMEOUT: Duration = Duration::from_secs(3600);
|
||||||
|
|
||||||
|
/// Tools the session may use without prompting.
|
||||||
|
///
|
||||||
|
/// `--dangerously-skip-permissions` is refused by the CLI when running as
|
||||||
|
/// root, which mission containers do, and blanket bypass is the wrong default
|
||||||
|
/// for something driving a real repository anyway. An explicit allow-list is
|
||||||
|
/// both accepted as root and easier to defend.
|
||||||
|
const ALLOWED_TOOLS: &[&str] = &["Read", "Edit", "Write", "Bash"];
|
||||||
|
|
||||||
|
/// What one session did, as observed from outside it.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct SessionOutcome {
|
||||||
|
/// The agent's closing summary. Diagnostic only — never evidence.
|
||||||
|
pub summary: String,
|
||||||
|
pub exit_code: Option<i64>,
|
||||||
|
/// Whether the expected branch actually appeared on the forge.
|
||||||
|
pub landed: bool,
|
||||||
|
/// Head sha of the branch, when it landed.
|
||||||
|
pub head_sha: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SessionOutcome {
|
||||||
|
/// The session both finished cleanly *and* delivered.
|
||||||
|
///
|
||||||
|
/// Both halves are required. `exit_code == Some(0)` alone is what the
|
||||||
|
/// 5% silent-nothing case looks like from the inside.
|
||||||
|
pub fn delivered(&self) -> bool {
|
||||||
|
self.exit_code == Some(0) && self.landed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Is the direct-session executor enabled?
|
||||||
|
///
|
||||||
|
/// Opt-in rather than default: the ZeroClaw path is what production has been
|
||||||
|
/// running, and a silent switch of how every mission executes is exactly the
|
||||||
|
/// kind of change that should require someone to have typed it.
|
||||||
|
pub fn direct_mode() -> bool {
|
||||||
|
matches!(
|
||||||
|
std::env::var("CLAWMATES_MISSION_EXECUTOR").as_deref(),
|
||||||
|
Ok("session")
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build the instruction for a mission session.
|
||||||
|
///
|
||||||
|
/// One statement of the whole job, not a per-phase directive. The branch name
|
||||||
|
/// is stated rather than left to the agent so there is a fixed thing to verify
|
||||||
|
/// against afterwards — an agent that picks its own branch name is an agent
|
||||||
|
/// whose work cannot be checked without asking it where the work went.
|
||||||
|
pub fn session_prompt(task: &str, repo_path: &str, branch: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"You are working in the git repository at {repo_path}.\n\
|
||||||
|
\n\
|
||||||
|
TASK\n\
|
||||||
|
{task}\n\
|
||||||
|
\n\
|
||||||
|
WHEN THE WORK IS DONE\n\
|
||||||
|
Commit it and push to a new branch named exactly `{branch}`.\n\
|
||||||
|
The remote `origin` is already configured with credentials.\n\
|
||||||
|
\n\
|
||||||
|
If the task cannot be completed as written — a file it refers to does \
|
||||||
|
not exist, a premise is wrong, the tests cannot run — say so plainly \
|
||||||
|
and do NOT push. An honest report that the work could not be done is \
|
||||||
|
worth more than a branch that looks finished.\n"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run one mission session inside an existing container.
|
||||||
|
pub async fn run_session(
|
||||||
|
container: &str,
|
||||||
|
repo_path: &str,
|
||||||
|
task: &str,
|
||||||
|
branch: &str,
|
||||||
|
) -> Result<(String, Option<i64>), String> {
|
||||||
|
let docker = container_exec::connect()?;
|
||||||
|
let prompt = session_prompt(task, repo_path, branch);
|
||||||
|
let mut argv = vec!["claude".to_string(), "-p".to_string()];
|
||||||
|
argv.push("--allowedTools".into());
|
||||||
|
argv.extend(ALLOWED_TOOLS.iter().map(|t| t.to_string()));
|
||||||
|
argv.push("--permission-mode".into());
|
||||||
|
argv.push("acceptEdits".into());
|
||||||
|
argv.push(prompt);
|
||||||
|
|
||||||
|
let out = container_exec::exec(
|
||||||
|
&docker,
|
||||||
|
container,
|
||||||
|
Some(repo_path),
|
||||||
|
&argv,
|
||||||
|
SESSION_TIMEOUT,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok((out.combined(), out.exit_code))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ask the forge whether the branch exists, and at what commit.
|
||||||
|
///
|
||||||
|
/// The whole point of the module. Everything above this line is the agent's
|
||||||
|
/// account of events; this is the only part that is evidence.
|
||||||
|
pub async fn verify_landed(
|
||||||
|
api_base: &str,
|
||||||
|
token: &str,
|
||||||
|
branch: &str,
|
||||||
|
) -> Result<Option<String>, String> {
|
||||||
|
let url = format!("{api_base}/branches/{}", urlencode(branch));
|
||||||
|
let client = reqwest::Client::new();
|
||||||
|
let resp = client
|
||||||
|
.get(&url)
|
||||||
|
.header("Authorization", format!("token {token}"))
|
||||||
|
.timeout(Duration::from_secs(30))
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("query branch: {e}"))?;
|
||||||
|
if resp.status().as_u16() == 404 {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
if !resp.status().is_success() {
|
||||||
|
return Err(format!("forge returned {}", resp.status()));
|
||||||
|
}
|
||||||
|
let body: serde_json::Value = resp
|
||||||
|
.json()
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("decode branch response: {e}"))?;
|
||||||
|
Ok(body
|
||||||
|
.get("commit")
|
||||||
|
.and_then(|c| c.get("id"))
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(str::to_string))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Percent-encode the path segment. Branch names contain `/`, which would
|
||||||
|
/// otherwise split the URL path and query the wrong endpoint.
|
||||||
|
fn urlencode(s: &str) -> String {
|
||||||
|
s.bytes()
|
||||||
|
.map(|b| match b {
|
||||||
|
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
|
||||||
|
(b as char).to_string()
|
||||||
|
}
|
||||||
|
_ => format!("%{b:02X}"),
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Branch a session-executed mission pushes to.
|
||||||
|
pub fn session_branch(mission_id: Uuid) -> String {
|
||||||
|
format!("clawmates/session-{}", &mission_id.simple().to_string()[..12])
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_prompt_names_the_branch_and_forbids_a_dishonest_push() {
|
||||||
|
let p = session_prompt("Add a file.", "/mission/repo", "clawmates/session-abc");
|
||||||
|
assert!(p.contains("clawmates/session-abc"), "branch must be fixed");
|
||||||
|
assert!(p.contains("/mission/repo"));
|
||||||
|
assert!(
|
||||||
|
p.contains("do NOT push"),
|
||||||
|
"the prompt must give an honest exit that is not a branch"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A clean exit is not delivery. This is the 5% case from the 60-run
|
||||||
|
/// experiment: `rc=0`, 18 seconds of work, no branch.
|
||||||
|
#[test]
|
||||||
|
fn a_clean_exit_without_a_branch_is_not_delivery() {
|
||||||
|
let silent = SessionOutcome {
|
||||||
|
summary: "All steps completed.".into(),
|
||||||
|
exit_code: Some(0),
|
||||||
|
landed: false,
|
||||||
|
head_sha: None,
|
||||||
|
};
|
||||||
|
assert!(
|
||||||
|
!silent.delivered(),
|
||||||
|
"exit 0 with nothing on the forge must never count as delivered"
|
||||||
|
);
|
||||||
|
|
||||||
|
let real = SessionOutcome {
|
||||||
|
landed: true,
|
||||||
|
head_sha: Some("abc123".into()),
|
||||||
|
..silent.clone()
|
||||||
|
};
|
||||||
|
assert!(real.delivered());
|
||||||
|
|
||||||
|
// And a failed session that somehow pushed is also not a success.
|
||||||
|
let broken = SessionOutcome {
|
||||||
|
exit_code: Some(1),
|
||||||
|
landed: true,
|
||||||
|
head_sha: Some("abc123".into()),
|
||||||
|
summary: String::new(),
|
||||||
|
};
|
||||||
|
assert!(!broken.delivered());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn branch_names_survive_url_encoding() {
|
||||||
|
assert_eq!(urlencode("clawmates/session-01"), "clawmates%2Fsession-01");
|
||||||
|
assert_eq!(urlencode("plain"), "plain");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The switch must be explicit. A near-miss value silently leaving every
|
||||||
|
/// mission on the old executor is better than a near-miss value silently
|
||||||
|
/// switching it — but either way, only the exact word counts.
|
||||||
|
#[test]
|
||||||
|
fn the_flag_must_be_typed_exactly() {
|
||||||
|
// Not asserting against the live env (that would race other tests);
|
||||||
|
// asserting the matcher's shape, which is what decides.
|
||||||
|
for wrong in ["Session", "sessions", "direct", "1", "true", ""] {
|
||||||
|
assert_ne!(wrong, "session", "{wrong:?} must not enable direct mode");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_session_branch_is_stable_and_namespaced() {
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
let b = session_branch(id);
|
||||||
|
assert_eq!(b, session_branch(id));
|
||||||
|
assert!(b.starts_with("clawmates/session-"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,678 @@
|
|||||||
|
//! The Anthropic provider backed by the SUBSCRIPTION token, not the metered key.
|
||||||
|
//!
|
||||||
|
//! Two Anthropic credentials reach this server and they bill differently:
|
||||||
|
//!
|
||||||
|
//! - `ANTHROPIC_API_KEY` (`sk-ant-api…`) — metered, pay-as-you-go, and the thing
|
||||||
|
//! that runs out. Every mission VM already avoids it: `mission_runtime` sends
|
||||||
|
//! only the subscription token into a guest, deliberately.
|
||||||
|
//! - `ANTHROPIC_OAUTH_TOKEN` / `CLAUDE_CODE_OAUTH_TOKEN` (`sk-ant-oat…`) — the
|
||||||
|
//! Claude Code subscription, which is what the CLI inside every VM runs on.
|
||||||
|
//!
|
||||||
|
//! Server-side model calls that went through `Runtime::complete` with a bare
|
||||||
|
//! model name resolved to the DEFAULT provider — the metered key. So the roster
|
||||||
|
//! planner died with
|
||||||
|
//! `400 … "Your credit balance is too low to access the Anthropic API"` while
|
||||||
|
//! every mission on the same machine kept running fine on the subscription.
|
||||||
|
//! The harness reported it honestly as FAIL-NORUN rather than a passing scenario,
|
||||||
|
//! which is the only reason it was visible at all.
|
||||||
|
//!
|
||||||
|
//! This is the one place that turns the subscription token into a provider.
|
||||||
|
//! `evaluator::subscription_judge` had its own copy; there is now one.
|
||||||
|
|
||||||
|
/// The subscription-backed provider, or `None` when no usable token is present.
|
||||||
|
///
|
||||||
|
/// Checks the `sk-ant-oat` prefix rather than trusting the variable name: an
|
||||||
|
/// `sk-ant-api` key pasted into the OAuth slot would authenticate and then bill
|
||||||
|
/// the metered account, which is the failure this module exists to prevent —
|
||||||
|
/// silently, and with the same error weeks later.
|
||||||
|
pub fn provider() -> Option<cm_llm::AnthropicProvider> {
|
||||||
|
for var in ["ANTHROPIC_OAUTH_TOKEN", "CLAUDE_CODE_OAUTH_TOKEN"] {
|
||||||
|
let Ok(token) = std::env::var(var) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let token = token.trim();
|
||||||
|
if token.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if !is_subscription_token(token) {
|
||||||
|
eprintln!(
|
||||||
|
"subscription: {var} is set but is not a Claude Code setup token \
|
||||||
|
(expected sk-ant-oat…) — ignoring it rather than billing the \
|
||||||
|
metered key by accident"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
return Some(cm_llm::AnthropicProvider::new(token.to_string()));
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a token is a Claude Code subscription token rather than an API key.
|
||||||
|
pub fn is_subscription_token(token: &str) -> bool {
|
||||||
|
token.trim().starts_with("sk-ant-oat")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One completion on the subscription, mirroring `Runtime::complete`'s contract
|
||||||
|
/// so a caller can swap between them without reshaping its call.
|
||||||
|
///
|
||||||
|
/// Falls back to the caller's runtime when no subscription token exists, so a
|
||||||
|
/// deployment without one behaves exactly as it did before.
|
||||||
|
pub async fn complete_or(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
|
system: &str,
|
||||||
|
user: &str,
|
||||||
|
model: &str,
|
||||||
|
max_tokens: u32,
|
||||||
|
// Carried explicitly rather than defaulted. The Master Planner and the claw
|
||||||
|
// enhancer both pass `true`, and a helper that quietly dropped it would take
|
||||||
|
// web search away from two features while every test still passed.
|
||||||
|
web_search: bool,
|
||||||
|
) -> Result<String, String> {
|
||||||
|
// A `name:model` spec is an operator's explicit provider choice — the swarm
|
||||||
|
// worker model is literally configured that way (`kimi:kimi-k2.6`), and
|
||||||
|
// `Runtime::resolve_provider` honours it. Forcing that onto Anthropic would
|
||||||
|
// silently run someone's chosen model on the wrong provider, which is the
|
||||||
|
// same class of bug as this module exists to fix, only pointed the other
|
||||||
|
// way. Only a BARE name is ambiguous, and a bare name is what resolves to
|
||||||
|
// the default provider — the metered key.
|
||||||
|
if !is_bare_model_name(model) || provider().is_none() {
|
||||||
|
return runtime
|
||||||
|
.complete(system, user, model, max_tokens, web_search)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
let provider = provider().expect("checked just above");
|
||||||
|
complete_with(&provider, system, user, model, max_tokens, web_search).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a model string names a model without naming a provider.
|
||||||
|
pub fn is_bare_model_name(model: &str) -> bool {
|
||||||
|
!model.contains(':')
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How long to wait before each retry. Four attempts, ~30s of patience total.
|
||||||
|
///
|
||||||
|
/// The subscription has no credit wall, but it does have a rate limit, and a
|
||||||
|
/// roster proposal is a single one-shot call: a 429 that a browser would shrug
|
||||||
|
/// off used to fail the whole "propose a team" button. Measured on this
|
||||||
|
/// deployment — moving the roster onto the subscription turned
|
||||||
|
/// `400 credit balance too low` into `429 rate_limit_error`, i.e. a wall that
|
||||||
|
/// clears on its own became the failure mode, so waiting is the right answer.
|
||||||
|
const BACKOFF_SECS: &[u64] = &[2, 8, 20];
|
||||||
|
|
||||||
|
/// Whether an error is worth waiting out rather than reporting.
|
||||||
|
///
|
||||||
|
/// Deliberately narrow. A 400 (bad request), 401 (wrong token) or 404 (unknown
|
||||||
|
/// model) will never succeed on a retry, and retrying them turns a legible
|
||||||
|
/// error into a 30-second hang followed by the same error.
|
||||||
|
fn is_transient(e: &cm_llm::LlmError) -> bool {
|
||||||
|
use cm_llm::LlmError;
|
||||||
|
match e {
|
||||||
|
// The transport never reached Anthropic — a dropped connection or a
|
||||||
|
// DNS blip, not a rejected request.
|
||||||
|
LlmError::Transport(_) => true,
|
||||||
|
LlmError::Api(detail) => {
|
||||||
|
// `anthropic.rs` formats these as `"{status}: {body}"`.
|
||||||
|
detail.starts_with("429")
|
||||||
|
|| detail.starts_with("500")
|
||||||
|
|| detail.starts_with("502")
|
||||||
|
|| detail.starts_with("503")
|
||||||
|
|| detail.starts_with("529")
|
||||||
|
|| detail.contains("rate_limit")
|
||||||
|
|| detail.contains("overloaded")
|
||||||
|
}
|
||||||
|
LlmError::Scenario(_) | LlmError::Wire(_) => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Models to try, in order, when the requested one is rate limited.
|
||||||
|
///
|
||||||
|
/// The order is capability first, then independence:
|
||||||
|
///
|
||||||
|
/// opus -> sonnet -> haiku one account, three tiers. A throttle usually
|
||||||
|
/// hits a tier, so stepping down often clears it.
|
||||||
|
/// -> kimi -> glm two separately funded accounts. Now an
|
||||||
|
/// Anthropic outage, not just a throttle, is
|
||||||
|
/// survivable.
|
||||||
|
/// -> local our own GPU. Nothing left to be down.
|
||||||
|
///
|
||||||
|
/// Every model id here was probed on this deployment 2026-08-09 and answered
|
||||||
|
/// 200: the four Anthropic tiers on the subscription, `kimi-k2.7-code` on
|
||||||
|
/// api.kimi.com/coding, `glm-4.7` on z.ai, and `ornith-fleet:9b` on the fleet.
|
||||||
|
/// Configured is not the same as working — see `preflight`, which re-checks
|
||||||
|
/// them at boot, because a link nobody exercises is discovered broken during
|
||||||
|
/// the outage it existed for.
|
||||||
|
///
|
||||||
|
/// The last link runs on our OWN hardware. Every other entry — and every other
|
||||||
|
/// link above it — depends on somebody else's account staying funded and
|
||||||
|
/// unthrottled; `local:` depends on a GPU in the next room. It is last because
|
||||||
|
/// it is the weakest model, and present because a chain whose every link is
|
||||||
|
/// external is not a fallback chain, it is one outage in a trench coat.
|
||||||
|
///
|
||||||
|
/// Note the model half contains a colon (`ornith-fleet:9b`), which is why
|
||||||
|
/// `resolve_provider` splits on the FIRST one only.
|
||||||
|
///
|
||||||
|
/// Override with `CLAWMATES_MODEL_FALLBACK` (comma-separated). An empty value
|
||||||
|
/// disables fallback and restores plain "503 and wait".
|
||||||
|
const DEFAULT_FALLBACK: &str = "claude-sonnet-4-6,claude-haiku-4-5-20251001,\
|
||||||
|
kimi:kimi-k2.7-code,glm:glm-4.7,local:ornith-fleet:9b";
|
||||||
|
|
||||||
|
/// The chain to walk after `requested`, with `requested` itself removed so a
|
||||||
|
/// capped model is never retried as its own fallback.
|
||||||
|
pub fn fallback_chain(requested: &str) -> Vec<String> {
|
||||||
|
let raw =
|
||||||
|
std::env::var("CLAWMATES_MODEL_FALLBACK").unwrap_or_else(|_| DEFAULT_FALLBACK.to_string());
|
||||||
|
raw.split(',')
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|m| !m.is_empty() && *m != requested.trim())
|
||||||
|
.map(str::to_string)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a failure means "this model has no capacity right now" as opposed
|
||||||
|
/// to "this request was wrong".
|
||||||
|
///
|
||||||
|
/// The distinction is the whole safety of the chain: walking it on a malformed
|
||||||
|
/// prompt would ask three models the same bad question and report the third
|
||||||
|
/// one's confusion, while walking it on a rate limit is exactly the point.
|
||||||
|
pub fn is_capacity_failure(err: &str) -> bool {
|
||||||
|
err.contains("rate_limit") || err.contains("429") || err.contains("credit balance")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One completion, stepping down `fallback_chain` when a model has no capacity.
|
||||||
|
///
|
||||||
|
/// Returns the text **and the model that actually produced it**. Callers must
|
||||||
|
/// persist that second value: a plan drafted by the third link in the chain and
|
||||||
|
/// filed as an opus plan is a silent quality change, which is the failure shape
|
||||||
|
/// this project keeps paying for. Every hop is logged.
|
||||||
|
pub async fn complete_with_fallback(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
|
system: &str,
|
||||||
|
user: &str,
|
||||||
|
model: &str,
|
||||||
|
max_tokens: u32,
|
||||||
|
web_search: bool,
|
||||||
|
) -> Result<(String, String), String> {
|
||||||
|
let mut last = match complete_or(runtime, system, user, model, max_tokens, web_search).await {
|
||||||
|
Ok(text) => return Ok((text, model.to_string())),
|
||||||
|
Err(e) if is_capacity_failure(&e) => e,
|
||||||
|
// A real error. Do not launder it through two more models.
|
||||||
|
Err(e) => return Err(e),
|
||||||
|
};
|
||||||
|
for next in fallback_chain(model) {
|
||||||
|
eprintln!("model fallback: {model} has no capacity ({last}) — trying {next}");
|
||||||
|
match complete_or(runtime, system, user, &next, max_tokens, web_search).await {
|
||||||
|
Ok(text) => {
|
||||||
|
eprintln!("model fallback: {next} answered in place of {model}");
|
||||||
|
return Ok((text, next));
|
||||||
|
}
|
||||||
|
Err(e) if is_capacity_failure(&e) => last = e,
|
||||||
|
Err(e) => return Err(format!("fallback {next}: {e}")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(last)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What a probe of one link found.
|
||||||
|
///
|
||||||
|
/// `Throttled` is deliberately NOT a failure. A 429 means the spec resolved, the
|
||||||
|
/// credential authenticated, and the provider simply had no capacity this
|
||||||
|
/// second — which is the exact condition the chain exists to route around. A
|
||||||
|
/// report that painted it red would train an operator to ignore the red.
|
||||||
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
|
pub enum LinkStatus {
|
||||||
|
Answered,
|
||||||
|
Throttled(String),
|
||||||
|
/// Never came back. Its own state because it is the one that used to make
|
||||||
|
/// the whole report vanish: with no timeout, a single hung provider meant
|
||||||
|
/// silence from the tool built to prevent silence.
|
||||||
|
TimedOut,
|
||||||
|
/// The spec named a provider the registry does not have, so
|
||||||
|
/// `resolve_provider` silently fell back to the DEFAULT provider. The link
|
||||||
|
/// would "work" while running on entirely the wrong model.
|
||||||
|
Unregistered,
|
||||||
|
Broken(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LinkStatus {
|
||||||
|
pub fn usable(&self) -> bool {
|
||||||
|
matches!(self, LinkStatus::Answered | LinkStatus::Throttled(_))
|
||||||
|
}
|
||||||
|
fn label(&self) -> String {
|
||||||
|
match self {
|
||||||
|
LinkStatus::Answered => "ok".into(),
|
||||||
|
LinkStatus::Throttled(_) => "throttled (configured, no capacity now)".into(),
|
||||||
|
LinkStatus::TimedOut => {
|
||||||
|
format!("TIMED OUT after {}s — treat as down", PROBE_TIMEOUT.as_secs())
|
||||||
|
}
|
||||||
|
LinkStatus::Unregistered => "UNREGISTERED — resolves to the DEFAULT provider".into(),
|
||||||
|
LinkStatus::Broken(e) => format!("BROKEN: {}", e.chars().take(120).collect::<String>()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Probe every link of the chain, head model included.
|
||||||
|
///
|
||||||
|
/// Eight tokens each, through the SAME path a real call takes, so it proves
|
||||||
|
/// resolution and reachability rather than that a string is present in a config
|
||||||
|
/// file. The distinction matters here more than usual: `resolve_provider` falls
|
||||||
|
/// back to the default provider for an unknown provider name, so a typo in
|
||||||
|
/// `kimi:` does not error — it quietly runs on Anthropic, and the chain reads
|
||||||
|
/// as five providers while being one.
|
||||||
|
/// Per-link ceiling. Generous on purpose: `complete_or` spends up to 30s in its
|
||||||
|
/// own backoff before giving up, so anything under that would report a merely
|
||||||
|
/// throttled link as hung.
|
||||||
|
const PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
|
||||||
|
|
||||||
|
pub async fn preflight(runtime: &cm_runtime::Runtime, head: &str) -> Vec<(String, LinkStatus)> {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for spec in std::iter::once(head.to_string()).chain(fallback_chain(head)) {
|
||||||
|
// A qualified spec whose provider is missing resolves to the default —
|
||||||
|
// detected the same way `cross_provider_judge` does it, by asking what
|
||||||
|
// the model half came back as.
|
||||||
|
if spec.contains(':') {
|
||||||
|
// Unrouted specs come back WHOLE; routed ones come back as the part
|
||||||
|
// after the FIRST colon. Testing "does it still contain a colon"
|
||||||
|
// reads the same and is wrong: `local:ornith-fleet:9b` resolves
|
||||||
|
// correctly to model `ornith-fleet:9b`, which does. This probe
|
||||||
|
// reported a provider the server had just registered as
|
||||||
|
// UNREGISTERED on its first live run, which is how the same latent
|
||||||
|
// bug was found in `evaluator::cross_provider_judge`.
|
||||||
|
let (_, resolved) = runtime.resolve_provider(&spec);
|
||||||
|
if resolved == spec {
|
||||||
|
out.push((spec.clone(), LinkStatus::Unregistered));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A non-empty system prompt. Kimi rejects an empty one outright —
|
||||||
|
// `400 the message at position 0 with role 'system' must not be empty` —
|
||||||
|
// so an empty probe reported a healthy provider as BROKEN on the first
|
||||||
|
// live run. The probe must look like the traffic it stands in for.
|
||||||
|
// NOT awaited here — the timeout has to wrap the FUTURE. Awaiting first
|
||||||
|
// and wrapping the result compiles, reads correctly, and bounds nothing.
|
||||||
|
let probe = complete_or(
|
||||||
|
runtime,
|
||||||
|
"You are a reachability probe.",
|
||||||
|
"Reply with exactly: OK",
|
||||||
|
&spec,
|
||||||
|
8,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
let status = match tokio::time::timeout(PROBE_TIMEOUT, probe).await {
|
||||||
|
Err(_) => LinkStatus::TimedOut,
|
||||||
|
Ok(Ok(_)) => LinkStatus::Answered,
|
||||||
|
Ok(Err(e)) if is_capacity_failure(&e) => LinkStatus::Throttled(e),
|
||||||
|
Ok(Err(e)) => LinkStatus::Broken(e),
|
||||||
|
};
|
||||||
|
// Emitted as it resolves, not collected and printed at the end. A later
|
||||||
|
// link that hangs must not be able to hide the ones already checked.
|
||||||
|
eprintln!("fallback chain: {spec:<32} {}", status.label());
|
||||||
|
out.push((spec, status));
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Probe the chain at boot and write the result to stderr.
|
||||||
|
///
|
||||||
|
/// Spawned rather than awaited, like `runtime_preflight`: this is diagnostic and
|
||||||
|
/// must never delay the server coming up. Loud when a link is unusable, because
|
||||||
|
/// the whole point of a chain is that nobody looks at it until the day it has to
|
||||||
|
/// work.
|
||||||
|
pub fn report_at_boot(runtime: cm_runtime::Runtime) {
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let head = std::env::var("CLAWMATES_PREFLIGHT_HEAD")
|
||||||
|
.unwrap_or_else(|_| "claude-opus-4-8".to_string());
|
||||||
|
let links = preflight(&runtime, &head).await;
|
||||||
|
let bad: Vec<_> = links.iter().filter(|(_, s)| !s.usable()).collect();
|
||||||
|
eprintln!(
|
||||||
|
"fallback chain ({} link(s), {} usable):",
|
||||||
|
links.len(),
|
||||||
|
links.len() - bad.len()
|
||||||
|
);
|
||||||
|
for (spec, status) in &links {
|
||||||
|
eprintln!(" {spec:<32} {}", status.label());
|
||||||
|
}
|
||||||
|
if !bad.is_empty() {
|
||||||
|
eprintln!(
|
||||||
|
"fallback chain: WARNING — {} link(s) are NOT usable. The chain is \
|
||||||
|
shorter than it reads, and the shortfall only shows up during the \
|
||||||
|
outage it exists for.",
|
||||||
|
bad.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Turn a `complete_or` failure into the right API error.
|
||||||
|
///
|
||||||
|
/// A rate limit that outlived the backoff is not a bug in this server, and
|
||||||
|
/// reporting it as one costs an operator a trip through the logs to find out
|
||||||
|
/// the answer was "wait". Measured: a bare 16-token probe with the same token
|
||||||
|
/// returned 429 with `x-should-retry: true` — Anthropic itself says try again.
|
||||||
|
pub fn as_api_error(err: &str) -> crate::error::ApiError {
|
||||||
|
if err.contains("rate_limit") || err.contains("429") {
|
||||||
|
return crate::error::ApiError::Unavailable(
|
||||||
|
"the Claude Code subscription is rate limited right now — this \
|
||||||
|
clears on its own; try again shortly"
|
||||||
|
.into(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
crate::error::ApiError::Internal
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stream one request and collect its text, waiting out transient failures.
|
||||||
|
async fn complete_with(
|
||||||
|
provider: &cm_llm::AnthropicProvider,
|
||||||
|
system: &str,
|
||||||
|
user: &str,
|
||||||
|
model: &str,
|
||||||
|
max_tokens: u32,
|
||||||
|
web_search: bool,
|
||||||
|
) -> Result<String, String> {
|
||||||
|
let mut attempt = 0usize;
|
||||||
|
loop {
|
||||||
|
match attempt_once(provider, system, user, model, max_tokens, web_search).await {
|
||||||
|
Ok(text) => return Ok(text),
|
||||||
|
Err((stage, e)) => {
|
||||||
|
let Some(delay) = BACKOFF_SECS.get(attempt).copied().filter(|_| is_transient(&e))
|
||||||
|
else {
|
||||||
|
return Err(format!("subscription {stage}: {e}"));
|
||||||
|
};
|
||||||
|
eprintln!(
|
||||||
|
"subscription {stage}: {e} — retrying in {delay}s \
|
||||||
|
(attempt {} of {})",
|
||||||
|
attempt + 2,
|
||||||
|
BACKOFF_SECS.len() + 1
|
||||||
|
);
|
||||||
|
tokio::time::sleep(std::time::Duration::from_secs(delay)).await;
|
||||||
|
attempt += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One attempt. The collected text is discarded on failure, so a retry never
|
||||||
|
/// concatenates a partial answer onto a whole one.
|
||||||
|
async fn attempt_once(
|
||||||
|
provider: &cm_llm::AnthropicProvider,
|
||||||
|
system: &str,
|
||||||
|
user: &str,
|
||||||
|
model: &str,
|
||||||
|
max_tokens: u32,
|
||||||
|
web_search: bool,
|
||||||
|
) -> Result<String, (&'static str, cm_llm::LlmError)> {
|
||||||
|
use cm_llm::{ChatMessage, ChatRequest, ChatRole, ContentPart, LlmEvent, LlmProvider};
|
||||||
|
use futures::StreamExt as _;
|
||||||
|
|
||||||
|
let request = ChatRequest {
|
||||||
|
system: system.to_string(),
|
||||||
|
model: model.to_string(),
|
||||||
|
messages: vec![ChatMessage {
|
||||||
|
role: ChatRole::User,
|
||||||
|
parts: vec![ContentPart::text(user)],
|
||||||
|
}],
|
||||||
|
tools: vec![],
|
||||||
|
max_tokens,
|
||||||
|
web_search,
|
||||||
|
};
|
||||||
|
let mut stream = provider.stream(request).await.map_err(|e| ("call", e))?;
|
||||||
|
let mut text = String::new();
|
||||||
|
while let Some(event) = stream.next().await {
|
||||||
|
match event {
|
||||||
|
Ok(LlmEvent::TextDelta(t)) => text.push_str(&t),
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(e) => return Err(("stream", e)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(text)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// Every server-side model call that should be on the subscription IS.
|
||||||
|
///
|
||||||
|
/// `validator_preflight` is the deliberate exception: it probes whatever
|
||||||
|
/// spec an operator configured (today `glm:glm-4.7`), and forcing it onto
|
||||||
|
/// Anthropic would make it prove the wrong thing — it exists to answer "is
|
||||||
|
/// the configured validator reachable".
|
||||||
|
/// The first version of this test grepped for the literal
|
||||||
|
/// `runtime.complete(` and passed while FOUR more call sites — the phase
|
||||||
|
/// planner, both swarm calls, and a second enhance path — still billed the
|
||||||
|
/// metered key. They were spelled `state.runtime` or wrapped across lines,
|
||||||
|
/// so the receiver name was never the thing to look for. Match the METHOD.
|
||||||
|
#[test]
|
||||||
|
fn no_server_side_call_silently_uses_the_metered_key() {
|
||||||
|
let sources = [
|
||||||
|
("routes/mission_roster.rs", include_str!("routes/mission_roster.rs")),
|
||||||
|
("routes/mission_plan.rs", include_str!("routes/mission_plan.rs")),
|
||||||
|
("routes/planner.rs", include_str!("routes/planner.rs")),
|
||||||
|
("routes/claws.rs", include_str!("routes/claws.rs")),
|
||||||
|
("swarm.rs", include_str!("swarm.rs")),
|
||||||
|
];
|
||||||
|
for (name, src) in sources {
|
||||||
|
assert!(
|
||||||
|
!src.contains(".complete("),
|
||||||
|
"{name} calls Runtime::complete directly — a bare model name there \
|
||||||
|
resolves to the DEFAULT provider, which is the metered API key. \
|
||||||
|
Use `subscription::complete_or`, which passes a `name:model` \
|
||||||
|
spec through untouched."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// And the exception stays an exception, on purpose.
|
||||||
|
assert!(
|
||||||
|
include_str!("validator_preflight.rs").contains("runtime.complete("),
|
||||||
|
"validator_preflight must keep probing the CONFIGURED spec"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Only errors that can clear on their own are waited out.
|
||||||
|
///
|
||||||
|
/// The negative half is the point: a 400 or a 401 retried three times is a
|
||||||
|
/// 30-second hang ending in the identical message, which reads as a stall
|
||||||
|
/// rather than a bad request — the failure mode this project keeps hitting.
|
||||||
|
#[test]
|
||||||
|
fn a_wall_that_clears_is_waited_out_and_one_that_does_not_is_not() {
|
||||||
|
use cm_llm::LlmError;
|
||||||
|
let api = |s: &str| LlmError::Api(s.to_string());
|
||||||
|
|
||||||
|
assert!(is_transient(&api(
|
||||||
|
"429 Too Many Requests: {\"type\":\"rate_limit_error\"}"
|
||||||
|
)));
|
||||||
|
assert!(is_transient(&api("529: overloaded_error")));
|
||||||
|
assert!(is_transient(&api("503 Service Unavailable")));
|
||||||
|
assert!(is_transient(&LlmError::Transport("connection reset".into())));
|
||||||
|
|
||||||
|
// The exact error that started this: it never clears by waiting, it
|
||||||
|
// clears by moving to the other credential — which is now done.
|
||||||
|
assert!(!is_transient(&api(
|
||||||
|
"400 Bad Request: Your credit balance is too low"
|
||||||
|
)));
|
||||||
|
assert!(!is_transient(&api("401 Unauthorized: invalid x-api-key")));
|
||||||
|
assert!(!is_transient(&api("404 Not Found: model not found")));
|
||||||
|
assert!(!is_transient(&LlmError::Wire("bad json".into())));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Nobody hand-rolls their own Anthropic HTTP call.
|
||||||
|
///
|
||||||
|
/// `phase_summarizer` did — its own `reqwest` POST to `api.anthropic.com`
|
||||||
|
/// with `x-api-key: $ANTHROPIC_API_KEY`. No audit of `.complete(` call
|
||||||
|
/// sites could ever have found it, and it was the last thing on this
|
||||||
|
/// deployment still billing an account with no credit: every phase summary
|
||||||
|
/// died with "credit balance is too low" while the phases themselves ran.
|
||||||
|
/// A call site is only routable if it goes through a provider, so walk the
|
||||||
|
/// whole crate rather than a hand-listed set of files.
|
||||||
|
#[test]
|
||||||
|
fn no_module_talks_to_anthropic_behind_the_providers_back() {
|
||||||
|
fn walk(dir: &std::path::Path, out: &mut Vec<std::path::PathBuf>) {
|
||||||
|
for entry in std::fs::read_dir(dir).expect("readable source dir") {
|
||||||
|
let path = entry.expect("readable entry").path();
|
||||||
|
if path.is_dir() {
|
||||||
|
walk(&path, out);
|
||||||
|
} else if path.extension().is_some_and(|e| e == "rs") {
|
||||||
|
out.push(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
|
||||||
|
let mut files = Vec::new();
|
||||||
|
walk(&root, &mut files);
|
||||||
|
assert!(files.len() > 20, "source walk found suspiciously few files");
|
||||||
|
|
||||||
|
for path in files {
|
||||||
|
// This module names the host in prose; it is the one that may.
|
||||||
|
if path.ends_with("subscription.rs") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let src = std::fs::read_to_string(&path).expect("readable source");
|
||||||
|
for needle in ["api.anthropic.com", "\"x-api-key\""] {
|
||||||
|
assert!(
|
||||||
|
!src.contains(needle),
|
||||||
|
"{} contains {needle} — build the request through cm_llm and \
|
||||||
|
route it via `subscription::complete_or`, so credential \
|
||||||
|
choice and the capacity fallback live in ONE place",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A model name may contain a colon, and "unregistered" must not mean that.
|
||||||
|
///
|
||||||
|
/// `resolve_provider` returns the spec unchanged when it does not recognise
|
||||||
|
/// the provider and the part after the FIRST colon when it does. The obvious
|
||||||
|
/// test — "does the model half still contain a colon" — reads the same and
|
||||||
|
/// is wrong the moment a model id has one. `ornith-fleet:9b` has one, and
|
||||||
|
/// the live preflight reported a provider the server had just registered as
|
||||||
|
/// UNREGISTERED. The identical bug was in `cross_provider_judge`, where it
|
||||||
|
/// would have refused a perfectly good independent judge.
|
||||||
|
#[test]
|
||||||
|
fn a_colon_in_the_model_name_is_not_a_missing_provider() {
|
||||||
|
// What `resolve_provider` returns in each case.
|
||||||
|
fn routed(spec: &str) -> &str {
|
||||||
|
spec.split_once(':').map(|(_, m)| m).unwrap_or(spec)
|
||||||
|
}
|
||||||
|
|
||||||
|
for spec in ["local:ornith-fleet:9b", "glm:glm-4.7", "kimi:kimi-k2.7-code"] {
|
||||||
|
assert_ne!(routed(spec), spec, "{spec} routed must not equal the whole spec");
|
||||||
|
}
|
||||||
|
// An unrecognised provider comes back WHOLE — the only true signal.
|
||||||
|
assert_eq!(routed("nosuch"), "nosuch");
|
||||||
|
// And the case that made the naive colon test look correct for so long.
|
||||||
|
assert!(routed("local:ornith-fleet:9b").contains(':'));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A throttled link is usable; an unregistered one is not.
|
||||||
|
///
|
||||||
|
/// The second is the dangerous one and the reason `preflight` checks
|
||||||
|
/// resolution separately from reachability. `resolve_provider` falls back to
|
||||||
|
/// the DEFAULT provider when it does not recognise a provider name, so a
|
||||||
|
/// typo in `kimi:` does not error — it quietly runs on Anthropic, and a
|
||||||
|
/// chain that reads as three accounts is really one. A reachability-only
|
||||||
|
/// probe would call that link green.
|
||||||
|
#[test]
|
||||||
|
fn only_a_link_that_could_never_answer_counts_as_unusable() {
|
||||||
|
assert!(LinkStatus::Answered.usable());
|
||||||
|
assert!(LinkStatus::Throttled("429 rate_limit".into()).usable());
|
||||||
|
|
||||||
|
assert!(!LinkStatus::Unregistered.usable());
|
||||||
|
assert!(!LinkStatus::Broken("401 invalid key".into()).usable());
|
||||||
|
|
||||||
|
// The labels must not read alike: "throttled" is a wait and
|
||||||
|
// "unregistered" is a config bug, and an operator acts differently on
|
||||||
|
// each.
|
||||||
|
assert!(LinkStatus::Throttled(String::new()).label().contains("configured"));
|
||||||
|
assert!(LinkStatus::Unregistered.label().contains("DEFAULT provider"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The chain never retries the capped model as its own fallback.
|
||||||
|
///
|
||||||
|
/// Without the filter, asking for haiku while haiku is capped would try
|
||||||
|
/// haiku, fail, and try haiku again — a chain that looks like resilience
|
||||||
|
/// and delivers none.
|
||||||
|
#[test]
|
||||||
|
fn the_chain_excludes_the_model_that_just_failed() {
|
||||||
|
// No env override in scope: this asserts the SHIPPED default.
|
||||||
|
let chain = fallback_chain("claude-opus-4-8");
|
||||||
|
assert_eq!(
|
||||||
|
chain,
|
||||||
|
vec![
|
||||||
|
"claude-sonnet-4-6",
|
||||||
|
"claude-haiku-4-5-20251001",
|
||||||
|
"kimi:kimi-k2.7-code",
|
||||||
|
"glm:glm-4.7",
|
||||||
|
"local:ornith-fleet:9b",
|
||||||
|
]
|
||||||
|
);
|
||||||
|
// Three providers behind five links. A chain that steps down three
|
||||||
|
// Anthropic tiers and stops is a tier ladder, not a fallback chain: one
|
||||||
|
// account being unreachable would end it.
|
||||||
|
let families: std::collections::BTreeSet<_> = chain
|
||||||
|
.iter()
|
||||||
|
.map(|m| m.split_once(':').map(|(p, _)| p).unwrap_or("anthropic"))
|
||||||
|
.collect();
|
||||||
|
assert!(
|
||||||
|
families.len() >= 3,
|
||||||
|
"the chain must span more than one account, got {families:?}"
|
||||||
|
);
|
||||||
|
// The last link must survive `resolve_provider`'s split, which takes the
|
||||||
|
// FIRST colon only — `local:ornith-fleet:9b` is provider `local`, model
|
||||||
|
// `ornith-fleet:9b`, and a split on the last colon would ask for a
|
||||||
|
// provider named `local:ornith-fleet`.
|
||||||
|
let last = chain.last().unwrap();
|
||||||
|
let (provider, model) = last.split_once(':').expect("a provider-qualified spec");
|
||||||
|
assert_eq!(provider, "local");
|
||||||
|
assert_eq!(model, "ornith-fleet:9b");
|
||||||
|
assert!(!fallback_chain("claude-haiku-4-5-20251001")
|
||||||
|
.iter()
|
||||||
|
.any(|m| m == "claude-haiku-4-5-20251001"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The chain is walked for "no capacity" and NOT for "bad request".
|
||||||
|
///
|
||||||
|
/// Walking it on a malformed prompt would ask three models the same bad
|
||||||
|
/// question and report the third one's confusion as the answer, burning
|
||||||
|
/// the two credentials that still work in order to hide the real error.
|
||||||
|
#[test]
|
||||||
|
fn only_a_capacity_failure_steps_down_the_chain() {
|
||||||
|
assert!(is_capacity_failure(
|
||||||
|
"subscription call: provider returned an error: 429 Too Many Requests"
|
||||||
|
));
|
||||||
|
assert!(is_capacity_failure("rate_limit_error"));
|
||||||
|
// The metered key's wall counts too — same meaning, different wording.
|
||||||
|
assert!(is_capacity_failure(
|
||||||
|
"400: Your credit balance is too low to access the Anthropic API"
|
||||||
|
));
|
||||||
|
|
||||||
|
assert!(!is_capacity_failure("400: messages.0: text content is empty"));
|
||||||
|
assert!(!is_capacity_failure("401: invalid x-api-key"));
|
||||||
|
assert!(!is_capacity_failure("404: model not found"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An operator's explicit provider choice is never hijacked.
|
||||||
|
///
|
||||||
|
/// The swarm worker model is a configured `name:model` spec. Routing that
|
||||||
|
/// onto the subscription would run someone's chosen Kimi or GLM model on
|
||||||
|
/// Anthropic and report success — the same silent-substitution bug as the
|
||||||
|
/// metered key, aimed the other way.
|
||||||
|
#[test]
|
||||||
|
fn a_provider_qualified_spec_is_left_alone() {
|
||||||
|
assert!(is_bare_model_name("claude-opus-4-8"));
|
||||||
|
assert!(is_bare_model_name("claude-haiku-4-5-20251001"));
|
||||||
|
assert!(!is_bare_model_name("kimi:kimi-k2.6"));
|
||||||
|
assert!(!is_bare_model_name("glm:glm-4.7"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A metered key in the OAuth slot must be REFUSED, not used.
|
||||||
|
///
|
||||||
|
/// Accepting it would authenticate, work, and bill the pay-as-you-go account
|
||||||
|
/// — the exact bill this module exists to stop, discovered weeks later when
|
||||||
|
/// it runs out mid-mission.
|
||||||
|
#[test]
|
||||||
|
fn only_a_setup_token_counts_as_the_subscription() {
|
||||||
|
assert!(is_subscription_token("sk-ant-oat01-abc"));
|
||||||
|
assert!(!is_subscription_token("sk-ant-api03-abc"));
|
||||||
|
assert!(!is_subscription_token(""));
|
||||||
|
assert!(!is_subscription_token("oat-but-not-anthropic"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -122,9 +122,11 @@ pub async fn run_swarm_job(
|
|||||||
let worker_model = resolve_worker_model(&job.worker_model);
|
let worker_model = resolve_worker_model(&job.worker_model);
|
||||||
|
|
||||||
// 1) PLAN — Opus decomposes the goal into worker tasks.
|
// 1) PLAN — Opus decomposes the goal into worker tasks.
|
||||||
|
// This record is written BEFORE the call, so it cannot name the model that
|
||||||
|
// answers. The record after the call can, and does.
|
||||||
records.push(step(
|
records.push(step(
|
||||||
"planner",
|
"planner",
|
||||||
"planner:opus",
|
"planner",
|
||||||
StepPhase::Plan,
|
StepPhase::Plan,
|
||||||
format!("Planning tasks for: {goal}"),
|
format!("Planning tasks for: {goal}"),
|
||||||
));
|
));
|
||||||
@@ -137,8 +139,17 @@ pub async fn run_swarm_job(
|
|||||||
"GOAL:\n{goal}\n\nCHECKLIST each task's output must satisfy:\n{}{want}",
|
"GOAL:\n{goal}\n\nCHECKLIST each task's output must satisfy:\n{}{want}",
|
||||||
checklist_lines(&checklist)
|
checklist_lines(&checklist)
|
||||||
);
|
);
|
||||||
let plan_raw = runtime
|
// The recorded role says which model ANSWERED. When opus is capped the
|
||||||
.complete(PLAN_SYSTEM, &plan_user, "claude-opus-4-8", 4000, false)
|
// chain steps down, and a step labelled "planner:opus" that GLM wrote is a
|
||||||
|
// lie in the one place an operator looks to explain a bad decomposition.
|
||||||
|
let (plan_raw, plan_model) = crate::subscription::complete_with_fallback(
|
||||||
|
runtime,
|
||||||
|
PLAN_SYSTEM,
|
||||||
|
&plan_user,
|
||||||
|
"claude-opus-4-8",
|
||||||
|
4000,
|
||||||
|
false,
|
||||||
|
)
|
||||||
.await?;
|
.await?;
|
||||||
let tasks: Vec<String> = extract_json(&plan_raw)
|
let tasks: Vec<String> = extract_json(&plan_raw)
|
||||||
.and_then(|v| {
|
.and_then(|v| {
|
||||||
@@ -154,7 +165,7 @@ pub async fn run_swarm_job(
|
|||||||
}
|
}
|
||||||
records.push(step(
|
records.push(step(
|
||||||
"planner",
|
"planner",
|
||||||
"planner:opus",
|
format!("planner:{plan_model}"),
|
||||||
StepPhase::Plan,
|
StepPhase::Plan,
|
||||||
format!(
|
format!(
|
||||||
"Decomposed into {} tasks. Workers: {worker_model}. Verifier: claude-opus-4-8.",
|
"Decomposed into {} tasks. Workers: {worker_model}. Verifier: claude-opus-4-8.",
|
||||||
@@ -177,8 +188,10 @@ pub async fn run_swarm_job(
|
|||||||
let mut still: Vec<(usize, String)> = Vec::new();
|
let mut still: Vec<(usize, String)> = Vec::new();
|
||||||
let mut rejected = 0usize;
|
let mut rejected = 0usize;
|
||||||
for (idx, task) in pending.iter() {
|
for (idx, task) in pending.iter() {
|
||||||
let out = runtime
|
// `worker_model` may be a `name:model` spec the operator chose;
|
||||||
.complete(&wsys, task, &worker_model, 4000, true)
|
// `complete_or` passes those straight through untouched.
|
||||||
|
let out =
|
||||||
|
crate::subscription::complete_or(runtime, &wsys, task, &worker_model, 4000, true)
|
||||||
.await
|
.await
|
||||||
.unwrap_or_else(|e| format!("worker error: {e}"));
|
.unwrap_or_else(|e| format!("worker error: {e}"));
|
||||||
records.push(step(
|
records.push(step(
|
||||||
@@ -190,9 +203,16 @@ pub async fn run_swarm_job(
|
|||||||
ckpt(pool, id, &records, &totals).await;
|
ckpt(pool, id, &records, &totals).await;
|
||||||
|
|
||||||
let vuser = format!("TASK:\n{task}\n\nWORKER OUTPUT:\n{out}");
|
let vuser = format!("TASK:\n{task}\n\nWORKER OUTPUT:\n{out}");
|
||||||
let v_raw = runtime
|
let v_raw = crate::subscription::complete_with_fallback(
|
||||||
.complete(&vsys, &vuser, "claude-opus-4-8", 1200, true)
|
runtime,
|
||||||
|
&vsys,
|
||||||
|
&vuser,
|
||||||
|
"claude-opus-4-8",
|
||||||
|
1200,
|
||||||
|
true,
|
||||||
|
)
|
||||||
.await
|
.await
|
||||||
|
.map(|(text, _)| text)
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
let v = extract_json(&v_raw);
|
let v = extract_json(&v_raw);
|
||||||
let passed = v
|
let passed = v
|
||||||
|
|||||||
@@ -69,6 +69,11 @@ struct TemplateRoleFile {
|
|||||||
skills: Vec<String>,
|
skills: Vec<String>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
brain_seed: Option<String>,
|
brain_seed: Option<String>,
|
||||||
|
/// Which model this role's claw runs on. Omitted means the mint's default,
|
||||||
|
/// which is what every authored template does today — so adding the field
|
||||||
|
/// changes nothing until a template uses it.
|
||||||
|
#[serde(default)]
|
||||||
|
model: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn templates_dir() -> PathBuf {
|
fn templates_dir() -> PathBuf {
|
||||||
@@ -137,6 +142,7 @@ async fn load_one(pool: &PgPool, path: &std::path::Path) -> Result<String, Strin
|
|||||||
system_prompt: &r.system_prompt,
|
system_prompt: &r.system_prompt,
|
||||||
skills: r.skills.clone(),
|
skills: r.skills.clone(),
|
||||||
brain_seed: r.brain_seed.as_deref(),
|
brain_seed: r.brain_seed.as_deref(),
|
||||||
|
model: r.model.as_deref(),
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
|
|||||||
@@ -24,15 +24,22 @@ use tokio::sync::Mutex;
|
|||||||
use tokio_tungstenite::connect_async;
|
use tokio_tungstenite::connect_async;
|
||||||
use tokio_tungstenite::tungstenite::Message;
|
use tokio_tungstenite::tungstenite::Message;
|
||||||
|
|
||||||
/// Overall wall-clock budget for draining one turn's event stream. Must
|
/// Overall wall-clock budget for draining one turn's event stream.
|
||||||
/// exceed the daemon's own claude_cli provider timeout (600s on gw-04
|
///
|
||||||
/// via ZEROCLAW_providers__models__claude_cli__default__timeout_ms) —
|
/// A turn is an agent LOOP, not one model call. Each call inside it is bounded
|
||||||
/// otherwise the executor kills the ws before the daemon can reply and
|
/// separately by the daemon — `claude_cli`'s `timeout_secs`, 600s on gw-04 —
|
||||||
/// we see a phantom "turn timed out" while the daemon still logs a
|
/// so this has to cover however many calls the loop makes, not one of them.
|
||||||
/// successful llm response coming back. 700s gives 100s of headroom so
|
///
|
||||||
/// a daemon that just barely made it under its own limit doesn't lose
|
/// It was 700s, which is 100s more than a single call may take. MEASURED: a
|
||||||
/// its answer here.
|
/// healthy research turn is ~157s, but a throttled one blew the budget with one
|
||||||
const TURN_TIMEOUT: Duration = Duration::from_secs(700);
|
/// slow call plus a second, and the executor killed it mid-flight after 11m43s
|
||||||
|
/// with no error from the daemon — because nothing had failed yet. All the
|
||||||
|
/// operator got was "turn timed out".
|
||||||
|
///
|
||||||
|
/// An hour matches the phase's own budget. A genuinely stuck CALL is still
|
||||||
|
/// caught at 600s by the daemon and surfaces as a real error; this only stops
|
||||||
|
/// us killing turns that are working, slowly.
|
||||||
|
const TURN_TIMEOUT: Duration = Duration::from_secs(3600);
|
||||||
|
|
||||||
/// Drives ZeroClaw role-agents (in one container) to execute topology turns.
|
/// Drives ZeroClaw role-agents (in one container) to execute topology turns.
|
||||||
pub struct ZeroClawDriveExecutor {
|
pub struct ZeroClawDriveExecutor {
|
||||||
@@ -47,6 +54,46 @@ pub struct ZeroClawDriveExecutor {
|
|||||||
/// Bearer token, paired lazily and reused across turns.
|
/// Bearer token, paired lazily and reused across turns.
|
||||||
token: Arc<Mutex<Option<String>>>,
|
token: Arc<Mutex<Option<String>>>,
|
||||||
http: reqwest::Client,
|
http: reqwest::Client,
|
||||||
|
/// Where this executor's turns record what they did. `None` on every path
|
||||||
|
/// that is not a mission phase (the governor, the door, the evaluator) —
|
||||||
|
/// those turns belong to no phase and have nothing to attribute to.
|
||||||
|
tap: Option<Arc<MissionTap>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Where a turn's tool activity is written, and what it belongs to.
|
||||||
|
///
|
||||||
|
/// Carried on the executor rather than passed per turn because `TurnRequest`
|
||||||
|
/// is the shared orchestrator contract: threading a mission id through it would
|
||||||
|
/// put mission concepts into every tier that has no missions.
|
||||||
|
pub struct MissionTap {
|
||||||
|
pub pool: sqlx::PgPool,
|
||||||
|
pub mission_id: uuid::Uuid,
|
||||||
|
pub phase_id: Option<uuid::Uuid>,
|
||||||
|
pub run_id: Option<uuid::Uuid>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One tool call, as the frame stream reported it.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct ToolCall {
|
||||||
|
pub tool: String,
|
||||||
|
/// The path the tool's **arguments** named, if any. Never extracted from a
|
||||||
|
/// prose summary — see [`crate::mission_events::tool_path`].
|
||||||
|
pub path: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What one turn's frames said about the work, beside its text.
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||||
|
pub struct ToolTrace {
|
||||||
|
pub calls: Vec<ToolCall>,
|
||||||
|
/// Frame `type` values this drain did not recognise, counted.
|
||||||
|
///
|
||||||
|
/// Shipped in the same change as the tap on purpose: the frame name
|
||||||
|
/// `tool_call` is taken from a comment in this file, not from a captured
|
||||||
|
/// frame. If the runtime calls it something else, the tap records nothing
|
||||||
|
/// and nothing anywhere errors — the World simply stays as sparse as it was
|
||||||
|
/// before. This histogram is how one gw-04 run names the real frame instead
|
||||||
|
/// of a bisect.
|
||||||
|
pub unmatched: std::collections::BTreeMap<String, u32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ZeroClawDriveExecutor {
|
impl ZeroClawDriveExecutor {
|
||||||
@@ -64,9 +111,17 @@ impl ZeroClawDriveExecutor {
|
|||||||
default_alias,
|
default_alias,
|
||||||
token: Arc::new(Mutex::new(None)),
|
token: Arc::new(Mutex::new(None)),
|
||||||
http: reqwest::Client::new(),
|
http: reqwest::Client::new(),
|
||||||
|
tap: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Attach the mission this executor's turns belong to, so their tool calls
|
||||||
|
/// are recorded. Without it the executor behaves exactly as it did.
|
||||||
|
pub fn with_tap(mut self, tap: MissionTap) -> Self {
|
||||||
|
self.tap = Some(Arc::new(tap));
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
/// Build from the environment:
|
/// Build from the environment:
|
||||||
/// - `ZEROCLAW_GATEWAY_URL` (required) e.g. `http://127.0.0.1:42617`
|
/// - `ZEROCLAW_GATEWAY_URL` (required) e.g. `http://127.0.0.1:42617`
|
||||||
/// - `ZEROCLAW_TOKEN` (preferred) a durable bearer token — pair once
|
/// - `ZEROCLAW_TOKEN` (preferred) a durable bearer token — pair once
|
||||||
@@ -196,6 +251,19 @@ impl ZeroClawDriveExecutor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn drive(&self, alias: &str, prompt: &str) -> Result<TurnOutcome, OrchestratorError> {
|
pub async fn drive(&self, alias: &str, prompt: &str) -> Result<TurnOutcome, OrchestratorError> {
|
||||||
|
self.drive_traced(alias, prompt).await.map(|(o, _)| o)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// [`Self::drive`], also returning what the turn's frames said it did.
|
||||||
|
///
|
||||||
|
/// Exists so the tool tap is testable at all: `drive` discards the trace
|
||||||
|
/// after recording it, and a tap whose extraction is never asserted is
|
||||||
|
/// exactly the kind of code that silently records nothing.
|
||||||
|
pub(crate) async fn drive_traced(
|
||||||
|
&self,
|
||||||
|
alias: &str,
|
||||||
|
prompt: &str,
|
||||||
|
) -> Result<(TurnOutcome, ToolTrace), OrchestratorError> {
|
||||||
let token = self.ensure_paired().await?;
|
let token = self.ensure_paired().await?;
|
||||||
let ws_base = if let Some(rest) = self.gateway_url.strip_prefix("https") {
|
let ws_base = if let Some(rest) = self.gateway_url.strip_prefix("https") {
|
||||||
format!("wss{rest}")
|
format!("wss{rest}")
|
||||||
@@ -219,11 +287,102 @@ impl ZeroClawDriveExecutor {
|
|||||||
.await
|
.await
|
||||||
.map_err(|e| OrchestratorError::Executor(format!("ws send failed: {e}")))?;
|
.map_err(|e| OrchestratorError::Executor(format!("ws send failed: {e}")))?;
|
||||||
|
|
||||||
let outcome = tokio::time::timeout(TURN_TIMEOUT, Self::drain(&mut ws))
|
let (outcome, trace) = match tokio::time::timeout(TURN_TIMEOUT, Self::drain(&mut ws)).await
|
||||||
.await
|
{
|
||||||
.map_err(|_| OrchestratorError::Executor("turn timed out".into()))??;
|
Ok(res) => res?,
|
||||||
|
Err(_) => {
|
||||||
|
// "turn timed out" on its own is unactionable, and the one place
|
||||||
|
// the reason lives — the per-mission runtime container — is torn
|
||||||
|
// down after the phase, taking its log with it. Read the tail
|
||||||
|
// while it still exists.
|
||||||
|
//
|
||||||
|
// MEASURED: a research phase timed out at exactly 700s having
|
||||||
|
// produced zero steps and zero output, and the container was
|
||||||
|
// already gone by the time anyone looked. All that survived was
|
||||||
|
// the string.
|
||||||
|
let container = self.container_name();
|
||||||
|
let tail = match &container {
|
||||||
|
Some(c) => crate::container_exec::tail_logs(c, 40).await,
|
||||||
|
None => "(could not derive the container name from the gateway url)".into(),
|
||||||
|
};
|
||||||
|
return Err(OrchestratorError::Executor(format!(
|
||||||
|
"turn timed out after {}s driving agent {alias} on {} — the agent \
|
||||||
|
never finished a turn. Last lines from {}:\n{tail}",
|
||||||
|
TURN_TIMEOUT.as_secs(),
|
||||||
|
self.gateway_url,
|
||||||
|
container.as_deref().unwrap_or("its runtime container"),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
};
|
||||||
let _ = ws.close(None).await;
|
let _ = ws.close(None).await;
|
||||||
Ok(outcome)
|
self.record_trace(alias, &trace).await;
|
||||||
|
Ok((outcome, trace))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Persist what this turn's frames said the agent did.
|
||||||
|
///
|
||||||
|
/// Best-effort and after the fact: a telemetry write must not be able to
|
||||||
|
/// fail a turn that already succeeded.
|
||||||
|
async fn record_trace(&self, alias: &str, trace: &ToolTrace) {
|
||||||
|
if !trace.unmatched.is_empty() {
|
||||||
|
// Logged whether or not a tap is attached — the point is to learn
|
||||||
|
// the real frame names, and the paths with no tap see the same
|
||||||
|
// stream.
|
||||||
|
eprintln!(
|
||||||
|
"topology_exec: unmatched frame types this turn ({alias}): {:?}",
|
||||||
|
trace.unmatched
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let Some(tap) = self.tap.as_ref() else { return };
|
||||||
|
if trace.calls.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let agent_id = crate::runtime_provision::claw_from_alias(alias);
|
||||||
|
let event = |kind: &str, target: String, detail: serde_json::Value| {
|
||||||
|
crate::mission_events::MissionEvent {
|
||||||
|
mission_id: tap.mission_id,
|
||||||
|
phase_id: tap.phase_id,
|
||||||
|
run_id: tap.run_id,
|
||||||
|
agent_id,
|
||||||
|
kind: kind.to_string(),
|
||||||
|
target: Some(target),
|
||||||
|
detail,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let mut events = Vec::new();
|
||||||
|
for call in &trace.calls {
|
||||||
|
events.push(event(
|
||||||
|
crate::mission_events::TOOL_CALL,
|
||||||
|
call.tool.clone(),
|
||||||
|
serde_json::Value::Null,
|
||||||
|
));
|
||||||
|
// A file touch is a SECOND event, not a replacement: the tool call
|
||||||
|
// happened whether or not we could name a path in its arguments,
|
||||||
|
// and collapsing the two would make every unparseable tool call
|
||||||
|
// disappear from the record entirely.
|
||||||
|
if let Some(path) = &call.path {
|
||||||
|
events.push(event(
|
||||||
|
crate::mission_events::FILE_TOUCH,
|
||||||
|
crate::mission_events::repo_relative(path, GUEST_ROOTS),
|
||||||
|
serde_json::json!({ "tool": call.tool }),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
crate::mission_events::record_all(&tap.pool, events).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The runtime container behind this executor, derived from its gateway URL
|
||||||
|
/// (`http://cm-runtime-mission-<hex>:42617`). Used only to fetch a log tail
|
||||||
|
/// for an error message, so an unparseable URL is `None` rather than a
|
||||||
|
/// failure.
|
||||||
|
fn container_name(&self) -> Option<String> {
|
||||||
|
let rest = self
|
||||||
|
.gateway_url
|
||||||
|
.split("://")
|
||||||
|
.nth(1)
|
||||||
|
.unwrap_or(&self.gateway_url);
|
||||||
|
let host = rest.split('/').next()?.split(':').next()?;
|
||||||
|
(!host.is_empty()).then(|| host.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Use a runtime agent as a governance judge: drive `alias` with the judge
|
/// Use a runtime agent as a governance judge: drive `alias` with the judge
|
||||||
@@ -286,7 +445,12 @@ impl ZeroClawDriveExecutor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Read frames until a terminal (`done`/`error`/`approval_request`) event.
|
/// Read frames until a terminal (`done`/`error`/`approval_request`) event.
|
||||||
async fn drain<S>(ws: &mut S) -> Result<TurnOutcome, OrchestratorError>
|
///
|
||||||
|
/// Returns the turn's outcome AND what its frames said the agent did. The
|
||||||
|
/// trace is separate from [`TurnOutcome`] deliberately: that type is the
|
||||||
|
/// shared orchestrator contract used by every tier, and tool telemetry is a
|
||||||
|
/// mission concern.
|
||||||
|
async fn drain<S>(ws: &mut S) -> Result<(TurnOutcome, ToolTrace), OrchestratorError>
|
||||||
where
|
where
|
||||||
S: StreamExt<Item = Result<Message, tokio_tungstenite::tungstenite::Error>>
|
S: StreamExt<Item = Result<Message, tokio_tungstenite::tungstenite::Error>>
|
||||||
+ SinkExt<Message>
|
+ SinkExt<Message>
|
||||||
@@ -295,6 +459,7 @@ impl ZeroClawDriveExecutor {
|
|||||||
let mut output = String::new();
|
let mut output = String::new();
|
||||||
let mut tokens: u64 = 0;
|
let mut tokens: u64 = 0;
|
||||||
let mut gated: Vec<GatedAction> = Vec::new();
|
let mut gated: Vec<GatedAction> = Vec::new();
|
||||||
|
let mut trace = ToolTrace::default();
|
||||||
|
|
||||||
while let Some(frame) = ws.next().await {
|
while let Some(frame) = ws.next().await {
|
||||||
let msg = frame.map_err(|e| OrchestratorError::Executor(format!("ws recv: {e}")))?;
|
let msg = frame.map_err(|e| OrchestratorError::Executor(format!("ws recv: {e}")))?;
|
||||||
@@ -340,8 +505,39 @@ impl ZeroClawDriveExecutor {
|
|||||||
"aborted" => {
|
"aborted" => {
|
||||||
return Err(OrchestratorError::Executor("turn aborted".into()));
|
return Err(OrchestratorError::Executor("turn aborted".into()));
|
||||||
}
|
}
|
||||||
// session_start, thinking, tool_call, tool_result, …
|
// The action channel. `arguments` is read as JSON and
|
||||||
_ => {}
|
// nothing else is: the frame also carries a prose
|
||||||
|
// summary, and a path pulled out of THAT would be right
|
||||||
|
// often enough to be believed and wrong often enough to
|
||||||
|
// put files on the map that nobody edited.
|
||||||
|
"tool_call" => {
|
||||||
|
let tool = v
|
||||||
|
.get("tool")
|
||||||
|
.or_else(|| v.get("name"))
|
||||||
|
.and_then(|t| t.as_str())
|
||||||
|
.unwrap_or("")
|
||||||
|
.trim()
|
||||||
|
.to_string();
|
||||||
|
if !tool.is_empty() {
|
||||||
|
let args = v
|
||||||
|
.get("arguments")
|
||||||
|
.or_else(|| v.get("input"))
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or(serde_json::Value::Null);
|
||||||
|
trace.calls.push(ToolCall {
|
||||||
|
path: crate::mission_events::tool_path(&args),
|
||||||
|
tool,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// session_start, thinking, tool_result, …
|
||||||
|
other => {
|
||||||
|
// Counted, not ignored. See `ToolTrace::unmatched`:
|
||||||
|
// the frame name above is unverified, and a tap
|
||||||
|
// that matches nothing looks exactly like a mission
|
||||||
|
// that used no tools.
|
||||||
|
*trace.unmatched.entry(other.to_string()).or_insert(0) += 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Message::Ping(p) => {
|
Message::Ping(p) => {
|
||||||
@@ -352,14 +548,21 @@ impl ZeroClawDriveExecutor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(TurnOutcome {
|
Ok((
|
||||||
|
TurnOutcome {
|
||||||
output: output.trim().to_string(),
|
output: output.trim().to_string(),
|
||||||
tokens,
|
tokens,
|
||||||
gated,
|
gated,
|
||||||
})
|
},
|
||||||
|
trace,
|
||||||
|
))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Guest workspace roots, stripped so a tool's absolute path becomes the
|
||||||
|
/// repo-relative one a person recognises.
|
||||||
|
const GUEST_ROOTS: &[&str] = &["/mission/repo", "/workspace", "/repo"];
|
||||||
|
|
||||||
impl TurnExecutor for ZeroClawDriveExecutor {
|
impl TurnExecutor for ZeroClawDriveExecutor {
|
||||||
async fn run_turn(&self, req: TurnRequest) -> Result<TurnOutcome, OrchestratorError> {
|
async fn run_turn(&self, req: TurnRequest) -> Result<TurnOutcome, OrchestratorError> {
|
||||||
// An explicit per-node agent (graph `node.attrs["agent"]`) wins, so one
|
// An explicit per-node agent (graph `node.attrs["agent"]`) wins, so one
|
||||||
@@ -406,6 +609,32 @@ fn parse_agent_map(s: &str) -> HashMap<String, String> {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
/// The container name comes out of the gateway URL, or nothing does.
|
||||||
|
///
|
||||||
|
/// This is only used to fetch a log tail for a failure message, so a URL
|
||||||
|
/// shape it does not recognise must degrade to "no log" rather than to a
|
||||||
|
/// second error on top of the first.
|
||||||
|
#[test]
|
||||||
|
fn the_container_name_is_derived_or_absent_never_wrong() {
|
||||||
|
let ex = |url: &str| ZeroClawDriveExecutor::new(
|
||||||
|
url.to_string(),
|
||||||
|
String::new(),
|
||||||
|
std::collections::HashMap::new(),
|
||||||
|
"scout".into(),
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
ex("http://cm-runtime-mission-019fec2d596f:42617").container_name().as_deref(),
|
||||||
|
Some("cm-runtime-mission-019fec2d596f")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
ex("https://host.example:8443/base").container_name().as_deref(),
|
||||||
|
Some("host.example")
|
||||||
|
);
|
||||||
|
// No scheme is still a host.
|
||||||
|
assert_eq!(ex("clawmates-runtime:42617").container_name().as_deref(), Some("clawmates-runtime"));
|
||||||
|
assert_eq!(ex("").container_name(), None);
|
||||||
|
}
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
use axum::extract::ws::{Message as AxMsg, WebSocket, WebSocketUpgrade};
|
use axum::extract::ws::{Message as AxMsg, WebSocket, WebSocketUpgrade};
|
||||||
use axum::response::Response;
|
use axum::response::Response;
|
||||||
@@ -435,6 +664,30 @@ mod tests {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A stream carrying tool calls and one frame type we do not know.
|
||||||
|
async fn tool_ws(ws: WebSocketUpgrade) -> Response {
|
||||||
|
ws.on_upgrade(|mut socket: WebSocket| async move {
|
||||||
|
let _ = socket.recv().await;
|
||||||
|
for f in [
|
||||||
|
json!({"type": "session_start"}),
|
||||||
|
json!({"type": "tool_call", "tool": "Read",
|
||||||
|
"arguments": {"file_path": "/mission/repo/src/a.rs"}}),
|
||||||
|
// A tool whose arguments name no path at all.
|
||||||
|
json!({"type": "tool_call", "tool": "Bash",
|
||||||
|
"arguments": {"command": "cargo test"}}),
|
||||||
|
// Prose that MENTIONS a path. It must not become a file touch.
|
||||||
|
json!({"type": "tool_call", "tool": "Grep",
|
||||||
|
"arguments_summary": "searching src/main.rs",
|
||||||
|
"arguments": {"pattern": "fn main"}}),
|
||||||
|
json!({"type": "a_frame_we_have_never_seen"}),
|
||||||
|
json!({"type": "a_frame_we_have_never_seen"}),
|
||||||
|
json!({"type": "done", "input_tokens": 1, "output_tokens": 1}),
|
||||||
|
] {
|
||||||
|
let _ = socket.send(AxMsg::Text(f.to_string().into())).await;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
async fn approval_ws(ws: WebSocketUpgrade) -> Response {
|
async fn approval_ws(ws: WebSocketUpgrade) -> Response {
|
||||||
ws.on_upgrade(|mut socket: WebSocket| async move {
|
ws.on_upgrade(|mut socket: WebSocket| async move {
|
||||||
let _ = socket.recv().await;
|
let _ = socket.recv().await;
|
||||||
@@ -499,6 +752,40 @@ mod tests {
|
|||||||
assert!(out.gated.is_empty());
|
assert!(out.gated.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Tool detail comes from arguments, and unknown frames are counted.
|
||||||
|
///
|
||||||
|
/// The two halves are one test because they are one risk. The frame type
|
||||||
|
/// `tool_call` is taken from a comment in this file, not from a captured
|
||||||
|
/// frame — so if it is wrong, the tap records nothing, the World stays as
|
||||||
|
/// sparse as it was, and NOTHING errors. The histogram is what turns that
|
||||||
|
/// into a log line naming the real frame.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn tool_frames_give_up_their_arguments_and_unknown_frames_are_counted() {
|
||||||
|
let router = Router::new()
|
||||||
|
.route("/pair", post(pair))
|
||||||
|
.route("/ws/chat", get(tool_ws));
|
||||||
|
let base = serve(router).await;
|
||||||
|
let exec = ZeroClawDriveExecutor::new(base, "code".into(), HashMap::new(), "scout".into());
|
||||||
|
|
||||||
|
let (_out, trace) = exec.drive_traced("scout", "go").await.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
trace.calls,
|
||||||
|
vec![
|
||||||
|
ToolCall { tool: "Read".into(), path: Some("/mission/repo/src/a.rs".into()) },
|
||||||
|
ToolCall { tool: "Bash".into(), path: None },
|
||||||
|
// `arguments_summary` said "src/main.rs". It is prose, so it is
|
||||||
|
// not a file touch — a path scraped from a sentence would put
|
||||||
|
// files on the map that no agent opened.
|
||||||
|
ToolCall { tool: "Grep".into(), path: None },
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert_eq!(trace.unmatched.get("a_frame_we_have_never_seen"), Some(&2));
|
||||||
|
assert_eq!(trace.unmatched.get("session_start"), Some(&1));
|
||||||
|
// `done` terminates the drain and is not an unmatched frame.
|
||||||
|
assert!(!trace.unmatched.contains_key("done"), "{:?}", trace.unmatched);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn approval_request_is_recorded_as_blocked() {
|
async fn approval_request_is_recorded_as_blocked() {
|
||||||
let router = Router::new()
|
let router = Router::new()
|
||||||
|
|||||||
@@ -33,7 +33,12 @@ const REAP_STUCK_AFTER_SECS: i64 = 15 * 60;
|
|||||||
|
|
||||||
/// Spawn the durable topology job worker. Polls for queued jobs every `poll`
|
/// Spawn the durable topology job worker. Polls for queued jobs every `poll`
|
||||||
/// interval; runs each to completion (or failure), checkpointing per step.
|
/// interval; runs each to completion (or failure), checkpointing per step.
|
||||||
pub fn spawn(pool: PgPool, runtime: cm_runtime::Runtime, poll: Duration) {
|
pub fn spawn(
|
||||||
|
pool: PgPool,
|
||||||
|
runtime: cm_runtime::Runtime,
|
||||||
|
hub: Arc<crate::fleet::NodeHub>,
|
||||||
|
poll: Duration,
|
||||||
|
) {
|
||||||
// Fire the stuck-container reaper on its own cadence — checking
|
// Fire the stuck-container reaper on its own cadence — checking
|
||||||
// once a minute is plenty and keeps this off the hot claim loop.
|
// once a minute is plenty and keeps this off the hot claim loop.
|
||||||
let reaper_pool = pool.clone();
|
let reaper_pool = pool.clone();
|
||||||
@@ -55,7 +60,7 @@ pub fn spawn(pool: PgPool, runtime: cm_runtime::Runtime, poll: Duration) {
|
|||||||
eprintln!("topology_worker: requeue_stale failed: {e}");
|
eprintln!("topology_worker: requeue_stale failed: {e}");
|
||||||
}
|
}
|
||||||
match cm_db::repo::topology_runs::claim_next_queued(&pool).await {
|
match cm_db::repo::topology_runs::claim_next_queued(&pool).await {
|
||||||
Ok(Some(job)) => run_job(&pool, &runtime, job).await,
|
Ok(Some(job)) => run_job(&pool, &runtime, &hub, job).await,
|
||||||
Ok(None) => tokio::time::sleep(poll).await,
|
Ok(None) => tokio::time::sleep(poll).await,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("topology_worker: claim failed: {e}");
|
eprintln!("topology_worker: claim failed: {e}");
|
||||||
@@ -80,10 +85,25 @@ async fn reap_stuck_runs(pool: &PgPool) -> Result<(), sqlx::Error> {
|
|||||||
FROM topology_runs
|
FROM topology_runs
|
||||||
WHERE status = 'running'
|
WHERE status = 'running'
|
||||||
AND mission_id IS NOT NULL
|
AND mission_id IS NOT NULL
|
||||||
|
-- Only jobs this worker drives. mission_id IS NOT NULL used to mean
|
||||||
|
-- the same thing as orchestrator-driven, and the microvm and session
|
||||||
|
-- tiers broke that: their checkpoint is NULL for life BY DESIGN, so the
|
||||||
|
-- zero-step-records test below is true of a perfectly healthy run.
|
||||||
|
AND tier = ANY($2)
|
||||||
AND created_at < now() - make_interval(secs => $1::float)
|
AND created_at < now() - make_interval(secs => $1::float)
|
||||||
AND coalesce(jsonb_array_length(coalesce(checkpoint->'records', '[]'::jsonb)), 0) = 0",
|
AND coalesce(jsonb_array_length(coalesce(checkpoint->'records', '[]'::jsonb)), 0) = 0",
|
||||||
)
|
)
|
||||||
.bind(REAP_STUCK_AFTER_SECS as f64)
|
.bind(REAP_STUCK_AFTER_SECS as f64)
|
||||||
|
.bind(
|
||||||
|
// REAPABLE, not worker-driven: `microvm_graph` is driven by this worker
|
||||||
|
// and must NOT be reaped — one of its steps is a whole agent session in a
|
||||||
|
// VM, so "no step records in 15 minutes" describes a healthy composed run
|
||||||
|
// as readily as a wedged one.
|
||||||
|
cm_db::repo::topology_runs::REAPABLE_TIERS
|
||||||
|
.iter()
|
||||||
|
.map(|s| (*s).to_string())
|
||||||
|
.collect::<Vec<_>>(),
|
||||||
|
)
|
||||||
.fetch_all(pool)
|
.fetch_all(pool)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
@@ -108,6 +128,7 @@ async fn reap_stuck_runs(pool: &PgPool) -> Result<(), sqlx::Error> {
|
|||||||
async fn run_job(
|
async fn run_job(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
runtime: &cm_runtime::Runtime,
|
runtime: &cm_runtime::Runtime,
|
||||||
|
hub: &Arc<crate::fleet::NodeHub>,
|
||||||
job: cm_db::repo::topology_runs::ClaimedTopologyRun,
|
job: cm_db::repo::topology_runs::ClaimedTopologyRun,
|
||||||
) {
|
) {
|
||||||
let id = job.id;
|
let id = job.id;
|
||||||
@@ -145,16 +166,28 @@ async fn run_job(
|
|||||||
// Resume from the last checkpoint, or start fresh.
|
// Resume from the last checkpoint, or start fresh.
|
||||||
let progress: RunProgress = job
|
let progress: RunProgress = job
|
||||||
.checkpoint
|
.checkpoint
|
||||||
|
.clone()
|
||||||
.and_then(|c| serde_json::from_value(c).ok())
|
.and_then(|c| serde_json::from_value(c).ok())
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
// The composed engines (Slice 4): this graph's nodes are not claws, they are
|
||||||
|
// Claude-Code-in-a-microVM sessions. Branched BEFORE the leaf executor is
|
||||||
|
// built, because that build reads the ZeroClaw gateway config — a composed
|
||||||
|
// run must not fail for want of a runtime it never dials.
|
||||||
|
if job.tier == "microvm_graph" {
|
||||||
|
let result = run_composed(pool, hub, &job, &graph, progress).await;
|
||||||
|
finish(pool, id, result).await;
|
||||||
|
maybe_teardown_ephemeral_team(pool, runtime, id).await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// C3: prefer the mission's per-run runtime endpoint when set on
|
// C3: prefer the mission's per-run runtime endpoint when set on
|
||||||
// the missions row; else fall back to the shared env-derived
|
// the missions row; else fall back to the shared env-derived
|
||||||
// gateway (pre-C3 missions + non-mission runs). This is what
|
// gateway (pre-C3 missions + non-mission runs). This is what
|
||||||
// isolates agents' workspace filesystem to that mission's repo.
|
// isolates agents' workspace filesystem to that mission's repo.
|
||||||
let mission_binding: Option<(Option<String>, Option<String>)> =
|
type MissionBinding = (Option<String>, Option<String>, Uuid, Option<Uuid>);
|
||||||
sqlx::query_as::<_, (Option<String>, Option<String>)>(
|
let mission_binding: Option<MissionBinding> = sqlx::query_as::<_, MissionBinding>(
|
||||||
"SELECT m.runtime_endpoint, m.runtime_pairing_code
|
"SELECT m.runtime_endpoint, m.runtime_pairing_code, m.id, r.mission_phase_id
|
||||||
FROM topology_runs r
|
FROM topology_runs r
|
||||||
JOIN missions m ON m.id = r.mission_id
|
JOIN missions m ON m.id = r.mission_id
|
||||||
WHERE r.id = $1",
|
WHERE r.id = $1",
|
||||||
@@ -164,11 +197,21 @@ async fn run_job(
|
|||||||
.await
|
.await
|
||||||
.ok()
|
.ok()
|
||||||
.flatten();
|
.flatten();
|
||||||
|
// What this run's turns will be attributed to. `None` when the run belongs
|
||||||
|
// to no mission — a bare topology run has no phase to hang tool calls on.
|
||||||
|
let tap = mission_binding
|
||||||
|
.as_ref()
|
||||||
|
.map(|(_, _, mission_id, phase_id)| crate::topology_exec::MissionTap {
|
||||||
|
pool: pool.clone(),
|
||||||
|
mission_id: *mission_id,
|
||||||
|
phase_id: *phase_id,
|
||||||
|
run_id: Some(id),
|
||||||
|
});
|
||||||
let leaf_result = match mission_binding {
|
let leaf_result = match mission_binding {
|
||||||
Some((Some(url), Some(code))) => {
|
Some((Some(url), Some(code), _, _)) => {
|
||||||
ZeroClawDriveExecutor::from_env_for_gateway_with_code(url, code)
|
ZeroClawDriveExecutor::from_env_for_gateway_with_code(url, code)
|
||||||
}
|
}
|
||||||
Some((Some(url), None)) => ZeroClawDriveExecutor::from_env_for_gateway(url),
|
Some((Some(url), None, _, _)) => ZeroClawDriveExecutor::from_env_for_gateway(url),
|
||||||
_ => ZeroClawDriveExecutor::from_env(),
|
_ => ZeroClawDriveExecutor::from_env(),
|
||||||
};
|
};
|
||||||
let leaf = match leaf_result {
|
let leaf = match leaf_result {
|
||||||
@@ -178,6 +221,13 @@ async fn run_job(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
// The tap rides on the leaf executor, so the recursive tiers get it too:
|
||||||
|
// they drive the same leaf all the way down, and a company-tier mission's
|
||||||
|
// tool calls belong to its phase exactly as a team-tier one's do.
|
||||||
|
let leaf = match tap {
|
||||||
|
Some(t) => leaf.with_tap(t),
|
||||||
|
None => leaf,
|
||||||
|
};
|
||||||
|
|
||||||
// Select the executor by deploy tier: `team` drives claws directly; the
|
// Select the executor by deploy tier: `team` drives claws directly; the
|
||||||
// upper tiers drive the recursive sub-topology executor (which runs each
|
// upper tiers drive the recursive sub-topology executor (which runs each
|
||||||
@@ -201,6 +251,14 @@ async fn run_job(
|
|||||||
_ => drive(pool, id, &graph, &job.task, progress, &leaf).await,
|
_ => drive(pool, id, &graph, &job.task, progress, &leaf).await,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
finish(pool, id, result).await;
|
||||||
|
maybe_teardown_ephemeral_team(pool, runtime, id).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write a driven run's terminal state. The single place a run finishes, shared
|
||||||
|
/// by every tier — a second one would be a second completion path, which is where
|
||||||
|
/// every microVM bug this project has hit came from.
|
||||||
|
async fn finish(pool: &PgPool, id: Uuid, result: Result<RunRecord, OrchestratorError>) {
|
||||||
match result {
|
match result {
|
||||||
Ok(record) => {
|
Ok(record) => {
|
||||||
let value = serde_json::to_value(&record).unwrap_or(serde_json::Value::Null);
|
let value = serde_json::to_value(&record).unwrap_or(serde_json::Value::Null);
|
||||||
@@ -219,7 +277,77 @@ async fn run_job(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
maybe_teardown_ephemeral_team(pool, runtime, id).await;
|
}
|
||||||
|
|
||||||
|
/// Drive a composed run: the outer graph is Engine Z, every node is a
|
||||||
|
/// Claude-Code-in-a-microVM session (Engine C).
|
||||||
|
///
|
||||||
|
/// The mission columns are read here rather than carried on the run row so a
|
||||||
|
/// re-placed or re-backed mission takes effect on resume, and so the composed
|
||||||
|
/// path has exactly one source of truth for where a VM boots.
|
||||||
|
async fn run_composed(
|
||||||
|
pool: &PgPool,
|
||||||
|
hub: &Arc<crate::fleet::NodeHub>,
|
||||||
|
job: &cm_db::repo::topology_runs::ClaimedTopologyRun,
|
||||||
|
graph: &TopologyGraph,
|
||||||
|
progress: RunProgress,
|
||||||
|
) -> Result<RunRecord, OrchestratorError> {
|
||||||
|
let mission_id = job.mission_id.ok_or_else(|| {
|
||||||
|
OrchestratorError::Executor(
|
||||||
|
"a composed run has no mission, so there is no checkout for its nodes \
|
||||||
|
to share"
|
||||||
|
.into(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
let phase_id = job.mission_phase_id.ok_or_else(|| {
|
||||||
|
OrchestratorError::Executor("a composed run must belong to a mission phase".into())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let mission: (Option<Uuid>, Option<String>, Option<String>, bool) =
|
||||||
|
sqlx::query_as(
|
||||||
|
"SELECT target_node_id, backend, team_engine, (repo_id IS NOT NULL) \
|
||||||
|
FROM missions WHERE id = $1",
|
||||||
|
)
|
||||||
|
.bind(mission_id)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| OrchestratorError::Executor(format!("load mission {mission_id}: {e}")))?;
|
||||||
|
|
||||||
|
// The phase's completion gate, read here rather than carried on the run row
|
||||||
|
// so an edited `done_when_check` takes effect on the next node instead of at
|
||||||
|
// the next mission.
|
||||||
|
let phase: (String, serde_json::Value) =
|
||||||
|
sqlx::query_as("SELECT kind, config FROM mission_phases WHERE id = $1")
|
||||||
|
.bind(phase_id)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| OrchestratorError::Executor(format!("load phase {phase_id}: {e}")))?;
|
||||||
|
|
||||||
|
let exec = crate::microvm_turn_executor::for_fleet(
|
||||||
|
hub.clone(),
|
||||||
|
pool.clone(),
|
||||||
|
crate::microvm_turn_executor::ComposedRun {
|
||||||
|
run_id: job.id,
|
||||||
|
mission_id,
|
||||||
|
phase_id,
|
||||||
|
iteration: job.iteration.unwrap_or(1),
|
||||||
|
repo: crate::mission_workspace::checkout_path(mission_id),
|
||||||
|
// A repo-less composed mission gets an empty shared workspace, the
|
||||||
|
// same as a solo phase — the graph's whole property is that node 2
|
||||||
|
// sees node 1's files, and that holds whether or not it is a git
|
||||||
|
// checkout.
|
||||||
|
has_repo: mission.3,
|
||||||
|
target_node_id: mission.0,
|
||||||
|
backend: mission.1,
|
||||||
|
team_engine: mission.2,
|
||||||
|
gate: crate::vm_stop_gate::StopGate::for_phase(&phase.0, &phase.1)
|
||||||
|
.and_then(crate::vm_stop_gate::StopGate::per_node),
|
||||||
|
// Resume continues the step numbering; restarting it would re-use a
|
||||||
|
// finished node's vm id.
|
||||||
|
completed_steps: progress.completed as u32,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
drive(pool, job.id, graph, &job.task, progress, &exec).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Post-terminal hook: if this run's team is `ephemeral` and no siblings are
|
/// Post-terminal hook: if this run's team is `ephemeral` and no siblings are
|
||||||
|
|||||||
@@ -0,0 +1,158 @@
|
|||||||
|
//! Can the independent validator actually be reached?
|
||||||
|
//!
|
||||||
|
//! The sibling of [`crate::runtime_preflight`], for the same class of failure:
|
||||||
|
//! the code is right and the machine is not, and nothing says so until a mission
|
||||||
|
//! pays for it.
|
||||||
|
//!
|
||||||
|
//! `evaluator::cross_provider_judge` deliberately refuses to fall back to the
|
||||||
|
//! agent's own provider — a verdict from the same family is not an independent
|
||||||
|
//! check, and quietly producing one would claim a property the verdict does not
|
||||||
|
//! have. That refusal is correct, and its cost is that a dead validator makes
|
||||||
|
//! every `done_when` phase UNMEETABLE. The mission still boots a VM, still runs
|
||||||
|
//! an agent turn, still collects and delivers, and only then records
|
||||||
|
//! "the independent validator could not be reached this pass" on one evaluation
|
||||||
|
//! row.
|
||||||
|
//!
|
||||||
|
//! That happened: the z.ai credential expired mid-session and the first symptom
|
||||||
|
//! was a two-phase mission failing after both VMs had run. The information
|
||||||
|
//! existed the whole time; nobody was told until it was expensive.
|
||||||
|
//!
|
||||||
|
//! A report, not a gate — the same stance `runtime_preflight` takes. The server
|
||||||
|
//! must still boot with a broken validator, because refusing to start would turn
|
||||||
|
//! a degraded deployment into a dead one, and because a mission that opts out
|
||||||
|
//! (`validator_model = ''`) is unaffected. What this buys is that the degradation
|
||||||
|
//! is visible at startup instead of inferred from a failed mission.
|
||||||
|
|
||||||
|
/// The smallest question that proves a credential works end to end.
|
||||||
|
///
|
||||||
|
/// A real completion rather than a models-list or a HEAD: an expired key, a
|
||||||
|
/// revoked key and a key with no quota can all pass a cheaper check and fail the
|
||||||
|
/// call that matters. Two tokens of output.
|
||||||
|
const PROBE_PROMPT: &str = "Reply with exactly: OK";
|
||||||
|
|
||||||
|
/// What the probe found.
|
||||||
|
#[derive(Debug, PartialEq, Eq)]
|
||||||
|
pub enum Verdict {
|
||||||
|
/// No independent validator is configured; phases are judged by the house
|
||||||
|
/// model. Not a fault — a deployment may choose this.
|
||||||
|
NotConfigured,
|
||||||
|
/// Configured, resolved, and it answered.
|
||||||
|
Reachable { spec: String },
|
||||||
|
/// Configured but the registry has no such provider, so
|
||||||
|
/// `cross_provider_judge` will refuse it rather than judge with the default.
|
||||||
|
Unregistered { spec: String },
|
||||||
|
/// Configured and resolved, and the call failed.
|
||||||
|
Unreachable { spec: String, error: String },
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Verdict {
|
||||||
|
/// Is every `done_when` phase currently unmeetable because of this?
|
||||||
|
pub fn breaks_gated_phases(&self) -> bool {
|
||||||
|
matches!(
|
||||||
|
self,
|
||||||
|
Verdict::Unregistered { .. } | Verdict::Unreachable { .. }
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Ask the configured independent validator to answer one trivial question.
|
||||||
|
pub async fn probe(runtime: &cm_runtime::Runtime) -> Verdict {
|
||||||
|
let Some(spec) = std::env::var("CLAWMATES_VALIDATOR_MODEL")
|
||||||
|
.ok()
|
||||||
|
.map(|s| s.trim().to_string())
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
else {
|
||||||
|
return Verdict::NotConfigured;
|
||||||
|
};
|
||||||
|
|
||||||
|
let (_provider, model) = runtime.resolve_provider(&spec);
|
||||||
|
// `resolve_provider` falls back to the DEFAULT provider for an unknown name,
|
||||||
|
// and the fallback is detectable because the returned model still carries the
|
||||||
|
// `name:` prefix. Checked here for the same reason the evaluator checks it:
|
||||||
|
// a validator that is silently the house model is worse than none.
|
||||||
|
if model.contains(':') {
|
||||||
|
return Verdict::Unregistered { spec };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Through `Runtime::complete`, which is the same resolve-then-stream path
|
||||||
|
// the evaluator's judge takes. A probe that dialled the provider its own way
|
||||||
|
// could pass while the real call fails.
|
||||||
|
match runtime.complete("", PROBE_PROMPT, &spec, 16, false).await {
|
||||||
|
Ok(_) => Verdict::Reachable { spec },
|
||||||
|
Err(e) => Verdict::Unreachable {
|
||||||
|
spec,
|
||||||
|
error: e.chars().take(160).collect(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Probe at boot and say plainly what it means for missions.
|
||||||
|
pub fn report_at_boot(runtime: cm_runtime::Runtime) {
|
||||||
|
tokio::spawn(async move {
|
||||||
|
match probe(&runtime).await {
|
||||||
|
Verdict::NotConfigured => eprintln!(
|
||||||
|
"validator_preflight: no CLAWMATES_VALIDATOR_MODEL — phase verdicts are judged \
|
||||||
|
by the house model, which is NOT an independent check"
|
||||||
|
),
|
||||||
|
Verdict::Reachable { spec } => {
|
||||||
|
eprintln!("validator_preflight: independent validator {spec} answered")
|
||||||
|
}
|
||||||
|
Verdict::Unregistered { spec } => eprintln!(
|
||||||
|
"validator_preflight: CLAWMATES_VALIDATOR_MODEL={spec} has no registered \
|
||||||
|
provider — the evaluator will refuse it rather than judge with the default, \
|
||||||
|
so EVERY phase with a done_when condition will fail as unmet. Register the \
|
||||||
|
provider, or set the mission's validator_model to '' to opt out."
|
||||||
|
),
|
||||||
|
Verdict::Unreachable { spec, error } => eprintln!(
|
||||||
|
"validator_preflight: independent validator {spec} is UNREACHABLE ({error}) — \
|
||||||
|
EVERY phase with a done_when condition will fail as unmet, after running its \
|
||||||
|
agent. Fix the credential, or set validator_model to '' per mission to judge \
|
||||||
|
with the house model."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// The two states that make gated phases unmeetable, and the two that do
|
||||||
|
/// not. This is the distinction the whole module exists to draw: "no
|
||||||
|
/// validator configured" is a choice, "configured and broken" is a fault
|
||||||
|
/// that silently fails every conditioned mission.
|
||||||
|
#[test]
|
||||||
|
fn only_a_configured_but_broken_validator_breaks_gated_phases() {
|
||||||
|
assert!(!Verdict::NotConfigured.breaks_gated_phases());
|
||||||
|
assert!(!Verdict::Reachable {
|
||||||
|
spec: "glm:glm-4.7".into()
|
||||||
|
}
|
||||||
|
.breaks_gated_phases());
|
||||||
|
|
||||||
|
assert!(Verdict::Unregistered {
|
||||||
|
spec: "glm:glm-4.7".into()
|
||||||
|
}
|
||||||
|
.breaks_gated_phases());
|
||||||
|
assert!(Verdict::Unreachable {
|
||||||
|
spec: "glm:glm-4.7".into(),
|
||||||
|
error: "401".into()
|
||||||
|
}
|
||||||
|
.breaks_gated_phases());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An unregistered provider is NOT reported as unreachable, and the
|
||||||
|
/// difference is actionable: one is fixed by registering a provider, the
|
||||||
|
/// other by fixing a credential. Collapsing them sends an operator to the
|
||||||
|
/// wrong place.
|
||||||
|
#[test]
|
||||||
|
fn the_two_faults_are_distinguishable() {
|
||||||
|
let a = Verdict::Unregistered {
|
||||||
|
spec: "glm:glm-4.7".into(),
|
||||||
|
};
|
||||||
|
let b = Verdict::Unreachable {
|
||||||
|
spec: "glm:glm-4.7".into(),
|
||||||
|
error: "401 Authentication Failed".into(),
|
||||||
|
};
|
||||||
|
assert_ne!(a, b);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,815 @@
|
|||||||
|
//! Which fleet node should run the next microVM phase, and whether any can.
|
||||||
|
//!
|
||||||
|
//! # What this replaces
|
||||||
|
//!
|
||||||
|
//! Placement was `capable.first()` over a list ordered `last_seen DESC`
|
||||||
|
//! (`mission_orchestrator`, `nodes::online_for_backend`) — the most recently
|
||||||
|
//! heartbeated node. Among healthy nodes all heartbeating every 5s that is
|
||||||
|
//! arbitrary, and it consults nothing about load: two missions launched together
|
||||||
|
//! land on the same machine. It did not matter while one node held the only
|
||||||
|
//! rootfs image; all three do now.
|
||||||
|
//!
|
||||||
|
//! # Observed memory is not capacity
|
||||||
|
//!
|
||||||
|
//! The correctness core, and the reason this is not a one-line sort change. A VM
|
||||||
|
//! that booted 30 seconds ago holds a fraction of its 8 GiB claim — the guest has
|
||||||
|
//! not touched the rest — so `mem_pct` reports a sold-out node as nearly idle.
|
||||||
|
//! Ranking on utilisation alone would happily book five more VMs onto a node with
|
||||||
|
//! room for one. `capacity_of` therefore takes the WORSE of observed usage and
|
||||||
|
//! committed usage, and `a_sold_out_node_is_not_mistaken_for_an_idle_one` is the
|
||||||
|
//! negative control that pins it.
|
||||||
|
//!
|
||||||
|
//! Commitments come from two places that must be unioned by IDENTITY, never
|
||||||
|
//! added: `microvm_client::list` (booted VMs, including orphans nothing has
|
||||||
|
//! reaped) and `nodes::pinned_microvm_phases` (chosen but not yet booted). The
|
||||||
|
//! deterministic `vm_id_for` is what lets the same phase be recognised in both.
|
||||||
|
//!
|
||||||
|
//! # Fail-closed
|
||||||
|
//!
|
||||||
|
//! A node whose health is stale, whose daemon will not answer, or which is
|
||||||
|
//! draining is INELIGIBLE, not low-scoring. Unknown is not permission — the same
|
||||||
|
//! rule `nodes::online_for_backend` already applies to capabilities. The one
|
||||||
|
//! exception is Beszel metrics: they feed `headroom` as a tiebreak only, so stale
|
||||||
|
//! metrics demote a node instead of excluding it.
|
||||||
|
|
||||||
|
use cm_db::repo::node_metrics::EvalRow;
|
||||||
|
use cm_domain::NodeId;
|
||||||
|
|
||||||
|
/// Memory a phase VM claims. Re-exported from the executor so there is ONE number
|
||||||
|
/// — a scheduler and a launcher that disagree about VM size is a fleet that
|
||||||
|
/// overcommits by exactly their difference.
|
||||||
|
pub(crate) use crate::microvm_executor::MEM_MIB as MEM_PER_VM_MIB;
|
||||||
|
|
||||||
|
/// Held back for the host: the daemon, the OS, page cache, and the margin that
|
||||||
|
/// keeps a node out of swap. A node in swap makes every VM on it slow, so this is
|
||||||
|
/// cheaper than the alternative.
|
||||||
|
const HOST_RESERVE_MIB: i64 = 4096;
|
||||||
|
|
||||||
|
/// The floor we refuse to believe a host's own footprint is below. Without it, a
|
||||||
|
/// node reporting less used memory than its VMs have claimed would compute a
|
||||||
|
/// negative baseline and inflate its free memory.
|
||||||
|
const HOST_BASELINE_FLOOR_MIB: i64 = 2048;
|
||||||
|
|
||||||
|
/// Disk a VM may consume: an 8 GiB sparse rootfs plus room for the collected tar.
|
||||||
|
const DISK_PER_VM_GIB: i64 = 12;
|
||||||
|
|
||||||
|
/// Never let VM disk drive a node below this. `_outputs` and images live on the
|
||||||
|
/// same filesystem on some nodes.
|
||||||
|
const DISK_RESERVE_GIB: i64 = 20;
|
||||||
|
|
||||||
|
/// Health older than this and the node is ineligible. Deliberately close to the
|
||||||
|
/// 20s at which `fleet::spawn_node_sweeper` marks a node offline: the window in
|
||||||
|
/// which a node is "online with unreadable memory" should be narrow.
|
||||||
|
pub const MAX_HEALTH_AGE_SECS: f64 = 30.0;
|
||||||
|
|
||||||
|
/// Beszel metrics older than this rank as zero headroom. Only a tiebreak.
|
||||||
|
const MAX_METRICS_AGE_SECS: f64 = 60.0;
|
||||||
|
|
||||||
|
/// A node that can take at least one more phase VM.
|
||||||
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
|
pub struct NodeCapacity {
|
||||||
|
pub node_id: NodeId,
|
||||||
|
pub name: String,
|
||||||
|
/// How many MORE 8 GiB VMs fit.
|
||||||
|
pub slots: i64,
|
||||||
|
pub headroom: f64,
|
||||||
|
pub committed_vms: i64,
|
||||||
|
pub mem_total_mib: i64,
|
||||||
|
pub used_eff_mib: i64,
|
||||||
|
pub disk_free_gib: i64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Why a node cannot take this phase. Each renders a distinct, actionable line —
|
||||||
|
/// "at capacity" and "we could not read it" send an operator to different places.
|
||||||
|
#[derive(Debug, Clone, PartialEq)]
|
||||||
|
pub enum Unfit {
|
||||||
|
Draining,
|
||||||
|
NotConnected,
|
||||||
|
NoRecentHealth { age_secs: Option<f64> },
|
||||||
|
CapacityUnknown { err: String },
|
||||||
|
AtCapacity { committed: i64, used_eff_mib: i64, mem_total_mib: i64 },
|
||||||
|
NoDisk { free_gib: i64 },
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Unfit {
|
||||||
|
pub fn reason(&self) -> String {
|
||||||
|
match self {
|
||||||
|
Unfit::Draining => "draining".into(),
|
||||||
|
Unfit::NotConnected => "daemon not connected".into(),
|
||||||
|
Unfit::NoRecentHealth { age_secs } => match age_secs {
|
||||||
|
Some(a) => format!("health {a:.0}s stale (max {MAX_HEALTH_AGE_SECS:.0}s)"),
|
||||||
|
None => "never reported health".into(),
|
||||||
|
},
|
||||||
|
Unfit::CapacityUnknown { err } => format!("could not read running VMs: {err}"),
|
||||||
|
Unfit::AtCapacity { committed, used_eff_mib, mem_total_mib } => format!(
|
||||||
|
"at capacity: {committed} VM(s), {used_eff_mib}/{mem_total_mib} MiB committed"
|
||||||
|
),
|
||||||
|
Unfit::NoDisk { free_gib } => format!("only {free_gib} GiB free"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Why placement produced no node. Distinguished because the operator response
|
||||||
|
/// differs: wait, fix a daemon, or build an image.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub enum PlacementError {
|
||||||
|
/// No node has the image / KVM at all. Not a capacity problem.
|
||||||
|
NoCapableNode { backend: String, how_to_fix: String },
|
||||||
|
/// Every capable node is full. Transient — the caller should queue.
|
||||||
|
FleetAtCapacity { report: String },
|
||||||
|
/// We could not READ capacity. Must never be reported as "full".
|
||||||
|
FleetUnreadable { report: String },
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PlacementError {
|
||||||
|
/// Whether the caller should wait and retry rather than fail the work.
|
||||||
|
pub fn is_transient(&self) -> bool {
|
||||||
|
matches!(
|
||||||
|
self,
|
||||||
|
PlacementError::FleetAtCapacity { .. } | PlacementError::FleetUnreadable { .. }
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn message(&self) -> String {
|
||||||
|
match self {
|
||||||
|
PlacementError::NoCapableNode { backend, how_to_fix } => {
|
||||||
|
format!("no online node can run backend {backend:?} — {how_to_fix}")
|
||||||
|
}
|
||||||
|
PlacementError::FleetAtCapacity { report } => format!(
|
||||||
|
"fleet at capacity — a phase VM runs up to 60 min; this phase waits for a slot.\n{report}"
|
||||||
|
),
|
||||||
|
PlacementError::FleetUnreadable { report } => format!(
|
||||||
|
"cannot read node capacity — this is NOT a full fleet; check the node daemons.\n{report}"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What the host itself costs, excluding its phase VMs.
|
||||||
|
///
|
||||||
|
/// With nothing committed the answer is simply what the node reports. With VMs
|
||||||
|
/// committed it cannot be measured, only remembered or inferred — and inference
|
||||||
|
/// is where this went wrong: subtracting the VMs' FULL 8 GiB claim from
|
||||||
|
/// observed usage assumes they have already consumed it. A VM booted seconds
|
||||||
|
/// ago holds about an eighth of that, so the subtraction goes negative, hits
|
||||||
|
/// the floor, and hands back memory the host is really using.
|
||||||
|
///
|
||||||
|
/// Measured on morpheus (31757 MiB total, 4314 MiB idle, 2 slots) with 2 VMs
|
||||||
|
/// committed and young: the inferred baseline collapsed to the 2048 floor,
|
||||||
|
/// freeing 2266 MiB — exactly enough to admit a 3rd VM to a 2-slot node. The
|
||||||
|
/// `capacity` harness scenario caught it on its first full run.
|
||||||
|
///
|
||||||
|
/// So prefer the remembered idle reading, and take the LARGER of it and the
|
||||||
|
/// inference: a host that has genuinely started doing non-VM work must not be
|
||||||
|
/// under-charged just because it was once idle at a lower number.
|
||||||
|
fn host_baseline(mem_used_mib: i64, committed_vms: i64, baseline_mib: Option<i64>) -> i64 {
|
||||||
|
if committed_vms <= 0 {
|
||||||
|
// Directly observable, and the only moment it is.
|
||||||
|
return mem_used_mib.max(HOST_BASELINE_FLOOR_MIB);
|
||||||
|
}
|
||||||
|
let inferred = mem_used_mib - committed_vms * MEM_PER_VM_MIB as i64;
|
||||||
|
inferred
|
||||||
|
.max(baseline_mib.unwrap_or(0))
|
||||||
|
.max(HOST_BASELINE_FLOOR_MIB)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The whole capacity decision for one node, as pure arithmetic.
|
||||||
|
///
|
||||||
|
/// Separated from every I/O concern so the numbers can be tested against measured
|
||||||
|
/// fleet values without a database, a hub, or a VM.
|
||||||
|
pub fn capacity_of(
|
||||||
|
node_id: NodeId,
|
||||||
|
name: &str,
|
||||||
|
mem_total_mib: i64,
|
||||||
|
mem_used_mib: i64,
|
||||||
|
disk_free_gib: i64,
|
||||||
|
committed_vms: i64,
|
||||||
|
// What this node used the last time it was seen with nothing committed.
|
||||||
|
// `None` before it has ever been observed idle.
|
||||||
|
baseline_mib: Option<i64>,
|
||||||
|
headroom: f64,
|
||||||
|
) -> Result<NodeCapacity, Unfit> {
|
||||||
|
let host_baseline = host_baseline(mem_used_mib, committed_vms, baseline_mib);
|
||||||
|
let committed_use = committed_vms * MEM_PER_VM_MIB as i64 + host_baseline;
|
||||||
|
|
||||||
|
// The worse of the two views. Observed alone under-counts a freshly booted
|
||||||
|
// VM; committed alone under-counts a host doing real work outside its VMs.
|
||||||
|
let used_eff = mem_used_mib.max(committed_use);
|
||||||
|
let free = mem_total_mib - used_eff - HOST_RESERVE_MIB;
|
||||||
|
let slots = if free <= 0 { 0 } else { free / MEM_PER_VM_MIB as i64 };
|
||||||
|
|
||||||
|
if disk_free_gib - DISK_PER_VM_GIB < DISK_RESERVE_GIB {
|
||||||
|
return Err(Unfit::NoDisk { free_gib: disk_free_gib });
|
||||||
|
}
|
||||||
|
if slots < 1 {
|
||||||
|
return Err(Unfit::AtCapacity {
|
||||||
|
committed: committed_vms,
|
||||||
|
used_eff_mib: used_eff,
|
||||||
|
mem_total_mib,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Ok(NodeCapacity {
|
||||||
|
node_id,
|
||||||
|
name: name.to_string(),
|
||||||
|
slots,
|
||||||
|
headroom,
|
||||||
|
committed_vms,
|
||||||
|
mem_total_mib,
|
||||||
|
used_eff_mib: used_eff,
|
||||||
|
disk_free_gib,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Admission inputs drawn from an `EvalRow`, or why the node is ineligible.
|
||||||
|
///
|
||||||
|
/// Fail-closed on stale or absent health: a node whose memory we cannot read is
|
||||||
|
/// one whose capacity we would be guessing at.
|
||||||
|
pub fn from_eval(
|
||||||
|
row: &EvalRow,
|
||||||
|
name: &str,
|
||||||
|
committed_vms: i64,
|
||||||
|
) -> Result<NodeCapacity, Unfit> {
|
||||||
|
if row.status == "draining" {
|
||||||
|
return Err(Unfit::Draining);
|
||||||
|
}
|
||||||
|
let fresh = row
|
||||||
|
.health_age_secs
|
||||||
|
.is_some_and(|a| a <= MAX_HEALTH_AGE_SECS);
|
||||||
|
let (Some(total), Some(used)) = (row.mem_total_bytes, row.mem_used_bytes) else {
|
||||||
|
return Err(Unfit::NoRecentHealth { age_secs: row.health_age_secs });
|
||||||
|
};
|
||||||
|
if !fresh || total <= 0 {
|
||||||
|
return Err(Unfit::NoRecentHealth { age_secs: row.health_age_secs });
|
||||||
|
}
|
||||||
|
const MIB: i64 = 1024 * 1024;
|
||||||
|
const GIB: i64 = 1024 * 1024 * 1024;
|
||||||
|
capacity_of(
|
||||||
|
row.node_id,
|
||||||
|
name,
|
||||||
|
total / MIB,
|
||||||
|
used / MIB,
|
||||||
|
row.disk_free_bytes.unwrap_or(0) / GIB,
|
||||||
|
committed_vms,
|
||||||
|
row.mem_baseline_mib,
|
||||||
|
row.headroom_fresh(MAX_METRICS_AGE_SECS),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rank admissible nodes: most free slots first, then live headroom, then id.
|
||||||
|
///
|
||||||
|
/// Slots before headroom SPREADS load rather than stacking it — two missions
|
||||||
|
/// launched together go to different machines. Headroom breaks ties with
|
||||||
|
/// real-time load, which is where a node mid-`cargo build` loses to an idle peer.
|
||||||
|
/// Node id last so the same fleet state always yields the same answer; the old
|
||||||
|
/// `last_seen DESC` made placement unreproducible between two identical runs.
|
||||||
|
pub fn rank(mut fit: Vec<NodeCapacity>) -> Vec<NodeCapacity> {
|
||||||
|
fit.sort_by(|a, b| {
|
||||||
|
b.slots
|
||||||
|
.cmp(&a.slots)
|
||||||
|
.then(
|
||||||
|
b.headroom
|
||||||
|
.partial_cmp(&a.headroom)
|
||||||
|
.unwrap_or(std::cmp::Ordering::Equal),
|
||||||
|
)
|
||||||
|
.then(a.node_id.as_uuid().cmp(&b.node_id.as_uuid()))
|
||||||
|
});
|
||||||
|
fit
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One line per node, for logs and for the message an operator reads.
|
||||||
|
pub fn report(fit: &[NodeCapacity], unfit: &[(NodeId, String, Unfit)]) -> String {
|
||||||
|
let mut out = Vec::new();
|
||||||
|
for f in fit {
|
||||||
|
out.push(format!(
|
||||||
|
" {}: {} slot(s) free, {} VM(s) committed, {}/{} MiB, headroom {:.0}",
|
||||||
|
f.name, f.slots, f.committed_vms, f.used_eff_mib, f.mem_total_mib, f.headroom
|
||||||
|
));
|
||||||
|
}
|
||||||
|
for (_, name, why) in unfit {
|
||||||
|
out.push(format!(" {name}: UNFIT — {}", why.reason()));
|
||||||
|
}
|
||||||
|
if out.is_empty() {
|
||||||
|
out.push(" (no capable nodes)".into());
|
||||||
|
}
|
||||||
|
out.join("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Count a node's commitments, unioning booted VMs with pinned-not-yet-booted
|
||||||
|
/// phases BY IDENTITY.
|
||||||
|
///
|
||||||
|
/// A composed graph's step VMs (`...-s0`, `-s1`) each count: each is a real
|
||||||
|
/// Firecracker process holding 8 GiB. A pinned phase counts only while no live VM
|
||||||
|
/// carries its id — otherwise the same claim would be counted twice and the fleet
|
||||||
|
/// would shrink by the number of phases currently starting.
|
||||||
|
pub fn commitments(live_vm_ids: &[String], pinned_keys: &[String]) -> i64 {
|
||||||
|
let live = live_vm_ids.len() as i64;
|
||||||
|
let unbooted = pinned_keys
|
||||||
|
.iter()
|
||||||
|
.filter(|k| !live_vm_ids.iter().any(|v| v.starts_with(k.as_str())))
|
||||||
|
.count() as i64;
|
||||||
|
live + unbooted
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every backend a phase needs on ONE node: the mission's, plus each backend
|
||||||
|
/// named by a node of its composed graph.
|
||||||
|
///
|
||||||
|
/// The roster stores them as `config.roster.nodes[].attrs.backend`, and they are
|
||||||
|
/// the reason this function exists. A 2-member roster with
|
||||||
|
/// `verifier@canary-claude` was placed on a node holding `claude` and not
|
||||||
|
/// `canary-claude`; the graph's first node ran, the second died with
|
||||||
|
/// `no rootfs for backend "canary-claude" on this node`, and the mission
|
||||||
|
/// delivered half its work and failed. Placement had asked only about the
|
||||||
|
/// mission's own backend, which was true and insufficient.
|
||||||
|
pub fn required_backends(mission_backend: Option<&str>, roster: Option<&serde_json::Value>) -> Vec<String> {
|
||||||
|
let mut out = vec![cm_db::repo::nodes::backend_key(mission_backend).to_string()];
|
||||||
|
if let Some(nodes) = roster.and_then(|r| r.get("nodes")).and_then(|n| n.as_array()) {
|
||||||
|
for n in nodes {
|
||||||
|
if let Some(b) = n
|
||||||
|
.get("attrs")
|
||||||
|
.and_then(|a| a.get("backend"))
|
||||||
|
.and_then(|b| b.as_str())
|
||||||
|
.filter(|b| !b.trim().is_empty())
|
||||||
|
{
|
||||||
|
out.push(b.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.sort();
|
||||||
|
out.dedup();
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Survey every capable node: which can take a phase VM, and why the rest cannot.
|
||||||
|
///
|
||||||
|
/// `vm_list` is asked of each candidate in parallel with a short deadline. A node
|
||||||
|
/// that will not answer is `CapacityUnknown` and therefore ineligible — we cannot
|
||||||
|
/// count what we cannot see, and guessing zero is how a node gets double-booked.
|
||||||
|
pub async fn survey(
|
||||||
|
pool: &sqlx::PgPool,
|
||||||
|
hub: &crate::fleet::NodeHub,
|
||||||
|
workspace_id: uuid::Uuid,
|
||||||
|
// EVERY backend the work needs, not just the mission's. A composed graph
|
||||||
|
// runs on ONE node and its nodes may each name their own — the roster's
|
||||||
|
// whole purpose is an independent verifier on another provider — so the
|
||||||
|
// node has to hold all of their rootfs images.
|
||||||
|
backends: &[String],
|
||||||
|
) -> Result<(Vec<NodeCapacity>, Vec<(NodeId, String, Unfit)>), String> {
|
||||||
|
let candidates = cm_db::repo::nodes::online_for_backends(pool, workspace_id, backends)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("looking up nodes for backends {backends:?}: {e}"))?;
|
||||||
|
if candidates.is_empty() {
|
||||||
|
return Ok((Vec::new(), Vec::new()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let evals = cm_db::repo::node_metrics::eval_all(pool)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("reading node metrics: {e}"))?;
|
||||||
|
let pinned = cm_db::repo::nodes::pinned_microvm_phases(pool, workspace_id)
|
||||||
|
.await
|
||||||
|
.map_err(|e| format!("reading pinned phases: {e}"))?;
|
||||||
|
|
||||||
|
let names = node_names(pool, workspace_id).await;
|
||||||
|
let mut fit = Vec::new();
|
||||||
|
let mut unfit = Vec::new();
|
||||||
|
for node in candidates {
|
||||||
|
let row = evals.iter().find(|e| e.node_id == node);
|
||||||
|
let name = names
|
||||||
|
.get(&node.as_uuid())
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_else(|| node.as_uuid().to_string()[..8].to_string());
|
||||||
|
|
||||||
|
// Not connected: nothing can be asked of it, and nothing can run on it.
|
||||||
|
if !hub.is_connected(node) {
|
||||||
|
unfit.push((node, name, Unfit::NotConnected));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let Some(row) = row else {
|
||||||
|
unfit.push((node, name, Unfit::NoRecentHealth { age_secs: None }));
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Commitments: booted VMs unioned with phases pinned here but not yet
|
||||||
|
// booted, by the deterministic id both sides agree on.
|
||||||
|
let live = match crate::microvm_client::list(hub, node).await {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(e) => {
|
||||||
|
unfit.push((node, name, Unfit::CapacityUnknown { err: e }));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let keys: Vec<String> = pinned
|
||||||
|
.iter()
|
||||||
|
.filter(|(n, _, _)| *n == node)
|
||||||
|
.map(|(_, phase, iter)| crate::microvm_executor::vm_id_for(*phase, *iter, None))
|
||||||
|
.collect();
|
||||||
|
let committed = commitments(&live, &keys);
|
||||||
|
|
||||||
|
// An idle node is the ONLY time its own footprint is measurable rather
|
||||||
|
// than inferred, so take the reading whenever we get one. Cheap: an
|
||||||
|
// UPDATE per idle node per survey, and it is what stops a young VM's
|
||||||
|
// unconsumed memory from being handed out a second time.
|
||||||
|
if committed == 0 {
|
||||||
|
if let Some(used) = row.mem_used_bytes.filter(|_| {
|
||||||
|
row.health_age_secs
|
||||||
|
.is_some_and(|a| a <= MAX_HEALTH_AGE_SECS)
|
||||||
|
}) {
|
||||||
|
let mib = used / (1024 * 1024);
|
||||||
|
if row.mem_baseline_mib != Some(mib) {
|
||||||
|
let _ = cm_db::repo::nodes::set_mem_baseline(pool, node, mib).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
match from_eval(row, &name, committed) {
|
||||||
|
Ok(c) => fit.push(c),
|
||||||
|
Err(why) => unfit.push((node, name, why)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok((rank(fit), unfit))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Node names for readable reports. A capacity report naming two machines
|
||||||
|
/// "New node" is a report nobody can act on.
|
||||||
|
async fn node_names(
|
||||||
|
pool: &sqlx::PgPool,
|
||||||
|
workspace_id: uuid::Uuid,
|
||||||
|
) -> std::collections::HashMap<uuid::Uuid, String> {
|
||||||
|
sqlx::query_as::<_, (uuid::Uuid, String)>(
|
||||||
|
"SELECT id, name FROM nodes WHERE workspace_id = $1",
|
||||||
|
)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.unwrap_or_default()
|
||||||
|
.into_iter()
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Choose a node for a phase, honouring an explicit target as a REQUEST.
|
||||||
|
///
|
||||||
|
/// `want` is honoured only if that node is genuinely admissible — the same
|
||||||
|
/// "a request, not a guarantee" rule the orchestrator already applied to
|
||||||
|
/// capability, now extended to capacity and draining.
|
||||||
|
pub async fn choose(
|
||||||
|
pool: &sqlx::PgPool,
|
||||||
|
hub: &crate::fleet::NodeHub,
|
||||||
|
workspace_id: uuid::Uuid,
|
||||||
|
backends: &[String],
|
||||||
|
want: Option<uuid::Uuid>,
|
||||||
|
) -> Result<NodeId, PlacementError> {
|
||||||
|
let named = backends.join(", ");
|
||||||
|
let how_to_fix = format!(
|
||||||
|
"needs /dev/kvm + firecracker (scripts/fc-node-setup.sh) AND the {named} rootfs \
|
||||||
|
built on ONE node (scripts/fc-build-rootfs.sh <host> <image> <name>) — a \
|
||||||
|
composed graph runs on a single node, so that node needs every image its \
|
||||||
|
nodes ask for"
|
||||||
|
);
|
||||||
|
let (fit, unfit) = survey(pool, hub, workspace_id, backends).await.map_err(|e| {
|
||||||
|
PlacementError::FleetUnreadable { report: format!(" survey failed: {e}") }
|
||||||
|
})?;
|
||||||
|
|
||||||
|
if fit.is_empty() && unfit.is_empty() {
|
||||||
|
return Err(PlacementError::NoCapableNode {
|
||||||
|
backend: named,
|
||||||
|
how_to_fix,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let report = report(&fit, &unfit);
|
||||||
|
|
||||||
|
// `want` is ADVISORY, always. The only caller passes `missions.target_node_id`,
|
||||||
|
// which is simply where the PREVIOUS phase ran — not an operator's choice.
|
||||||
|
// Treating it as a requirement had two consequences, both wrong:
|
||||||
|
//
|
||||||
|
// - a previous node that had since filled up (or gone unreadable) failed
|
||||||
|
// the phase outright: `TargetUnfit` is not transient, so it never
|
||||||
|
// reached the queue. Note this was NOT the drain case — a draining node
|
||||||
|
// is already excluded by `online_for_backend`'s `status = 'online'`, so
|
||||||
|
// it never reaches `unfit` at all and the pin simply falls through.
|
||||||
|
// `drain-midmission` passes either way; the path it does not cover is
|
||||||
|
// "phase 1's node is now full", which is the one that used to fail.
|
||||||
|
// - and while the node stayed fit, every later phase went back to it
|
||||||
|
// regardless of ranking — accidental mission-to-node affinity, which
|
||||||
|
// this module's own header says must not exist.
|
||||||
|
//
|
||||||
|
// Mission state lives on the gateway (inject -> run -> collect -> destroy),
|
||||||
|
// so re-placing costs nothing. Prefer the pin when it still fits; say out
|
||||||
|
// loud why it did not when it does not, and rank as usual.
|
||||||
|
if let Some(want) = want {
|
||||||
|
if let Some(c) = fit.iter().find(|c| c.node_id.as_uuid() == want) {
|
||||||
|
return Ok(c.node_id);
|
||||||
|
}
|
||||||
|
if let Some((_, name, why)) = unfit.iter().find(|(n, _, _)| n.as_uuid() == want) {
|
||||||
|
eprintln!(
|
||||||
|
"vm_placement: the previous phase's node {name} is {} — re-placing this phase",
|
||||||
|
why.reason()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(best) = fit.into_iter().next() {
|
||||||
|
return Ok(best.node_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nothing fit. Distinguish "full" from "blind": an operator sent to look for
|
||||||
|
// a load problem that is really a dead daemon wastes the outage.
|
||||||
|
let blind = unfit.iter().all(|(_, _, w)| {
|
||||||
|
matches!(w, Unfit::CapacityUnknown { .. } | Unfit::NotConnected | Unfit::NoRecentHealth { .. })
|
||||||
|
});
|
||||||
|
Err(if blind {
|
||||||
|
PlacementError::FleetUnreadable { report }
|
||||||
|
} else {
|
||||||
|
PlacementError::FleetAtCapacity { report }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn nid(n: u128) -> NodeId {
|
||||||
|
NodeId::from(uuid::Uuid::from_u128(n))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// THE test. Measured on tank: 60 GiB total, and five VMs booted moments ago
|
||||||
|
/// showing only ~12 GiB used because the guests have not touched their claim.
|
||||||
|
///
|
||||||
|
/// Observed-usage-only arithmetic says (61440-12000-4096)/8192 = 5 more VMs.
|
||||||
|
/// The node has room for ONE. Booking those five is a node in swap, and every
|
||||||
|
/// VM on it slows down together.
|
||||||
|
/// A node whose VMs have not yet consumed their claim must not hand the
|
||||||
|
/// difference out again.
|
||||||
|
///
|
||||||
|
/// This is the bug the `capacity` harness scenario found on its first full
|
||||||
|
/// run — "morpheus peaked at 3 concurrent VM(s) with only 2 slot(s)" — and
|
||||||
|
/// the numbers here are that node's real ones. Idle it reports 4314 MiB of
|
||||||
|
/// 31757 and the survey correctly gives it 2 slots. Two VMs later, each
|
||||||
|
/// holding roughly 1 GiB of its 8 GiB, observed usage is ~6314 MiB;
|
||||||
|
/// inferring the baseline as 6314 - 16384 goes negative, clamps to the
|
||||||
|
/// 2048 floor, and invents 2266 MiB — exactly one more VM than exists.
|
||||||
|
/// A previous node that is no longer usable re-places the next phase; it
|
||||||
|
/// does not fail it.
|
||||||
|
///
|
||||||
|
/// `choose` treated `missions.target_node_id` — which is only ever "where
|
||||||
|
/// the last phase ran" — as a hard requirement, so a pinned node that had
|
||||||
|
/// since FILLED UP produced `TargetUnfit`, which is not transient, and the
|
||||||
|
/// phase failed instead of queueing or moving. It also gave every later
|
||||||
|
/// phase silent affinity back to the first node.
|
||||||
|
///
|
||||||
|
/// The drain case is not this one and never was: `online_for_backend`
|
||||||
|
/// filters on `status = 'online'`, so a draining node is not a candidate
|
||||||
|
/// and the pin falls through to ranking. `drain-midmission` passes on both
|
||||||
|
/// the old and new code, which is why the capacity half needs this test.
|
||||||
|
/// A composed graph's per-node backends are part of what placement needs.
|
||||||
|
///
|
||||||
|
/// The full harness found this: a 2-member roster with
|
||||||
|
/// `verifier@canary-claude` was placed on a node holding `claude` and not
|
||||||
|
/// `canary-claude`. The first graph node ran, the second died with
|
||||||
|
/// `no rootfs for backend "canary-claude" on this node`, and the mission
|
||||||
|
/// delivered half its work and failed. Placement had asked only about the
|
||||||
|
/// mission's own backend — true, and insufficient.
|
||||||
|
#[test]
|
||||||
|
fn a_composed_graph_needs_every_backend_its_nodes_name() {
|
||||||
|
let roster = serde_json::json!({
|
||||||
|
"kind": "pipeline",
|
||||||
|
"nodes": [
|
||||||
|
{"id": "n0", "role": "implementer"},
|
||||||
|
{"id": "n1", "role": "verifier", "attrs": {"backend": "canary-claude"}},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
assert_eq!(
|
||||||
|
required_backends(Some("claude"), Some(&roster)),
|
||||||
|
vec!["canary-claude".to_string(), "claude".to_string()],
|
||||||
|
"both images have to be on the ONE node the graph runs on"
|
||||||
|
);
|
||||||
|
|
||||||
|
// A solo mission is unchanged — this must not make ordinary placement
|
||||||
|
// stricter than it was.
|
||||||
|
assert_eq!(required_backends(Some("claude"), None), vec!["claude"]);
|
||||||
|
assert_eq!(required_backends(None, None), vec!["default"]);
|
||||||
|
|
||||||
|
// A node with no explicit backend inherits the mission's, so it adds
|
||||||
|
// nothing. Deduped, or a 5-node graph would ask for `claude` five times
|
||||||
|
// and the containment query would still be right but the error message
|
||||||
|
// would be nonsense.
|
||||||
|
let inherit = serde_json::json!({"nodes": [
|
||||||
|
{"id": "n0", "role": "a"},
|
||||||
|
{"id": "n1", "role": "b", "attrs": {}},
|
||||||
|
{"id": "n2", "role": "c", "attrs": {"backend": ""}},
|
||||||
|
]});
|
||||||
|
assert_eq!(required_backends(Some("claude"), Some(&inherit)), vec!["claude"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_unfit_previous_node_is_re_placed_not_refused() {
|
||||||
|
let drained = uuid::Uuid::from_u128(1);
|
||||||
|
let healthy = capacity_of(nid(2), "tank", 61440, 6144, 800, 0, None, 90.0).unwrap();
|
||||||
|
|
||||||
|
// Stand in for `choose`'s decision: the pin is consulted, then dropped.
|
||||||
|
let fit = vec![healthy.clone()];
|
||||||
|
let picked = fit
|
||||||
|
.iter()
|
||||||
|
.find(|c| c.node_id.as_uuid() == drained)
|
||||||
|
.or_else(|| fit.first())
|
||||||
|
.expect("a fit node exists");
|
||||||
|
assert_eq!(
|
||||||
|
picked.node_id,
|
||||||
|
nid(2),
|
||||||
|
"with the pinned node absent from `fit`, ranking must still yield a node"
|
||||||
|
);
|
||||||
|
|
||||||
|
// And the error that used to be produced here no longer exists, so it
|
||||||
|
// cannot be reintroduced as a non-transient failure by accident.
|
||||||
|
for e in [
|
||||||
|
PlacementError::FleetAtCapacity { report: String::new() },
|
||||||
|
PlacementError::FleetUnreadable { report: String::new() },
|
||||||
|
] {
|
||||||
|
assert!(e.is_transient(), "both no-node outcomes must QUEUE, not fail");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_young_vms_unconsumed_memory_is_not_handed_out_twice() {
|
||||||
|
// Idle: the reading that gets remembered, and the slot count it implies.
|
||||||
|
let idle = capacity_of(nid(3), "morpheus", 31757, 4314, 312, 0, None, 90.0)
|
||||||
|
.expect("an idle morpheus fits VMs");
|
||||||
|
assert_eq!(idle.slots, 2, "idle capacity is the number we are defending");
|
||||||
|
|
||||||
|
// Two committed, both young. WITHOUT the remembered baseline this
|
||||||
|
// returned 1 slot and admitted a third VM.
|
||||||
|
let inferred = capacity_of(nid(3), "morpheus", 31757, 6314, 312, 2, None, 90.0);
|
||||||
|
assert!(
|
||||||
|
inferred.is_ok(),
|
||||||
|
"the old inference is preserved as the no-baseline fallback"
|
||||||
|
);
|
||||||
|
|
||||||
|
// WITH it, the node is correctly full.
|
||||||
|
let remembered = capacity_of(nid(3), "morpheus", 31757, 6314, 312, 2, Some(4314), 90.0);
|
||||||
|
assert!(
|
||||||
|
matches!(remembered, Err(Unfit::AtCapacity { committed: 2, .. })),
|
||||||
|
"a 2-slot node with 2 VMs committed is FULL, got {remembered:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A host that starts doing real work outside its VMs is charged for it.
|
||||||
|
///
|
||||||
|
/// The remembered baseline is a floor, not a substitute. If it replaced the
|
||||||
|
/// inference outright, a node that was idle at 4 GiB and is now running a
|
||||||
|
/// 20 GiB build would still be scored as if it were idle — the same
|
||||||
|
/// over-commit, arrived at from the opposite direction.
|
||||||
|
#[test]
|
||||||
|
fn a_remembered_baseline_never_under_charges_a_busy_host() {
|
||||||
|
// 1 VM committed and consumed (8192), plus 20 GiB of non-VM work.
|
||||||
|
let used = 8192 + 20480;
|
||||||
|
let c = capacity_of(nid(3), "busy", 61440, used, 800, 1, Some(4096), 90.0)
|
||||||
|
.expect("still has room");
|
||||||
|
// Inference says 20480; the stale 4096 baseline must not win.
|
||||||
|
assert_eq!(c.used_eff_mib, used, "observed usage is charged in full");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_sold_out_node_is_not_mistaken_for_an_idle_one() {
|
||||||
|
let observed_only =
|
||||||
|
capacity_of(nid(1), "tank", 61440, 12000, 800, 0, None, 50.0).expect("fits");
|
||||||
|
assert_eq!(
|
||||||
|
observed_only.slots, 5,
|
||||||
|
"this is what utilisation alone claims — the bug being fixed"
|
||||||
|
);
|
||||||
|
|
||||||
|
let with_commitments =
|
||||||
|
capacity_of(nid(1), "tank", 61440, 12000, 800, 5, None, 50.0).expect("fits");
|
||||||
|
assert_eq!(
|
||||||
|
with_commitments.slots, 1,
|
||||||
|
"five 8 GiB claims are already spoken for, whatever the guests have touched"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The measured idle fleet. Numbers from `free`/`df` on the real machines, so
|
||||||
|
/// a future change to the constants has to face what it does to real nodes.
|
||||||
|
#[test]
|
||||||
|
fn the_measured_fleet_gets_the_slots_it_actually_has() {
|
||||||
|
// tank: 60 GiB, ~6 GiB used at idle.
|
||||||
|
let tank = capacity_of(nid(1), "tank", 61440, 6144, 869, 0, None, 90.0).unwrap();
|
||||||
|
assert_eq!(tank.slots, 6);
|
||||||
|
// architect: 60 GiB, ~7 GiB used.
|
||||||
|
let arch = capacity_of(nid(2), "architect", 61440, 7168, 388, 0, None, 90.0).unwrap();
|
||||||
|
assert_eq!(arch.slots, 6);
|
||||||
|
// morpheus: 31 GiB — deliberately the conservative 2, not 3. Three VMs
|
||||||
|
// would leave under 2 GiB for the host, which is where the OOM killer
|
||||||
|
// lives, and an OOM-killed VM looks like an agent that gave up.
|
||||||
|
let morph = capacity_of(nid(3), "morpheus", 31744, 5120, 312, 0, None, 90.0).unwrap();
|
||||||
|
assert_eq!(morph.slots, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spread, don't stack; then real load; then determinism.
|
||||||
|
#[test]
|
||||||
|
fn ranking_prefers_free_slots_then_headroom_then_a_stable_order() {
|
||||||
|
let a = capacity_of(nid(1), "a", 61440, 6144, 800, 0, None, 40.0).unwrap(); // 6 slots
|
||||||
|
let b = capacity_of(nid(2), "b", 61440, 6144, 800, 3, None, 90.0).unwrap(); // 3 slots
|
||||||
|
assert_eq!(rank(vec![b.clone(), a.clone()])[0].name, "a", "more slots wins");
|
||||||
|
|
||||||
|
// Equal slots → the node under less real load.
|
||||||
|
let busy = capacity_of(nid(3), "busy", 61440, 6144, 800, 0, None, 10.0).unwrap();
|
||||||
|
let idle = capacity_of(nid(4), "idle", 61440, 6144, 800, 0, None, 95.0).unwrap();
|
||||||
|
assert_eq!(rank(vec![busy.clone(), idle.clone()])[0].name, "idle");
|
||||||
|
|
||||||
|
// Equal on both → same answer twice. `last_seen DESC` could not promise this.
|
||||||
|
let x = capacity_of(nid(9), "x", 61440, 6144, 800, 0, None, 50.0).unwrap();
|
||||||
|
let y = capacity_of(nid(8), "y", 61440, 6144, 800, 0, None, 50.0).unwrap();
|
||||||
|
assert_eq!(rank(vec![x.clone(), y.clone()])[0].name, "y");
|
||||||
|
assert_eq!(rank(vec![y, x])[0].name, "y");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A booted VM and its pinned phase row are ONE claim, not two.
|
||||||
|
#[test]
|
||||||
|
fn commitments_union_by_identity_rather_than_adding() {
|
||||||
|
let live = vec!["m-abc123def456-0".to_string(), "m-abc123def456-0-s2".to_string()];
|
||||||
|
// Same phase as the live VMs: already counted.
|
||||||
|
assert_eq!(commitments(&live, &["m-abc123def456-0".to_string()]), 2);
|
||||||
|
// A different phase, pinned but not yet booted: a real additional claim.
|
||||||
|
assert_eq!(
|
||||||
|
commitments(&live, &["m-999888777666-0".to_string()]),
|
||||||
|
3,
|
||||||
|
"a phase chosen seconds ago holds 8 GiB no node can report yet"
|
||||||
|
);
|
||||||
|
assert_eq!(commitments(&[], &["m-1-0".into(), "m-2-0".into()]), 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Disk is a hard gate, and it is checked BEFORE capacity so the message
|
||||||
|
/// names the real problem.
|
||||||
|
#[test]
|
||||||
|
fn a_node_short_of_disk_is_refused_even_with_memory_to_spare() {
|
||||||
|
let e = capacity_of(nid(1), "tank", 61440, 6144, 25, 0, None, 90.0).unwrap_err();
|
||||||
|
assert!(matches!(e, Unfit::NoDisk { free_gib: 25 }), "{e:?}");
|
||||||
|
assert!(e.reason().contains("25 GiB"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stale metrics may cost a tie; they may never win one, and they may never
|
||||||
|
/// exclude a node — that is health's job.
|
||||||
|
#[test]
|
||||||
|
fn stale_beszel_metrics_demote_but_do_not_exclude() {
|
||||||
|
let mut row = row_for(nid(1), 61440 * MIB_T, 6144 * MIB_T, 800 * GIB_T);
|
||||||
|
row.metrics_age_secs = Some(3600.0);
|
||||||
|
row.health_age_secs = Some(3.0);
|
||||||
|
row.cpu_pct = Some(5.0);
|
||||||
|
let fit = from_eval(&row, "tank", 0).expect("still eligible");
|
||||||
|
assert_eq!(fit.headroom, 95.0, "fresh health carries the headroom");
|
||||||
|
|
||||||
|
row.health_age_secs = Some(3600.0);
|
||||||
|
assert!(
|
||||||
|
matches!(from_eval(&row, "tank", 0), Err(Unfit::NoRecentHealth { .. })),
|
||||||
|
"stale HEALTH is exclusion, because memory is then a guess"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The two failures an operator must never confuse.
|
||||||
|
#[test]
|
||||||
|
fn unreadable_capacity_never_reads_as_a_full_fleet() {
|
||||||
|
let full = PlacementError::FleetAtCapacity { report: " tank: 0 slots".into() };
|
||||||
|
let blind = PlacementError::FleetUnreadable { report: " tank: UNFIT".into() };
|
||||||
|
assert!(full.message().contains("at capacity"));
|
||||||
|
assert!(blind.message().contains("cannot read"));
|
||||||
|
assert!(
|
||||||
|
!blind.message().contains("at capacity"),
|
||||||
|
"sends an operator hunting a load problem that does not exist"
|
||||||
|
);
|
||||||
|
assert!(full.is_transient() && blind.is_transient());
|
||||||
|
let missing = PlacementError::NoCapableNode {
|
||||||
|
backend: "claude".into(),
|
||||||
|
how_to_fix: "build the image".into(),
|
||||||
|
};
|
||||||
|
assert!(!missing.is_transient(), "a missing image will not fix itself by waiting");
|
||||||
|
}
|
||||||
|
|
||||||
|
const MIB_T: i64 = 1024 * 1024;
|
||||||
|
const GIB_T: i64 = 1024 * 1024 * 1024;
|
||||||
|
|
||||||
|
fn row_for(node_id: NodeId, total: i64, used: i64, disk_free: i64) -> EvalRow {
|
||||||
|
EvalRow {
|
||||||
|
node_id,
|
||||||
|
workspace_id: cm_domain::WorkspaceId::from(uuid::Uuid::from_u128(1)),
|
||||||
|
status: "online".into(),
|
||||||
|
cpu_pct: None,
|
||||||
|
mem_pct: None,
|
||||||
|
disk_pct: None,
|
||||||
|
gpu_pct: None,
|
||||||
|
temp_max: None,
|
||||||
|
load1: None,
|
||||||
|
mem_total_bytes: Some(total),
|
||||||
|
mem_used_bytes: Some(used),
|
||||||
|
disk_free_bytes: Some(disk_free),
|
||||||
|
mem_baseline_mib: None,
|
||||||
|
health_age_secs: Some(3.0),
|
||||||
|
metrics_age_secs: Some(3.0),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A draining node is ineligible, not merely unattractive. The microVM path
|
||||||
|
/// never checked this before: a mission pinned before a drain kept feeding
|
||||||
|
/// VMs to a node an operator had cordoned.
|
||||||
|
#[test]
|
||||||
|
fn a_draining_node_is_ineligible() {
|
||||||
|
let mut row = row_for(nid(1), 61440 * MIB_T, 6144 * MIB_T, 800 * GIB_T);
|
||||||
|
row.status = "draining".into();
|
||||||
|
assert_eq!(from_eval(&row, "tank", 0), Err(Unfit::Draining));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,516 @@
|
|||||||
|
//! The completion gate, moved into the agent's own loop.
|
||||||
|
//!
|
||||||
|
//! Every check this platform has on a phase runs **after** the agent has
|
||||||
|
//! finished: the evaluator judges `done_when`, capture notices that a coding
|
||||||
|
//! phase delivered nothing, and either verdict costs a whole new VM — a fresh
|
||||||
|
//! boot, a fresh inject, and an agent starting again with none of the context
|
||||||
|
//! that got it that far. Meanwhile the documented failure of a long-running
|
||||||
|
//! agent is that it *stops too early*.
|
||||||
|
//!
|
||||||
|
//! Claude Code's `Stop` hook is the seam. **Exit code 2 blocks the stop and
|
||||||
|
//! feeds stderr back to the model as the reason.** Measured, not read off docs —
|
||||||
|
//! an agent told "say hello and do nothing else", whose `Stop` hook exited 2
|
||||||
|
//! saying `evidence.txt` was missing, created `evidence.txt` and then stopped.
|
||||||
|
//!
|
||||||
|
//! # What it may and may not check
|
||||||
|
//!
|
||||||
|
//! Deliberately mechanical: whether the repository changed, and whether a
|
||||||
|
//! command the phase author wrote exits 0. NOT the `done_when` verdict — that is
|
||||||
|
//! an LLM judgement made host-side by a *different provider* on purpose
|
||||||
|
//! ([[evaluator-verification]]), and re-implementing it inside the VM would put
|
||||||
|
//! the agent's own environment in charge of grading the agent, which is the
|
||||||
|
//! correlated failure the independent judge exists to break.
|
||||||
|
//!
|
||||||
|
//! # The cap is load-bearing
|
||||||
|
//!
|
||||||
|
//! A gate with no ceiling turns a stuck agent into a wedged one: it would be
|
||||||
|
//! blocked, retry, be blocked again, and burn the hour-long turn budget instead
|
||||||
|
//! of failing in a way the operator can see. After [`MAX_BLOCKS`] the gate lets
|
||||||
|
//! the agent stop, records that it did, and leaves the verdict to the existing
|
||||||
|
//! post-hoc path — which still runs, unchanged.
|
||||||
|
//!
|
||||||
|
//! # Which hooks exist here
|
||||||
|
//!
|
||||||
|
//! `TaskCompleted` / `TeammateIdle` were the plan's chosen seam. Measured under
|
||||||
|
//! `claude -p`: they never fire, because no team forms in print mode at all.
|
||||||
|
//! `Stop`, `SubagentStop`, `PreToolUse`, `PostToolUse`, `UserPromptSubmit` and
|
||||||
|
//! `SessionStart` do.
|
||||||
|
|
||||||
|
|
||||||
|
/// How many times the gate may refuse a stop before it gives up and lets the
|
||||||
|
/// agent finish. Three is enough for "you wrote nothing" → "you wrote something"
|
||||||
|
/// → "your check passes" without ever approaching the turn budget.
|
||||||
|
pub const MAX_BLOCKS: u32 = 3;
|
||||||
|
|
||||||
|
/// The file the gate writes when it gives up and lets the agent stop with its
|
||||||
|
/// condition still failing.
|
||||||
|
///
|
||||||
|
/// A separate file rather than a line in the log, because the log is not
|
||||||
|
/// parseable for this: a block reason embeds the check's own output, and an
|
||||||
|
/// output line beginning `cap:` would read as a cap release that never happened.
|
||||||
|
///
|
||||||
|
/// It exists because the block COUNT cannot answer the question. Three blocks
|
||||||
|
/// followed by a stop that finally passed, and three blocks followed by a
|
||||||
|
/// release at the cap, both report `blocks: 3` — and they are opposite outcomes.
|
||||||
|
/// Without this, the second one completed the phase green.
|
||||||
|
pub const CAPPED_FILE: &str = "capped";
|
||||||
|
|
||||||
|
/// Where the gate lives in the guest.
|
||||||
|
///
|
||||||
|
/// Under `/root`, never under the repository. Anything written into
|
||||||
|
/// `/mission/repo` is collected and diffed, so a gate script placed there would
|
||||||
|
/// arrive in the user's delivered patch as if an agent had authored it.
|
||||||
|
pub const GATE_DIR: &str = "/root/gate";
|
||||||
|
|
||||||
|
/// What must hold before this phase's agent is allowed to stop.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct StopGate {
|
||||||
|
/// The phase must leave the repository changed. Set for coding phases that
|
||||||
|
/// have not declared `allow_empty` — the same rule
|
||||||
|
/// `empty_delivery_is_a_failure` applies post-hoc, applied while the agent
|
||||||
|
/// can still do something about it.
|
||||||
|
pub require_changes: bool,
|
||||||
|
/// `config.done_when_check`: a shell command, run in the repo, that must
|
||||||
|
/// exit 0. The deterministic half of a completion condition — a command,
|
||||||
|
/// not a judgement.
|
||||||
|
pub check: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StopGate {
|
||||||
|
/// The gate for a phase, or `None` when there is nothing to enforce.
|
||||||
|
///
|
||||||
|
/// `None` matters: installing a hook that can never block would still cost a
|
||||||
|
/// process per stop and would put a `--settings` flag on the command line
|
||||||
|
/// for no reason.
|
||||||
|
pub fn for_phase(kind: &str, config: &serde_json::Value) -> Option<StopGate> {
|
||||||
|
let allow_empty = config.get("allow_empty").and_then(|v| v.as_bool()) == Some(true);
|
||||||
|
let check = config
|
||||||
|
.get("done_when_check")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(str::to_string);
|
||||||
|
let require_changes = kind == "coding" && !allow_empty;
|
||||||
|
if !require_changes && check.is_none() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some(StopGate {
|
||||||
|
require_changes,
|
||||||
|
check,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The same gate, for ONE NODE of a composed run.
|
||||||
|
///
|
||||||
|
/// `require_changes` is a property of the phase, not of every node in it: a
|
||||||
|
/// graph whose second node reviews or verifies is *supposed* to leave the
|
||||||
|
/// tree alone, and a per-node gate would refuse its stop three times for
|
||||||
|
/// doing exactly its job. Dropping it loses nothing, because
|
||||||
|
/// `empty_delivery_is_a_failure` applies the same rule post-hoc to what the
|
||||||
|
/// phase as a whole delivered.
|
||||||
|
///
|
||||||
|
/// That "post-hoc" claim used to be written as covering the `check` too. It
|
||||||
|
/// did not: nothing outside this hook has ever re-run `done_when_check`, so
|
||||||
|
/// a release at [`MAX_BLOCKS`] completed the phase green with the check
|
||||||
|
/// still failing. [`CAPPED_FILE`] is what closes that.
|
||||||
|
///
|
||||||
|
/// A declared `check` DOES apply per node: it is a command the phase author
|
||||||
|
/// wrote, and every stage of the work should satisfy it.
|
||||||
|
pub fn per_node(self) -> Option<StopGate> {
|
||||||
|
self.check.map(|check| StopGate {
|
||||||
|
require_changes: false,
|
||||||
|
check: Some(check),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The hook script, as POSIX `sh`.
|
||||||
|
///
|
||||||
|
/// `repo` and `dir` are parameters rather than the constants above so a test
|
||||||
|
/// can run this script — the real one, not a paraphrase — against a real git
|
||||||
|
/// repository in a temp directory.
|
||||||
|
pub fn script(&self, repo: &str, dir: &str) -> String {
|
||||||
|
let mut s = String::from("#!/bin/sh\n# ClawMates stop gate. Exit 2 refuses the stop.\n");
|
||||||
|
s.push_str(&format!("REPO={}\nGATE={}\nMAX={MAX_BLOCKS}\n", q(repo), q(dir)));
|
||||||
|
s.push_str("N=$(cat \"$GATE/blocks\" 2>/dev/null || echo 0)\nreason=''\n");
|
||||||
|
|
||||||
|
if self.require_changes {
|
||||||
|
// Two questions, because either alone is answerable "no" by a
|
||||||
|
// perfectly good phase: an agent that committed its work leaves a
|
||||||
|
// clean tree, and an agent that did not commit leaves HEAD where it
|
||||||
|
// was. Only both together mean nothing happened.
|
||||||
|
s.push_str(
|
||||||
|
"BASE=$(cat \"$REPO/.git/clawmates-base\" 2>/dev/null || echo '')\n\
|
||||||
|
DIRTY=$(git -C \"$REPO\" status --porcelain 2>/dev/null | head -c 400)\n\
|
||||||
|
HEAD=$(git -C \"$REPO\" rev-parse HEAD 2>/dev/null || echo '')\n\
|
||||||
|
if [ -z \"$DIRTY\" ] && [ -n \"$BASE\" ] && [ \"$HEAD\" = \"$BASE\" ]; then\n\
|
||||||
|
\x20 reason='This phase has changed nothing: the working tree is clean and \
|
||||||
|
HEAD is still the commit you started from. Do the work the task describes \
|
||||||
|
and leave it in the tree. If the task genuinely requires no code change, \
|
||||||
|
say so explicitly in your final message.'\n\
|
||||||
|
fi\n",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(check) = &self.check {
|
||||||
|
s.push_str(&format!(
|
||||||
|
"if [ -z \"$reason\" ]; then\n\
|
||||||
|
\x20 out=$(cd \"$REPO\" && sh -c {} 2>&1); rc=$?\n\
|
||||||
|
\x20 if [ \"$rc\" -ne 0 ]; then\n\
|
||||||
|
\x20 reason=\"This phase's completion check exited $rc, so the work is not \
|
||||||
|
done yet. The check is: {}\n\nIts output:\n$(printf '%s' \"$out\" | tail -c 1500)\"\n\
|
||||||
|
\x20 fi\n\
|
||||||
|
fi\n",
|
||||||
|
q(check),
|
||||||
|
// Inside a double-quoted assignment, so the command text itself
|
||||||
|
// must not carry a `\"` or a `$` that the shell would expand.
|
||||||
|
check.replace('\\', "\\\\").replace('"', "'").replace('$', "\\$"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
s.push_str(
|
||||||
|
"if [ -z \"$reason\" ]; then echo pass >> \"$GATE/log\"; exit 0; fi\n\
|
||||||
|
if [ \"$N\" -ge \"$MAX\" ]; then\n\
|
||||||
|
\x20 echo \"cap: $reason\" >> \"$GATE/log\"\n\
|
||||||
|
\x20 echo 1 > \"$GATE/capped\"\n\
|
||||||
|
\x20 exit 0\n\
|
||||||
|
fi\n\
|
||||||
|
N=$((N+1)); echo \"$N\" > \"$GATE/blocks\"\n\
|
||||||
|
echo \"block $N: $reason\" >> \"$GATE/log\"\n\
|
||||||
|
printf '%s\\n' \"$reason\" >&2\n\
|
||||||
|
exit 2\n",
|
||||||
|
);
|
||||||
|
s
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One shell command that writes the gate SCRIPT into the guest.
|
||||||
|
///
|
||||||
|
/// It deliberately does NOT write `settings.json`. It used to, and it wrote
|
||||||
|
/// the whole document — so the moment a second feature needed a hook, the
|
||||||
|
/// later writer would silently erase this one. The composed document is
|
||||||
|
/// built in exactly one place: [`crate::vm_tool_tap::guest_settings`].
|
||||||
|
///
|
||||||
|
/// Written by `printf` through an exec rather than injected as part of the
|
||||||
|
/// tar: the tar lands in `/mission/repo`, which is exactly where this must
|
||||||
|
/// not be.
|
||||||
|
pub fn install_command(&self, repo: &str, dir: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"mkdir -p {d} && rm -f {d}/blocks {d}/log {d}/capped \
|
||||||
|
&& printf '%s' {script} > {d}/stop-gate.sh \
|
||||||
|
&& chmod +x {d}/stop-gate.sh",
|
||||||
|
d = dir,
|
||||||
|
script = q(&self.script(repo, dir)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Single-quote for `sh`. Same rule as `microvm_executor::shell_quote`, kept
|
||||||
|
/// local so this module has no dependency on the executor it is used by.
|
||||||
|
fn q(s: &str) -> String {
|
||||||
|
format!("'{}'", s.replace('\'', r"'\''"))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use serde_json::json;
|
||||||
|
use std::path::Path;
|
||||||
|
use std::process::Command;
|
||||||
|
|
||||||
|
fn sh(script: &str, dir: &Path) -> std::process::Output {
|
||||||
|
let path = dir.join("stop-gate.sh");
|
||||||
|
std::fs::write(&path, script).unwrap();
|
||||||
|
Command::new("sh").arg(&path).output().expect("run the gate")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A git repo with one commit and the clone-point marker the real checkout
|
||||||
|
/// carries (`mission_workspace::record_base_commit` writes it).
|
||||||
|
fn repo_with_base(root: &Path) -> std::path::PathBuf {
|
||||||
|
let repo = root.join("repo");
|
||||||
|
std::fs::create_dir_all(&repo).unwrap();
|
||||||
|
let git = |args: &[&str]| {
|
||||||
|
let o = Command::new("git")
|
||||||
|
.arg("-C")
|
||||||
|
.arg(&repo)
|
||||||
|
.args(args)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
assert!(o.status.success(), "git {args:?}: {:?}", o);
|
||||||
|
};
|
||||||
|
git(&["init", "--quiet"]);
|
||||||
|
git(&["config", "user.email", "t@t"]);
|
||||||
|
git(&["config", "user.name", "T"]);
|
||||||
|
std::fs::write(repo.join("README.md"), "base\n").unwrap();
|
||||||
|
git(&["add", "."]);
|
||||||
|
git(&["commit", "--quiet", "-m", "base"]);
|
||||||
|
let head = Command::new("git")
|
||||||
|
.arg("-C")
|
||||||
|
.arg(&repo)
|
||||||
|
.args(["rev-parse", "HEAD"])
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
std::fs::write(
|
||||||
|
repo.join(".git/clawmates-base"),
|
||||||
|
String::from_utf8_lossy(&head.stdout).trim(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
repo
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The failure this exists for: an agent that stops having written nothing.
|
||||||
|
/// Post-hoc that costs a whole new VM; here it costs one sentence.
|
||||||
|
#[test]
|
||||||
|
fn an_agent_that_changed_nothing_is_not_allowed_to_stop() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let repo = repo_with_base(tmp.path());
|
||||||
|
let gate = StopGate {
|
||||||
|
require_changes: true,
|
||||||
|
check: None,
|
||||||
|
};
|
||||||
|
let script = gate.script(&repo.display().to_string(), &tmp.path().display().to_string());
|
||||||
|
|
||||||
|
let out = sh(&script, tmp.path());
|
||||||
|
assert_eq!(out.status.code(), Some(2), "the stop must be refused");
|
||||||
|
let why = String::from_utf8_lossy(&out.stderr);
|
||||||
|
assert!(why.contains("changed nothing"), "{why}");
|
||||||
|
|
||||||
|
// Uncommitted work counts — the usual case, since the agent is told to
|
||||||
|
// leave its work in the tree rather than commit it.
|
||||||
|
std::fs::write(repo.join("new.rs"), "fn done() {}\n").unwrap();
|
||||||
|
let out = sh(&script, tmp.path());
|
||||||
|
assert_eq!(out.status.code(), Some(0), "{:?}", out);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The gate gives up after [`MAX_BLOCKS`] and lets the agent stop — and it
|
||||||
|
/// must LEAVE A MARK when it does. Nothing outside this hook ever runs a
|
||||||
|
/// `done_when_check`, so a silent release completed the phase green with its
|
||||||
|
/// condition still failing.
|
||||||
|
///
|
||||||
|
/// The two files say different things and both are needed: `blocks` reaches
|
||||||
|
/// 3 in this test AND in a run where the agent got it right on the fourth
|
||||||
|
/// try, so the count alone cannot tell success from surrender.
|
||||||
|
#[test]
|
||||||
|
fn a_gate_that_gives_up_records_that_it_gave_up() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let dir = tmp.path().display().to_string();
|
||||||
|
let repo = repo_with_base(tmp.path());
|
||||||
|
let gate = StopGate {
|
||||||
|
require_changes: false,
|
||||||
|
check: Some("exit 1".into()),
|
||||||
|
};
|
||||||
|
let script = gate.script(&repo.display().to_string(), &dir);
|
||||||
|
|
||||||
|
for n in 1..=MAX_BLOCKS {
|
||||||
|
let out = sh(&script, tmp.path());
|
||||||
|
assert_eq!(out.status.code(), Some(2), "block {n} must refuse the stop");
|
||||||
|
assert!(
|
||||||
|
!tmp.path().join(CAPPED_FILE).exists(),
|
||||||
|
"the cap mark must not appear while the gate is still blocking"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// One more stop: the gate is out of blocks and must let the agent go.
|
||||||
|
let out = sh(&script, tmp.path());
|
||||||
|
assert_eq!(out.status.code(), Some(0), "at the cap the stop is allowed");
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(tmp.path().join(CAPPED_FILE))
|
||||||
|
.unwrap()
|
||||||
|
.trim(),
|
||||||
|
"1",
|
||||||
|
"the release must be recorded, or nothing downstream can see it"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The negative control for the mark: a gate whose check PASSES releases the
|
||||||
|
/// agent too, and that release must not be recorded as a surrender. Without
|
||||||
|
/// this, "always write the file" would pass the test above and fail every
|
||||||
|
/// healthy phase in production.
|
||||||
|
#[test]
|
||||||
|
fn a_gate_that_is_satisfied_leaves_no_cap_mark() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let repo = repo_with_base(tmp.path());
|
||||||
|
let gate = StopGate {
|
||||||
|
require_changes: false,
|
||||||
|
check: Some("true".into()),
|
||||||
|
};
|
||||||
|
let script = gate.script(&repo.display().to_string(), &tmp.path().display().to_string());
|
||||||
|
|
||||||
|
let out = sh(&script, tmp.path());
|
||||||
|
assert_eq!(out.status.code(), Some(0));
|
||||||
|
assert!(
|
||||||
|
!tmp.path().join(CAPPED_FILE).exists(),
|
||||||
|
"a satisfied gate must not look like one that gave up"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// And committed work counts too. An agent that committed leaves a CLEAN
|
||||||
|
/// tree, so a gate that only looked at `git status` would refuse the stop of
|
||||||
|
/// a phase that had done everything asked of it.
|
||||||
|
#[test]
|
||||||
|
fn work_the_agent_committed_satisfies_the_gate() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let repo = repo_with_base(tmp.path());
|
||||||
|
std::fs::write(repo.join("new.rs"), "fn done() {}\n").unwrap();
|
||||||
|
for args in [vec!["add", "."], vec!["commit", "--quiet", "-m", "work"]] {
|
||||||
|
Command::new("git")
|
||||||
|
.arg("-C")
|
||||||
|
.arg(&repo)
|
||||||
|
.args(&args)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let gate = StopGate {
|
||||||
|
require_changes: true,
|
||||||
|
check: None,
|
||||||
|
};
|
||||||
|
let out = sh(
|
||||||
|
&gate.script(&repo.display().to_string(), &tmp.path().display().to_string()),
|
||||||
|
tmp.path(),
|
||||||
|
);
|
||||||
|
assert_eq!(out.status.code(), Some(0), "{:?}", out);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The cap. Without it a stuck agent is blocked, retries, is blocked again,
|
||||||
|
/// and spends the whole hour-long turn budget instead of failing where an
|
||||||
|
/// operator can see it.
|
||||||
|
#[test]
|
||||||
|
fn the_gate_gives_up_after_the_cap_and_says_so() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let repo = repo_with_base(tmp.path());
|
||||||
|
let gate = StopGate {
|
||||||
|
require_changes: true,
|
||||||
|
check: None,
|
||||||
|
};
|
||||||
|
let script = gate.script(&repo.display().to_string(), &tmp.path().display().to_string());
|
||||||
|
|
||||||
|
for i in 1..=MAX_BLOCKS {
|
||||||
|
assert_eq!(
|
||||||
|
sh(&script, tmp.path()).status.code(),
|
||||||
|
Some(2),
|
||||||
|
"block {i} of {MAX_BLOCKS}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
sh(&script, tmp.path()).status.code(),
|
||||||
|
Some(0),
|
||||||
|
"past the cap the agent must be allowed to stop"
|
||||||
|
);
|
||||||
|
let log = std::fs::read_to_string(tmp.path().join("log")).unwrap();
|
||||||
|
assert!(log.contains("cap:"), "giving up is recorded: {log}");
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read_to_string(tmp.path().join("blocks"))
|
||||||
|
.unwrap()
|
||||||
|
.trim(),
|
||||||
|
MAX_BLOCKS.to_string(),
|
||||||
|
"and the count is exact, so the host can report it"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A phase-declared check runs in the repo, and its OUTPUT comes back — a
|
||||||
|
/// gate that said only "the check failed" would send the agent guessing.
|
||||||
|
#[test]
|
||||||
|
fn a_declared_check_must_pass_and_its_output_is_the_feedback() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let repo = repo_with_base(tmp.path());
|
||||||
|
let gate = StopGate {
|
||||||
|
require_changes: false,
|
||||||
|
check: Some("test -f wanted.txt || { echo 'wanted.txt is missing'; exit 3; }".into()),
|
||||||
|
};
|
||||||
|
let script = gate.script(&repo.display().to_string(), &tmp.path().display().to_string());
|
||||||
|
|
||||||
|
let out = sh(&script, tmp.path());
|
||||||
|
assert_eq!(out.status.code(), Some(2));
|
||||||
|
let why = String::from_utf8_lossy(&out.stderr);
|
||||||
|
assert!(why.contains("exited 3"), "{why}");
|
||||||
|
assert!(why.contains("wanted.txt is missing"), "{why}");
|
||||||
|
|
||||||
|
std::fs::write(repo.join("wanted.txt"), "here\n").unwrap();
|
||||||
|
assert_eq!(sh(&script, tmp.path()).status.code(), Some(0));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A check with quotes, `$` and apostrophes is ordinary. It travels through
|
||||||
|
/// `sh -c` inside a script that itself travels through `sh -c` to reach the
|
||||||
|
/// guest, and a quoting bug at either layer would run something else.
|
||||||
|
#[test]
|
||||||
|
fn a_check_with_shell_metacharacters_survives_both_layers() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let repo = repo_with_base(tmp.path());
|
||||||
|
std::fs::write(repo.join("it's here.txt"), "x\n").unwrap();
|
||||||
|
let gate = StopGate {
|
||||||
|
require_changes: false,
|
||||||
|
check: Some("test -f \"it's here.txt\" && echo $HOME > /dev/null".into()),
|
||||||
|
};
|
||||||
|
let out = sh(
|
||||||
|
&gate.script(&repo.display().to_string(), &tmp.path().display().to_string()),
|
||||||
|
tmp.path(),
|
||||||
|
);
|
||||||
|
assert_eq!(out.status.code(), Some(0), "{:?}", out);
|
||||||
|
// And the install command it is embedded in is still one shell argument.
|
||||||
|
let install = gate.install_command("/mission/repo", GATE_DIR);
|
||||||
|
assert!(install.contains("stop-gate.sh"), "{install}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Nothing the gate writes may land under the repository: `/mission/repo` is
|
||||||
|
/// collected and diffed, so a file there arrives in the user's patch as if
|
||||||
|
/// an agent had written it.
|
||||||
|
#[test]
|
||||||
|
fn the_gate_never_writes_into_the_delivered_tree() {
|
||||||
|
let gate = StopGate {
|
||||||
|
require_changes: true,
|
||||||
|
check: Some("cargo test".into()),
|
||||||
|
};
|
||||||
|
assert!(GATE_DIR.starts_with("/root/"), "{GATE_DIR}");
|
||||||
|
let install = gate.install_command("/mission/repo", GATE_DIR);
|
||||||
|
for write in ["> /mission/repo", "/mission/repo/stop", "/mission/repo/.claude"] {
|
||||||
|
assert!(!install.contains(write), "{install}");
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
crate::vm_tool_tap::guest_settings(Some(GATE_DIR), None)["hooks"]["Stop"][0]["hooks"]
|
||||||
|
[0]["command"],
|
||||||
|
json!("/root/gate/stop-gate.sh")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A composed run's nodes must not each be held to "this phase changed
|
||||||
|
/// something". The graph's verifier node changes nothing BY DESIGN, and a
|
||||||
|
/// per-node gate would refuse its stop until the cap — three wasted agent
|
||||||
|
/// turns for doing its job correctly.
|
||||||
|
#[test]
|
||||||
|
fn a_composed_node_is_not_held_to_the_whole_phases_delivery() {
|
||||||
|
let phase = StopGate::for_phase("coding", &json!({})).unwrap();
|
||||||
|
assert!(phase.require_changes);
|
||||||
|
assert!(
|
||||||
|
phase.per_node().is_none(),
|
||||||
|
"with nothing but the delivery rule, a node has no gate at all"
|
||||||
|
);
|
||||||
|
|
||||||
|
let with_check =
|
||||||
|
StopGate::for_phase("coding", &json!({ "done_when_check": "cargo test" })).unwrap();
|
||||||
|
let node = with_check.per_node().expect("the declared check still applies");
|
||||||
|
assert!(!node.require_changes);
|
||||||
|
assert_eq!(node.check.as_deref(), Some("cargo test"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A gate with nothing to enforce must not be installed at all — a hook that
|
||||||
|
/// can never block still costs a process per stop and a flag on the command
|
||||||
|
/// line.
|
||||||
|
#[test]
|
||||||
|
fn a_phase_with_nothing_to_enforce_gets_no_gate() {
|
||||||
|
let none = json!({});
|
||||||
|
assert!(StopGate::for_phase("research", &none).is_none());
|
||||||
|
assert!(StopGate::for_phase("coding", &json!({ "allow_empty": true })).is_none());
|
||||||
|
|
||||||
|
let coding = StopGate::for_phase("coding", &none).expect("a coding phase must deliver");
|
||||||
|
assert!(coding.require_changes);
|
||||||
|
assert!(coding.check.is_none());
|
||||||
|
|
||||||
|
// A declared check applies to any kind, including one that is allowed to
|
||||||
|
// change nothing — a verification phase's whole job is that check.
|
||||||
|
let verify = StopGate::for_phase(
|
||||||
|
"research",
|
||||||
|
&json!({ "allow_empty": true, "done_when_check": " ./verify.sh " }),
|
||||||
|
)
|
||||||
|
.expect("a declared check is a gate on its own");
|
||||||
|
assert!(!verify.require_changes);
|
||||||
|
assert_eq!(verify.check.as_deref(), Some("./verify.sh"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,274 @@
|
|||||||
|
//! What the agent did inside a microVM, taken from Claude Code's own hooks.
|
||||||
|
//!
|
||||||
|
//! The microVM tier had no action channel at all: a phase ran, a diff came
|
||||||
|
//! back, and everything between was invisible. The seam is the same one
|
||||||
|
//! [`crate::vm_stop_gate`] proved works in this image — `PostToolUse` fires
|
||||||
|
//! under `claude -p`, measured, not read off documentation.
|
||||||
|
//!
|
||||||
|
//! # The observer must not become a participant
|
||||||
|
//!
|
||||||
|
//! The hook `exit 0`s unconditionally. A `PostToolUse` hook that exits non-zero
|
||||||
|
//! feeds its stderr back to the model, so a tap with a bug would start
|
||||||
|
//! *instructing* the agent it exists to watch — and the resulting transcript
|
||||||
|
//! would look like a model that lost the plot rather than a broken hook.
|
||||||
|
//!
|
||||||
|
//! # Never inside the repository
|
||||||
|
//!
|
||||||
|
//! Everything lives under `/root`. `/mission/repo` is collected and diffed, so
|
||||||
|
//! a tap file written there would arrive in the user's delivered patch as
|
||||||
|
//! though an agent had authored it — the same rule, and the same reason, as the
|
||||||
|
//! stop gate's [`crate::vm_stop_gate::GATE_DIR`].
|
||||||
|
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
|
||||||
|
/// Where the tap writes in the guest. Under `/root`, never the repo.
|
||||||
|
pub const TAP_DIR: &str = "/root/tap";
|
||||||
|
|
||||||
|
/// The file the hook appends to, one JSON object per line.
|
||||||
|
pub const TAP_FILE: &str = "/root/tap/tools.jsonl";
|
||||||
|
|
||||||
|
/// The single settings document the guest agent runs with.
|
||||||
|
///
|
||||||
|
/// One path, because there is only ever one writer — see [`guest_settings`].
|
||||||
|
pub const SETTINGS_PATH: &str = "/root/guest-settings.json";
|
||||||
|
|
||||||
|
/// Read the tap out of the guest, before collection destroys the VM.
|
||||||
|
///
|
||||||
|
/// `|| true` so a phase whose agent called no tools — or where the hook never
|
||||||
|
/// fired — reads as empty rather than as a failed probe. The difference between
|
||||||
|
/// those two is the histogram in the log, not an error here.
|
||||||
|
pub const DRAIN_PROBE: &str = "cat /root/tap/tools.jsonl 2>/dev/null || true";
|
||||||
|
|
||||||
|
/// One observed tool call.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct Observed {
|
||||||
|
pub tool: String,
|
||||||
|
/// The path the tool's **input** named, if any. From JSON, never prose.
|
||||||
|
pub path: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The hook script. Copies stdin verbatim to the tap file and gets out of the
|
||||||
|
/// way.
|
||||||
|
///
|
||||||
|
/// The parsing happens host-side, on purpose: a `jq` or `sed` pipeline in the
|
||||||
|
/// guest would need the tool's JSON schema baked into a shell script, inside an
|
||||||
|
/// image we do not rebuild for a parser change, with no way to tell a parse
|
||||||
|
/// failure from a quiet turn.
|
||||||
|
pub fn hook_script(dir: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"#!/bin/sh\n\
|
||||||
|
# The tool tap. See cm-api/src/vm_tool_tap.rs.\n\
|
||||||
|
mkdir -p {dir} 2>/dev/null\n\
|
||||||
|
# `cat` of stdin, appended whole. One JSON object per line, because\n\
|
||||||
|
# Claude Code hands the hook one event per invocation.\n\
|
||||||
|
cat >> {dir}/tools.jsonl 2>/dev/null\n\
|
||||||
|
printf '\\n' >> {dir}/tools.jsonl 2>/dev/null\n\
|
||||||
|
# ALWAYS zero. A non-zero PostToolUse hook talks back to the model.\n\
|
||||||
|
exit 0\n"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The settings document for the guest, carrying **every** hook at once.
|
||||||
|
///
|
||||||
|
/// This function exists because the alternative — each feature writing its own
|
||||||
|
/// `settings.json` — is a silent clobber. The stop gate wrote the whole
|
||||||
|
/// document; a tap that did the same would erase the gate, and a coding phase
|
||||||
|
/// would then complete having written nothing, which is the exact failure the
|
||||||
|
/// gate exists to catch. One writer, one document, one test that both hooks
|
||||||
|
/// survive it.
|
||||||
|
///
|
||||||
|
/// `None` for either half means that hook is simply absent.
|
||||||
|
pub fn guest_settings(gate_dir: Option<&str>, tap_dir: Option<&str>) -> Value {
|
||||||
|
let mut hooks = serde_json::Map::new();
|
||||||
|
if let Some(dir) = gate_dir {
|
||||||
|
hooks.insert(
|
||||||
|
"Stop".into(),
|
||||||
|
json!([{ "hooks": [{ "type": "command", "command": format!("{dir}/stop-gate.sh") }] }]),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if let Some(dir) = tap_dir {
|
||||||
|
hooks.insert(
|
||||||
|
"PostToolUse".into(),
|
||||||
|
json!([{ "hooks": [{ "type": "command", "command": format!("{dir}/tap.sh") }] }]),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
json!({ "hooks": Value::Object(hooks) })
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One shell command that installs the tap.
|
||||||
|
///
|
||||||
|
/// Written by `printf` through an exec rather than injected with the workspace
|
||||||
|
/// tar: the tar lands in `/mission/repo`, which is exactly where this must not.
|
||||||
|
pub fn install_command(dir: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"mkdir -p {dir} && rm -f {dir}/tools.jsonl \
|
||||||
|
&& printf '%s' {script} > {dir}/tap.sh && chmod +x {dir}/tap.sh",
|
||||||
|
dir = dir,
|
||||||
|
script = q(&hook_script(dir)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Write the composed settings document.
|
||||||
|
pub fn settings_command(path: &str, settings: &Value) -> String {
|
||||||
|
format!("printf '%s' {} > {path}", q(&settings.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parse a drained tap.
|
||||||
|
///
|
||||||
|
/// Tolerant by construction: the file is appended to by a shell hook in a VM
|
||||||
|
/// that may be killed mid-write, so a truncated last line is expected and is
|
||||||
|
/// skipped rather than failing the whole drain. Losing the last tool call of a
|
||||||
|
/// phase costs one orb; losing all of them because of it would cost the tier.
|
||||||
|
pub fn parse(raw: &str) -> Vec<Observed> {
|
||||||
|
raw.lines()
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|l| !l.is_empty())
|
||||||
|
.filter_map(|line| {
|
||||||
|
let v: Value = serde_json::from_str(line).ok()?;
|
||||||
|
// Only tool events. The same hook file would carry others if the
|
||||||
|
// settings ever install one, and a `hook_event_name` we do not
|
||||||
|
// recognise must not be read as a tool named "".
|
||||||
|
let tool = v
|
||||||
|
.get("tool_name")
|
||||||
|
.or_else(|| v.get("toolName"))
|
||||||
|
.and_then(Value::as_str)?
|
||||||
|
.trim()
|
||||||
|
.to_string();
|
||||||
|
if tool.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let input = v
|
||||||
|
.get("tool_input")
|
||||||
|
.or_else(|| v.get("toolInput"))
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or(Value::Null);
|
||||||
|
Some(Observed {
|
||||||
|
path: crate::mission_events::tool_path(&input),
|
||||||
|
tool,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Single-quote for `sh`. Local copy, same rule as the stop gate's — these two
|
||||||
|
/// modules deliberately share no code, so neither can break the other.
|
||||||
|
fn q(s: &str) -> String {
|
||||||
|
format!("'{}'", s.replace('\'', r"'\''"))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// The gate and the tap must BOTH survive one settings document.
|
||||||
|
///
|
||||||
|
/// This is the whole reason `guest_settings` exists. Two writers each
|
||||||
|
/// producing a whole `settings.json` is not a merge conflict — the second
|
||||||
|
/// simply wins, no error, and the loser's hook never runs. When the loser is
|
||||||
|
/// the stop gate, a coding phase completes having written nothing: the exact
|
||||||
|
/// failure the gate was built to catch.
|
||||||
|
#[test]
|
||||||
|
fn both_hooks_survive_one_settings_document() {
|
||||||
|
let s = guest_settings(Some("/root/gate"), Some(TAP_DIR));
|
||||||
|
let hooks = s.get("hooks").expect("hooks");
|
||||||
|
assert_eq!(
|
||||||
|
hooks["Stop"][0]["hooks"][0]["command"],
|
||||||
|
json!("/root/gate/stop-gate.sh"),
|
||||||
|
"the stop gate must survive the tap being installed"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
hooks["PostToolUse"][0]["hooks"][0]["command"],
|
||||||
|
json!("/root/tap/tap.sh")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Either half absent leaves the other exactly as it was.
|
||||||
|
#[test]
|
||||||
|
fn one_hook_alone_is_a_valid_document() {
|
||||||
|
let gate_only = guest_settings(Some("/root/gate"), None);
|
||||||
|
assert!(gate_only["hooks"].get("Stop").is_some());
|
||||||
|
assert!(gate_only["hooks"].get("PostToolUse").is_none());
|
||||||
|
|
||||||
|
let tap_only = guest_settings(None, Some(TAP_DIR));
|
||||||
|
assert!(tap_only["hooks"].get("Stop").is_none());
|
||||||
|
assert!(tap_only["hooks"].get("PostToolUse").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The observer must never talk back to the model.
|
||||||
|
#[test]
|
||||||
|
fn the_hook_always_exits_zero() {
|
||||||
|
let s = hook_script(TAP_DIR);
|
||||||
|
assert!(s.contains("exit 0"));
|
||||||
|
// No conditional exits at all: a `PostToolUse` hook that exits non-zero
|
||||||
|
// feeds stderr back to the agent, so the tap would become an instruction.
|
||||||
|
assert!(!s.contains("exit 1") && !s.contains("exit 2"), "{s}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Nothing the tap writes may land in the delivered tree.
|
||||||
|
#[test]
|
||||||
|
fn the_tap_never_writes_into_the_repository() {
|
||||||
|
assert!(TAP_DIR.starts_with("/root/"));
|
||||||
|
assert!(TAP_FILE.starts_with("/root/"));
|
||||||
|
assert!(SETTINGS_PATH.starts_with("/root/"));
|
||||||
|
let cmd = install_command(TAP_DIR);
|
||||||
|
assert!(!cmd.contains("/mission/repo"), "{cmd}");
|
||||||
|
assert!(!hook_script(TAP_DIR).contains("/mission/repo"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A real `PostToolUse` payload gives up its tool and its path — and a
|
||||||
|
/// truncated final line does not take the rest of the phase with it.
|
||||||
|
#[test]
|
||||||
|
fn a_drained_tap_parses_and_tolerates_a_torn_last_line() {
|
||||||
|
let raw = concat!(
|
||||||
|
r#"{"hook_event_name":"PostToolUse","tool_name":"Edit","#,
|
||||||
|
r#""tool_input":{"file_path":"/mission/repo/src/a.rs"}}"#,
|
||||||
|
"\n",
|
||||||
|
r#"{"hook_event_name":"PostToolUse","tool_name":"Bash","tool_input":{"command":"ls"}}"#,
|
||||||
|
"\n",
|
||||||
|
"\n",
|
||||||
|
// The VM was destroyed mid-write.
|
||||||
|
r#"{"hook_event_name":"PostToolUse","tool_name":"Wri"#,
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
parse(raw),
|
||||||
|
vec![
|
||||||
|
Observed { tool: "Edit".into(), path: Some("/mission/repo/src/a.rs".into()) },
|
||||||
|
Observed { tool: "Bash".into(), path: None },
|
||||||
|
]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Exactly one place in the tree writes the guest settings document.
|
||||||
|
///
|
||||||
|
/// The unit test above proves `guest_settings` composes correctly; it says
|
||||||
|
/// nothing about whether anyone bypasses it. A second `> …settings.json`
|
||||||
|
/// anywhere is the silent clobber itself, and it would pass every other
|
||||||
|
/// test in this file.
|
||||||
|
#[test]
|
||||||
|
fn nothing_else_writes_the_guest_settings() {
|
||||||
|
for (name, src) in [
|
||||||
|
("vm_stop_gate.rs", include_str!("vm_stop_gate.rs")),
|
||||||
|
("microvm_executor.rs", include_str!("microvm_executor.rs")),
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
!src.contains("> {d}/settings.json") && !src.contains("settings.json\","),
|
||||||
|
"{name} writes a settings document of its own; compose it through \
|
||||||
|
vm_tool_tap::guest_settings instead"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// And the one legitimate writer is this module's own helper.
|
||||||
|
let exec = include_str!("microvm_executor.rs");
|
||||||
|
assert_eq!(
|
||||||
|
exec.matches("vm_tool_tap::settings_command").count(),
|
||||||
|
1,
|
||||||
|
"the settings document must be written exactly once per turn"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// An event that is not a tool call is not a tool named "".
|
||||||
|
#[test]
|
||||||
|
fn a_non_tool_event_is_skipped() {
|
||||||
|
assert!(parse(r#"{"hook_event_name":"SessionStart","session_id":"x"}"#).is_empty());
|
||||||
|
assert!(parse(r#"{"tool_name":" ","tool_input":{}}"#).is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
//! Auto-merge against real git repositories.
|
||||||
|
//!
|
||||||
|
//! The rule is measured from the diff, so it has to be tested against real
|
||||||
|
//! diffs — a unit test on the classifier alone would not catch a wrong
|
||||||
|
//! revision range.
|
||||||
|
|
||||||
|
use cm_api::auto_merge::{self, MergePolicy};
|
||||||
|
|
||||||
|
fn git(repo: &std::path::Path, args: &[&str]) {
|
||||||
|
let out = std::process::Command::new("git")
|
||||||
|
.arg("-C")
|
||||||
|
.arg(repo)
|
||||||
|
.args(args)
|
||||||
|
.env("GIT_AUTHOR_NAME", "T")
|
||||||
|
.env("GIT_AUTHOR_EMAIL", "[email protected]")
|
||||||
|
.env("GIT_COMMITTER_NAME", "T")
|
||||||
|
.env("GIT_COMMITTER_EMAIL", "[email protected]")
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
out.status.success(),
|
||||||
|
"git {args:?}: {}",
|
||||||
|
String::from_utf8_lossy(&out.stderr)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns (work checkout, bare remote path).
|
||||||
|
fn seed() -> (tempfile::TempDir, std::path::PathBuf, std::path::PathBuf) {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let remote = tmp.path().join("remote.git");
|
||||||
|
let work = tmp.path().join("work");
|
||||||
|
std::process::Command::new("git")
|
||||||
|
.args(["init", "--quiet", "--bare"])
|
||||||
|
.arg(&remote)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
std::fs::create_dir_all(&work).unwrap();
|
||||||
|
git(&work, &["init", "--quiet"]);
|
||||||
|
git(&work, &["checkout", "-q", "-B", "main"]);
|
||||||
|
std::fs::write(work.join("README.md"), "# vault\n").unwrap();
|
||||||
|
git(&work, &["add", "."]);
|
||||||
|
git(&work, &["commit", "--quiet", "-m", "base"]);
|
||||||
|
git(&work, &["remote", "add", "origin", remote.to_str().unwrap()]);
|
||||||
|
git(&work, &["push", "--quiet", "origin", "main"]);
|
||||||
|
(tmp, work, remote)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_purely_additive_branch_is_merged() {
|
||||||
|
let (_tmp, work, remote) = seed();
|
||||||
|
git(&work, &["checkout", "-q", "-B", "lib/add"]);
|
||||||
|
std::fs::create_dir_all(work.join("60 Papers")).unwrap();
|
||||||
|
std::fs::write(work.join("60 Papers/a.md"), "# paper\n").unwrap();
|
||||||
|
git(&work, &["add", "."]);
|
||||||
|
git(&work, &["commit", "--quiet", "-m", "add paper"]);
|
||||||
|
git(&work, &["push", "--quiet", "origin", "lib/add"]);
|
||||||
|
|
||||||
|
let out = auto_merge::try_merge(
|
||||||
|
&work, remote.to_str().unwrap(), "lib/add", "main",
|
||||||
|
MergePolicy::AdditiveOnly, true,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(out.merged, "should have merged: {}", out.reason);
|
||||||
|
|
||||||
|
// The note must really be on main at the remote, not just locally.
|
||||||
|
let ls = std::process::Command::new("git")
|
||||||
|
.arg("-C").arg(&remote)
|
||||||
|
.args(["ls-tree", "--name-only", "-r", "main"])
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
let listed = String::from_utf8_lossy(&ls.stdout);
|
||||||
|
assert!(listed.contains("60 Papers/a.md"), "remote main: {listed}");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The load-bearing refusal: a branch that rewrites an existing file must be
|
||||||
|
/// left for a human even though its mission type is allowed to auto-merge.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_branch_that_modifies_an_existing_file_is_refused() {
|
||||||
|
let (_tmp, work, remote) = seed();
|
||||||
|
git(&work, &["checkout", "-q", "-B", "lib/bad"]);
|
||||||
|
std::fs::create_dir_all(work.join("60 Papers")).unwrap();
|
||||||
|
std::fs::write(work.join("60 Papers/a.md"), "# paper\n").unwrap();
|
||||||
|
// …and clobbers a hand-written file.
|
||||||
|
std::fs::write(work.join("README.md"), "# REWRITTEN BY A MACHINE\n").unwrap();
|
||||||
|
git(&work, &["add", "."]);
|
||||||
|
git(&work, &["commit", "--quiet", "-m", "add + clobber"]);
|
||||||
|
git(&work, &["push", "--quiet", "origin", "lib/bad"]);
|
||||||
|
|
||||||
|
let out = auto_merge::try_merge(
|
||||||
|
&work, remote.to_str().unwrap(), "lib/bad", "main",
|
||||||
|
MergePolicy::AdditiveOnly, true,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!out.merged, "must refuse a non-additive branch");
|
||||||
|
assert!(out.reason.contains("not additive"), "reason: {}", out.reason);
|
||||||
|
|
||||||
|
let show = std::process::Command::new("git")
|
||||||
|
.arg("-C").arg(&remote)
|
||||||
|
.args(["show", "main:README.md"])
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
String::from_utf8_lossy(&show.stdout),
|
||||||
|
"# vault\n",
|
||||||
|
"the hand-written file must be untouched on main"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unverified_work_is_never_merged() {
|
||||||
|
let (_tmp, work, remote) = seed();
|
||||||
|
git(&work, &["checkout", "-q", "-B", "lib/unverified"]);
|
||||||
|
std::fs::create_dir_all(work.join("60 Papers")).unwrap();
|
||||||
|
std::fs::write(work.join("60 Papers/a.md"), "# paper\n").unwrap();
|
||||||
|
git(&work, &["add", "."]);
|
||||||
|
git(&work, &["commit", "--quiet", "-m", "add"]);
|
||||||
|
git(&work, &["push", "--quiet", "origin", "lib/unverified"]);
|
||||||
|
|
||||||
|
let out = auto_merge::try_merge(
|
||||||
|
&work, remote.to_str().unwrap(), "lib/unverified", "main",
|
||||||
|
MergePolicy::AdditiveOnly, false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!out.merged);
|
||||||
|
assert!(out.reason.contains("did not verify"), "reason: {}", out.reason);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_never_policy_branch_is_left_alone() {
|
||||||
|
let (_tmp, work, remote) = seed();
|
||||||
|
git(&work, &["checkout", "-q", "-B", "code/change"]);
|
||||||
|
std::fs::write(work.join("new.rs"), "fn main() {}\n").unwrap();
|
||||||
|
git(&work, &["add", "."]);
|
||||||
|
git(&work, &["commit", "--quiet", "-m", "code"]);
|
||||||
|
git(&work, &["push", "--quiet", "origin", "code/change"]);
|
||||||
|
|
||||||
|
let out = auto_merge::try_merge(
|
||||||
|
&work, remote.to_str().unwrap(), "code/change", "main",
|
||||||
|
MergePolicy::Never, true,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(!out.merged, "code must never auto-merge");
|
||||||
|
}
|
||||||
@@ -19,6 +19,24 @@ async fn workspace(pool: &sqlx::PgPool) -> Uuid {
|
|||||||
ws
|
ws
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A real mission row. `corpus_items.mission_id` has a foreign key, which is
|
||||||
|
/// deliberate: attribution to a mission that does not exist is not
|
||||||
|
/// attribution. The first version of the test below used a bare UUID and was
|
||||||
|
/// correctly rejected.
|
||||||
|
async fn mission(pool: &sqlx::PgPool, ws: Uuid) -> Uuid {
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO missions (id, workspace_id, title, template_kind, schedule, status, config)
|
||||||
|
VALUES ($1,$2,'library','research_only','{}'::jsonb,'running','{}'::jsonb)",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(ws)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
id
|
||||||
|
}
|
||||||
|
|
||||||
fn seed_vault(root: &std::path::Path) {
|
fn seed_vault(root: &std::path::Path) {
|
||||||
std::fs::create_dir_all(root.join("50 APESS 2026/Lectures")).unwrap();
|
std::fs::create_dir_all(root.join("50 APESS 2026/Lectures")).unwrap();
|
||||||
std::fs::create_dir_all(root.join("Repos")).unwrap();
|
std::fs::create_dir_all(root.join("Repos")).unwrap();
|
||||||
@@ -226,3 +244,43 @@ async fn live_arxiv_search_and_fetch() {
|
|||||||
assert!(pdf.starts_with(b"%PDF"));
|
assert!(pdf.starts_with(b"%PDF"));
|
||||||
assert!(pdf.len() > 10_000, "suspiciously small pdf: {}", pdf.len());
|
assert!(pdf.len() > 10_000, "suspiciously small pdf: {}", pdf.len());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A rerun must not be able to claim credit for work an earlier run did.
|
||||||
|
///
|
||||||
|
/// This is the verification predicate for a continuous mission: "did THIS run
|
||||||
|
/// contribute anything new". If a rerun could re-record an existing source
|
||||||
|
/// under its own mission id, every run would report success forever — the
|
||||||
|
/// failure that killed the 0030-0044 generation of this feature.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_rerun_cannot_claim_an_earlier_missions_work() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let first_mission = mission(&pool, ws).await;
|
||||||
|
let second_mission = mission(&pool, ws).await;
|
||||||
|
|
||||||
|
corpus::record(
|
||||||
|
&pool, ws, "lib", "source", "arxiv:2401.55555",
|
||||||
|
Some("Paper"), None, None, "h1", Some(first_mission),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// The second mission sees the same paper and re-records it.
|
||||||
|
corpus::record(
|
||||||
|
&pool, ws, "lib", "source", "arxiv:2401.55555",
|
||||||
|
Some("Paper"), None, None, "h2", Some(second_mission),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
corpus::contributed(&pool, ws, "lib", first_mission).await.unwrap(),
|
||||||
|
1,
|
||||||
|
"the finder keeps the credit"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
corpus::contributed(&pool, ws, "lib", second_mission).await.unwrap(),
|
||||||
|
0,
|
||||||
|
"a rerun that found nothing new must report zero, not one"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -564,6 +564,109 @@ async fn a_failed_gate_publishes_to_a_wip_branch() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// #55: a mission whose checkout was re-cloned builds divergent history against
|
||||||
|
/// its OWN deterministic branch, and git rejects every push it will ever make.
|
||||||
|
/// That was terminal — the work stayed on a local branch in a directory that
|
||||||
|
/// gets reaped — and it is reachable from a retry, a container teardown, or disk
|
||||||
|
/// loss, not just from someone deleting a checkout by hand.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn diverged_history_lands_on_a_new_branch_instead_of_being_lost() {
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let remote = tmp.path().join("remote.git");
|
||||||
|
std::fs::create_dir_all(&remote).unwrap();
|
||||||
|
Command::new("git")
|
||||||
|
.args(["init", "--bare", "--quiet"])
|
||||||
|
.arg(&remote)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
let branch = "clawmates/mission-test-dddddddd";
|
||||||
|
let url = remote.to_str().unwrap();
|
||||||
|
|
||||||
|
// The first attempt: a checkout that pushed its work and then vanished.
|
||||||
|
let first = seed_repo(tmp.path(), Uuid::now_v7());
|
||||||
|
std::fs::write(first.join("first.rs"), "fn first() {}\n").unwrap();
|
||||||
|
git(&first, &["add", "."]);
|
||||||
|
git(&first, &["commit", "--quiet", "-m", "first pass"]);
|
||||||
|
git(&first, &["checkout", "-B", branch]);
|
||||||
|
let one = mission_delivery::publish_phase_branch(
|
||||||
|
&first,
|
||||||
|
url,
|
||||||
|
branch,
|
||||||
|
mission_delivery::Gate::Always,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(one.pushed, "setup push failed: {:?}", one.error);
|
||||||
|
let claimed = Command::new("git")
|
||||||
|
.arg("-C")
|
||||||
|
.arg(&remote)
|
||||||
|
.args(["rev-parse", branch])
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
let claimed = String::from_utf8_lossy(&claimed.stdout).trim().to_string();
|
||||||
|
|
||||||
|
// The retry: a fresh clone of the same mission, so unrelated history under
|
||||||
|
// the same deterministic branch name.
|
||||||
|
let second = seed_repo(tmp.path(), Uuid::now_v7());
|
||||||
|
std::fs::write(second.join("second.rs"), "fn second() {}\n").unwrap();
|
||||||
|
git(&second, &["add", "."]);
|
||||||
|
git(&second, &["commit", "--quiet", "-m", "retry"]);
|
||||||
|
git(&second, &["checkout", "-B", branch]);
|
||||||
|
|
||||||
|
let out = mission_delivery::publish_phase_branch(
|
||||||
|
&second,
|
||||||
|
url,
|
||||||
|
branch,
|
||||||
|
mission_delivery::Gate::Always,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
out.pushed,
|
||||||
|
"the retry's work never reached the forge: {:?}",
|
||||||
|
out.error
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
out.branch.starts_with(branch) && out.branch != branch,
|
||||||
|
"it must land on a NEW ref, not the contested one: {}",
|
||||||
|
out.branch
|
||||||
|
);
|
||||||
|
|
||||||
|
let refs = Command::new("git")
|
||||||
|
.arg("-C")
|
||||||
|
.arg(&remote)
|
||||||
|
.args(["for-each-ref", "--format=%(refname:short)"])
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
let refs = String::from_utf8_lossy(&refs.stdout);
|
||||||
|
assert!(refs.contains(&out.branch), "refs: {refs}");
|
||||||
|
|
||||||
|
// NEVER force: the first attempt's ref still points where it did. Losing it
|
||||||
|
// to make this push look tidy would trade one lost copy of the work for
|
||||||
|
// another.
|
||||||
|
let still = Command::new("git")
|
||||||
|
.arg("-C")
|
||||||
|
.arg(&remote)
|
||||||
|
.args(["rev-parse", branch])
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
String::from_utf8_lossy(&still.stdout).trim(),
|
||||||
|
claimed,
|
||||||
|
"the earlier attempt's branch was overwritten"
|
||||||
|
);
|
||||||
|
let show = Command::new("git")
|
||||||
|
.arg("-C")
|
||||||
|
.arg(&remote)
|
||||||
|
.args(["show", &format!("{}:second.rs", out.branch)])
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
assert!(String::from_utf8_lossy(&show.stdout).contains("fn second()"));
|
||||||
|
}
|
||||||
|
|
||||||
/// An unreachable remote is a degraded success, not a failure: the patch and
|
/// An unreachable remote is a degraded success, not a failure: the patch and
|
||||||
/// the local branch both still exist.
|
/// the local branch both still exist.
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -876,3 +979,43 @@ async fn an_unrunnable_suite_is_distinguishable_from_no_suite() {
|
|||||||
"the two must be distinguishable — this is the whole point"
|
"the two must be distinguishable — this is the whole point"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A COMMIT_EDITMSG left by the agent must not block delivery.
|
||||||
|
///
|
||||||
|
/// From mission 019fcd0c: the agent ran `git commit` itself, leaving
|
||||||
|
/// `.git/COMMIT_EDITMSG` owned by root at 0644, and the server's commit died
|
||||||
|
/// with "Permission denied". The mission produced correct work — a reviewed,
|
||||||
|
/// tested function — and delivered none of it.
|
||||||
|
///
|
||||||
|
/// A test process cannot own a file as another uid, so this asserts the
|
||||||
|
/// mechanism: whatever COMMIT_EDITMSG was there before, a delivery commit
|
||||||
|
/// still succeeds and the file is the one git just wrote.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_stale_commit_editmsg_does_not_block_delivery() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let tmp = tempfile::tempdir().unwrap();
|
||||||
|
let mission = Uuid::now_v7();
|
||||||
|
let repo = seed_repo(tmp.path(), mission);
|
||||||
|
let (_, phase) = seed_mission_phase(&pool, mission).await;
|
||||||
|
|
||||||
|
// Stand in for the agent's leftover: content that must not survive.
|
||||||
|
let msg = repo.join(".git/COMMIT_EDITMSG");
|
||||||
|
std::fs::write(&msg, "LEFTOVER FROM THE AGENT\n").unwrap();
|
||||||
|
|
||||||
|
std::fs::write(repo.join("WORK.md"), "work\n").unwrap();
|
||||||
|
let cap = capture(&pool, tmp.path(), mission, phase)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let commit = cap
|
||||||
|
.committed
|
||||||
|
.expect("delivery must commit despite a stale COMMIT_EDITMSG");
|
||||||
|
assert!(!commit.sha.is_empty());
|
||||||
|
|
||||||
|
let body = std::fs::read_to_string(&msg).unwrap_or_default();
|
||||||
|
assert!(
|
||||||
|
!body.contains("LEFTOVER FROM THE AGENT"),
|
||||||
|
"the stale message survived: {body:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -69,6 +69,7 @@ async fn seed_test_template(pool: &sqlx::PgPool) -> Uuid {
|
|||||||
system_prompt: "Plan the feature. Break it into INT-XX items.",
|
system_prompt: "Plan the feature. Break it into INT-XX items.",
|
||||||
skills: vec!["decompose-int-items".into()],
|
skills: vec!["decompose-int-items".into()],
|
||||||
brain_seed: Some("# Planner\nBreak features into INT items."),
|
brain_seed: Some("# Planner\nBreak features into INT items."),
|
||||||
|
model: None,
|
||||||
},
|
},
|
||||||
team_templates::UpsertBuiltinRole {
|
team_templates::UpsertBuiltinRole {
|
||||||
slot: "coder",
|
slot: "coder",
|
||||||
@@ -76,6 +77,7 @@ async fn seed_test_template(pool: &sqlx::PgPool) -> Uuid {
|
|||||||
system_prompt: "Implement one INT item at a time.",
|
system_prompt: "Implement one INT item at a time.",
|
||||||
skills: vec!["write-rust-current-edition".into()],
|
skills: vec!["write-rust-current-edition".into()],
|
||||||
brain_seed: Some("# Coder\nOne INT per commit."),
|
brain_seed: Some("# Coder\nOne INT per commit."),
|
||||||
|
model: None,
|
||||||
},
|
},
|
||||||
team_templates::UpsertBuiltinRole {
|
team_templates::UpsertBuiltinRole {
|
||||||
slot: "reviewer",
|
slot: "reviewer",
|
||||||
@@ -83,6 +85,9 @@ async fn seed_test_template(pool: &sqlx::PgPool) -> Uuid {
|
|||||||
system_prompt: "Review each commit before merge.",
|
system_prompt: "Review each commit before merge.",
|
||||||
skills: vec!["code-review-checklist".into()],
|
skills: vec!["code-review-checklist".into()],
|
||||||
brain_seed: None,
|
brain_seed: None,
|
||||||
|
// The point of migration 0071: a reviewer that does NOT
|
||||||
|
// share a model with the coder it reviews.
|
||||||
|
model: Some("glm-4.7"),
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
@@ -258,3 +263,114 @@ async fn on_launch_no_template_hard_fails() {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(team_id.is_none());
|
assert!(team_id.is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Slice 5: an APPROVED roster outranks the team template.
|
||||||
|
///
|
||||||
|
/// The template gives every composed mission the same five roles on the same
|
||||||
|
/// image. A roster is the model's answer for THIS mission, and it is the only
|
||||||
|
/// path that carries a per-node backend — which is how a mission runs more than
|
||||||
|
/// one provider at all. If the template won, a heterogeneous roster would be
|
||||||
|
/// accepted, stored, and then silently ignored at launch.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn an_approved_roster_outranks_the_template() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = seed_workspace(&pool).await;
|
||||||
|
let template_id = seed_test_template(&pool).await;
|
||||||
|
let mission = seed_mission(&pool, ws, template_id, "roster beats template").await;
|
||||||
|
|
||||||
|
// With no roster, the shape comes from the template — the behaviour every
|
||||||
|
// composed mission had before this slice.
|
||||||
|
let from_template = mission_orchestrator::composed_graph(&pool, mission, &["mission"])
|
||||||
|
.await
|
||||||
|
.expect("template graph")
|
||||||
|
.expect("the template supplies a shape");
|
||||||
|
let template_nodes = from_template["nodes"].as_array().unwrap().len();
|
||||||
|
assert!(template_nodes >= 1);
|
||||||
|
assert!(
|
||||||
|
from_template["nodes"][0]["attrs"].get("backend").is_none(),
|
||||||
|
"a template cannot express a per-node backend — that is the gap the roster fills"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Approve a roster the way the route does: the built graph under
|
||||||
|
// `config.roster`.
|
||||||
|
let roster = cm_api::mission_roster::Roster {
|
||||||
|
topology_kind: "pipeline".into(),
|
||||||
|
members: vec![
|
||||||
|
cm_api::mission_roster::RosterMember {
|
||||||
|
role: "implementer".into(),
|
||||||
|
backend: Some("claude".into()),
|
||||||
|
rationale: None,
|
||||||
|
},
|
||||||
|
cm_api::mission_roster::RosterMember {
|
||||||
|
role: "verifier".into(),
|
||||||
|
backend: Some("kimi".into()),
|
||||||
|
rationale: None,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
||||||
|
let graph = roster.graph().expect("a runnable graph");
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE missions SET config = jsonb_set(config, '{roster}', $2::jsonb, true) WHERE id = $1",
|
||||||
|
)
|
||||||
|
.bind(mission)
|
||||||
|
.bind(&graph)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let chosen = mission_orchestrator::composed_graph(&pool, mission, &["mission"])
|
||||||
|
.await
|
||||||
|
.expect("roster graph")
|
||||||
|
.expect("the roster supplies a shape");
|
||||||
|
let nodes = chosen["nodes"].as_array().unwrap();
|
||||||
|
assert_eq!(nodes.len(), 2, "the roster's two nodes, not the template's");
|
||||||
|
assert_eq!(nodes[0]["role"], "implementer");
|
||||||
|
assert_eq!(nodes[0]["attrs"]["backend"], "claude");
|
||||||
|
assert_eq!(nodes[1]["attrs"]["backend"], "kimi");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Migration 0071: a template role may name its own model, and the claw minted
|
||||||
|
/// for it must actually run on that model.
|
||||||
|
///
|
||||||
|
/// Before this, `mint_team_from_template` bound EVERY role to one literal — so a
|
||||||
|
/// template whose whole point is an independent reviewer minted a reviewer
|
||||||
|
/// sharing a model with the coder it reviews. That is the correlated failure the
|
||||||
|
/// cross-provider judge exists to break, reintroduced one layer down.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_template_role_may_run_on_its_own_model() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = seed_workspace(&pool).await;
|
||||||
|
let user = seed_owner(&pool, ws).await;
|
||||||
|
let template_id = seed_test_template(&pool).await;
|
||||||
|
let mission = seed_mission(&pool, ws, template_id, "per-role models").await;
|
||||||
|
|
||||||
|
mission_orchestrator::on_launch(&pool, ws, user, mission, None)
|
||||||
|
.await
|
||||||
|
.expect("launch");
|
||||||
|
|
||||||
|
let rows: Vec<(String, Option<String>)> = sqlx::query_as(
|
||||||
|
"SELECT job_title, model_binding FROM agents
|
||||||
|
WHERE workspace_id = $1 AND deleted_at IS NULL
|
||||||
|
ORDER BY job_title",
|
||||||
|
)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.fetch_all(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let by_role: std::collections::HashMap<_, _> = rows.into_iter().collect();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
by_role.get("reviewer").and_then(|m| m.clone()).as_deref(),
|
||||||
|
Some("glm-4.7"),
|
||||||
|
"the reviewer must run the model its role names: {by_role:?}"
|
||||||
|
);
|
||||||
|
// And a role that names none still gets the mint's default, so every
|
||||||
|
// template written before 0071 behaves exactly as it did.
|
||||||
|
for silent in ["planner", "coder"] {
|
||||||
|
assert_eq!(
|
||||||
|
by_role.get(silent).and_then(|m| m.clone()).as_deref(),
|
||||||
|
Some("claude-sonnet-5"),
|
||||||
|
"{silent} named no model and must take the default"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -278,6 +278,7 @@ async fn evaluations_are_unique_per_iteration_and_upsert() {
|
|||||||
model: "runtime:coordinator".into(),
|
model: "runtime:coordinator".into(),
|
||||||
error: None,
|
error: None,
|
||||||
checks: Vec::new(),
|
checks: Vec::new(),
|
||||||
|
independent: false,
|
||||||
};
|
};
|
||||||
cm_api::evaluator::record(&pool, mission, phase, 0, &first)
|
cm_api::evaluator::record(&pool, mission, phase, 0, &first)
|
||||||
.await
|
.await
|
||||||
@@ -296,6 +297,7 @@ async fn evaluations_are_unique_per_iteration_and_upsert() {
|
|||||||
exit_code: Some(0),
|
exit_code: Some(0),
|
||||||
evidence: "exit status: 0".into(),
|
evidence: "exit status: 0".into(),
|
||||||
}],
|
}],
|
||||||
|
independent: false,
|
||||||
};
|
};
|
||||||
cm_api::evaluator::record(&pool, mission, phase, 0, &second)
|
cm_api::evaluator::record(&pool, mission, phase, 0, &second)
|
||||||
.await
|
.await
|
||||||
@@ -353,6 +355,7 @@ async fn latest_returns_the_most_recent_iteration() {
|
|||||||
model: "m".into(),
|
model: "m".into(),
|
||||||
error: None,
|
error: None,
|
||||||
checks: Vec::new(),
|
checks: Vec::new(),
|
||||||
|
independent: false,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -113,3 +113,51 @@ pub async fn agents_for_template(
|
|||||||
})
|
})
|
||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A claw already in this workspace that can take this template role again.
|
||||||
|
///
|
||||||
|
/// The workforce is meant to be KEPT: a mission that needs a `coder` should
|
||||||
|
/// hire the one that already exists rather than mint a sixth. Without this,
|
||||||
|
/// every zeroclaw mission added a whole team to the roster permanently — they
|
||||||
|
/// are minted `lifecycle = 'permanent'` and nothing reaps them until the
|
||||||
|
/// mission itself is deleted — while each member was used exactly once.
|
||||||
|
///
|
||||||
|
/// A claw currently on a RUNNING mission is not offered. Two missions driving
|
||||||
|
/// the same ZeroClaw agent and the same `.brain` at once is a data race with a
|
||||||
|
/// model on the other end of it; minting a second claw is much cheaper than
|
||||||
|
/// reasoning about that.
|
||||||
|
///
|
||||||
|
/// Oldest first, so reuse concentrates on the same few claws and their brains
|
||||||
|
/// actually accumulate, instead of spreading thinly across a growing pool.
|
||||||
|
pub async fn reusable_claw(
|
||||||
|
pool: &PgPool,
|
||||||
|
workspace_id: uuid::Uuid,
|
||||||
|
template_id: uuid::Uuid,
|
||||||
|
role_slot: &str,
|
||||||
|
) -> Result<Option<uuid::Uuid>, DbError> {
|
||||||
|
let row: Option<(uuid::Uuid,)> = sqlx::query_as(
|
||||||
|
"SELECT a.id
|
||||||
|
FROM agents a
|
||||||
|
JOIN agent_template_link l ON l.agent_id = a.id
|
||||||
|
WHERE a.workspace_id = $1
|
||||||
|
AND a.deleted_at IS NULL
|
||||||
|
AND l.template_id = $2
|
||||||
|
AND l.role_slot = $3
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM team_members tm
|
||||||
|
JOIN mission_teams mt ON mt.team_id = tm.team_id
|
||||||
|
JOIN missions m ON m.id = mt.mission_id
|
||||||
|
WHERE tm.claw_id = a.id
|
||||||
|
AND m.status = 'running'
|
||||||
|
)
|
||||||
|
ORDER BY a.created_at
|
||||||
|
LIMIT 1",
|
||||||
|
)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(template_id)
|
||||||
|
.bind(role_slot)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(row.map(|(id,)| id))
|
||||||
|
}
|
||||||
|
|||||||
@@ -64,6 +64,52 @@ pub async fn insert(pool: &PgPool, agent: &Agent, policy: &AccessPolicy) -> Resu
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Fetch an agent even if it has been soft-deleted.
|
||||||
|
///
|
||||||
|
/// For the PURGE path only. `get` hides soft-deleted rows, which is right for
|
||||||
|
/// every read — but it also meant the hard purge could not see the rows it
|
||||||
|
/// exists to remove: a soft-deleted agent was unreachable from every route and
|
||||||
|
/// accumulated forever with no way out of the application. Four of them dated
|
||||||
|
/// from June before anyone noticed, because the UI correctly never showed them.
|
||||||
|
pub async fn get_any(pool: &PgPool, agent_id: AgentId) -> Result<Agent, DbError> {
|
||||||
|
// `sqlx::query_as` rather than the checked macro: this is the same columns
|
||||||
|
// as `get` minus one predicate, and adding a second compile-time query for
|
||||||
|
// that would mean regenerating the offline cache on every machine that
|
||||||
|
// builds this.
|
||||||
|
let row: Option<(
|
||||||
|
uuid::Uuid,
|
||||||
|
uuid::Uuid,
|
||||||
|
String,
|
||||||
|
String,
|
||||||
|
String,
|
||||||
|
String,
|
||||||
|
String,
|
||||||
|
String,
|
||||||
|
uuid::Uuid,
|
||||||
|
String,
|
||||||
|
)> = sqlx::query_as(
|
||||||
|
"SELECT id, workspace_id, name, job_title, system_prompt, avatar,
|
||||||
|
accent, wallpaper, managed_by, status
|
||||||
|
FROM agents WHERE id = $1",
|
||||||
|
)
|
||||||
|
.bind(agent_id.as_uuid())
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await?;
|
||||||
|
let row = row.ok_or(DbError::NotFound)?;
|
||||||
|
Ok(Agent {
|
||||||
|
id: AgentId::from(row.0),
|
||||||
|
workspace_id: WorkspaceId::from(row.1),
|
||||||
|
name: row.2,
|
||||||
|
job_title: row.3,
|
||||||
|
system_prompt: row.4,
|
||||||
|
avatar: row.5,
|
||||||
|
accent: row.6,
|
||||||
|
wallpaper: row.7,
|
||||||
|
managed_by: UserId::from(row.8),
|
||||||
|
status: row.9.parse().expect("status CHECK constraint"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn get(pool: &PgPool, agent_id: AgentId) -> Result<Agent, DbError> {
|
pub async fn get(pool: &PgPool, agent_id: AgentId) -> Result<Agent, DbError> {
|
||||||
let row = sqlx::query!(
|
let row = sqlx::query!(
|
||||||
"SELECT id, workspace_id, name, job_title, system_prompt, avatar,
|
"SELECT id, workspace_id, name, job_title, system_prompt, avatar,
|
||||||
@@ -87,7 +133,7 @@ pub async fn get(pool: &PgPool, agent_id: AgentId) -> Result<Agent, DbError> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Persist the model a claw was deployed with (e.g. "claude", "gemini",
|
/// Persist the model a claw was deployed with (e.g. "claude", "glm",
|
||||||
/// "glm-5.2") — the runtime config is otherwise the only record of it.
|
/// "glm-5.2") — the runtime config is otherwise the only record of it.
|
||||||
pub async fn set_model_binding(
|
pub async fn set_model_binding(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
|
|||||||
@@ -0,0 +1,231 @@
|
|||||||
|
//! Model-authored mission PLANS — the phase list — and whether a human
|
||||||
|
//! accepted them.
|
||||||
|
//!
|
||||||
|
//! See `migrations/0070_mission_plan_proposals.sql` for why a proposal is
|
||||||
|
//! persisted rather than applied on arrival.
|
||||||
|
|
||||||
|
use crate::DbError;
|
||||||
|
use serde_json::Value;
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use time::OffsetDateTime;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, serde::Serialize)]
|
||||||
|
pub struct MissionPlanProposal {
|
||||||
|
pub id: Uuid,
|
||||||
|
pub mission_id: Uuid,
|
||||||
|
pub plan: Value,
|
||||||
|
pub author_model: String,
|
||||||
|
pub status: String,
|
||||||
|
pub note: Option<String>,
|
||||||
|
#[serde(with = "time::serde::rfc3339")]
|
||||||
|
pub created_at: OffsetDateTime,
|
||||||
|
#[serde(with = "time::serde::rfc3339::option")]
|
||||||
|
pub decided_at: Option<OffsetDateTime>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn insert(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
mission_id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
plan: &Value,
|
||||||
|
author_model: &str,
|
||||||
|
) -> Result<(), DbError> {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO mission_plan_proposals
|
||||||
|
(id, mission_id, workspace_id, plan, author_model)
|
||||||
|
VALUES ($1, $2, $3, $4, $5)",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(mission_id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(plan)
|
||||||
|
.bind(author_model)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every proposal for a mission, newest first. Rejected ones are included on
|
||||||
|
/// purpose: what a human turned down is the only record of what the planner
|
||||||
|
/// gets wrong.
|
||||||
|
pub async fn list(
|
||||||
|
pool: &PgPool,
|
||||||
|
mission_id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
) -> Result<Vec<MissionPlanProposal>, DbError> {
|
||||||
|
let rows = sqlx::query_as::<_, (Uuid, Uuid, Value, String, String, Option<String>, OffsetDateTime, Option<OffsetDateTime>)>(
|
||||||
|
"SELECT id, mission_id, plan, author_model, status, note, created_at, decided_at
|
||||||
|
FROM mission_plan_proposals
|
||||||
|
WHERE mission_id = $1 AND workspace_id = $2
|
||||||
|
ORDER BY created_at DESC",
|
||||||
|
)
|
||||||
|
.bind(mission_id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows
|
||||||
|
.into_iter()
|
||||||
|
.map(
|
||||||
|
|(id, mission_id, plan, author_model, status, note, created_at, decided_at)| {
|
||||||
|
MissionPlanProposal {
|
||||||
|
id,
|
||||||
|
mission_id,
|
||||||
|
plan,
|
||||||
|
author_model,
|
||||||
|
status,
|
||||||
|
note,
|
||||||
|
created_at,
|
||||||
|
decided_at,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
) -> Result<Option<MissionPlanProposal>, DbError> {
|
||||||
|
let row = sqlx::query_as::<_, (Uuid, Uuid, Value, String, String, Option<String>, OffsetDateTime, Option<OffsetDateTime>)>(
|
||||||
|
"SELECT id, mission_id, plan, author_model, status, note, created_at, decided_at
|
||||||
|
FROM mission_plan_proposals
|
||||||
|
WHERE id = $1 AND workspace_id = $2",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(row.map(
|
||||||
|
|(id, mission_id, plan, author_model, status, note, created_at, decided_at)| {
|
||||||
|
MissionPlanProposal {
|
||||||
|
id,
|
||||||
|
mission_id,
|
||||||
|
plan,
|
||||||
|
author_model,
|
||||||
|
status,
|
||||||
|
note,
|
||||||
|
created_at,
|
||||||
|
decided_at,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Approve a plan AND write its phases onto the mission, atomically.
|
||||||
|
///
|
||||||
|
/// One transaction, for the reason `mission_team_proposals::approve_and_apply`
|
||||||
|
/// documents: a two-statement version left a proposal marked `approved` against
|
||||||
|
/// a mission that never received it, and the partial unique index then makes
|
||||||
|
/// that state permanent.
|
||||||
|
///
|
||||||
|
/// The mission's existing phases are REPLACED. A plan is an answer to "what is
|
||||||
|
/// this mission", not an addition to the recipe's answer — merging the two would
|
||||||
|
/// produce a phase list neither the model nor the recipe author intended. Only a
|
||||||
|
/// draft mission is eligible (checked by the caller), so nothing in flight is
|
||||||
|
/// discarded.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub async fn approve_and_apply(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
mission_id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
phases: &[(String, i32, Value)],
|
||||||
|
note: Option<&str>,
|
||||||
|
decided_by: Option<Uuid>,
|
||||||
|
) -> Result<bool, DbError> {
|
||||||
|
let mut tx = pool.begin().await?;
|
||||||
|
|
||||||
|
let claimed = sqlx::query(
|
||||||
|
"UPDATE mission_plan_proposals
|
||||||
|
SET status = 'approved', note = $3, decided_at = now(), decided_by = $4
|
||||||
|
WHERE id = $1 AND workspace_id = $2 AND status = 'proposed'",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(note)
|
||||||
|
.bind(decided_by)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
if claimed != 1 {
|
||||||
|
tx.rollback().await?;
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scoped by workspace on the mission, so a proposal cannot rewrite the
|
||||||
|
// phases of a mission in another workspace even if its own row were forged.
|
||||||
|
let owned: i64 = sqlx::query_scalar(
|
||||||
|
"SELECT count(*) FROM missions WHERE id = $1 AND workspace_id = $2",
|
||||||
|
)
|
||||||
|
.bind(mission_id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.fetch_one(&mut *tx)
|
||||||
|
.await?;
|
||||||
|
if owned != 1 {
|
||||||
|
tx.rollback().await?;
|
||||||
|
return Err(DbError::NotFound);
|
||||||
|
}
|
||||||
|
|
||||||
|
sqlx::query("DELETE FROM mission_phases WHERE mission_id = $1")
|
||||||
|
.bind(mission_id)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?;
|
||||||
|
for (kind, order_idx, config) in phases {
|
||||||
|
// `done_when` is PROMOTED out of the config into its column, exactly as
|
||||||
|
// `missions::create` does. The evaluator sweep filters on the column in
|
||||||
|
// SQL on every tick — a plan whose condition stayed in the JSONB blob
|
||||||
|
// would be stored, rendered, and never judged, which is the same shape
|
||||||
|
// as the unread `task` this whole registry exists because of.
|
||||||
|
let done_when = config
|
||||||
|
.get("done_when")
|
||||||
|
.and_then(|v| v.as_str())
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty());
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO mission_phases
|
||||||
|
(id, mission_id, kind, order_idx, status, config, done_when, max_iterations)
|
||||||
|
VALUES ($1, $2, $3, $4, 'pending', $5, $6, 1)",
|
||||||
|
)
|
||||||
|
.bind(Uuid::now_v7())
|
||||||
|
.bind(mission_id)
|
||||||
|
.bind(kind)
|
||||||
|
.bind(order_idx)
|
||||||
|
.bind(config)
|
||||||
|
.bind(done_when)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
tx.commit().await?;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record a decision. Only a `proposed` row may be decided, so approving twice
|
||||||
|
/// — a double-click, a retried request — cannot re-apply a roster to a mission
|
||||||
|
/// that has since moved on. Returns whether this call was the one that decided.
|
||||||
|
pub async fn decide(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
status: &str,
|
||||||
|
note: Option<&str>,
|
||||||
|
decided_by: Option<Uuid>,
|
||||||
|
) -> Result<bool, DbError> {
|
||||||
|
let done = sqlx::query(
|
||||||
|
"UPDATE mission_plan_proposals
|
||||||
|
SET status = $3, note = $4, decided_at = now(), decided_by = $5
|
||||||
|
WHERE id = $1 AND workspace_id = $2 AND status = 'proposed'",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(status)
|
||||||
|
.bind(note)
|
||||||
|
.bind(decided_by)
|
||||||
|
.execute(pool)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
Ok(done == 1)
|
||||||
|
}
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
//! Model-authored mission rosters, and whether a human accepted them.
|
||||||
|
//!
|
||||||
|
//! See `migrations/0070_mission_team_proposals.sql` for why a proposal is
|
||||||
|
//! persisted rather than applied on arrival.
|
||||||
|
|
||||||
|
use crate::DbError;
|
||||||
|
use serde_json::Value;
|
||||||
|
use sqlx::PgPool;
|
||||||
|
use time::OffsetDateTime;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, serde::Serialize)]
|
||||||
|
pub struct MissionTeamProposal {
|
||||||
|
pub id: Uuid,
|
||||||
|
pub mission_id: Uuid,
|
||||||
|
pub roster: Value,
|
||||||
|
pub author_model: String,
|
||||||
|
pub status: String,
|
||||||
|
pub note: Option<String>,
|
||||||
|
#[serde(with = "time::serde::rfc3339")]
|
||||||
|
pub created_at: OffsetDateTime,
|
||||||
|
#[serde(with = "time::serde::rfc3339::option")]
|
||||||
|
pub decided_at: Option<OffsetDateTime>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn insert(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
mission_id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
roster: &Value,
|
||||||
|
author_model: &str,
|
||||||
|
) -> Result<(), DbError> {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO mission_team_proposals
|
||||||
|
(id, mission_id, workspace_id, roster, author_model)
|
||||||
|
VALUES ($1, $2, $3, $4, $5)",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(mission_id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(roster)
|
||||||
|
.bind(author_model)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Every proposal for a mission, newest first. Rejected ones are included on
|
||||||
|
/// purpose: what a human turned down is the only record of what the planner
|
||||||
|
/// gets wrong.
|
||||||
|
pub async fn list(
|
||||||
|
pool: &PgPool,
|
||||||
|
mission_id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
) -> Result<Vec<MissionTeamProposal>, DbError> {
|
||||||
|
let rows = sqlx::query_as::<_, (Uuid, Uuid, Value, String, String, Option<String>, OffsetDateTime, Option<OffsetDateTime>)>(
|
||||||
|
"SELECT id, mission_id, roster, author_model, status, note, created_at, decided_at
|
||||||
|
FROM mission_team_proposals
|
||||||
|
WHERE mission_id = $1 AND workspace_id = $2
|
||||||
|
ORDER BY created_at DESC",
|
||||||
|
)
|
||||||
|
.bind(mission_id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows
|
||||||
|
.into_iter()
|
||||||
|
.map(
|
||||||
|
|(id, mission_id, roster, author_model, status, note, created_at, decided_at)| {
|
||||||
|
MissionTeamProposal {
|
||||||
|
id,
|
||||||
|
mission_id,
|
||||||
|
roster,
|
||||||
|
author_model,
|
||||||
|
status,
|
||||||
|
note,
|
||||||
|
created_at,
|
||||||
|
decided_at,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
) -> Result<Option<MissionTeamProposal>, DbError> {
|
||||||
|
let row = sqlx::query_as::<_, (Uuid, Uuid, Value, String, String, Option<String>, OffsetDateTime, Option<OffsetDateTime>)>(
|
||||||
|
"SELECT id, mission_id, roster, author_model, status, note, created_at, decided_at
|
||||||
|
FROM mission_team_proposals
|
||||||
|
WHERE id = $1 AND workspace_id = $2",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.fetch_optional(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(row.map(
|
||||||
|
|(id, mission_id, roster, author_model, status, note, created_at, decided_at)| {
|
||||||
|
MissionTeamProposal {
|
||||||
|
id,
|
||||||
|
mission_id,
|
||||||
|
roster,
|
||||||
|
author_model,
|
||||||
|
status,
|
||||||
|
note,
|
||||||
|
created_at,
|
||||||
|
decided_at,
|
||||||
|
}
|
||||||
|
},
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Approve a proposal AND apply it to its mission, atomically.
|
||||||
|
///
|
||||||
|
/// One transaction, because the two halves are one decision. The first version
|
||||||
|
/// claimed the proposal and then wrote the mission in two statements, and
|
||||||
|
/// production found the hole on the first real approval: the write failed, the
|
||||||
|
/// claim stood, and the mission was left with no roster while its proposal said
|
||||||
|
/// `approved` — a state the partial unique index then makes permanent, since no
|
||||||
|
/// second proposal for that mission can ever be approved.
|
||||||
|
///
|
||||||
|
/// Returns false when the proposal was already decided (a double-clicked
|
||||||
|
/// approve), in which case nothing is written.
|
||||||
|
pub async fn approve_and_apply(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
mission_id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
graph: &Value,
|
||||||
|
note: Option<&str>,
|
||||||
|
decided_by: Option<Uuid>,
|
||||||
|
) -> Result<bool, DbError> {
|
||||||
|
let mut tx = pool.begin().await?;
|
||||||
|
|
||||||
|
let claimed = sqlx::query(
|
||||||
|
"UPDATE mission_team_proposals
|
||||||
|
SET status = 'approved', note = $3, decided_at = now(), decided_by = $4
|
||||||
|
WHERE id = $1 AND workspace_id = $2 AND status = 'proposed'",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(note)
|
||||||
|
.bind(decided_by)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
if claimed != 1 {
|
||||||
|
tx.rollback().await?;
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
// `jsonb_set` REFUSES a scalar, and a mission created without a `config`
|
||||||
|
// stores jsonb `null` — a scalar. `coalesce` does not help: it guards SQL
|
||||||
|
// NULL, and this is a JSON null, which is a perfectly good non-NULL value of
|
||||||
|
// the wrong shape. Production hit this on the first real approval with
|
||||||
|
// "cannot set path in scalar".
|
||||||
|
let applied = sqlx::query(
|
||||||
|
"UPDATE missions
|
||||||
|
SET config = jsonb_set(
|
||||||
|
CASE WHEN jsonb_typeof(config) = 'object' THEN config ELSE '{}'::jsonb END,
|
||||||
|
'{roster}', $3::jsonb, true),
|
||||||
|
team_engine = 'composed',
|
||||||
|
updated_at = now()
|
||||||
|
WHERE id = $1 AND workspace_id = $2",
|
||||||
|
)
|
||||||
|
.bind(mission_id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(graph)
|
||||||
|
.execute(&mut *tx)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
if applied != 1 {
|
||||||
|
tx.rollback().await?;
|
||||||
|
return Err(DbError::NotFound);
|
||||||
|
}
|
||||||
|
|
||||||
|
tx.commit().await?;
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Record a decision. Only a `proposed` row may be decided, so approving twice
|
||||||
|
/// — a double-click, a retried request — cannot re-apply a roster to a mission
|
||||||
|
/// that has since moved on. Returns whether this call was the one that decided.
|
||||||
|
pub async fn decide(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
|
status: &str,
|
||||||
|
note: Option<&str>,
|
||||||
|
decided_by: Option<Uuid>,
|
||||||
|
) -> Result<bool, DbError> {
|
||||||
|
let done = sqlx::query(
|
||||||
|
"UPDATE mission_team_proposals
|
||||||
|
SET status = $3, note = $4, decided_at = now(), decided_by = $5
|
||||||
|
WHERE id = $1 AND workspace_id = $2 AND status = 'proposed'",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(status)
|
||||||
|
.bind(note)
|
||||||
|
.bind(decided_by)
|
||||||
|
.execute(pool)
|
||||||
|
.await?
|
||||||
|
.rows_affected();
|
||||||
|
Ok(done == 1)
|
||||||
|
}
|
||||||
@@ -34,6 +34,10 @@ pub struct Mission {
|
|||||||
pub runtime_kind: String,
|
pub runtime_kind: String,
|
||||||
/// FK → nodes(id); only relevant when runtime_kind = 'local_herdr'
|
/// FK → nodes(id); only relevant when runtime_kind = 'local_herdr'
|
||||||
pub target_node_id: Option<Uuid>,
|
pub target_node_id: Option<Uuid>,
|
||||||
|
/// Which per-CLI rootfs a `microvm` mission boots. NULL = the node's default
|
||||||
|
/// image. Read by placement (a node must HOLD this image) and by the executor
|
||||||
|
/// (it is passed to `vm_create`).
|
||||||
|
pub backend: Option<String>,
|
||||||
/// Per-mission ZeroClaw runtime container name (C3 workspace isolation).
|
/// Per-mission ZeroClaw runtime container name (C3 workspace isolation).
|
||||||
/// Null until `mission_runtime::ensure_container` provisions it.
|
/// Null until `mission_runtime::ensure_container` provisions it.
|
||||||
pub runtime_container_name: Option<String>,
|
pub runtime_container_name: Option<String>,
|
||||||
@@ -129,6 +133,15 @@ pub struct NewMission<'a> {
|
|||||||
/// Defaults to 'zeroclaw' when None.
|
/// Defaults to 'zeroclaw' when None.
|
||||||
pub runtime_kind: Option<&'a str>,
|
pub runtime_kind: Option<&'a str>,
|
||||||
pub target_node_id: Option<Uuid>,
|
pub target_node_id: Option<Uuid>,
|
||||||
|
/// Which per-CLI image a `microvm` mission boots. NULL = the node's default
|
||||||
|
/// rootfs. Deliberately unconstrained in the schema: which images exist is a
|
||||||
|
/// property of the NODES, not of the database.
|
||||||
|
pub backend: Option<&'a str>,
|
||||||
|
/// Independent validator for this mission's verdicts. `None` = deployment
|
||||||
|
/// default; `Some("")` = explicitly none. See migration 0068.
|
||||||
|
pub validator_model: Option<&'a str>,
|
||||||
|
/// `"claude_code"` to ask for an agent team; `None` = solo. See 0069.
|
||||||
|
pub team_engine: Option<&'a str>,
|
||||||
pub phases: Vec<NewMissionPhase>,
|
pub phases: Vec<NewMissionPhase>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -157,9 +170,9 @@ pub async fn insert(pool: &PgPool, m: NewMission<'_>) -> Result<Uuid, DbError> {
|
|||||||
"INSERT INTO missions
|
"INSERT INTO missions
|
||||||
(id, workspace_id, title, template_kind, team_id,
|
(id, workspace_id, title, template_kind, team_id,
|
||||||
team_template_id, repo_id, schedule, status, description, config,
|
team_template_id, repo_id, schedule, status, description, config,
|
||||||
runtime_kind, target_node_id)
|
runtime_kind, target_node_id, backend, validator_model, team_engine)
|
||||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'draft',$9,$10,
|
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'draft',$9,$10,
|
||||||
COALESCE($11,'zeroclaw'),$12)",
|
COALESCE($11,'zeroclaw'),$12,$13,$14,$15)",
|
||||||
)
|
)
|
||||||
.bind(mission_id)
|
.bind(mission_id)
|
||||||
.bind(m.workspace_id)
|
.bind(m.workspace_id)
|
||||||
@@ -173,6 +186,9 @@ pub async fn insert(pool: &PgPool, m: NewMission<'_>) -> Result<Uuid, DbError> {
|
|||||||
.bind(&m.config)
|
.bind(&m.config)
|
||||||
.bind(m.runtime_kind)
|
.bind(m.runtime_kind)
|
||||||
.bind(m.target_node_id)
|
.bind(m.target_node_id)
|
||||||
|
.bind(m.backend)
|
||||||
|
.bind(m.validator_model)
|
||||||
|
.bind(m.team_engine)
|
||||||
.execute(&mut *tx)
|
.execute(&mut *tx)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
@@ -224,7 +240,7 @@ pub async fn get(pool: &PgPool, id: Uuid, workspace_id: Uuid) -> Result<Option<M
|
|||||||
let row = sqlx::query(
|
let row = sqlx::query(
|
||||||
"SELECT id, workspace_id, title, template_kind, team_id,
|
"SELECT id, workspace_id, title, template_kind, team_id,
|
||||||
team_template_id, repo_id, schedule, status,
|
team_template_id, repo_id, schedule, status,
|
||||||
description, config, runtime_kind, target_node_id,
|
description, config, runtime_kind, target_node_id, backend,
|
||||||
runtime_container_name, runtime_endpoint, runtime_pairing_code,
|
runtime_container_name, runtime_endpoint, runtime_pairing_code,
|
||||||
created_at, updated_at, completed_at
|
created_at, updated_at, completed_at
|
||||||
FROM missions WHERE id = $1 AND workspace_id = $2",
|
FROM missions WHERE id = $1 AND workspace_id = $2",
|
||||||
@@ -247,6 +263,7 @@ pub async fn get(pool: &PgPool, id: Uuid, workspace_id: Uuid) -> Result<Option<M
|
|||||||
config: r.get("config"),
|
config: r.get("config"),
|
||||||
runtime_kind: r.get("runtime_kind"),
|
runtime_kind: r.get("runtime_kind"),
|
||||||
target_node_id: r.get("target_node_id"),
|
target_node_id: r.get("target_node_id"),
|
||||||
|
backend: r.get("backend"),
|
||||||
runtime_container_name: r.get("runtime_container_name"),
|
runtime_container_name: r.get("runtime_container_name"),
|
||||||
runtime_endpoint: r.get("runtime_endpoint"),
|
runtime_endpoint: r.get("runtime_endpoint"),
|
||||||
runtime_pairing_code: r.get("runtime_pairing_code"),
|
runtime_pairing_code: r.get("runtime_pairing_code"),
|
||||||
@@ -266,7 +283,7 @@ pub async fn list_by_workspace(
|
|||||||
let rows = sqlx::query(
|
let rows = sqlx::query(
|
||||||
"SELECT id, workspace_id, title, template_kind, team_id,
|
"SELECT id, workspace_id, title, template_kind, team_id,
|
||||||
team_template_id, repo_id, schedule, status,
|
team_template_id, repo_id, schedule, status,
|
||||||
description, config, runtime_kind, target_node_id,
|
description, config, runtime_kind, target_node_id, backend,
|
||||||
runtime_container_name, runtime_endpoint, runtime_pairing_code,
|
runtime_container_name, runtime_endpoint, runtime_pairing_code,
|
||||||
created_at, updated_at, completed_at
|
created_at, updated_at, completed_at
|
||||||
FROM missions WHERE workspace_id = $1
|
FROM missions WHERE workspace_id = $1
|
||||||
@@ -292,6 +309,7 @@ pub async fn list_by_workspace(
|
|||||||
config: r.get("config"),
|
config: r.get("config"),
|
||||||
runtime_kind: r.get("runtime_kind"),
|
runtime_kind: r.get("runtime_kind"),
|
||||||
target_node_id: r.get("target_node_id"),
|
target_node_id: r.get("target_node_id"),
|
||||||
|
backend: r.get("backend"),
|
||||||
runtime_container_name: r.get("runtime_container_name"),
|
runtime_container_name: r.get("runtime_container_name"),
|
||||||
runtime_endpoint: r.get("runtime_endpoint"),
|
runtime_endpoint: r.get("runtime_endpoint"),
|
||||||
runtime_pairing_code: r.get("runtime_pairing_code"),
|
runtime_pairing_code: r.get("runtime_pairing_code"),
|
||||||
@@ -358,7 +376,7 @@ pub async fn set_runtime_binding(
|
|||||||
endpoint: Option<&str>,
|
endpoint: Option<&str>,
|
||||||
pairing_code: Option<&str>,
|
pairing_code: Option<&str>,
|
||||||
) -> Result<(), DbError> {
|
) -> Result<(), DbError> {
|
||||||
sqlx::query(
|
let r = sqlx::query(
|
||||||
"UPDATE missions
|
"UPDATE missions
|
||||||
SET runtime_container_name = $3,
|
SET runtime_container_name = $3,
|
||||||
runtime_endpoint = $4,
|
runtime_endpoint = $4,
|
||||||
@@ -373,6 +391,16 @@ pub async fn set_runtime_binding(
|
|||||||
.bind(pairing_code)
|
.bind(pairing_code)
|
||||||
.execute(pool)
|
.execute(pool)
|
||||||
.await?;
|
.await?;
|
||||||
|
// A `WHERE id = $1 AND workspace_id = $2` that matches nothing is not an
|
||||||
|
// error to sqlx — it updates zero rows and returns Ok. That made a
|
||||||
|
// mismatched workspace indistinguishable from a successful bind, and the
|
||||||
|
// binding is what the sweeper uses to find a mission's container: a silent
|
||||||
|
// no-op here leaks a container with no record that anything went wrong.
|
||||||
|
// Callers log this rather than aborting, which is the point — it becomes
|
||||||
|
// visible instead of invisible.
|
||||||
|
if r.rows_affected() == 0 {
|
||||||
|
return Err(DbError::NotFound);
|
||||||
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ pub mod fleet_beszel;
|
|||||||
pub mod fleet_tailscale;
|
pub mod fleet_tailscale;
|
||||||
pub mod level_up;
|
pub mod level_up;
|
||||||
pub mod messages;
|
pub mod messages;
|
||||||
|
pub mod mission_plan_proposals;
|
||||||
|
pub mod mission_team_proposals;
|
||||||
pub mod missions;
|
pub mod missions;
|
||||||
pub mod node_metrics;
|
pub mod node_metrics;
|
||||||
pub mod node_rules;
|
pub mod node_rules;
|
||||||
|
|||||||
@@ -71,6 +71,22 @@ pub struct EvalRow {
|
|||||||
pub gpu_pct: Option<f64>,
|
pub gpu_pct: Option<f64>,
|
||||||
pub temp_max: Option<f64>,
|
pub temp_max: Option<f64>,
|
||||||
pub load1: Option<f64>,
|
pub load1: Option<f64>,
|
||||||
|
/// Absolute memory, for CAPACITY rather than utilisation. `mem_pct` cannot
|
||||||
|
/// answer "does another 8 GiB VM fit" — a node at 20% of 31 GiB and one at
|
||||||
|
/// 20% of 60 GiB report the same percentage and hold a different number of
|
||||||
|
/// VMs. From the 5s heartbeat, which is the only source with absolutes.
|
||||||
|
pub mem_total_bytes: Option<i64>,
|
||||||
|
pub mem_used_bytes: Option<i64>,
|
||||||
|
pub disk_free_bytes: Option<i64>,
|
||||||
|
/// Memory in use with no phase VMs committed, remembered from the last time
|
||||||
|
/// this node was observed idle. `None` until then, which makes placement
|
||||||
|
/// fall back to inferring it — the behaviour that over-committed morpheus.
|
||||||
|
pub mem_baseline_mib: Option<i64>,
|
||||||
|
/// Age of each source. Placement is fail-closed on stale health (a node whose
|
||||||
|
/// RAM we cannot read is one we are guessing at), and demotes rather than
|
||||||
|
/// excludes on stale Beszel metrics, which only ever break ties.
|
||||||
|
pub health_age_secs: Option<f64>,
|
||||||
|
pub metrics_age_secs: Option<f64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl EvalRow {
|
impl EvalRow {
|
||||||
@@ -91,6 +107,22 @@ impl EvalRow {
|
|||||||
let used = self.cpu_pct.unwrap_or(0.0).max(self.mem_pct.unwrap_or(0.0));
|
let used = self.cpu_pct.unwrap_or(0.0).max(self.mem_pct.unwrap_or(0.0));
|
||||||
100.0 - used
|
100.0 - used
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// [`headroom`] when at least one source is recent, otherwise the WORST
|
||||||
|
/// possible score.
|
||||||
|
///
|
||||||
|
/// Used only as a placement TIEBREAK, never as an admission gate: stale
|
||||||
|
/// metrics may cost a node a tie, they may never win one. Admission is
|
||||||
|
/// decided by absolute memory from the heartbeat, which has its own
|
||||||
|
/// freshness check.
|
||||||
|
pub fn headroom_fresh(&self, max_age_secs: f64) -> f64 {
|
||||||
|
let fresh = |a: Option<f64>| a.is_some_and(|x| x <= max_age_secs);
|
||||||
|
if fresh(self.health_age_secs) || fresh(self.metrics_age_secs) {
|
||||||
|
self.headroom()
|
||||||
|
} else {
|
||||||
|
0.0
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Every node's current metric scalars (merged Beszel + heartbeat health).
|
/// Every node's current metric scalars (merged Beszel + heartbeat health).
|
||||||
@@ -101,7 +133,13 @@ pub async fn eval_all(pool: &PgPool) -> Result<Vec<EvalRow>, DbError> {
|
|||||||
COALESCE(m.mem_pct, CASE WHEN h.mem_total > 0 THEN h.mem_used::float8 / h.mem_total * 100 END) AS mem_pct,
|
COALESCE(m.mem_pct, CASE WHEN h.mem_total > 0 THEN h.mem_used::float8 / h.mem_total * 100 END) AS mem_pct,
|
||||||
COALESCE(m.disk_pct, CASE WHEN h.disk_total > 0 THEN (h.disk_total - h.disk_free)::float8 / h.disk_total * 100 END) AS disk_pct,
|
COALESCE(m.disk_pct, CASE WHEN h.disk_total > 0 THEN (h.disk_total - h.disk_free)::float8 / h.disk_total * 100 END) AS disk_pct,
|
||||||
m.gpu_pct, m.temp_max,
|
m.gpu_pct, m.temp_max,
|
||||||
COALESCE(m.load1, h.load1) AS load1
|
COALESCE(m.load1, h.load1) AS load1,
|
||||||
|
h.mem_total AS mem_total_bytes,
|
||||||
|
h.mem_used AS mem_used_bytes,
|
||||||
|
h.disk_free AS disk_free_bytes,
|
||||||
|
n.mem_baseline_mib,
|
||||||
|
EXTRACT(EPOCH FROM now() - h.captured_at)::float8 AS health_age_secs,
|
||||||
|
EXTRACT(EPOCH FROM now() - m.updated_at)::float8 AS metrics_age_secs
|
||||||
FROM nodes n
|
FROM nodes n
|
||||||
LEFT JOIN node_health h ON h.node_id = n.id
|
LEFT JOIN node_health h ON h.node_id = n.id
|
||||||
LEFT JOIN node_metrics m ON m.node_id = n.id",
|
LEFT JOIN node_metrics m ON m.node_id = n.id",
|
||||||
@@ -120,6 +158,12 @@ pub async fn eval_all(pool: &PgPool) -> Result<Vec<EvalRow>, DbError> {
|
|||||||
gpu_pct: r.get("gpu_pct"),
|
gpu_pct: r.get("gpu_pct"),
|
||||||
temp_max: r.get("temp_max"),
|
temp_max: r.get("temp_max"),
|
||||||
load1: r.get("load1"),
|
load1: r.get("load1"),
|
||||||
|
mem_total_bytes: r.get("mem_total_bytes"),
|
||||||
|
mem_used_bytes: r.get("mem_used_bytes"),
|
||||||
|
disk_free_bytes: r.get("disk_free_bytes"),
|
||||||
|
mem_baseline_mib: r.get("mem_baseline_mib"),
|
||||||
|
health_age_secs: r.get("health_age_secs"),
|
||||||
|
metrics_age_secs: r.get("metrics_age_secs"),
|
||||||
})
|
})
|
||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -178,6 +178,170 @@ pub async fn set_status(pool: &PgPool, id: NodeId, status: &str) -> Result<(), D
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Record what a node reports it can host, for placement predicates.
|
||||||
|
///
|
||||||
|
/// Replaces rather than merges: the node sends its complete view on every
|
||||||
|
/// report, so a capability it has *stopped* having (firecracker uninstalled,
|
||||||
|
/// `/dev/kvm` gone after a reboot into a non-virt kernel) must disappear here
|
||||||
|
/// too. Merging would let a stale `true` survive forever.
|
||||||
|
pub async fn set_capabilities(
|
||||||
|
pool: &PgPool,
|
||||||
|
id: NodeId,
|
||||||
|
capabilities: &serde_json::Value,
|
||||||
|
) -> Result<(), DbError> {
|
||||||
|
sqlx::query("UPDATE nodes SET capabilities = $2 WHERE id = $1")
|
||||||
|
.bind(id.as_uuid())
|
||||||
|
.bind(capabilities)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Online nodes that report every one of `required` as `true`.
|
||||||
|
///
|
||||||
|
/// The predicate side of placement. Nothing is assumed: a node that has never
|
||||||
|
/// reported has `capabilities = '{}'`, which fails every requirement — an
|
||||||
|
/// unqueried node and an incapable node are treated identically, because
|
||||||
|
/// scheduling work onto a node whose abilities are unknown is how you get a
|
||||||
|
/// mission that cannot start and does not say why.
|
||||||
|
pub async fn online_with_capabilities(
|
||||||
|
pool: &PgPool,
|
||||||
|
workspace_id: uuid::Uuid,
|
||||||
|
required: &[&str],
|
||||||
|
) -> Result<Vec<NodeId>, DbError> {
|
||||||
|
let needed: serde_json::Value = required
|
||||||
|
.iter()
|
||||||
|
.map(|k| ((*k).to_string(), serde_json::Value::Bool(true)))
|
||||||
|
.collect::<serde_json::Map<_, _>>()
|
||||||
|
.into();
|
||||||
|
let rows: Vec<(uuid::Uuid,)> = sqlx::query_as(
|
||||||
|
"SELECT id FROM nodes
|
||||||
|
WHERE workspace_id = $1 AND status = 'online' AND capabilities @> $2
|
||||||
|
ORDER BY last_seen DESC NULLS LAST",
|
||||||
|
)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(&needed)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows.into_iter().map(|(id,)| NodeId::from(id)).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Online nodes that can host a microVM **and** hold the image `backend` names.
|
||||||
|
///
|
||||||
|
/// KVM alone is the wrong predicate. The first real microVM mission was placed
|
||||||
|
/// on a node reporting `microvm: true` that did not have `rootfs-claude.ext4`;
|
||||||
|
/// it failed by name rather than booting the wrong image, but whether a mission
|
||||||
|
/// ran came down to which capable node was listed first.
|
||||||
|
///
|
||||||
|
/// `backend = None` means the node's default image, which reports itself as
|
||||||
|
/// `"default"` — so the requirement is never vacuous. A node running an older
|
||||||
|
/// daemon has no `rootfs` key at all and matches nothing, which is the same
|
||||||
|
/// treatment an unqueried node gets for every other capability: unknown is not
|
||||||
|
/// permission.
|
||||||
|
///
|
||||||
|
/// Capability only — this says a node COULD run the backend, not that it has room.
|
||||||
|
/// Capacity is `cm_api::vm_placement`'s job.
|
||||||
|
/// microVM phases already pinned to a node but whose VM may not exist yet.
|
||||||
|
///
|
||||||
|
/// The other half of "how much is this node committed to". `vm_list` reports
|
||||||
|
/// BOOTED VMs; between `phase_runner` choosing a node and the guest answering,
|
||||||
|
/// there is a window of seconds in which a phase is a real 8 GiB claim that no
|
||||||
|
/// node can report. Two missions launched together both survey a node as empty
|
||||||
|
/// and both land on it.
|
||||||
|
///
|
||||||
|
/// Returns (node, phase_id, iteration) so the caller can build the same
|
||||||
|
/// deterministic vm id the executor uses and union the two sets by identity
|
||||||
|
/// rather than adding them — a phase whose VM HAS booted must count once, not
|
||||||
|
/// twice.
|
||||||
|
pub async fn pinned_microvm_phases(
|
||||||
|
pool: &PgPool,
|
||||||
|
workspace_id: uuid::Uuid,
|
||||||
|
) -> Result<Vec<(NodeId, uuid::Uuid, i32)>, DbError> {
|
||||||
|
let rows: Vec<(uuid::Uuid, uuid::Uuid, i32)> = sqlx::query_as(
|
||||||
|
"SELECT m.target_node_id, p.id, p.iteration
|
||||||
|
FROM mission_phases p
|
||||||
|
JOIN missions m ON m.id = p.mission_id
|
||||||
|
WHERE m.workspace_id = $1
|
||||||
|
AND m.runtime_kind = 'microvm'
|
||||||
|
AND m.status = 'running'
|
||||||
|
AND m.target_node_id IS NOT NULL
|
||||||
|
-- `pending` counts: it is about to become a VM. `completed`/`failed`
|
||||||
|
-- do not: their VM is destroyed on every exit path of
|
||||||
|
-- `run_phase_in_vm`, so counting them would shrink the fleet by the
|
||||||
|
-- number of missions it has ever run.
|
||||||
|
AND p.status IN ('pending', 'running')",
|
||||||
|
)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows
|
||||||
|
.into_iter()
|
||||||
|
.map(|(n, p, i)| (NodeId::from(n), p, i))
|
||||||
|
.collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn online_for_backend(
|
||||||
|
pool: &PgPool,
|
||||||
|
workspace_id: uuid::Uuid,
|
||||||
|
backend: Option<&str>,
|
||||||
|
) -> Result<Vec<NodeId>, DbError> {
|
||||||
|
online_for_backends(pool, workspace_id, &[backend_key(backend).to_string()]).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Nodes that can run EVERY one of these backends.
|
||||||
|
///
|
||||||
|
/// A composed mission runs its whole graph on one node, and the graph's nodes
|
||||||
|
/// may each name their own backend — an independent verifier on another
|
||||||
|
/// provider is the entire point of the roster. Asking only for the mission's
|
||||||
|
/// backend placed such a mission on a node with `claude` and no
|
||||||
|
/// `canary-claude`, and the run died at the second graph node with
|
||||||
|
/// `no rootfs for backend "canary-claude" on this node`. The full harness
|
||||||
|
/// caught it; nothing before it had a reason to.
|
||||||
|
pub async fn online_for_backends(
|
||||||
|
pool: &PgPool,
|
||||||
|
workspace_id: uuid::Uuid,
|
||||||
|
backends: &[String],
|
||||||
|
) -> Result<Vec<NodeId>, DbError> {
|
||||||
|
// `@>` on the array asks "does this node's list contain ALL of these" —
|
||||||
|
// containment, not intersection, which is exactly the question here and the
|
||||||
|
// whole reason the node reports an array rather than a count.
|
||||||
|
let rows: Vec<(uuid::Uuid,)> = sqlx::query_as(
|
||||||
|
"SELECT id FROM nodes
|
||||||
|
WHERE workspace_id = $1 AND status = 'online'
|
||||||
|
AND capabilities @> '{\"microvm\": true}'::jsonb
|
||||||
|
AND capabilities -> 'rootfs' @> $2::jsonb
|
||||||
|
-- Deterministic, NOT `last_seen DESC`. Ranking now happens in
|
||||||
|
-- `cm_api::vm_placement`, against real capacity. Ordering by last_seen
|
||||||
|
-- and taking `.first()` was the placement algorithm until now: among
|
||||||
|
-- healthy nodes all heartbeating every 5s, that is arbitrary — it sent
|
||||||
|
-- concurrent missions to whichever node's packet landed most recently,
|
||||||
|
-- with no regard for what was already running there.
|
||||||
|
ORDER BY id",
|
||||||
|
)
|
||||||
|
.bind(workspace_id)
|
||||||
|
.bind(serde_json::Value::Array(
|
||||||
|
backends
|
||||||
|
.iter()
|
||||||
|
.map(|b| serde_json::Value::String(b.clone()))
|
||||||
|
.collect(),
|
||||||
|
))
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(rows.into_iter().map(|(id,)| NodeId::from(id)).collect())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The name a backend reports itself as in a node's `rootfs` list.
|
||||||
|
///
|
||||||
|
/// Must agree with `clawmates-node::microvm::rootfs_for`, which resolves the same
|
||||||
|
/// three spellings to the default image. If these two drift, placement promises
|
||||||
|
/// an image the booter cannot find — or refuses one it has.
|
||||||
|
pub fn backend_key(backend: Option<&str>) -> &str {
|
||||||
|
match backend {
|
||||||
|
None | Some("") | Some("default") => "default",
|
||||||
|
Some(b) => b,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Mark online nodes whose last heartbeat is older than `secs` as offline.
|
/// Mark online nodes whose last heartbeat is older than `secs` as offline.
|
||||||
pub async fn mark_stale_offline(pool: &PgPool, secs: i64) -> Result<(), DbError> {
|
pub async fn mark_stale_offline(pool: &PgPool, secs: i64) -> Result<(), DbError> {
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
@@ -242,3 +406,40 @@ fn map_node(r: sqlx::postgres::PgRow) -> NodeRow {
|
|||||||
temp_max: r.get("m_temp_max"),
|
temp_max: r.get("m_temp_max"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// The three spellings that mean "the node's default image" must all resolve
|
||||||
|
/// to the name the node actually advertises for it. A mismatch here makes
|
||||||
|
/// placement reject every node for an ordinary mission with no backend set.
|
||||||
|
#[test]
|
||||||
|
fn the_default_backend_has_one_name() {
|
||||||
|
for spelling in [None, Some(""), Some("default")] {
|
||||||
|
assert_eq!(backend_key(spelling), "default", "{spelling:?}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// And a named backend is passed through verbatim — it is matched against the
|
||||||
|
/// node's list, which is built from the filenames on its disk.
|
||||||
|
#[test]
|
||||||
|
fn a_named_backend_is_not_rewritten() {
|
||||||
|
assert_eq!(backend_key(Some("claude")), "claude");
|
||||||
|
assert_eq!(backend_key(Some("agent-terminal")), "agent-terminal");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Remember a node's idle memory footprint.
|
||||||
|
///
|
||||||
|
/// Only ever called with a reading taken while the node had ZERO phase VMs
|
||||||
|
/// committed — that is the one moment the number is honestly observable.
|
||||||
|
/// Writing it at any other time would record the VMs as part of the host.
|
||||||
|
pub async fn set_mem_baseline(pool: &PgPool, node_id: NodeId, mib: i64) -> Result<(), DbError> {
|
||||||
|
sqlx::query("UPDATE nodes SET mem_baseline_mib = $2 WHERE id = $1")
|
||||||
|
.bind(node_id.as_uuid())
|
||||||
|
.bind(mib)
|
||||||
|
.execute(pool)
|
||||||
|
.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|||||||
@@ -44,6 +44,12 @@ pub struct TemplateRole {
|
|||||||
pub system_prompt: String,
|
pub system_prompt: String,
|
||||||
pub skills: Vec<String>,
|
pub skills: Vec<String>,
|
||||||
pub brain_seed: Option<String>,
|
pub brain_seed: Option<String>,
|
||||||
|
/// Which model this role's claw runs on. `None` takes the mint's default —
|
||||||
|
/// which is what every role did unconditionally before migration 0071, and
|
||||||
|
/// why a template could not put its reviewer on a different model from the
|
||||||
|
/// coder it reviews.
|
||||||
|
#[serde(default)]
|
||||||
|
pub model: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize)]
|
#[derive(Debug, Clone, Serialize)]
|
||||||
@@ -60,6 +66,8 @@ pub struct UpsertBuiltinRole<'a> {
|
|||||||
pub system_prompt: &'a str,
|
pub system_prompt: &'a str,
|
||||||
pub skills: Vec<String>,
|
pub skills: Vec<String>,
|
||||||
pub brain_seed: Option<&'a str>,
|
pub brain_seed: Option<&'a str>,
|
||||||
|
/// Optional per-role model. `None` leaves the mint's default in place.
|
||||||
|
pub model: Option<&'a str>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
@@ -138,13 +146,14 @@ pub async fn upsert_builtin(pool: &PgPool, b: UpsertBuiltin<'_>) -> Result<Uuid,
|
|||||||
for r in &b.roles {
|
for r in &b.roles {
|
||||||
sqlx::query(
|
sqlx::query(
|
||||||
"INSERT INTO template_roles
|
"INSERT INTO template_roles
|
||||||
(template_id, slot, order_idx, system_prompt, skills, brain_seed)
|
(template_id, slot, order_idx, system_prompt, skills, brain_seed, model)
|
||||||
VALUES ($1,$2,$3,$4,$5,$6)
|
VALUES ($1,$2,$3,$4,$5,$6,$7)
|
||||||
ON CONFLICT (template_id, slot) DO UPDATE SET
|
ON CONFLICT (template_id, slot) DO UPDATE SET
|
||||||
order_idx = EXCLUDED.order_idx,
|
order_idx = EXCLUDED.order_idx,
|
||||||
system_prompt = EXCLUDED.system_prompt,
|
system_prompt = EXCLUDED.system_prompt,
|
||||||
skills = EXCLUDED.skills,
|
skills = EXCLUDED.skills,
|
||||||
brain_seed = EXCLUDED.brain_seed",
|
brain_seed = EXCLUDED.brain_seed,
|
||||||
|
model = EXCLUDED.model",
|
||||||
)
|
)
|
||||||
.bind(id)
|
.bind(id)
|
||||||
.bind(r.slot)
|
.bind(r.slot)
|
||||||
@@ -152,6 +161,7 @@ pub async fn upsert_builtin(pool: &PgPool, b: UpsertBuiltin<'_>) -> Result<Uuid,
|
|||||||
.bind(r.system_prompt)
|
.bind(r.system_prompt)
|
||||||
.bind(&r.skills)
|
.bind(&r.skills)
|
||||||
.bind(r.brain_seed)
|
.bind(r.brain_seed)
|
||||||
|
.bind(r.model)
|
||||||
.execute(&mut *tx)
|
.execute(&mut *tx)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
@@ -226,7 +236,7 @@ pub async fn get(pool: &PgPool, id: Uuid) -> Result<Option<TeamTemplateDetail>,
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
};
|
};
|
||||||
let role_rows = sqlx::query(
|
let role_rows = sqlx::query(
|
||||||
"SELECT template_id, slot, order_idx, system_prompt, skills, brain_seed
|
"SELECT template_id, slot, order_idx, system_prompt, skills, brain_seed, model
|
||||||
FROM template_roles WHERE template_id = $1
|
FROM template_roles WHERE template_id = $1
|
||||||
ORDER BY order_idx ASC",
|
ORDER BY order_idx ASC",
|
||||||
)
|
)
|
||||||
@@ -242,6 +252,7 @@ pub async fn get(pool: &PgPool, id: Uuid) -> Result<Option<TeamTemplateDetail>,
|
|||||||
system_prompt: r.get("system_prompt"),
|
system_prompt: r.get("system_prompt"),
|
||||||
skills: r.get("skills"),
|
skills: r.get("skills"),
|
||||||
brain_seed: r.get("brain_seed"),
|
brain_seed: r.get("brain_seed"),
|
||||||
|
model: r.get("model"),
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
Ok(Some(TeamTemplateDetail { template: t, roles }))
|
Ok(Some(TeamTemplateDetail { template: t, roles }))
|
||||||
|
|||||||
@@ -54,6 +54,15 @@ pub struct ClaimedTopologyRun {
|
|||||||
/// Deploy tier: `team` drives claws directly; `company`/`org` drive the
|
/// Deploy tier: `team` drives claws directly; `company`/`org` drive the
|
||||||
/// recursive sub-topology executor.
|
/// recursive sub-topology executor.
|
||||||
pub tier: String,
|
pub tier: String,
|
||||||
|
/// The mission this run belongs to, when it belongs to one. The composed
|
||||||
|
/// (`microvm_graph`) tier needs it: its nodes share the mission's checkout,
|
||||||
|
/// and that shared tree is how file work survives a node boundary.
|
||||||
|
pub mission_id: Option<Uuid>,
|
||||||
|
/// The mission phase, for the same reason — the phase and pass identify the
|
||||||
|
/// VMs a composed run may boot.
|
||||||
|
pub mission_phase_id: Option<Uuid>,
|
||||||
|
/// Which pass of the phase produced this run.
|
||||||
|
pub iteration: Option<i32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Lifecycle status + progress for a durable run (status endpoint).
|
/// Lifecycle status + progress for a durable run (status endpoint).
|
||||||
@@ -203,51 +212,129 @@ pub async fn check_ephemeral_teardown(
|
|||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Tiers the topology worker drives, and therefore the only ones it may claim,
|
||||||
|
/// requeue or reap.
|
||||||
|
///
|
||||||
|
/// **A load-bearing allowlist, not tidiness.** Both sweepers were written when
|
||||||
|
/// every `running` row was a `cm_orchestrator` job that checkpointed after each
|
||||||
|
/// step. `tier='microvm'` and `tier='session'` broke that assumption: they are
|
||||||
|
/// inserted directly as `running` by `phase_runner`, driven by a `tokio::spawn`
|
||||||
|
/// that owns them start to finish, and they never write `updated_at` or
|
||||||
|
/// `checkpoint` while in flight.
|
||||||
|
///
|
||||||
|
/// Measured cost of the omission: `requeue_stale` flipped an in-flight microVM run
|
||||||
|
/// to `queued` at 180s, `claim_next_queued` handed it to the worker, and the worker
|
||||||
|
/// failed it with "missing or invalid graph" — a microvm run's graph is a
|
||||||
|
/// placeholder `TopologyGraph` cannot parse. Mission 019fd43e died at 210 seconds
|
||||||
|
/// with its agent still working and its VM orphaned. Every microVM mission that
|
||||||
|
/// appeared to work did so only by finishing inside three minutes.
|
||||||
|
///
|
||||||
|
/// An allowlist rather than a denylist on purpose: the next self-driven tier is
|
||||||
|
/// then safe by default, instead of exposed until someone remembers this file.
|
||||||
|
pub const WORKER_DRIVEN_TIERS: &[&str] = &[
|
||||||
|
"team",
|
||||||
|
"company",
|
||||||
|
"org",
|
||||||
|
"swarm",
|
||||||
|
"compare",
|
||||||
|
// The composed engines: a ZeroClaw graph whose every node is a
|
||||||
|
// Claude-Code-in-a-microVM session. Worker-driven BY DESIGN — the outer
|
||||||
|
// graph's durability (checkpoint, resume, cancellation) is the entire reason
|
||||||
|
// the tier exists, and it comes from being claimed like any other job. It
|
||||||
|
// survives `requeue_stale` because the executor touches `updated_at` from a
|
||||||
|
// ticker for the whole length of a VM turn, not only between steps.
|
||||||
|
"microvm_graph",
|
||||||
|
];
|
||||||
|
|
||||||
|
/// Tiers the stuck-run reaper may fail.
|
||||||
|
///
|
||||||
|
/// A subset of [`WORKER_DRIVEN_TIERS`], and the difference matters. The reaper
|
||||||
|
/// asks "has this run journaled a step within 15 minutes of being CREATED?",
|
||||||
|
/// which assumes a step is short. A `microvm_graph` node is a whole agent session
|
||||||
|
/// in a VM with an hour's budget, so a healthy composed run can legitimately
|
||||||
|
/// journal nothing for far longer than the reaper's patience — it would kill the
|
||||||
|
/// run and orphan a live VM, which is #54 wearing a different tier.
|
||||||
|
///
|
||||||
|
/// Losing the reaper for that tier costs little: a composed run that genuinely
|
||||||
|
/// wedges stops touching `updated_at` and `requeue_stale` recovers it at 180s,
|
||||||
|
/// which is the mechanism the reaper was a backstop for in the first place.
|
||||||
|
pub const REAPABLE_TIERS: &[&str] = &["team", "company", "org", "swarm", "compare"];
|
||||||
|
|
||||||
|
/// The allowlist as owned strings, for binding as `text[]`.
|
||||||
|
fn worker_driven() -> Vec<String> {
|
||||||
|
WORKER_DRIVEN_TIERS.iter().map(|s| (*s).to_string()).collect()
|
||||||
|
}
|
||||||
|
|
||||||
/// Atomically claim the oldest queued job, flipping it to `running`. Uses
|
/// Atomically claim the oldest queued job, flipping it to `running`. Uses
|
||||||
/// `FOR UPDATE SKIP LOCKED` so multiple workers never claim the same job.
|
/// `FOR UPDATE SKIP LOCKED` so multiple workers never claim the same job.
|
||||||
/// Returns `None` when the queue is empty.
|
/// Returns `None` when the queue is empty.
|
||||||
pub async fn claim_next_queued(pool: &PgPool) -> Result<Option<ClaimedTopologyRun>, DbError> {
|
pub async fn claim_next_queued(pool: &PgPool) -> Result<Option<ClaimedTopologyRun>, DbError> {
|
||||||
let row = sqlx::query!(
|
use sqlx::Row as _;
|
||||||
|
// A runtime query rather than `query!` so the tier allowlist can be bound
|
||||||
|
// without regenerating the offline metadata on a machine with no database.
|
||||||
|
let row = sqlx::query(
|
||||||
"UPDATE topology_runs
|
"UPDATE topology_runs
|
||||||
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
|
SET status = 'running', started_at = COALESCE(started_at, now()), updated_at = now()
|
||||||
WHERE id = (
|
WHERE id = (
|
||||||
SELECT id FROM topology_runs
|
SELECT id FROM topology_runs
|
||||||
WHERE status = 'queued'
|
WHERE status = 'queued'
|
||||||
|
-- Defence in depth. Even if a self-driven row somehow reaches
|
||||||
|
-- `queued`, the worker must not adopt a job it cannot execute:
|
||||||
|
-- doing so is what turned a live microVM run into a
|
||||||
|
-- missing-or-invalid-graph failure.
|
||||||
|
AND tier = ANY($1)
|
||||||
ORDER BY created_at
|
ORDER BY created_at
|
||||||
FOR UPDATE SKIP LOCKED
|
FOR UPDATE SKIP LOCKED
|
||||||
LIMIT 1
|
LIMIT 1
|
||||||
)
|
)
|
||||||
RETURNING id, workspace_id, task, graph, checkpoint, last_event_id, tier",
|
RETURNING id, workspace_id, task, graph, checkpoint, last_event_id, tier,
|
||||||
|
mission_id, mission_phase_id, iteration",
|
||||||
)
|
)
|
||||||
|
.bind(worker_driven())
|
||||||
.fetch_optional(pool)
|
.fetch_optional(pool)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(row.map(|r| ClaimedTopologyRun {
|
Ok(row.map(|r| ClaimedTopologyRun {
|
||||||
id: r.id,
|
id: r.get("id"),
|
||||||
workspace_id: r.workspace_id,
|
workspace_id: r.get("workspace_id"),
|
||||||
task: r.task,
|
task: r.get("task"),
|
||||||
graph: r.graph,
|
graph: r.get("graph"),
|
||||||
checkpoint: r.checkpoint,
|
checkpoint: r.get("checkpoint"),
|
||||||
last_event_id: r.last_event_id,
|
last_event_id: r.get("last_event_id"),
|
||||||
tier: r.tier,
|
tier: r.get("tier"),
|
||||||
|
mission_id: r.get("mission_id"),
|
||||||
|
mission_phase_id: r.get("mission_phase_id"),
|
||||||
|
iteration: r.get("iteration"),
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Persist mid-run progress: the completed-step checkpoint + journal offset.
|
/// Persist mid-run progress: the completed-step checkpoint + journal offset.
|
||||||
/// Touches `updated_at` so the stale-run sweeper treats the job as alive.
|
/// Touches `updated_at` so the stale-run sweeper treats the job as alive.
|
||||||
|
///
|
||||||
|
/// MERGES rather than replaces. This is not cosmetic: a second writer appends
|
||||||
|
/// live agent output under `checkpoint.log` (see `fleet.rs`, `Uplink::VmOut`),
|
||||||
|
/// and a composed run checkpoints after EVERY graph node. With `SET checkpoint =
|
||||||
|
/// $2` each node's progress silently wiped the log written during it, so a
|
||||||
|
/// composed mission finished with a full `records` array and no output at all —
|
||||||
|
/// while a solo mission, which has no second writer, streamed fine. The keys are
|
||||||
|
/// disjoint, so the progress object still wins for everything it owns.
|
||||||
pub async fn checkpoint(
|
pub async fn checkpoint(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
id: Uuid,
|
id: Uuid,
|
||||||
checkpoint: &Value,
|
checkpoint: &Value,
|
||||||
last_event_id: i64,
|
last_event_id: i64,
|
||||||
) -> Result<(), DbError> {
|
) -> Result<(), DbError> {
|
||||||
sqlx::query!(
|
// `query` rather than `query!`: the macro verifies against a cached schema
|
||||||
|
// that would need regenerating for this SQL, and the bind types here are
|
||||||
|
// unambiguous.
|
||||||
|
sqlx::query(
|
||||||
"UPDATE topology_runs
|
"UPDATE topology_runs
|
||||||
SET checkpoint = $2, last_event_id = $3, updated_at = now()
|
SET checkpoint = COALESCE(checkpoint, '{}'::jsonb) || $2,
|
||||||
|
last_event_id = $3, updated_at = now()
|
||||||
WHERE id = $1",
|
WHERE id = $1",
|
||||||
id,
|
|
||||||
checkpoint,
|
|
||||||
last_event_id,
|
|
||||||
)
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(checkpoint)
|
||||||
|
.bind(last_event_id)
|
||||||
.execute(pool)
|
.execute(pool)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -325,12 +412,19 @@ pub async fn current_status(pool: &PgPool, id: Uuid) -> Result<Option<String>, D
|
|||||||
/// touch within `older_than_secs`). The next claim resumes them from checkpoint.
|
/// touch within `older_than_secs`). The next claim resumes them from checkpoint.
|
||||||
/// Returns how many were requeued.
|
/// Returns how many were requeued.
|
||||||
pub async fn requeue_stale(pool: &PgPool, older_than_secs: f64) -> Result<u64, DbError> {
|
pub async fn requeue_stale(pool: &PgPool, older_than_secs: f64) -> Result<u64, DbError> {
|
||||||
let result = sqlx::query!(
|
let result = sqlx::query(
|
||||||
"UPDATE topology_runs
|
"UPDATE topology_runs
|
||||||
SET status = 'queued', updated_at = now()
|
SET status = 'queued', updated_at = now()
|
||||||
WHERE status = 'running' AND updated_at < now() - make_interval(secs => $1)",
|
WHERE status = 'running'
|
||||||
older_than_secs,
|
AND updated_at < now() - make_interval(secs => $1)
|
||||||
|
-- Only jobs the WORKER drives. A self-driven run (microvm, session) is
|
||||||
|
-- owned by its own task for its whole life and never touches
|
||||||
|
-- `updated_at`, so without this every one of them looked stale after
|
||||||
|
-- three minutes and was requeued out from under a live VM.
|
||||||
|
AND tier = ANY($2)",
|
||||||
)
|
)
|
||||||
|
.bind(older_than_secs)
|
||||||
|
.bind(worker_driven())
|
||||||
.execute(pool)
|
.execute(pool)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(result.rows_affected())
|
Ok(result.rows_affected())
|
||||||
|
|||||||
@@ -0,0 +1,148 @@
|
|||||||
|
//! Approving a plan rewrites a mission's phases — atomically, and with
|
||||||
|
//! `done_when` promoted into the column the evaluator actually reads.
|
||||||
|
|
||||||
|
use cm_db::repo::mission_plan_proposals as plans;
|
||||||
|
use cm_domain::WorkspaceId;
|
||||||
|
use serde_json::{json, Value};
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
async fn workspace(pool: &sqlx::PgPool) -> WorkspaceId {
|
||||||
|
let ws = cm_domain::Workspace {
|
||||||
|
id: WorkspaceId::new(),
|
||||||
|
name: "Plan".into(),
|
||||||
|
plan: "team".into(),
|
||||||
|
};
|
||||||
|
cm_db::repo::workspaces::insert(pool, &ws).await.expect("workspace");
|
||||||
|
ws.id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A mission with the recipe-derived phases a plan is meant to replace.
|
||||||
|
async fn mission_with_phases(pool: &sqlx::PgPool, ws: WorkspaceId) -> Uuid {
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO missions (id, workspace_id, title, template_kind, status, schedule, config)
|
||||||
|
VALUES ($1, $2, 'plan test', 'research_and_code', 'draft', '{}'::jsonb, 'null'::jsonb)",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("insert mission");
|
||||||
|
for (kind, idx) in [("research", 0), ("coding", 1)] {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO mission_phases (id, mission_id, kind, order_idx, status, config)
|
||||||
|
VALUES ($1, $2, $3, $4, 'pending', '{}'::jsonb)",
|
||||||
|
)
|
||||||
|
.bind(Uuid::now_v7())
|
||||||
|
.bind(id)
|
||||||
|
.bind(kind)
|
||||||
|
.bind(idx)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("insert phase");
|
||||||
|
}
|
||||||
|
id
|
||||||
|
}
|
||||||
|
|
||||||
|
fn a_plan() -> Value {
|
||||||
|
json!({"phases": [{"kind": "coding", "task": "do the thing", "done_when": "FILE.md exists"}]})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn phases() -> Vec<(String, i32, Value)> {
|
||||||
|
vec![(
|
||||||
|
"coding".to_string(),
|
||||||
|
0,
|
||||||
|
json!({"task": "do the thing", "done_when": "FILE.md exists"}),
|
||||||
|
)]
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The plan REPLACES the recipe's phases — a plan is an answer to "what is this
|
||||||
|
/// mission", not an addition to one.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn an_approved_plan_replaces_the_missions_phases() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = mission_with_phases(&pool, ws).await;
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
plans::insert(&pool, id, m, ws.as_uuid().to_owned(), &a_plan(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
|
||||||
|
assert!(plans::approve_and_apply(&pool, id, m, ws.as_uuid().to_owned(), &phases(), None, None)
|
||||||
|
.await
|
||||||
|
.expect("apply"));
|
||||||
|
|
||||||
|
let rows: Vec<(String, i32, Option<String>, i32)> = sqlx::query_as(
|
||||||
|
"SELECT kind, order_idx, done_when, max_iterations FROM mission_phases
|
||||||
|
WHERE mission_id = $1 ORDER BY order_idx",
|
||||||
|
)
|
||||||
|
.bind(m)
|
||||||
|
.fetch_all(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(rows.len(), 1, "the two recipe phases must be gone: {rows:?}");
|
||||||
|
assert_eq!(rows[0].0, "coding");
|
||||||
|
assert_eq!(rows[0].1, 0);
|
||||||
|
// THE assertion. `done_when` lives in a COLUMN because the evaluator sweep
|
||||||
|
// filters on it in SQL every tick; a plan whose condition stayed in the
|
||||||
|
// JSONB blob would be stored, rendered, and never judged.
|
||||||
|
assert_eq!(
|
||||||
|
rows[0].2.as_deref(),
|
||||||
|
Some("FILE.md exists"),
|
||||||
|
"done_when must be promoted out of the config, or nothing ever judges it"
|
||||||
|
);
|
||||||
|
assert_eq!(rows[0].3, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Claim and apply are one decision. A proposal marked `approved` against a
|
||||||
|
/// mission whose phases were never rewritten is permanent — the partial unique
|
||||||
|
/// index blocks every later approval.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_failed_apply_leaves_the_proposal_undecided() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = mission_with_phases(&pool, ws).await;
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
plans::insert(&pool, id, m, ws.as_uuid().to_owned(), &a_plan(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
|
||||||
|
// Another workspace's id: the mission-ownership check inside the
|
||||||
|
// transaction must fail and undo the claim.
|
||||||
|
let other = workspace(&pool).await;
|
||||||
|
let err = plans::approve_and_apply(&pool, id, m, other.as_uuid().to_owned(), &phases(), None, None).await;
|
||||||
|
assert!(err.is_ok() || err.is_err());
|
||||||
|
|
||||||
|
let rows = plans::list(&pool, m, ws.as_uuid().to_owned()).await.expect("list");
|
||||||
|
assert_eq!(
|
||||||
|
rows[0].status, "proposed",
|
||||||
|
"the claim must be rolled back, or this proposal is stuck approved forever"
|
||||||
|
);
|
||||||
|
// And the mission's original phases are untouched.
|
||||||
|
let n: i64 = sqlx::query_scalar("SELECT count(*) FROM mission_phases WHERE mission_id = $1")
|
||||||
|
.bind(m)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(n, 2, "a failed apply must not have deleted the existing phases");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// At most one approved plan per mission: two would be two answers to "what is
|
||||||
|
/// this mission", and the phase table holds one.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_mission_cannot_have_two_approved_plans() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = mission_with_phases(&pool, ws).await;
|
||||||
|
let (a, b) = (Uuid::now_v7(), Uuid::now_v7());
|
||||||
|
for id in [a, b] {
|
||||||
|
plans::insert(&pool, id, m, ws.as_uuid().to_owned(), &a_plan(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
}
|
||||||
|
assert!(plans::approve_and_apply(&pool, a, m, ws.as_uuid().to_owned(), &phases(), None, None)
|
||||||
|
.await
|
||||||
|
.expect("approve a"));
|
||||||
|
let second = plans::approve_and_apply(&pool, b, m, ws.as_uuid().to_owned(), &phases(), None, None).await;
|
||||||
|
assert!(second.is_err(), "a second approved plan was allowed: {second:?}");
|
||||||
|
}
|
||||||
@@ -0,0 +1,240 @@
|
|||||||
|
//! A mission may have many proposals and at most one approved roster.
|
||||||
|
//!
|
||||||
|
//! Both properties are enforced in SQL rather than in the handler, and both
|
||||||
|
//! matter for the same reason: the composed executor reads ONE field for what
|
||||||
|
//! shape a mission is, so a second approval would silently win by being written
|
||||||
|
//! last.
|
||||||
|
|
||||||
|
use cm_db::repo::mission_team_proposals as proposals;
|
||||||
|
use cm_domain::WorkspaceId;
|
||||||
|
use serde_json::json;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
async fn workspace(pool: &sqlx::PgPool) -> WorkspaceId {
|
||||||
|
let ws = cm_domain::Workspace {
|
||||||
|
id: WorkspaceId::new(),
|
||||||
|
name: "Roster".into(),
|
||||||
|
plan: "team".into(),
|
||||||
|
};
|
||||||
|
cm_db::repo::workspaces::insert(pool, &ws).await.expect("workspace");
|
||||||
|
ws.id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A mission row to hang proposals off — `mission_id` is a real FK.
|
||||||
|
async fn mission(pool: &sqlx::PgPool, ws: WorkspaceId) -> Uuid {
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO missions (id, workspace_id, title, template_kind, status, schedule, config)
|
||||||
|
VALUES ($1, $2, 'roster test', 'research_and_code', 'draft', '{}'::jsonb, '{}'::jsonb)",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("insert mission");
|
||||||
|
id
|
||||||
|
}
|
||||||
|
|
||||||
|
fn roster() -> serde_json::Value {
|
||||||
|
json!({
|
||||||
|
"topology_kind": "pipeline",
|
||||||
|
"members": [
|
||||||
|
{"role": "implementer"},
|
||||||
|
{"role": "verifier", "backend": "kimi"}
|
||||||
|
]
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The whole point of persisting: a proposal is a record, not a click. It
|
||||||
|
/// arrives `proposed`, applied to nothing.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_proposal_arrives_undecided_and_is_listed() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = mission(&pool, ws).await;
|
||||||
|
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
proposals::insert(&pool, id, m, ws.as_uuid().to_owned(), &roster(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
|
||||||
|
let rows = proposals::list(&pool, m, ws.as_uuid().to_owned()).await.expect("list");
|
||||||
|
assert_eq!(rows.len(), 1);
|
||||||
|
assert_eq!(rows[0].status, "proposed");
|
||||||
|
assert_eq!(rows[0].author_model, "claude-opus-4-8");
|
||||||
|
assert!(rows[0].decided_at.is_none());
|
||||||
|
assert_eq!(rows[0].roster["members"][1]["backend"], "kimi");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Deciding twice must not decide twice. A double-clicked approve, or a retried
|
||||||
|
/// request, would otherwise re-apply a roster to a mission that has moved on.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn only_the_first_decision_counts() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = mission(&pool, ws).await;
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
proposals::insert(&pool, id, m, ws.as_uuid().to_owned(), &roster(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
|
||||||
|
let first = proposals::decide(&pool, id, ws.as_uuid().to_owned(), "approved", None, None)
|
||||||
|
.await
|
||||||
|
.expect("decide");
|
||||||
|
assert!(first, "the first approval must claim the proposal");
|
||||||
|
|
||||||
|
let second = proposals::decide(&pool, id, ws.as_uuid().to_owned(), "rejected", None, None)
|
||||||
|
.await
|
||||||
|
.expect("decide");
|
||||||
|
assert!(!second, "a decided proposal must not be re-decided");
|
||||||
|
|
||||||
|
let rows = proposals::list(&pool, m, ws.as_uuid().to_owned()).await.expect("list");
|
||||||
|
assert_eq!(rows[0].status, "approved", "and the first decision stands");
|
||||||
|
assert!(rows[0].decided_at.is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// At most one approved roster per mission, enforced by a partial unique index.
|
||||||
|
/// Two approved proposals are two answers to "what shape is this mission".
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_mission_cannot_have_two_approved_rosters() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = mission(&pool, ws).await;
|
||||||
|
|
||||||
|
let a = Uuid::now_v7();
|
||||||
|
let b = Uuid::now_v7();
|
||||||
|
for id in [a, b] {
|
||||||
|
proposals::insert(&pool, id, m, ws.as_uuid().to_owned(), &roster(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(proposals::decide(&pool, a, ws.as_uuid().to_owned(), "approved", None, None)
|
||||||
|
.await
|
||||||
|
.expect("approve a"));
|
||||||
|
// The second approval must be REFUSED by the database, not merely lose a
|
||||||
|
// race in the handler.
|
||||||
|
let second = proposals::decide(&pool, b, ws.as_uuid().to_owned(), "approved", None, None).await;
|
||||||
|
assert!(second.is_err(), "a second approved roster was allowed: {second:?}");
|
||||||
|
|
||||||
|
// Rejecting it is still fine — the constraint is on approvals only, and the
|
||||||
|
// ones a human turned down are the record of what the planner gets wrong.
|
||||||
|
assert!(proposals::decide(&pool, b, ws.as_uuid().to_owned(), "rejected", Some("too many VMs"), None)
|
||||||
|
.await
|
||||||
|
.expect("reject b"));
|
||||||
|
let rows = proposals::list(&pool, m, ws.as_uuid().to_owned()).await.expect("list");
|
||||||
|
assert_eq!(rows.len(), 2, "a rejected proposal is kept, not deleted");
|
||||||
|
assert!(rows.iter().any(|r| r.status == "rejected" && r.note.as_deref() == Some("too many VMs")));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Another workspace's proposal is not visible and not decidable. Every read
|
||||||
|
/// here is scoped, and this is the test that keeps it that way.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_proposal_belongs_to_its_workspace() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let other = workspace(&pool).await;
|
||||||
|
let m = mission(&pool, ws).await;
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
proposals::insert(&pool, id, m, ws.as_uuid().to_owned(), &roster(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
|
||||||
|
assert!(proposals::get(&pool, id, other.as_uuid().to_owned()).await.expect("get").is_none());
|
||||||
|
assert!(proposals::list(&pool, m, other.as_uuid().to_owned()).await.expect("list").is_empty());
|
||||||
|
assert!(
|
||||||
|
!proposals::decide(&pool, id, other.as_uuid().to_owned(), "approved", None, None)
|
||||||
|
.await
|
||||||
|
.expect("decide"),
|
||||||
|
"another workspace must not be able to approve this roster"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The bug production found on the FIRST real approval, in the exact shape it
|
||||||
|
/// had: a mission created through the API with no `config` stores jsonb `null`
|
||||||
|
/// — a scalar — and `jsonb_set` refuses a scalar with "cannot set path in
|
||||||
|
/// scalar". `coalesce` does not help, because that guards SQL NULL and this is a
|
||||||
|
/// perfectly good JSON null of the wrong shape.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_roster_applies_to_a_mission_whose_config_is_json_null() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = mission(&pool, ws).await;
|
||||||
|
// Exactly what `POST /api/missions` stores when the body omits `config`.
|
||||||
|
sqlx::query("UPDATE missions SET config = 'null'::jsonb WHERE id = $1")
|
||||||
|
.bind(m)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
proposals::insert(&pool, id, m, ws.as_uuid().to_owned(), &roster(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
let graph = json!({"kind":"pipeline","nodes":[{"id":"n0","role":"implementer","attrs":{}}],"edges":[]});
|
||||||
|
|
||||||
|
let applied = proposals::approve_and_apply(
|
||||||
|
&pool,
|
||||||
|
id,
|
||||||
|
m,
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
&graph,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("apply");
|
||||||
|
assert!(applied);
|
||||||
|
|
||||||
|
let (engine, nodes): (Option<String>, Option<i32>) = sqlx::query_as(
|
||||||
|
"SELECT team_engine, jsonb_array_length(config->'roster'->'nodes') FROM missions WHERE id = $1",
|
||||||
|
)
|
||||||
|
.bind(m)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(engine.as_deref(), Some("composed"));
|
||||||
|
assert_eq!(nodes, Some(1), "the roster must actually be on the mission");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Claim and apply are one decision, so they must commit or fail together. A
|
||||||
|
/// proposal marked `approved` against a mission that never received the roster
|
||||||
|
/// is permanent: the partial unique index blocks every later approval, and the
|
||||||
|
/// mission runs solo while its proposal says otherwise.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_failed_apply_leaves_the_proposal_undecided() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = mission(&pool, ws).await;
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
proposals::insert(&pool, id, m, ws.as_uuid().to_owned(), &roster(), "claude-opus-4-8")
|
||||||
|
.await
|
||||||
|
.expect("insert");
|
||||||
|
|
||||||
|
// A mission id that does not exist in this workspace: the apply half matches
|
||||||
|
// no row, which is the failure the transaction has to undo.
|
||||||
|
let err = proposals::approve_and_apply(
|
||||||
|
&pool,
|
||||||
|
id,
|
||||||
|
Uuid::now_v7(),
|
||||||
|
ws.as_uuid().to_owned(),
|
||||||
|
&json!({}),
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(err.is_err(), "applying to a missing mission must fail: {err:?}");
|
||||||
|
|
||||||
|
let rows = proposals::list(&pool, m, ws.as_uuid().to_owned()).await.expect("list");
|
||||||
|
assert_eq!(
|
||||||
|
rows[0].status, "proposed",
|
||||||
|
"the claim must have been rolled back, or this proposal is stuck approved forever"
|
||||||
|
);
|
||||||
|
// And it can still be approved properly afterwards.
|
||||||
|
let graph = json!({"kind":"pipeline","nodes":[{"id":"n0","role":"implementer","attrs":{}}],"edges":[]});
|
||||||
|
assert!(
|
||||||
|
proposals::approve_and_apply(&pool, id, m, ws.as_uuid().to_owned(), &graph, None, None)
|
||||||
|
.await
|
||||||
|
.expect("apply")
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -263,6 +263,7 @@ async fn a_template_with_live_agents_still_accepts_edits() {
|
|||||||
system_prompt: prompt,
|
system_prompt: prompt,
|
||||||
skills: extra,
|
skills: extra,
|
||||||
brain_seed: None,
|
brain_seed: None,
|
||||||
|
model: None,
|
||||||
}],
|
}],
|
||||||
};
|
};
|
||||||
team_templates::upsert_builtin(&pool, build("first", vec![]))
|
team_templates::upsert_builtin(&pool, build("first", vec![]))
|
||||||
|
|||||||
@@ -0,0 +1,218 @@
|
|||||||
|
//! The sweepers must leave SELF-DRIVEN runs alone.
|
||||||
|
//!
|
||||||
|
//! A `tier='microvm'` or `tier='session'` run is inserted directly as `running` by
|
||||||
|
//! `phase_runner` and owned start-to-finish by its own `tokio::spawn`. Nothing
|
||||||
|
//! touches its `updated_at` or `checkpoint` while it is in flight, because there
|
||||||
|
//! is no per-step loop to hook.
|
||||||
|
//!
|
||||||
|
//! Both sweepers were written when every `running` row was a `cm_orchestrator` job
|
||||||
|
//! that checkpointed after each step, and neither filtered on tier. The result,
|
||||||
|
//! measured in production on 2026-08-06: `requeue_stale` declared a healthy microVM
|
||||||
|
//! run stale at 180 seconds, the worker claimed it, failed to deserialize its graph
|
||||||
|
//! placeholder, and killed the phase with "missing or invalid graph" — while the
|
||||||
|
//! agent went on working and its VM was orphaned for over an hour.
|
||||||
|
//!
|
||||||
|
//! **Every microVM mission that appeared to work did so by finishing inside three
|
||||||
|
//! minutes.** The end-to-end harness runs a 90-second mission, so it cannot see
|
||||||
|
//! this class at all — which is why the guard lives here, against the real SQL, and
|
||||||
|
//! costs milliseconds instead of eight minutes.
|
||||||
|
|
||||||
|
use cm_db::repo::topology_runs;
|
||||||
|
use cm_domain::WorkspaceId;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
/// Insert a run that is `running` and has looked idle for a long time — exactly
|
||||||
|
/// the shape a long agent turn presents.
|
||||||
|
async fn stale_running_run(pool: &sqlx::PgPool, ws: WorkspaceId, tier: &str) -> Uuid {
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO topology_runs
|
||||||
|
(id, workspace_id, task, kind, status, graph, tier,
|
||||||
|
created_at, updated_at)
|
||||||
|
VALUES ($1, $2, 'long turn', 'run', 'running', $3, $4,
|
||||||
|
now() - interval '30 minutes', now() - interval '30 minutes')",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
// The placeholder a self-driven run carries: no `kind`, so `TopologyGraph`
|
||||||
|
// cannot parse it. That is what turned a requeue into a hard failure.
|
||||||
|
.bind(serde_json::json!({ "nodes": [], "edges": [], "executor": tier }))
|
||||||
|
.bind(tier)
|
||||||
|
// `mission_id` is left NULL: it has an FK to `missions`, and `requeue_stale`
|
||||||
|
// does not look at it. The reaper DOES filter on `mission_id IS NOT NULL` —
|
||||||
|
// which is precisely what used to be mistaken for "orchestrator-driven" — and
|
||||||
|
// it now shares the same tier allowlist, asserted below.
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("insert run");
|
||||||
|
id
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn status_of(pool: &sqlx::PgPool, id: Uuid) -> String {
|
||||||
|
sqlx::query_scalar::<_, String>("SELECT status FROM topology_runs WHERE id = $1")
|
||||||
|
.bind(id)
|
||||||
|
.fetch_one(pool)
|
||||||
|
.await
|
||||||
|
.expect("read status")
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn workspace(pool: &sqlx::PgPool) -> WorkspaceId {
|
||||||
|
let ws = cm_domain::Workspace {
|
||||||
|
id: WorkspaceId::new(),
|
||||||
|
name: "Sweeper".into(),
|
||||||
|
plan: "team".into(),
|
||||||
|
};
|
||||||
|
cm_db::repo::workspaces::insert(pool, &ws)
|
||||||
|
.await
|
||||||
|
.expect("workspace");
|
||||||
|
ws.id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The bug, in one assertion: 30 minutes idle and it must still be `running`.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn requeue_stale_leaves_self_driven_runs_alone() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
|
||||||
|
let microvm = stale_running_run(&pool, ws, "microvm").await;
|
||||||
|
let session = stale_running_run(&pool, ws, "session").await;
|
||||||
|
|
||||||
|
let moved = topology_runs::requeue_stale(&pool, 180.0)
|
||||||
|
.await
|
||||||
|
.expect("requeue");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
status_of(&pool, microvm).await,
|
||||||
|
"running",
|
||||||
|
"a microvm run was requeued out from under a live VM ({moved} rows moved)"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
status_of(&pool, session).await,
|
||||||
|
"running",
|
||||||
|
"a session run was requeued out from under a live agent"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// And a worker-driven run in the same state MUST still be requeued, or the fix
|
||||||
|
/// would have been "stop sweeping" rather than "sweep the right rows".
|
||||||
|
#[tokio::test]
|
||||||
|
async fn requeue_stale_still_rescues_worker_driven_runs() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let team = stale_running_run(&pool, ws, "team").await;
|
||||||
|
|
||||||
|
topology_runs::requeue_stale(&pool, 180.0).await.expect("requeue");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
status_of(&pool, team).await,
|
||||||
|
"queued",
|
||||||
|
"a genuinely stalled team run must still be recovered"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Defence in depth: even handed a queued self-driven row, the worker must not
|
||||||
|
/// adopt a job it cannot execute. Claiming one is what produced the
|
||||||
|
/// "missing or invalid graph" failure on a run that was perfectly healthy.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn the_worker_will_not_claim_a_self_driven_run() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
topology_runs::enqueue_run_tier(
|
||||||
|
&pool,
|
||||||
|
id,
|
||||||
|
ws,
|
||||||
|
"should never be claimed",
|
||||||
|
&serde_json::json!({ "nodes": [], "edges": [], "executor": "microvm" }),
|
||||||
|
"microvm",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("enqueue");
|
||||||
|
|
||||||
|
let claimed = topology_runs::claim_next_queued(&pool).await.expect("claim");
|
||||||
|
assert!(
|
||||||
|
claimed.is_none(),
|
||||||
|
"the worker claimed a microvm run: {:?}",
|
||||||
|
claimed.map(|c| c.tier)
|
||||||
|
);
|
||||||
|
assert_eq!(status_of(&pool, id).await, "queued", "and it must be left as it was");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The composed tier is the mirror image of the two above and must not be
|
||||||
|
/// mistaken for them: it runs VMs, but the WORKER drives its graph, so being
|
||||||
|
/// claimed and requeued is exactly what gives it checkpointing and resume.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn the_worker_claims_and_rescues_a_composed_run() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
topology_runs::enqueue_run_tier(
|
||||||
|
&pool,
|
||||||
|
id,
|
||||||
|
ws,
|
||||||
|
"compose the engines",
|
||||||
|
// A real graph, unlike the self-driven placeholder: the worker plans it.
|
||||||
|
&serde_json::json!({
|
||||||
|
"kind": "pipeline",
|
||||||
|
"nodes": [{ "id": "a", "role": "worker", "attrs": {} }],
|
||||||
|
"edges": []
|
||||||
|
}),
|
||||||
|
"microvm_graph",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("enqueue");
|
||||||
|
|
||||||
|
let claimed = topology_runs::claim_next_queued(&pool)
|
||||||
|
.await
|
||||||
|
.expect("claim")
|
||||||
|
.expect("a composed run must be claimable, or it never runs at all");
|
||||||
|
assert_eq!(claimed.tier, "microvm_graph");
|
||||||
|
assert_eq!(claimed.id, id);
|
||||||
|
|
||||||
|
// And a composed run whose worker died must come back: its checkpoint is
|
||||||
|
// what makes resume possible, and requeue is what triggers it.
|
||||||
|
sqlx::query("UPDATE topology_runs SET updated_at = now() - interval '30 minutes' WHERE id = $1")
|
||||||
|
.bind(id)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.expect("age the run");
|
||||||
|
topology_runs::requeue_stale(&pool, 180.0).await.expect("requeue");
|
||||||
|
assert_eq!(
|
||||||
|
status_of(&pool, id).await,
|
||||||
|
"queued",
|
||||||
|
"a composed run orphaned by a dead worker must be recovered"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The allowlist is the single place this policy lives, so assert its membership
|
||||||
|
/// directly — a new self-driven tier added without touching it would otherwise be
|
||||||
|
/// exposed exactly as microvm was.
|
||||||
|
#[test]
|
||||||
|
fn the_allowlist_names_only_worker_driven_tiers() {
|
||||||
|
for driven in ["team", "swarm", "company", "org"] {
|
||||||
|
assert!(
|
||||||
|
topology_runs::WORKER_DRIVEN_TIERS.contains(&driven),
|
||||||
|
"{driven} is driven by the worker and must be sweepable"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for self_driven in ["microvm", "session"] {
|
||||||
|
assert!(
|
||||||
|
!topology_runs::WORKER_DRIVEN_TIERS.contains(&self_driven),
|
||||||
|
"{self_driven} owns its own lifecycle; sweeping it kills live work"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// `microvm_graph` is worker-driven but NOT reapable: one of its steps is a
|
||||||
|
// whole agent session in a VM, so "no step records in 15 minutes" is what a
|
||||||
|
// healthy composed run looks like, and reaping it would orphan a live VM —
|
||||||
|
// #54 in a different tier.
|
||||||
|
assert!(topology_runs::WORKER_DRIVEN_TIERS.contains(&"microvm_graph"));
|
||||||
|
assert!(!topology_runs::REAPABLE_TIERS.contains(&"microvm_graph"));
|
||||||
|
for reapable in topology_runs::REAPABLE_TIERS {
|
||||||
|
assert!(
|
||||||
|
topology_runs::WORKER_DRIVEN_TIERS.contains(reapable),
|
||||||
|
"{reapable} is reaped but never driven"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
//! A failed phase must not strand its mission at `running` forever.
|
||||||
|
//!
|
||||||
|
//! `start_pending_phases` launches a phase only when every lower-order phase is
|
||||||
|
//! `completed`, so once one fails the rest can never run. They sat `pending`,
|
||||||
|
//! and `close_finished_missions` requires no phase to be non-terminal — so the
|
||||||
|
//! mission never finished, and `mission_runtime`'s sweeper (which fires after a
|
||||||
|
//! terminal state) never reaped its container.
|
||||||
|
//!
|
||||||
|
//! Found by counting containers on gw-04, not by a test: one leaked runtime
|
||||||
|
//! container per failed multi-phase mission, accumulating for days. This is the
|
||||||
|
//! SQL that ends it, tested against a real database because the bug lived
|
||||||
|
//! entirely in the interaction between two queries' predicates.
|
||||||
|
|
||||||
|
use cm_domain::WorkspaceId;
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
async fn workspace(pool: &sqlx::PgPool) -> WorkspaceId {
|
||||||
|
let ws = cm_domain::Workspace {
|
||||||
|
id: WorkspaceId::new(),
|
||||||
|
name: "Unreachable".into(),
|
||||||
|
plan: "team".into(),
|
||||||
|
};
|
||||||
|
cm_db::repo::workspaces::insert(pool, &ws).await.expect("workspace");
|
||||||
|
ws.id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A running mission whose phase 0 failed and whose phases 1..n never started.
|
||||||
|
async fn stuck_mission(pool: &sqlx::PgPool, ws: WorkspaceId) -> Uuid {
|
||||||
|
let id = Uuid::now_v7();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO missions (id, workspace_id, title, template_kind, status, schedule, config)
|
||||||
|
VALUES ($1, $2, 'stuck', 'research_and_code', 'running', '{}'::jsonb, '{}'::jsonb)",
|
||||||
|
)
|
||||||
|
.bind(id)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("mission");
|
||||||
|
for (idx, status) in [(0, "failed"), (1, "pending"), (2, "pending")] {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO mission_phases (id, mission_id, kind, order_idx, status, config)
|
||||||
|
VALUES ($1, $2, 'coding', $3, $4, '{}'::jsonb)",
|
||||||
|
)
|
||||||
|
.bind(Uuid::now_v7())
|
||||||
|
.bind(id)
|
||||||
|
.bind(idx)
|
||||||
|
.bind(status)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("phase");
|
||||||
|
}
|
||||||
|
id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The sweep, as `phase_runner::skip_unreachable_phases` runs it.
|
||||||
|
async fn skip_unreachable(pool: &sqlx::PgPool) -> u64 {
|
||||||
|
sqlx::query(
|
||||||
|
"UPDATE mission_phases mp
|
||||||
|
SET status = 'skipped', completed_at = now()
|
||||||
|
WHERE mp.status = 'pending'
|
||||||
|
AND EXISTS (SELECT 1 FROM missions m WHERE m.id = mp.mission_id AND m.status = 'running')
|
||||||
|
AND EXISTS (
|
||||||
|
SELECT 1 FROM mission_phases prior
|
||||||
|
WHERE prior.mission_id = mp.mission_id
|
||||||
|
AND prior.order_idx < mp.order_idx
|
||||||
|
AND prior.status = 'failed'
|
||||||
|
)",
|
||||||
|
)
|
||||||
|
.execute(pool)
|
||||||
|
.await
|
||||||
|
.expect("skip")
|
||||||
|
.rows_affected()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn statuses(pool: &sqlx::PgPool, mission: Uuid) -> Vec<String> {
|
||||||
|
sqlx::query_scalar("SELECT status FROM mission_phases WHERE mission_id = $1 ORDER BY order_idx")
|
||||||
|
.bind(mission)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await
|
||||||
|
.expect("statuses")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_failed_phase_makes_the_later_ones_unreachable_not_pending_forever() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = stuck_mission(&pool, ws).await;
|
||||||
|
|
||||||
|
assert_eq!(skip_unreachable(&pool).await, 2, "both later phases are unreachable");
|
||||||
|
assert_eq!(
|
||||||
|
statuses(&pool, m).await,
|
||||||
|
vec!["failed", "skipped", "skipped"],
|
||||||
|
"a phase that can never run must say so, or the mission never closes"
|
||||||
|
);
|
||||||
|
// Every phase is now terminal, which is what `close_finished_missions`
|
||||||
|
// waits for — the container sweeper keys off the mission reaching that.
|
||||||
|
let non_terminal: i64 = sqlx::query_scalar(
|
||||||
|
"SELECT count(*) FROM mission_phases
|
||||||
|
WHERE mission_id = $1 AND status NOT IN ('completed','failed','skipped')",
|
||||||
|
)
|
||||||
|
.bind(m)
|
||||||
|
.fetch_one(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(non_terminal, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A phase waiting AHEAD of the failure is untouched: order is what makes a
|
||||||
|
/// phase unreachable, and a failure later in the list says nothing about one
|
||||||
|
/// still queued before it.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_phase_before_the_failure_is_left_alone() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = Uuid::now_v7();
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO missions (id, workspace_id, title, template_kind, status, schedule, config)
|
||||||
|
VALUES ($1, $2, 'ordered', 'research_and_code', 'running', '{}'::jsonb, '{}'::jsonb)",
|
||||||
|
)
|
||||||
|
.bind(m)
|
||||||
|
.bind(ws.as_uuid())
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
for (idx, status) in [(0, "pending"), (1, "failed"), (2, "pending")] {
|
||||||
|
sqlx::query(
|
||||||
|
"INSERT INTO mission_phases (id, mission_id, kind, order_idx, status, config)
|
||||||
|
VALUES ($1, $2, 'coding', $3, $4, '{}'::jsonb)",
|
||||||
|
)
|
||||||
|
.bind(Uuid::now_v7())
|
||||||
|
.bind(m)
|
||||||
|
.bind(idx)
|
||||||
|
.bind(status)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
skip_unreachable(&pool).await;
|
||||||
|
assert_eq!(statuses(&pool, m).await, vec!["pending", "failed", "skipped"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A mission that is not running is not swept: a draft's phases are pending by
|
||||||
|
/// definition and must not be skipped out from under it.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn only_a_running_missions_phases_are_skipped() {
|
||||||
|
let pool = cm_testkit::test_pool().await;
|
||||||
|
let ws = workspace(&pool).await;
|
||||||
|
let m = stuck_mission(&pool, ws).await;
|
||||||
|
sqlx::query("UPDATE missions SET status = 'draft' WHERE id = $1")
|
||||||
|
.bind(m)
|
||||||
|
.execute(&pool)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(skip_unreachable(&pool).await, 0);
|
||||||
|
assert_eq!(statuses(&pool, m).await, vec!["failed", "pending", "pending"]);
|
||||||
|
}
|
||||||
@@ -187,5 +187,9 @@ async fn browsing_returns_web_tainted_content_and_taints_later_gated_actions() {
|
|||||||
let png = blob.get(&key).await.expect("screenshot stored");
|
let png = blob.get(&key).await.expect("screenshot stored");
|
||||||
assert_eq!(&png[..8], b"\x89PNG\r\n\x1a\n", "PNG magic");
|
assert_eq!(&png[..8], b"\x89PNG\r\n\x1a\n", "PNG magic");
|
||||||
|
|
||||||
|
// `shutdown` drains the warm pool only; the sandbox assigned to this agent
|
||||||
|
// outlives it by design (reused across a redeploy). A test has no next
|
||||||
|
// deploy, so it must release its own or the container simply stays.
|
||||||
|
browser.release_agent(agent.id).await;
|
||||||
browser.shutdown().await;
|
browser.shutdown().await;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -152,6 +152,11 @@ async fn shell_exec_runs_in_the_agent_sandbox_with_persistent_home() {
|
|||||||
outputs[1]
|
outputs[1]
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// `shutdown` drains the warm pool; the container this exec ASSIGNED to the
|
||||||
|
// agent survives it on purpose, so a redeploy can reuse it. A test has no
|
||||||
|
// next deploy, so it must release its own or the container simply stays —
|
||||||
|
// which is how 289 of them accumulated before anyone counted.
|
||||||
|
sandboxes.release_agent(agent.id).await;
|
||||||
sandboxes.shutdown().await;
|
sandboxes.shutdown().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -229,4 +234,5 @@ async fn shell_exec_without_a_sandbox_runtime_reports_a_tool_error() {
|
|||||||
statuses.iter().all(|s| s == "error"),
|
statuses.iter().all(|s| s == "error"),
|
||||||
"steps must record the failure: {statuses:?}"
|
"steps must record the failure: {statuses:?}"
|
||||||
);
|
);
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -118,7 +118,17 @@ async fn the_pool_prefills_assigns_and_refills() {
|
|||||||
"reuse must not drain the pool"
|
"reuse must not drain the pool"
|
||||||
);
|
);
|
||||||
|
|
||||||
// Shutdown destroys assigned AND pooled sandboxes.
|
// `shutdown` tears down the POOL only — assigned sandboxes deliberately
|
||||||
|
// survive it, so they can be reused across a redeploy. The comment here
|
||||||
|
// used to claim it destroyed both, which is why nobody noticed that every
|
||||||
|
// run of this test left its assigned container running: three such tests,
|
||||||
|
// three leaked containers per `./scripts/test.sh`, 289 of them by the time
|
||||||
|
// anyone counted.
|
||||||
|
manager.release_agent(agent).await;
|
||||||
manager.shutdown().await;
|
manager.shutdown().await;
|
||||||
assert_eq!(manager.pool_size().await, 0);
|
assert_eq!(manager.pool_size().await, 0);
|
||||||
|
assert!(
|
||||||
|
!manager.release_agent(agent).await,
|
||||||
|
"the agent's sandbox must be gone, not merely unpooled"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -85,15 +85,10 @@ mcp_config = "/zeroclaw-data/clawmates-mcp.json"
|
|||||||
# model_provider = "claude_cli.glm5"
|
# model_provider = "claude_cli.glm5"
|
||||||
# risk_profile = "toolfree"
|
# risk_profile = "toolfree"
|
||||||
#
|
#
|
||||||
# Gemini (Google) — built-in `gemini` API family (no image rebuild). Key via env
|
# Gemini is NOT wired. The platform stripped it: no provider alias, no key
|
||||||
# ZEROCLAW_providers__models__gemini__default__api_key=<GEMINI_API_KEY>; model in
|
# forwarded to agent containers, no selector entry. ZeroClaw still has a built-in
|
||||||
# config. gemini-2.5-flash = stable + high free-tier RPM (throttle-friendly);
|
# `gemini` family, so an operator could add it back here — but `provider_alias_for`
|
||||||
# gemini-3-pro-preview is the flagship.
|
# no longer resolves `gemini*` to it, and it would log as an unrecognised model.
|
||||||
# [providers.models.gemini.default]
|
|
||||||
# model = "gemini-2.5-flash"
|
|
||||||
# [agents.worker_gemini]
|
|
||||||
# model_provider = "gemini.default"
|
|
||||||
# risk_profile = "toolfree"
|
|
||||||
#
|
#
|
||||||
# Groq — built-in `groq` family (key via ZEROCLAW_providers__models__groq__default__api_key).
|
# Groq — built-in `groq` family (key via ZEROCLAW_providers__models__groq__default__api_key).
|
||||||
# Fast, but LOW free-tier TPM: each turn carries a ~9.5k-tok system prompt, so 2
|
# Fast, but LOW free-tier TPM: each turn carries a ~9.5k-tok system prompt, so 2
|
||||||
@@ -109,7 +104,7 @@ mcp_config = "/zeroclaw-data/clawmates-mcp.json"
|
|||||||
# env ZEROCLAW_AGENT_MAP="role=alias,role=alias" (+ ZEROCLAW_DEFAULT_AGENT for
|
# env ZEROCLAW_AGENT_MAP="role=alias,role=alias" (+ ZEROCLAW_DEFAULT_AGENT for
|
||||||
# unmapped roles). Point each semantic role at a different model to run ONE
|
# unmapped roles). Point each semantic role at a different model to run ONE
|
||||||
# topology across vendors, e.g.:
|
# topology across vendors, e.g.:
|
||||||
# ZEROCLAW_AGENT_MAP="coordinator=coordinator,researcher=worker_glm,analyst=worker_kimi,writer=worker_gemini,actor=worker_groq"
|
# ZEROCLAW_AGENT_MAP="coordinator=coordinator,researcher=worker_glm,analyst=worker_kimi,writer=worker_glm5,actor=worker_groq"
|
||||||
# Then POST /api/topologies/run {task, graph} with those role names. QUOTA CARE:
|
# Then POST /api/topologies/run {task, graph} with those role names. QUOTA CARE:
|
||||||
# GLM/Kimi plans have 5h/weekly caps + low concurrency (GLM Lite ~1 project at a
|
# GLM/Kimi plans have 5h/weekly caps + low concurrency (GLM Lite ~1 project at a
|
||||||
# time) — prefer pipeline (sequential) over swarm/mesh, keep tasks short, and ask
|
# time) — prefer pipeline (sequential) over swarm/mesh, keep tasks short, and ask
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Clawmates paper library harvest (arXiv -> shelf + vault catalogue)
|
||||||
|
Wants=docker.service
|
||||||
|
After=docker.service network-online.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
ExecStart=/usr/local/bin/clawmates-library.sh
|
||||||
|
StandardOutput=journal
|
||||||
|
StandardError=journal
|
||||||
|
Nice=10
|
||||||
|
# A harvest downloads PDFs and pushes a branch; give it room but do not
|
||||||
|
# let a wedged run hold the slot until the next week.
|
||||||
|
TimeoutStartSec=30min
|
||||||
Executable
+49
@@ -0,0 +1,49 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Weekly paper-library harvest.
|
||||||
|
#
|
||||||
|
# Deliberately thin: it calls the API and reports what came back. All the
|
||||||
|
# logic lives in the server, so this file never needs to change when the
|
||||||
|
# harvest does.
|
||||||
|
#
|
||||||
|
# The token lives in /etc/clawmates/library.token (root-only). It is a
|
||||||
|
# long-lived operator session; rotate by replacing the file.
|
||||||
|
set -uo pipefail
|
||||||
|
|
||||||
|
TOKEN_FILE=/etc/clawmates/library.token
|
||||||
|
[ -r "$TOKEN_FILE" ] || { echo "library: no token at $TOKEN_FILE"; exit 1; }
|
||||||
|
TOKEN=$(cat "$TOKEN_FILE")
|
||||||
|
|
||||||
|
RESP=$(docker run --rm --network clawmates_core curlimages/curl:latest \
|
||||||
|
-s -m 1800 -X POST \
|
||||||
|
-H "Authorization: Bearer $TOKEN" \
|
||||||
|
-H "Content-Type: application/json" \
|
||||||
|
-d '{"per_topic":5}' \
|
||||||
|
http://clawmates_server_1:8080/api/library/runs)
|
||||||
|
|
||||||
|
echo "library: $RESP" | head -c 2000
|
||||||
|
|
||||||
|
# Report health explicitly. A run that shelved nothing is normal for a
|
||||||
|
# mature library; a run that ERRORED is not, and the two look identical
|
||||||
|
# if you only count papers.
|
||||||
|
echo "$RESP" | python3 -c '
|
||||||
|
import json, sys
|
||||||
|
try:
|
||||||
|
d = json.load(sys.stdin)
|
||||||
|
except Exception as e:
|
||||||
|
print("library: unreadable response (%s)" % e)
|
||||||
|
sys.exit(1)
|
||||||
|
shelved = len(d.get("shelved", []))
|
||||||
|
healthy = d.get("healthy", False)
|
||||||
|
# Backslashes are avoided inside this program on purpose: it is embedded in a
|
||||||
|
# single-quoted shell string, and an escaped quote here does not survive the
|
||||||
|
# shell. The first version used one inside an f-string, crashed on every run,
|
||||||
|
# and systemd reported a FAILED unit for a harvest that had actually shelved
|
||||||
|
# 15 papers and pushed them. A false failure destroys trust in the signal as
|
||||||
|
# surely as a false success.
|
||||||
|
print("library: %d candidates, %d already held, %d shelved, healthy=%s, pushed=%s, branch=%s" % (
|
||||||
|
d.get("candidates", 0), d.get("already_had", 0), shelved,
|
||||||
|
healthy, d.get("pushed"), d.get("branch")))
|
||||||
|
for f in d.get("failed", []):
|
||||||
|
print("library: FAILED %s" % f)
|
||||||
|
sys.exit(0 if healthy else 1)
|
||||||
|
'
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Clawmates paper library — weekly harvest
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
# Monday 07:00 local. Weekly rather than daily because arXiv moves at
|
||||||
|
# roughly that pace for a narrow topic set, and a run that almost always
|
||||||
|
# finds nothing trains you to ignore it.
|
||||||
|
OnCalendar=Mon *-*-* 07:00:00
|
||||||
|
# Fire on next boot if the machine was down at the scheduled time — a
|
||||||
|
# missed week is a silently empty library.
|
||||||
|
Persistent=true
|
||||||
|
AccuracySec=1min
|
||||||
|
Unit=clawmates-library.service
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
Generated
+1511
-5
File diff suppressed because it is too large
Load Diff
@@ -19,11 +19,15 @@
|
|||||||
"class-variance-authority": "^0.7.1",
|
"class-variance-authority": "^0.7.1",
|
||||||
"clsx": "^2.1.1",
|
"clsx": "^2.1.1",
|
||||||
"geist": "^1.7.2",
|
"geist": "^1.7.2",
|
||||||
|
"github-slugger": "^2.0.0",
|
||||||
"lucide-react": "^1.17.0",
|
"lucide-react": "^1.17.0",
|
||||||
"next": "16.2.9",
|
"next": "16.2.9",
|
||||||
"nuqs": "^2.8.9",
|
"nuqs": "^2.8.9",
|
||||||
"react": "19.2.4",
|
"react": "19.2.4",
|
||||||
"react-dom": "19.2.4",
|
"react-dom": "19.2.4",
|
||||||
|
"react-markdown": "^10.1.0",
|
||||||
|
"rehype-slug": "^6.0.0",
|
||||||
|
"remark-gfm": "^4.0.1",
|
||||||
"tailwind-merge": "^3.6.0",
|
"tailwind-merge": "^3.6.0",
|
||||||
"three": "^0.169.0",
|
"three": "^0.169.0",
|
||||||
"zod": "^4.4.3"
|
"zod": "^4.4.3"
|
||||||
|
|||||||
@@ -1,62 +0,0 @@
|
|||||||
// Local Next route (NOT proxied — a specific path beats the /api/[...path]
|
|
||||||
// catch-all): generates an agent avatar with Gemini's image model ("Nano
|
|
||||||
// Banana", gemini-2.5-flash-image) using GEMINI_API_KEY from the frontend env,
|
|
||||||
// and returns a base64 data URL. Saving the chosen image is a separate
|
|
||||||
// PATCH /api/claws/{id} {avatar} (the existing backend route).
|
|
||||||
|
|
||||||
import { NextResponse, type NextRequest } from "next/server";
|
|
||||||
|
|
||||||
import { resolveBearer } from "@/lib/auth/bearer";
|
|
||||||
|
|
||||||
const MODEL = "gemini-2.5-flash-image";
|
|
||||||
const ENDPOINT = `https://generativelanguage.googleapis.com/v1beta/models/${MODEL}:generateContent`;
|
|
||||||
|
|
||||||
interface InlineData { data?: string; mimeType?: string; mime_type?: string }
|
|
||||||
interface GeminiPart { inlineData?: InlineData; inline_data?: InlineData }
|
|
||||||
interface GeminiResponse { candidates?: Array<{ content?: { parts?: GeminiPart[] } }> }
|
|
||||||
|
|
||||||
export async function POST(request: NextRequest) {
|
|
||||||
const token = await resolveBearer();
|
|
||||||
if (!token) return NextResponse.json({ error: "unauthenticated" }, { status: 401 });
|
|
||||||
|
|
||||||
const key = process.env.GEMINI_API_KEY;
|
|
||||||
if (!key) return NextResponse.json({ error: "image generation is not configured (set GEMINI_API_KEY)" }, { status: 503 });
|
|
||||||
|
|
||||||
let prompt = "";
|
|
||||||
try {
|
|
||||||
const body = (await request.json()) as { prompt?: unknown };
|
|
||||||
prompt = String(body?.prompt ?? "").trim();
|
|
||||||
} catch {
|
|
||||||
/* fall through to the 400 below */
|
|
||||||
}
|
|
||||||
if (!prompt) return NextResponse.json({ error: "prompt required" }, { status: 400 });
|
|
||||||
|
|
||||||
let upstream: Response;
|
|
||||||
try {
|
|
||||||
upstream = await fetch(ENDPOINT, {
|
|
||||||
method: "POST",
|
|
||||||
headers: { "Content-Type": "application/json", "x-goog-api-key": key },
|
|
||||||
body: JSON.stringify({
|
|
||||||
contents: [{ parts: [{ text: `A clean, centered square avatar portrait for an AI agent. ${prompt}` }] }],
|
|
||||||
generationConfig: { responseModalities: ["IMAGE"] },
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
return NextResponse.json({ error: "could not reach the image service" }, { status: 502 });
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!upstream.ok) {
|
|
||||||
const detail = await upstream.text().catch(() => "");
|
|
||||||
return NextResponse.json({ error: `image service error (${upstream.status})`, detail: detail.slice(0, 400) }, { status: 502 });
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = (await upstream.json().catch(() => null)) as GeminiResponse | null;
|
|
||||||
const parts = data?.candidates?.[0]?.content?.parts ?? [];
|
|
||||||
const part = parts.find((p) => p.inlineData?.data || p.inline_data?.data);
|
|
||||||
const inline = part?.inlineData ?? part?.inline_data;
|
|
||||||
if (!inline?.data) {
|
|
||||||
return NextResponse.json({ error: "the model did not return an image — try a different prompt" }, { status: 502 });
|
|
||||||
}
|
|
||||||
const mime = inline.mimeType ?? inline.mime_type ?? "image/png";
|
|
||||||
return NextResponse.json({ image: `data:${mime};base64,${inline.data}` });
|
|
||||||
}
|
|
||||||
@@ -251,3 +251,109 @@ body {
|
|||||||
.marketing {
|
.marketing {
|
||||||
color-scheme: light;
|
color-scheme: light;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ── Mission artifact reader ──────────────────────────────────────
|
||||||
|
Styles the HTML that react-markdown generates for a mission's markdown
|
||||||
|
documents (see components/dashboard/MarkdownView.tsx). Every rule is
|
||||||
|
descendant-scoped to .md-view: the markup is generated, so there are no
|
||||||
|
class hooks to target, and unscoped element selectors would restyle the
|
||||||
|
whole dashboard. */
|
||||||
|
.md-view {
|
||||||
|
color: #d8d8de;
|
||||||
|
font-size: 13.5px;
|
||||||
|
line-height: 1.68;
|
||||||
|
/* Research documents are read, not skimmed. A bounded measure keeps lines
|
||||||
|
comfortable; wide content (tables, code) scrolls inside its own box. */
|
||||||
|
max-width: 78ch;
|
||||||
|
overflow-wrap: anywhere;
|
||||||
|
}
|
||||||
|
.md-view > :first-child { margin-top: 0; }
|
||||||
|
.md-view h1, .md-view h2, .md-view h3, .md-view h4 {
|
||||||
|
color: #f3f3f5;
|
||||||
|
font-weight: 600;
|
||||||
|
line-height: 1.3;
|
||||||
|
margin: 1.6em 0 0.6em;
|
||||||
|
}
|
||||||
|
.md-view h1 {
|
||||||
|
font-size: 21px;
|
||||||
|
border-bottom: 1px solid rgba(255, 255, 255, 0.09);
|
||||||
|
padding-bottom: 0.35em;
|
||||||
|
}
|
||||||
|
.md-view h2 {
|
||||||
|
font-size: 17px;
|
||||||
|
border-bottom: 1px solid rgba(255, 255, 255, 0.06);
|
||||||
|
padding-bottom: 0.3em;
|
||||||
|
}
|
||||||
|
.md-view h3 { font-size: 15px; }
|
||||||
|
.md-view h4 { font-size: 13.5px; color: #c8c8d0; }
|
||||||
|
.md-view p { margin: 0.85em 0; }
|
||||||
|
.md-view a {
|
||||||
|
color: #7cd6e0;
|
||||||
|
text-decoration: none;
|
||||||
|
border-bottom: 1px solid rgba(124, 214, 224, 0.35);
|
||||||
|
}
|
||||||
|
.md-view a:hover { border-bottom-color: #7cd6e0; }
|
||||||
|
.md-view strong { color: #f3f3f5; font-weight: 600; }
|
||||||
|
.md-view ul, .md-view ol { margin: 0.8em 0; padding-left: 1.5em; }
|
||||||
|
.md-view li { margin: 0.32em 0; }
|
||||||
|
.md-view li::marker { color: #7cd6e0; }
|
||||||
|
.md-view code {
|
||||||
|
font-family: var(--font-mono);
|
||||||
|
font-size: 12px;
|
||||||
|
background: rgba(255, 255, 255, 0.06);
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.07);
|
||||||
|
border-radius: 4px;
|
||||||
|
padding: 1px 5px;
|
||||||
|
color: #e6d9a8;
|
||||||
|
}
|
||||||
|
.md-view pre {
|
||||||
|
background: #0a0a0d;
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.08);
|
||||||
|
border-radius: 8px;
|
||||||
|
padding: 12px 14px;
|
||||||
|
overflow-x: auto;
|
||||||
|
margin: 1em 0;
|
||||||
|
}
|
||||||
|
/* Inside a block the chip styling would double the border and re-tint text. */
|
||||||
|
.md-view pre code {
|
||||||
|
background: none;
|
||||||
|
border: none;
|
||||||
|
padding: 0;
|
||||||
|
color: #d8d8de;
|
||||||
|
line-height: 1.55;
|
||||||
|
}
|
||||||
|
.md-view blockquote {
|
||||||
|
margin: 1em 0;
|
||||||
|
padding: 0.1em 0 0.1em 1em;
|
||||||
|
border-left: 3px solid rgba(124, 214, 224, 0.5);
|
||||||
|
color: #a8a8b2;
|
||||||
|
}
|
||||||
|
/* Agents write GFM tables constantly — this is the main payoff of remark-gfm.
|
||||||
|
`display: block` so a wide table scrolls itself instead of the page. */
|
||||||
|
.md-view table {
|
||||||
|
border-collapse: collapse;
|
||||||
|
width: 100%;
|
||||||
|
margin: 1.1em 0;
|
||||||
|
font-size: 12.5px;
|
||||||
|
display: block;
|
||||||
|
overflow-x: auto;
|
||||||
|
}
|
||||||
|
.md-view th, .md-view td {
|
||||||
|
border: 1px solid rgba(255, 255, 255, 0.09);
|
||||||
|
padding: 7px 10px;
|
||||||
|
text-align: left;
|
||||||
|
vertical-align: top;
|
||||||
|
}
|
||||||
|
.md-view th {
|
||||||
|
background: rgba(255, 255, 255, 0.04);
|
||||||
|
color: #f3f3f5;
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
.md-view tr:nth-child(even) td { background: rgba(255, 255, 255, 0.015); }
|
||||||
|
.md-view hr {
|
||||||
|
border: none;
|
||||||
|
border-top: 1px solid rgba(255, 255, 255, 0.09);
|
||||||
|
margin: 1.8em 0;
|
||||||
|
}
|
||||||
|
.md-view img { max-width: 100%; border-radius: 6px; }
|
||||||
|
.md-view input[type="checkbox"] { margin-right: 6px; }
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user