fix(fleet): placement requires the backend's rootfs image, not just KVM

The first real microVM mission was placed on morpheus because it reports
{"microvm": true}, while only tank had rootfs-claude.ext4. It failed by name
rather than booting the wrong image — but whether a mission ran came down to
which capable node was listed first, which is a coin flip dressed as scheduling.
`missions.backend` was invisible to the scheduler.

The node now enumerates the images on its disk and reports them as a `rootfs`
ARRAY. `microvm::available_backends` lives beside `rootfs_for`, its inverse,
because the two must agree on what a backend name means; split apart, one drifts
and the scheduler starts promising images the booter cannot find. It only
advertises names `rootfs_for` would accept, and reports an empty array rather than
omitting the key — set_capabilities REPLACES, so a deleted image stops being
advertised instead of leaving a stale claim.

`nodes::online_for_backend` requires microvm AND that the node's list contains the
mission's backend. A node on an older daemon has no `rootfs` key and matches
nothing: unknown is not permission, the same treatment every other capability
gets. `backend_key` maps the three spellings of "the default image" to the one
name the node advertises, and is tested — a mismatch there would reject every node
for an ordinary mission with no backend set.

The launch error now names both halves of the fix, since "no capable node" was
true but unhelpful when the node was capable and merely lacked the image.

Mission gains `backend` on the domain struct; it was a column the executor read
from the phase query while the struct that placement uses could not see it.

464 tests pass, clippy clean.
This commit is contained in:
Omar Sobh
2026-08-05 17:22:12 -07:00
parent 1cd81a8b2a
commit d9f53a3f96
5 changed files with 171 additions and 15 deletions
+39 -7
View File
@@ -299,7 +299,11 @@ fn probe_capabilities() -> Value {
.map(|l| l.trim().to_string()) .map(|l| l.trim().to_string())
}); });
capabilities_from(kvm, firecracker.as_deref()) // Which rootfs images are actually on this node's disk. Reported so
// placement can require the mission's backend rather than assuming any
// KVM-capable node can boot any image — see microvm::available_backends.
let backends = microvm::available_backends();
capabilities_from(kvm, firecracker.as_deref(), &backends)
} }
/// Shape the capability report from probe results. /// Shape the capability report from probe results.
@@ -307,10 +311,17 @@ fn probe_capabilities() -> Value {
/// Split from [`probe_capabilities`] so the rule can be tested without a /// Split from [`probe_capabilities`] so the rule can be tested without a
/// `/dev/kvm` to open — the machine running the tests is usually the one that /// `/dev/kvm` to open — the machine running the tests is usually the one that
/// cannot host a microVM. /// cannot host a microVM.
fn capabilities_from(kvm: bool, firecracker: Option<&str>) -> Value { fn capabilities_from(kvm: bool, firecracker: Option<&str>, backends: &[String]) -> Value {
json!({ json!({
"kvm": kvm, "kvm": kvm,
"firecracker": firecracker, "firecracker": firecracker,
// The backends this node can boot. An ARRAY, and empty when there are
// none: `set_capabilities` REPLACES, so an image that was deleted stops
// being advertised on the next report instead of leaving a stale claim.
//
// Reported even when `microvm` is false, because it is a fact about the
// disk rather than a promise — placement requires both.
"rootfs": backends,
// BOTH must hold. A node with KVM but no firecracker binary looks // BOTH must hold. A node with KVM but no firecracker binary looks
// capable by the obvious test and fails at launch; a node with the // capable by the obvious test and fails at launch; a node with the
// binary but no KVM is gw-04. Computed here rather than in the // binary but no KVM is gw-04. Computed here rather than in the
@@ -1378,20 +1389,20 @@ mod capability_tests {
#[test] #[test]
fn microvm_needs_both_kvm_and_firecracker() { fn microvm_needs_both_kvm_and_firecracker() {
assert_eq!( assert_eq!(
capabilities_from(true, Some("Firecracker v1.16.1"))["microvm"], capabilities_from(true, Some("Firecracker v1.16.1"), &[])["microvm"],
json!(true) json!(true)
); );
assert_eq!( assert_eq!(
capabilities_from(true, None)["microvm"], capabilities_from(true, None, &[])["microvm"],
json!(false), json!(false),
"KVM without firecracker cannot host a microVM" "KVM without firecracker cannot host a microVM"
); );
assert_eq!( assert_eq!(
capabilities_from(false, Some("Firecracker v1.16.1"))["microvm"], capabilities_from(false, Some("Firecracker v1.16.1"), &[])["microvm"],
json!(false), json!(false),
"firecracker without KVM is gw-04 — it can never host one" "firecracker without KVM is gw-04 — it can never host one"
); );
assert_eq!(capabilities_from(false, None)["microvm"], json!(false)); assert_eq!(capabilities_from(false, None, &[])["microvm"], json!(false));
} }
/// The report replaces rather than merges server-side, so a node that has /// The report replaces rather than merges server-side, so a node that has
@@ -1399,11 +1410,32 @@ mod capability_tests {
/// key and a false one must not be distinguishable to the predicate. /// key and a false one must not be distinguishable to the predicate.
#[test] #[test]
fn a_lost_capability_is_reported_false_not_omitted() { fn a_lost_capability_is_reported_false_not_omitted() {
let caps = capabilities_from(false, None); let caps = capabilities_from(false, None, &[]);
assert!(caps.get("kvm").is_some(), "kvm must always be present"); assert!(caps.get("kvm").is_some(), "kvm must always be present");
assert!( assert!(
caps.get("microvm").is_some(), caps.get("microvm").is_some(),
"microvm must always be present" "microvm must always be present"
); );
// Same reasoning for the image list: a node that deleted its last rootfs
// must report an empty ARRAY, not omit the key. Placement asks "does this
// node have backend X"; against a missing key that question has no
// answer, and a scheduler with no answer picks something.
assert_eq!(
caps.get("rootfs"),
Some(&json!([])),
"rootfs must always be present, empty when there are no images"
);
}
/// The list is what placement matches a mission's `backend` against, so it
/// must carry the names verbatim.
#[test]
fn reported_backends_are_the_names_placement_will_ask_for() {
let caps = capabilities_from(
true,
Some("Firecracker v1.16.1"),
&["claude".to_string(), "default".to_string()],
);
assert_eq!(caps["rootfs"], json!(["claude", "default"]));
} }
} }
+38
View File
@@ -161,6 +161,44 @@ fn rootfs_for(backend: Option<&str>) -> Result<PathBuf, String> {
Ok(path) Ok(path)
} }
/// The backend names this node can actually boot, derived from the images on
/// disk.
///
/// Reported as a capability so **placement can require it**. Without this,
/// `missions.backend` is invisible to the scheduler: the first real microVM
/// mission was placed on morpheus because it reports `microvm: true`, while only
/// tank had `rootfs-claude.ext4`. It failed by name rather than booting the wrong
/// image — but whether a mission ran came down to which capable node was picked
/// first, which is a coin flip dressed as scheduling.
///
/// Deliberately in this module: it is the inverse of [`rootfs_for`], and the two
/// must agree about what a backend name means. Split apart, one of them drifts
/// and the scheduler starts promising images the booter cannot find.
pub fn available_backends() -> Vec<String> {
let root = work_root();
let mut out = Vec::new();
// `rootfs.ext4` is what `None`/`""`/`"default"` resolve to.
if root.join("rootfs.ext4").is_file() {
out.push("default".to_string());
}
if let Ok(entries) = std::fs::read_dir(&root) {
for e in entries.flatten() {
let name = e.file_name().to_string_lossy().to_string();
if let Some(rest) = name.strip_prefix("rootfs-") {
if let Some(backend) = rest.strip_suffix(".ext4") {
// Only what `rootfs_for` would accept, so the list cannot
// advertise a name the booter would reject.
if check_id(backend).is_ok() && e.path().is_file() {
out.push(backend.to_string());
}
}
}
}
}
out.sort();
out
}
/// The agent CLI a backend image is named for, and the command that proves it /// The agent CLI a backend image is named for, and the command that proves it
/// is present. /// is present.
/// ///
+15 -6
View File
@@ -131,22 +131,31 @@ pub async fn on_launch(
// none, because the alternative is a mission that sits in 'running' having // none, because the alternative is a mission that sits in 'running' having
// never had anywhere to run. // never had anywhere to run.
if mission.runtime_kind == "microvm" { if mission.runtime_kind == "microvm" {
// Capable means BOTH: it can host a microVM, and it holds the image this
// mission's backend names. Asking only for `microvm` sent the first real
// microVM mission to a node without `rootfs-claude.ext4`.
let backend = mission.backend.as_deref();
let capable = let capable =
cm_db::repo::nodes::online_with_capabilities(pool, mission.workspace_id, &["microvm"]) cm_db::repo::nodes::online_for_backend(pool, mission.workspace_id, backend)
.await .await
.map_err(|e| format!("looking up microvm-capable nodes: {e}"))?; .map_err(|e| format!("looking up nodes for backend {backend:?}: {e}"))?;
const HOW_TO_FIX: &str = "needs /dev/kvm and firecracker installed — \ let how_to_fix = format!(
see scripts/fc-node-setup.sh"; "needs /dev/kvm + firecracker (scripts/fc-node-setup.sh) AND the {} rootfs \
built on that node (scripts/fc-build-rootfs.sh <host> <image> {})",
backend.unwrap_or("default"),
backend.unwrap_or("<name>")
);
let how_to_fix = how_to_fix.as_str();
let chosen = match mission.target_node_id { let chosen = match mission.target_node_id {
// An explicit target is a request, not a guarantee. Honour it only // An explicit target is a request, not a guarantee. Honour it only
// if the node actually reports the capability. // if the node actually reports the capability.
Some(want) => *capable Some(want) => *capable
.iter() .iter()
.find(|n| n.as_uuid() == want) .find(|n| n.as_uuid() == want)
.ok_or_else(|| format!("mission targets node {want}, which is not an online node reporting microvm capability ({HOW_TO_FIX})"))?, .ok_or_else(|| format!("mission targets node {want}, which is not an online node reporting microvm capability ({how_to_fix})"))?,
None => *capable None => *capable
.first() .first()
.ok_or_else(|| format!("no online node reports microvm capability ({HOW_TO_FIX})"))?, .ok_or_else(|| format!("no online node reports microvm capability ({how_to_fix})"))?,
}; };
sqlx::query("UPDATE missions SET target_node_id = $1, updated_at = now() WHERE id = $2") sqlx::query("UPDATE missions SET target_node_id = $1, updated_at = now() WHERE id = $2")
.bind(chosen.as_uuid()) .bind(chosen.as_uuid())
+8 -2
View File
@@ -34,6 +34,10 @@ pub struct Mission {
pub runtime_kind: String, pub runtime_kind: String,
/// FK → nodes(id); only relevant when runtime_kind = 'local_herdr' /// FK → nodes(id); only relevant when runtime_kind = 'local_herdr'
pub target_node_id: Option<Uuid>, pub target_node_id: Option<Uuid>,
/// Which per-CLI rootfs a `microvm` mission boots. NULL = the node's default
/// image. Read by placement (a node must HOLD this image) and by the executor
/// (it is passed to `vm_create`).
pub backend: Option<String>,
/// Per-mission ZeroClaw runtime container name (C3 workspace isolation). /// Per-mission ZeroClaw runtime container name (C3 workspace isolation).
/// Null until `mission_runtime::ensure_container` provisions it. /// Null until `mission_runtime::ensure_container` provisions it.
pub runtime_container_name: Option<String>, pub runtime_container_name: Option<String>,
@@ -229,7 +233,7 @@ pub async fn get(pool: &PgPool, id: Uuid, workspace_id: Uuid) -> Result<Option<M
let row = sqlx::query( let row = sqlx::query(
"SELECT id, workspace_id, title, template_kind, team_id, "SELECT id, workspace_id, title, template_kind, team_id,
team_template_id, repo_id, schedule, status, team_template_id, repo_id, schedule, status,
description, config, runtime_kind, target_node_id, description, config, runtime_kind, target_node_id, backend,
runtime_container_name, runtime_endpoint, runtime_pairing_code, runtime_container_name, runtime_endpoint, runtime_pairing_code,
created_at, updated_at, completed_at created_at, updated_at, completed_at
FROM missions WHERE id = $1 AND workspace_id = $2", FROM missions WHERE id = $1 AND workspace_id = $2",
@@ -252,6 +256,7 @@ pub async fn get(pool: &PgPool, id: Uuid, workspace_id: Uuid) -> Result<Option<M
config: r.get("config"), config: r.get("config"),
runtime_kind: r.get("runtime_kind"), runtime_kind: r.get("runtime_kind"),
target_node_id: r.get("target_node_id"), target_node_id: r.get("target_node_id"),
backend: r.get("backend"),
runtime_container_name: r.get("runtime_container_name"), runtime_container_name: r.get("runtime_container_name"),
runtime_endpoint: r.get("runtime_endpoint"), runtime_endpoint: r.get("runtime_endpoint"),
runtime_pairing_code: r.get("runtime_pairing_code"), runtime_pairing_code: r.get("runtime_pairing_code"),
@@ -271,7 +276,7 @@ pub async fn list_by_workspace(
let rows = sqlx::query( let rows = sqlx::query(
"SELECT id, workspace_id, title, template_kind, team_id, "SELECT id, workspace_id, title, template_kind, team_id,
team_template_id, repo_id, schedule, status, team_template_id, repo_id, schedule, status,
description, config, runtime_kind, target_node_id, description, config, runtime_kind, target_node_id, backend,
runtime_container_name, runtime_endpoint, runtime_pairing_code, runtime_container_name, runtime_endpoint, runtime_pairing_code,
created_at, updated_at, completed_at created_at, updated_at, completed_at
FROM missions WHERE workspace_id = $1 FROM missions WHERE workspace_id = $1
@@ -297,6 +302,7 @@ pub async fn list_by_workspace(
config: r.get("config"), config: r.get("config"),
runtime_kind: r.get("runtime_kind"), runtime_kind: r.get("runtime_kind"),
target_node_id: r.get("target_node_id"), target_node_id: r.get("target_node_id"),
backend: r.get("backend"),
runtime_container_name: r.get("runtime_container_name"), runtime_container_name: r.get("runtime_container_name"),
runtime_endpoint: r.get("runtime_endpoint"), runtime_endpoint: r.get("runtime_endpoint"),
runtime_pairing_code: r.get("runtime_pairing_code"), runtime_pairing_code: r.get("runtime_pairing_code"),
+71
View File
@@ -226,6 +226,54 @@ pub async fn online_with_capabilities(
Ok(rows.into_iter().map(|(id,)| NodeId::from(id)).collect()) Ok(rows.into_iter().map(|(id,)| NodeId::from(id)).collect())
} }
/// Online nodes that can host a microVM **and** hold the image `backend` names.
///
/// KVM alone is the wrong predicate. The first real microVM mission was placed
/// on a node reporting `microvm: true` that did not have `rootfs-claude.ext4`;
/// it failed by name rather than booting the wrong image, but whether a mission
/// ran came down to which capable node was listed first.
///
/// `backend = None` means the node's default image, which reports itself as
/// `"default"` — so the requirement is never vacuous. A node running an older
/// daemon has no `rootfs` key at all and matches nothing, which is the same
/// treatment an unqueried node gets for every other capability: unknown is not
/// permission.
pub async fn online_for_backend(
pool: &PgPool,
workspace_id: uuid::Uuid,
backend: Option<&str>,
) -> Result<Vec<NodeId>, DbError> {
let want = backend_key(backend);
// `@>` on the array asks "does this node's list contain that name" — the
// whole reason the node reports an array rather than a count.
let rows: Vec<(uuid::Uuid,)> = sqlx::query_as(
"SELECT id FROM nodes
WHERE workspace_id = $1 AND status = 'online'
AND capabilities @> '{\"microvm\": true}'::jsonb
AND capabilities -> 'rootfs' @> $2::jsonb
ORDER BY last_seen DESC NULLS LAST",
)
.bind(workspace_id)
.bind(serde_json::Value::Array(vec![serde_json::Value::String(
want.to_string(),
)]))
.fetch_all(pool)
.await?;
Ok(rows.into_iter().map(|(id,)| NodeId::from(id)).collect())
}
/// The name a backend reports itself as in a node's `rootfs` list.
///
/// Must agree with `clawmates-node::microvm::rootfs_for`, which resolves the same
/// three spellings to the default image. If these two drift, placement promises
/// an image the booter cannot find — or refuses one it has.
fn backend_key(backend: Option<&str>) -> &str {
match backend {
None | Some("") | Some("default") => "default",
Some(b) => b,
}
}
/// Mark online nodes whose last heartbeat is older than `secs` as offline. /// Mark online nodes whose last heartbeat is older than `secs` as offline.
pub async fn mark_stale_offline(pool: &PgPool, secs: i64) -> Result<(), DbError> { pub async fn mark_stale_offline(pool: &PgPool, secs: i64) -> Result<(), DbError> {
sqlx::query( sqlx::query(
@@ -290,3 +338,26 @@ fn map_node(r: sqlx::postgres::PgRow) -> NodeRow {
temp_max: r.get("m_temp_max"), temp_max: r.get("m_temp_max"),
} }
} }
#[cfg(test)]
mod tests {
use super::*;
/// The three spellings that mean "the node's default image" must all resolve
/// to the name the node actually advertises for it. A mismatch here makes
/// placement reject every node for an ordinary mission with no backend set.
#[test]
fn the_default_backend_has_one_name() {
for spelling in [None, Some(""), Some("default")] {
assert_eq!(backend_key(spelling), "default", "{spelling:?}");
}
}
/// And a named backend is passed through verbatim — it is matched against the
/// node's list, which is built from the filenames on its disk.
#[test]
fn a_named_backend_is_not_rewritten() {
assert_eq!(backend_key(Some("claude")), "claude");
assert_eq!(backend_key(Some("agent-terminal")), "agent-terminal");
}
}