fix(fleet): placement requires the backend's rootfs image, not just KVM
The first real microVM mission was placed on morpheus because it reports
{"microvm": true}, while only tank had rootfs-claude.ext4. It failed by name
rather than booting the wrong image — but whether a mission ran came down to
which capable node was listed first, which is a coin flip dressed as scheduling.
`missions.backend` was invisible to the scheduler.
The node now enumerates the images on its disk and reports them as a `rootfs`
ARRAY. `microvm::available_backends` lives beside `rootfs_for`, its inverse,
because the two must agree on what a backend name means; split apart, one drifts
and the scheduler starts promising images the booter cannot find. It only
advertises names `rootfs_for` would accept, and reports an empty array rather than
omitting the key — set_capabilities REPLACES, so a deleted image stops being
advertised instead of leaving a stale claim.
`nodes::online_for_backend` requires microvm AND that the node's list contains the
mission's backend. A node on an older daemon has no `rootfs` key and matches
nothing: unknown is not permission, the same treatment every other capability
gets. `backend_key` maps the three spellings of "the default image" to the one
name the node advertises, and is tested — a mismatch there would reject every node
for an ordinary mission with no backend set.
The launch error now names both halves of the fix, since "no capable node" was
true but unhelpful when the node was capable and merely lacked the image.
Mission gains `backend` on the domain struct; it was a column the executor read
from the phase query while the struct that placement uses could not see it.
464 tests pass, clippy clean.
This commit is contained in:
@@ -34,6 +34,10 @@ pub struct Mission {
|
||||
pub runtime_kind: String,
|
||||
/// FK → nodes(id); only relevant when runtime_kind = 'local_herdr'
|
||||
pub target_node_id: Option<Uuid>,
|
||||
/// Which per-CLI rootfs a `microvm` mission boots. NULL = the node's default
|
||||
/// image. Read by placement (a node must HOLD this image) and by the executor
|
||||
/// (it is passed to `vm_create`).
|
||||
pub backend: Option<String>,
|
||||
/// Per-mission ZeroClaw runtime container name (C3 workspace isolation).
|
||||
/// Null until `mission_runtime::ensure_container` provisions it.
|
||||
pub runtime_container_name: Option<String>,
|
||||
@@ -229,7 +233,7 @@ pub async fn get(pool: &PgPool, id: Uuid, workspace_id: Uuid) -> Result<Option<M
|
||||
let row = sqlx::query(
|
||||
"SELECT id, workspace_id, title, template_kind, team_id,
|
||||
team_template_id, repo_id, schedule, status,
|
||||
description, config, runtime_kind, target_node_id,
|
||||
description, config, runtime_kind, target_node_id, backend,
|
||||
runtime_container_name, runtime_endpoint, runtime_pairing_code,
|
||||
created_at, updated_at, completed_at
|
||||
FROM missions WHERE id = $1 AND workspace_id = $2",
|
||||
@@ -252,6 +256,7 @@ pub async fn get(pool: &PgPool, id: Uuid, workspace_id: Uuid) -> Result<Option<M
|
||||
config: r.get("config"),
|
||||
runtime_kind: r.get("runtime_kind"),
|
||||
target_node_id: r.get("target_node_id"),
|
||||
backend: r.get("backend"),
|
||||
runtime_container_name: r.get("runtime_container_name"),
|
||||
runtime_endpoint: r.get("runtime_endpoint"),
|
||||
runtime_pairing_code: r.get("runtime_pairing_code"),
|
||||
@@ -271,7 +276,7 @@ pub async fn list_by_workspace(
|
||||
let rows = sqlx::query(
|
||||
"SELECT id, workspace_id, title, template_kind, team_id,
|
||||
team_template_id, repo_id, schedule, status,
|
||||
description, config, runtime_kind, target_node_id,
|
||||
description, config, runtime_kind, target_node_id, backend,
|
||||
runtime_container_name, runtime_endpoint, runtime_pairing_code,
|
||||
created_at, updated_at, completed_at
|
||||
FROM missions WHERE workspace_id = $1
|
||||
@@ -297,6 +302,7 @@ pub async fn list_by_workspace(
|
||||
config: r.get("config"),
|
||||
runtime_kind: r.get("runtime_kind"),
|
||||
target_node_id: r.get("target_node_id"),
|
||||
backend: r.get("backend"),
|
||||
runtime_container_name: r.get("runtime_container_name"),
|
||||
runtime_endpoint: r.get("runtime_endpoint"),
|
||||
runtime_pairing_code: r.get("runtime_pairing_code"),
|
||||
|
||||
@@ -226,6 +226,54 @@ pub async fn online_with_capabilities(
|
||||
Ok(rows.into_iter().map(|(id,)| NodeId::from(id)).collect())
|
||||
}
|
||||
|
||||
/// Online nodes that can host a microVM **and** hold the image `backend` names.
|
||||
///
|
||||
/// KVM alone is the wrong predicate. The first real microVM mission was placed
|
||||
/// on a node reporting `microvm: true` that did not have `rootfs-claude.ext4`;
|
||||
/// it failed by name rather than booting the wrong image, but whether a mission
|
||||
/// ran came down to which capable node was listed first.
|
||||
///
|
||||
/// `backend = None` means the node's default image, which reports itself as
|
||||
/// `"default"` — so the requirement is never vacuous. A node running an older
|
||||
/// daemon has no `rootfs` key at all and matches nothing, which is the same
|
||||
/// treatment an unqueried node gets for every other capability: unknown is not
|
||||
/// permission.
|
||||
pub async fn online_for_backend(
|
||||
pool: &PgPool,
|
||||
workspace_id: uuid::Uuid,
|
||||
backend: Option<&str>,
|
||||
) -> Result<Vec<NodeId>, DbError> {
|
||||
let want = backend_key(backend);
|
||||
// `@>` on the array asks "does this node's list contain that name" — the
|
||||
// whole reason the node reports an array rather than a count.
|
||||
let rows: Vec<(uuid::Uuid,)> = sqlx::query_as(
|
||||
"SELECT id FROM nodes
|
||||
WHERE workspace_id = $1 AND status = 'online'
|
||||
AND capabilities @> '{\"microvm\": true}'::jsonb
|
||||
AND capabilities -> 'rootfs' @> $2::jsonb
|
||||
ORDER BY last_seen DESC NULLS LAST",
|
||||
)
|
||||
.bind(workspace_id)
|
||||
.bind(serde_json::Value::Array(vec![serde_json::Value::String(
|
||||
want.to_string(),
|
||||
)]))
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
Ok(rows.into_iter().map(|(id,)| NodeId::from(id)).collect())
|
||||
}
|
||||
|
||||
/// The name a backend reports itself as in a node's `rootfs` list.
|
||||
///
|
||||
/// Must agree with `clawmates-node::microvm::rootfs_for`, which resolves the same
|
||||
/// three spellings to the default image. If these two drift, placement promises
|
||||
/// an image the booter cannot find — or refuses one it has.
|
||||
fn backend_key(backend: Option<&str>) -> &str {
|
||||
match backend {
|
||||
None | Some("") | Some("default") => "default",
|
||||
Some(b) => b,
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark online nodes whose last heartbeat is older than `secs` as offline.
|
||||
pub async fn mark_stale_offline(pool: &PgPool, secs: i64) -> Result<(), DbError> {
|
||||
sqlx::query(
|
||||
@@ -290,3 +338,26 @@ fn map_node(r: sqlx::postgres::PgRow) -> NodeRow {
|
||||
temp_max: r.get("m_temp_max"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The three spellings that mean "the node's default image" must all resolve
|
||||
/// to the name the node actually advertises for it. A mismatch here makes
|
||||
/// placement reject every node for an ordinary mission with no backend set.
|
||||
#[test]
|
||||
fn the_default_backend_has_one_name() {
|
||||
for spelling in [None, Some(""), Some("default")] {
|
||||
assert_eq!(backend_key(spelling), "default", "{spelling:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// And a named backend is passed through verbatim — it is matched against the
|
||||
/// node's list, which is built from the filenames on its disk.
|
||||
#[test]
|
||||
fn a_named_backend_is_not_rewritten() {
|
||||
assert_eq!(backend_key(Some("claude")), "claude");
|
||||
assert_eq!(backend_key(Some("agent-terminal")), "agent-terminal");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user