Merge: B1 microvm placement — tank and morpheus report microvm:true

Co-Authored-By: Claude Opus 5 <[email protected]>

crates/bins/clawmates-node/src/main.rs

@@ -131,6 +131,14 @@ async fn run(ws_url: &str) -> Result<(), Box<dyn std::error::Error>> {
         if tools_tx.send(frame).is_err() {
             break;
         }
+        // What this node can HOST, as opposed to what it has installed. The
+        // scheduler needs it to place microVM missions, and the node is the
+        // only honest source: /dev/kvm either exists here or it does not, and
+        // no amount of configuration on the server can make it appear.
+        let caps = json!({ "t": "node_capabilities", "capabilities": probe_capabilities() });
+        if tools_tx.send(caps.to_string()).is_err() {
+            break;
+        }
         std::thread::sleep(Duration::from_secs(900));
     });
 
@@ -241,6 +249,59 @@ fn heartbeat(sys: &mut System) -> String {
 /// Probe installed dev-tool versions: for each tool, find its binary across the
 /// usual bin dirs and read `--version`. Returns `{ tool: "x.y.z", … }` for the
 /// ones found. Probes `kimi-cli` (the real uv tool), not the `kimi` API wrapper.
+/// What this node can HOST — the inputs to placement predicates.
+///
+/// Distinct from [`probe_tools`], which reports what is *installed* for the
+/// operator to see and update. This answers "may the scheduler put a microVM
+/// mission here", and the answer is a property of the hardware: gw-04 is
+/// itself a VM without nested virtualisation and has no `/dev/kvm`, so it can
+/// never host one however it is configured.
+///
+/// Every value is probed, never assumed. A capability that is merely expected
+/// is the same as a capability that is absent, right up until a mission is
+/// scheduled onto a node that cannot run it.
+fn probe_capabilities() -> Value {
+    // The device node is necessary but not sufficient — it can exist while
+    // being unopenable (wrong group, or a container without the device
+    // passed through). Try to open it, because that is what firecracker does.
+    let kvm = std::fs::OpenOptions::new()
+        .read(true)
+        .write(true)
+        .open("/dev/kvm")
+        .is_ok();
+
+    let firecracker = std::process::Command::new("firecracker")
+        .arg("--version")
+        .output()
+        .ok()
+        .filter(|o| o.status.success())
+        .and_then(|o| {
+            String::from_utf8_lossy(&o.stdout)
+                .lines()
+                .next()
+                .map(|l| l.trim().to_string())
+        });
+
+    capabilities_from(kvm, firecracker.as_deref())
+}
+
+/// Shape the capability report from probe results.
+///
+/// Split from [`probe_capabilities`] so the rule can be tested without a
+/// `/dev/kvm` to open — the machine running the tests is usually the one that
+/// cannot host a microVM.
+fn capabilities_from(kvm: bool, firecracker: Option<&str>) -> Value {
+    json!({
+        "kvm": kvm,
+        "firecracker": firecracker,
+        // BOTH must hold. A node with KVM but no firecracker binary looks
+        // capable by the obvious test and fails at launch; a node with the
+        // binary but no KVM is gw-04. Computed here rather than in the
+        // scheduler so the rule sits next to the probe that feeds it.
+        "microvm": kvm && firecracker.is_some(),
+    })
+}
+
 fn probe_tools() -> Value {
     let home = std::env::var("HOME").unwrap_or_default();
     let dirs = [
@@ -1274,3 +1335,40 @@ fn ensure_tmux() {
         eprintln!("tmux not found (auto-install unavailable) — host terminal will use a plain shell; `apt install tmux` for resumable sessions");
     }
 }
+
+#[cfg(test)]
+mod capability_tests {
+    use super::*;
+
+    #[test]
+    fn microvm_needs_both_kvm_and_firecracker() {
+        assert_eq!(
+            capabilities_from(true, Some("Firecracker v1.16.1"))["microvm"],
+            json!(true)
+        );
+        assert_eq!(
+            capabilities_from(true, None)["microvm"],
+            json!(false),
+            "KVM without firecracker cannot host a microVM"
+        );
+        assert_eq!(
+            capabilities_from(false, Some("Firecracker v1.16.1"))["microvm"],
+            json!(false),
+            "firecracker without KVM is gw-04 — it can never host one"
+        );
+        assert_eq!(capabilities_from(false, None)["microvm"], json!(false));
+    }
+
+    /// The report replaces rather than merges server-side, so a node that has
+    /// LOST a capability must say so rather than omitting the key — an absent
+    /// key and a false one must not be distinguishable to the predicate.
+    #[test]
+    fn a_lost_capability_is_reported_false_not_omitted() {
+        let caps = capabilities_from(false, None);
+        assert!(caps.get("kvm").is_some(), "kvm must always be present");
+        assert!(
+            caps.get("microvm").is_some(),
+            "microvm must always be present"
+        );
+    }
+}

crates/cm-api/src/fleet.rs

@@ -381,6 +381,11 @@ enum Uplink {
     NodeTools {
         tools: std::collections::HashMap<String, String>,
     },
+    /// What the node can HOST, as opposed to what it has installed — the
+    /// inputs to placement predicates. Free-form so a new predicate does not
+    /// need a migration; see `migrations/0065_microvm_placement.sql`.
+    #[serde(rename = "node_capabilities")]
+    NodeCapabilities { capabilities: serde_json::Value },
 }
 
 #[derive(Deserialize)]
@@ -539,6 +544,19 @@ pub async fn run_channel(pool: PgPool, hub: Arc<NodeHub>, node_id: NodeId, socke
                         let pairs: Vec<(String, String)> = tools.into_iter().collect();
                         let _ = cm_db::repo::node_tools::upsert(&pool, node_id, &pairs).await;
                     }
+                    Ok(Uplink::NodeCapabilities { capabilities }) => {
+                        if let Err(e) = nodes::set_capabilities(&pool, node_id, &capabilities).await
+                        {
+                            // Loud: a node whose capabilities never land looks
+                            // exactly like a node that has none, and will be
+                            // passed over for every microVM mission forever
+                            // while appearing perfectly healthy.
+                            eprintln!(
+                                "fleet: could not record capabilities for node {node_id} ({e}) — \
+                                 it will not be selected for microvm placement"
+                            );
+                        }
+                    }
                     Err(_) => {}
                 }
             },

crates/cm-api/src/mission_orchestrator.rs

@@ -260,6 +260,39 @@ pub async fn on_launch(
         }
     }
 
+    // microVM placement. KVM is a hard predicate, not a preference: gw-04 —
+    // where every mission runs today — is itself a VM without nested
+    // virtualisation and has no /dev/kvm, so a microvm mission landing there
+    // cannot start. Resolve a capable node now and fail the launch if there is
+    // none, because the alternative is a mission that sits in 'running' having
+    // never had anywhere to run.
+    if mission.runtime_kind == "microvm" {
+        let capable =
+            cm_db::repo::nodes::online_with_capabilities(pool, mission.workspace_id, &["microvm"])
+                .await
+                .map_err(|e| format!("looking up microvm-capable nodes: {e}"))?;
+        const HOW_TO_FIX: &str = "needs /dev/kvm and firecracker installed — \
+                                  see scripts/fc-node-setup.sh";
+        let chosen = match mission.target_node_id {
+            // An explicit target is a request, not a guarantee. Honour it only
+            // if the node actually reports the capability.
+            Some(want) => *capable
+                .iter()
+                .find(|n| n.as_uuid() == want)
+                .ok_or_else(|| format!("mission targets node {want}, which is not an online node reporting microvm capability ({HOW_TO_FIX})"))?,
+            None => *capable
+                .first()
+                .ok_or_else(|| format!("no online node reports microvm capability ({HOW_TO_FIX})"))?,
+        };
+        sqlx::query("UPDATE missions SET target_node_id = $1, updated_at = now() WHERE id = $2")
+            .bind(chosen.as_uuid())
+            .bind(mission_id)
+            .execute(pool)
+            .await
+            .map_err(|e| format!("pin mission {mission_id} to node {chosen:?}: {e}"))?;
+        eprintln!("mission_orchestrator: mission {mission_id} placed on microvm node {chosen:?}");
+    }
+
     // Herdr second-runtime: if runtime_kind='local_herdr', spawn a
     // pane on target_node running the first available local CLI.
     // Non-fatal on failure — the operator sees the error in server

crates/cm-db/src/repo/nodes.rs

@@ -178,6 +178,54 @@ pub async fn set_status(pool: &PgPool, id: NodeId, status: &str) -> Result<(), D
     Ok(())
 }
 
+/// Record what a node reports it can host, for placement predicates.
+///
+/// Replaces rather than merges: the node sends its complete view on every
+/// report, so a capability it has *stopped* having (firecracker uninstalled,
+/// `/dev/kvm` gone after a reboot into a non-virt kernel) must disappear here
+/// too. Merging would let a stale `true` survive forever.
+pub async fn set_capabilities(
+    pool: &PgPool,
+    id: NodeId,
+    capabilities: &serde_json::Value,
+) -> Result<(), DbError> {
+    sqlx::query("UPDATE nodes SET capabilities = $2 WHERE id = $1")
+        .bind(id.as_uuid())
+        .bind(capabilities)
+        .execute(pool)
+        .await?;
+    Ok(())
+}
+
+/// Online nodes that report every one of `required` as `true`.
+///
+/// The predicate side of placement. Nothing is assumed: a node that has never
+/// reported has `capabilities = '{}'`, which fails every requirement — an
+/// unqueried node and an incapable node are treated identically, because
+/// scheduling work onto a node whose abilities are unknown is how you get a
+/// mission that cannot start and does not say why.
+pub async fn online_with_capabilities(
+    pool: &PgPool,
+    workspace_id: uuid::Uuid,
+    required: &[&str],
+) -> Result<Vec<NodeId>, DbError> {
+    let needed: serde_json::Value = required
+        .iter()
+        .map(|k| ((*k).to_string(), serde_json::Value::Bool(true)))
+        .collect::<serde_json::Map<_, _>>()
+        .into();
+    let rows: Vec<(uuid::Uuid,)> = sqlx::query_as(
+        "SELECT id FROM nodes
+          WHERE workspace_id = $1 AND status = 'online' AND capabilities @> $2
+          ORDER BY last_seen DESC NULLS LAST",
+    )
+    .bind(workspace_id)
+    .bind(&needed)
+    .fetch_all(pool)
+    .await?;
+    Ok(rows.into_iter().map(|(id,)| NodeId::from(id)).collect())
+}
+
 /// Mark online nodes whose last heartbeat is older than `secs` as offline.
 pub async fn mark_stale_offline(pool: &PgPool, secs: i64) -> Result<(), DbError> {
     sqlx::query(

migrations/0065_microvm_placement.sql

@@ -0,0 +1,27 @@
+-- Phase B1: make 'microvm' a runtime a mission can ask for, and give nodes a
+-- place to declare what they can actually host.
+--
+-- Placement for microVMs is a HARD predicate, not a preference. gw-04 — where
+-- every mission runs today — is itself a VM without nested virtualisation, so
+-- it has no /dev/kvm and never will. tank / morpheus / architect do. A microvm
+-- mission scheduled onto a node without KVM cannot start, so the scheduler has
+-- to be able to tell the difference, which means the node has to report it.
+--
+-- `capabilities` is deliberately a jsonb blob rather than a `has_kvm boolean`:
+-- the next predicate (a baked rootfs present, a particular CLI image, a GPU)
+-- should not need a migration, and the node is the only honest source for any
+-- of them.
+
+ALTER TABLE missions DROP CONSTRAINT IF EXISTS missions_runtime_kind_check;
+ALTER TABLE missions ADD CONSTRAINT missions_runtime_kind_check
+  CHECK (runtime_kind IN ('zeroclaw', 'local_herdr', 'microvm'));
+
+-- Default '{}' and not null: a node that has never reported is "capable of
+-- nothing known", which is the safe reading. An absent capability must never
+-- be mistaken for an unqueried one — a node with no entry and a node that
+-- reported `kvm: false` should both fail a KVM predicate, and with this
+-- default they do.
+ALTER TABLE nodes ADD COLUMN IF NOT EXISTS capabilities jsonb NOT NULL DEFAULT '{}'::jsonb;
+
+-- Placement asks "which online nodes have KVM", so index the lookup.
+CREATE INDEX IF NOT EXISTS nodes_capabilities_idx ON nodes USING gin (capabilities);

scripts/fc-node-setup.sh

@@ -42,6 +42,21 @@ for host in "$@"; do
   fi
   pass "$host" "/dev/kvm present"
 
+  # Present is not the same as usable. /dev/kvm is `crw-rw---- root:kvm`, and
+  # the node daemon runs as an ordinary user — on tank and morpheus the kvm
+  # group was EMPTY, so the daemon reported `kvm: false` while the device sat
+  # right there. The B0 spike missed it entirely because it ran under sudo.
+  # This is why the capability probe opens the device instead of stat-ing it.
+  if ! ssh "$host" 'test -r /dev/kvm && test -w /dev/kvm' 2>/dev/null; then
+    if ssh "$host" 'sudo -n usermod -aG kvm $(id -un) && sudo -n systemctl restart clawmates-node' 2>/dev/null; then
+      pass "$host" "added $(ssh "$host" 'id -un') to the kvm group (daemon restarted)"
+    else
+      fail "$host" "/dev/kvm is not readable/writable by the daemon user and could not be fixed — run: sudo usermod -aG kvm <user> && sudo systemctl restart clawmates-node"
+    fi
+  else
+    pass "$host" "/dev/kvm is openable by the daemon user"
+  fi
+
   # 2. Install, verifying the checksum by hand.
   #
   # `sha256sum -c` against the published file compares by FILENAME, so saving