Omar SobhandClaude Opus 5
f56d41f5b7
Claude Code pointed at the Ollama already installed on every GPU node. Ollama has served a native Anthropic-compatible /v1/messages since v0.14, so this is an env contract rather than a translation layer — the fourth variation on the same idea as agent-glm and agent-kimi. The route is NOT the egress proxy, and that is the design. `egress` speaks CONNECT, takes a destination from the guest, resolves it and decides; every one of those powers is a liability, which is why it refuses non-443 ports and IP literals after a unit test caught them being bypassed. Routing a local model through it would have meant relaxing both. `local_model` is the opposite shape: there is no destination in the protocol. fcagent listens on guest 127.0.0.1:11434 and pumps to vsock 9003; the node splices that onto its own 127.0.0.1:11434 and copies bytes. A compromised guest cannot redirect it because there is nothing to redirect — it is a pipe, not a proxy, and strictly narrower than anything an allow-list could express. The bytes never touch a network, so there is no wire for TLS to protect, and Ollama stays bound to loopback rather than being exposed on the tailnet. The socket is bound only for a backend declared to use a local model, so a `local-ornith` VM reaches the forge through egress and nothing else, while every other backend's guest port simply refuses. Both halves have negative controls. `scripts/fleet-model-setup.sh` exists because of one measurement: stock ornith:9b reported input_tokens=2050 for a 48000-word prompt and answered as though nothing had been dropped. Ollama's default window is ~2K whatever the model card says, and it truncates silently — the exact failure an agent turn would hit and never report. The script pins num_ctx=131072 into a derived tag and then PROVES both the window and tool calling before declaring success. Verified on architect: ~65536 words -> 65604 input tokens, stop_reason=tool_use. Placement needs no new capability key: building the rootfs only on GPU nodes means `nodes::online_for_backend`'s existing `rootfs @> ["local-ornith"]` predicate does the affinity, so morpheus never offers the backend. Co-Authored-By: Claude Opus 5 <[email protected]>
1559 lines
60 KiB
Rust
1559 lines
60 KiB
Rust
//! clawmates-node — the fleet daemon a user installs on each local-hardware node.
|
|
//!
|
|
//! It dials home to the ClawMates gateway over an OUTBOUND WebSocket
|
|
//! (`/api/nodes/agent?token=…`), reports host health on a heartbeat, and runs the
|
|
//! commands the gateway sends (host verification today; container placement in a
|
|
//! later phase). Outbound-only → no inbound port, NAT-friendly.
|
|
|
|
use std::collections::HashMap;
|
|
use std::io::{Read, Write};
|
|
use std::sync::Arc;
|
|
use std::time::Duration;
|
|
|
|
use base64::Engine;
|
|
use cm_sandbox::{DockerDriver, SandboxDriver, SandboxHandle, SandboxSpec};
|
|
use futures::{SinkExt, StreamExt};
|
|
use portable_pty::{native_pty_system, CommandBuilder, MasterPty, PtySize};
|
|
use serde_json::{json, Value};
|
|
use sysinfo::{Disks, System};
|
|
use tokio::sync::{mpsc, Mutex};
|
|
use tokio_tungstenite::tungstenite::Message;
|
|
|
|
mod egress;
|
|
mod local_model;
|
|
mod microvm;
|
|
mod rtc;
|
|
|
|
const B64: base64::engine::general_purpose::GeneralPurpose =
|
|
base64::engine::general_purpose::STANDARD;
|
|
|
|
/// A live host-shell PTY the gateway opened (keyed by session id).
|
|
struct Pty {
|
|
master: Box<dyn MasterPty + Send>,
|
|
writer: Box<dyn Write + Send>,
|
|
child: Box<dyn portable_pty::Child + Send + Sync>,
|
|
}
|
|
type Ptys = Arc<Mutex<HashMap<u64, Pty>>>;
|
|
|
|
const VERSION: &str = env!("CARGO_PKG_VERSION");
|
|
|
|
#[tokio::main]
|
|
async fn main() {
|
|
// The dep graph enables both rustls crypto providers (tungstenite + bollard),
|
|
// so rustls can't auto-pick — install one explicitly before any TLS.
|
|
let _ = rustls::crypto::ring::default_provider().install_default();
|
|
if std::env::args().any(|a| a == "--selftest") {
|
|
selftest();
|
|
return;
|
|
}
|
|
// Exercise the microVM lifecycle against a real VM on this node. Separate
|
|
// from --selftest because it needs KVM, so it can only pass on a node that
|
|
// actually reports microvm capability.
|
|
if std::env::args().any(|a| a == "--vm-selftest") {
|
|
if !microvm::selftest().await {
|
|
std::process::exit(1);
|
|
}
|
|
return;
|
|
}
|
|
let (server, token, ts_authkey) = parse_args();
|
|
if server.is_empty() || token.is_empty() {
|
|
eprintln!("usage: clawmates-node --server <https://gateway> --token <token> [--tailscale-authkey <key>]");
|
|
eprintln!(" (or set CLAWMATES_SERVER / CLAWMATES_TOKEN / CLAWMATES_TS_AUTHKEY)");
|
|
std::process::exit(2);
|
|
}
|
|
tailscale_up(&ts_authkey);
|
|
ensure_tmux();
|
|
let ws_url = ws_url(&server, &token);
|
|
println!("clawmates-node {VERSION} connecting to {server}");
|
|
loop {
|
|
if let Err(e) = run(&ws_url).await {
|
|
eprintln!("channel ended: {e}; reconnecting in 5s");
|
|
}
|
|
tokio::time::sleep(Duration::from_secs(5)).await;
|
|
}
|
|
}
|
|
|
|
fn parse_args() -> (String, String, String) {
|
|
let mut server = String::new();
|
|
let mut token = String::new();
|
|
let mut ts_authkey = String::new();
|
|
let mut args = std::env::args().skip(1);
|
|
while let Some(a) = args.next() {
|
|
match a.as_str() {
|
|
"--server" => server = args.next().unwrap_or_default(),
|
|
"--token" => token = args.next().unwrap_or_default(),
|
|
"--tailscale-authkey" => ts_authkey = args.next().unwrap_or_default(),
|
|
_ => {}
|
|
}
|
|
}
|
|
if server.is_empty() {
|
|
server = std::env::var("CLAWMATES_SERVER").unwrap_or_default();
|
|
}
|
|
if token.is_empty() {
|
|
token = std::env::var("CLAWMATES_TOKEN").unwrap_or_default();
|
|
}
|
|
if ts_authkey.is_empty() {
|
|
ts_authkey = std::env::var("CLAWMATES_TS_AUTHKEY").unwrap_or_default();
|
|
}
|
|
(server, token, ts_authkey)
|
|
}
|
|
|
|
fn ws_url(server: &str, token: &str) -> String {
|
|
let base = server.trim_end_matches('/');
|
|
let base = base
|
|
.replacen("https://", "wss://", 1)
|
|
.replacen("http://", "ws://", 1);
|
|
let base = if base.starts_with("ws") {
|
|
base
|
|
} else {
|
|
format!("wss://{base}")
|
|
};
|
|
format!("{base}/api/nodes/agent?token={token}")
|
|
}
|
|
|
|
async fn run(ws_url: &str) -> Result<(), Box<dyn std::error::Error>> {
|
|
let (ws, _) = tokio_tungstenite::connect_async(ws_url).await?;
|
|
println!("connected; reporting health every 5s");
|
|
let (mut write, mut read) = ws.split();
|
|
// Everything outbound (heartbeats, command results, PTY output) funnels
|
|
// through one channel so PTY reader threads can push asynchronously.
|
|
let (out_tx, mut out_rx) = mpsc::unbounded_channel::<String>();
|
|
let ptys: Ptys = Arc::new(Mutex::new(HashMap::new()));
|
|
let peers: rtc::RtcPeers = Arc::new(Mutex::new(HashMap::new()));
|
|
// microVMs this connection started. Scoped to the connection deliberately:
|
|
// a reconnect must not inherit VMs it cannot prove are still alive, and
|
|
// `vm_destroy` cleans a workdir by path even for an unregistered id, so a
|
|
// VM from a previous incarnation is reapable rather than orphaned.
|
|
let vms = microvm::new_vms();
|
|
// Collect heartbeats on a dedicated thread: the metric helpers shell out to
|
|
// docker/tailscale and stat disks (blocking), which must never stall the
|
|
// async select loop (or heartbeats/pongs would starve during a slow op).
|
|
let hb_tx = out_tx.clone();
|
|
std::thread::spawn(move || {
|
|
let mut sys = System::new_all();
|
|
loop {
|
|
if hb_tx.send(heartbeat(&mut sys)).is_err() {
|
|
break; // connection gone
|
|
}
|
|
std::thread::sleep(Duration::from_secs(5));
|
|
}
|
|
});
|
|
|
|
// Probe installed dev-tool versions (docker/claude/kimi-cli/ollama) on a
|
|
// dedicated thread — the version commands shell out (blocking) — on connect,
|
|
// then every 15 min. The server flags updates against upstream.
|
|
let tools_tx = out_tx.clone();
|
|
std::thread::spawn(move || loop {
|
|
let frame = json!({ "t": "node_tools", "tools": probe_tools() }).to_string();
|
|
if tools_tx.send(frame).is_err() {
|
|
break;
|
|
}
|
|
// What this node can HOST, as opposed to what it has installed. The
|
|
// scheduler needs it to place microVM missions, and the node is the
|
|
// only honest source: /dev/kvm either exists here or it does not, and
|
|
// no amount of configuration on the server can make it appear.
|
|
let caps = json!({ "t": "node_capabilities", "capabilities": probe_capabilities() });
|
|
if tools_tx.send(caps.to_string()).is_err() {
|
|
break;
|
|
}
|
|
std::thread::sleep(Duration::from_secs(900));
|
|
});
|
|
|
|
// Liveness has two independent failure modes to catch, both of which we
|
|
// hit on architect during the Jul 5 2026 outage:
|
|
// (a) READ-side stall: server never sends anything (or the socket goes
|
|
// half-open on read). Caught by last_rx / 40s idle window below.
|
|
// (b) WRITE-side stall: peer's TCP stack has died but our OS buffer is
|
|
// still soaking heartbeat writes. write.send().await blocks
|
|
// indefinitely inside the select! branch — tokio::select doesn't
|
|
// preempt a running future, so the whole loop freezes; idle_tick
|
|
// never gets to fire. Wrapping the send in a timeout is the fix.
|
|
//
|
|
// WRITE_DEADLINE is short enough (10s) that a stuck send is caught before
|
|
// it can outlast the 40s read-idle threshold and leave the daemon spinning
|
|
// silently for hours (which is what happened pre-patch).
|
|
const WRITE_DEADLINE: Duration = Duration::from_secs(10);
|
|
let mut idle_tick = tokio::time::interval(Duration::from_secs(5));
|
|
let mut last_rx = std::time::Instant::now();
|
|
loop {
|
|
tokio::select! {
|
|
Some(frame) = out_rx.recv() => {
|
|
match tokio::time::timeout(
|
|
WRITE_DEADLINE,
|
|
write.send(Message::Text(frame.into())),
|
|
).await {
|
|
Ok(Ok(())) => {}
|
|
Ok(Err(e)) => return Err(e.into()),
|
|
Err(_) => {
|
|
eprintln!("write.send timeout > {}s — socket dead, reconnecting",
|
|
WRITE_DEADLINE.as_secs());
|
|
return Ok(());
|
|
}
|
|
}
|
|
}
|
|
_ = idle_tick.tick() => {
|
|
if last_rx.elapsed() > Duration::from_secs(40) {
|
|
return Ok(());
|
|
}
|
|
}
|
|
msg = read.next() => {
|
|
last_rx = std::time::Instant::now();
|
|
match msg {
|
|
Some(Ok(Message::Text(t))) => {
|
|
// Run each op concurrently so long docker/PTY work never
|
|
// blocks heartbeats, pongs, or other commands.
|
|
let out = out_tx.clone();
|
|
let ptys = ptys.clone();
|
|
let peers = peers.clone();
|
|
let vms = vms.clone();
|
|
let text = t.to_string();
|
|
tokio::spawn(async move { handle_frame(&text, &out, &ptys, &peers, &vms).await; });
|
|
}
|
|
Some(Ok(Message::Ping(p))) => {
|
|
match tokio::time::timeout(WRITE_DEADLINE, write.send(Message::Pong(p))).await {
|
|
Ok(Ok(())) => {}
|
|
Ok(Err(e)) => return Err(e.into()),
|
|
Err(_) => {
|
|
eprintln!("pong write timeout — socket dead, reconnecting");
|
|
return Ok(());
|
|
}
|
|
}
|
|
}
|
|
Some(Ok(Message::Close(_))) | None => return Ok(()),
|
|
Some(Err(e)) => return Err(e.into()),
|
|
_ => {}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Collect a host-health snapshot and serialize the heartbeat frame.
|
|
fn heartbeat(sys: &mut System) -> String {
|
|
sys.refresh_cpu_usage();
|
|
sys.refresh_memory();
|
|
let mem_total = sys.total_memory() as i64;
|
|
let mem_used = sys.used_memory() as i64;
|
|
let mem_pressure = if mem_total > 0 {
|
|
mem_used as f64 / mem_total as f64
|
|
} else {
|
|
0.0
|
|
};
|
|
let load = System::load_average();
|
|
let (disk_total, disk_free) = root_disk();
|
|
json!({
|
|
"t": "heartbeat",
|
|
"version": VERSION,
|
|
"tailscale_ip": tailscale_ip(),
|
|
"hostname": System::host_name(),
|
|
"local_ip": local_ip(),
|
|
"health": {
|
|
"cpu_pct": sys.global_cpu_usage() as f64,
|
|
"mem_total": mem_total,
|
|
"mem_used": mem_used,
|
|
"mem_pressure": mem_pressure,
|
|
"swap_used": sys.used_swap() as i64,
|
|
"disk_total": disk_total,
|
|
"disk_free": disk_free,
|
|
"load1": load.one,
|
|
"load5": load.five,
|
|
"load15": load.fifteen,
|
|
"container_count": docker_count(),
|
|
}
|
|
})
|
|
.to_string()
|
|
}
|
|
|
|
/// Probe installed dev-tool versions: for each tool, find its binary across the
|
|
/// usual bin dirs and read `--version`. Returns `{ tool: "x.y.z", … }` for the
|
|
/// ones found. Probes `kimi-cli` (the real uv tool), not the `kimi` API wrapper.
|
|
/// What this node can HOST — the inputs to placement predicates.
|
|
///
|
|
/// Distinct from [`probe_tools`], which reports what is *installed* for the
|
|
/// operator to see and update. This answers "may the scheduler put a microVM
|
|
/// mission here", and the answer is a property of the hardware: gw-04 is
|
|
/// itself a VM without nested virtualisation and has no `/dev/kvm`, so it can
|
|
/// never host one however it is configured.
|
|
///
|
|
/// Every value is probed, never assumed. A capability that is merely expected
|
|
/// is the same as a capability that is absent, right up until a mission is
|
|
/// scheduled onto a node that cannot run it.
|
|
fn probe_capabilities() -> Value {
|
|
// The device node is necessary but not sufficient — it can exist while
|
|
// being unopenable (wrong group, or a container without the device
|
|
// passed through). Try to open it, because that is what firecracker does.
|
|
let kvm = std::fs::OpenOptions::new()
|
|
.read(true)
|
|
.write(true)
|
|
.open("/dev/kvm")
|
|
.is_ok();
|
|
|
|
let firecracker = std::process::Command::new("firecracker")
|
|
.arg("--version")
|
|
.output()
|
|
.ok()
|
|
.filter(|o| o.status.success())
|
|
.and_then(|o| {
|
|
String::from_utf8_lossy(&o.stdout)
|
|
.lines()
|
|
.next()
|
|
.map(|l| l.trim().to_string())
|
|
});
|
|
|
|
// Which rootfs images are actually on this node's disk. Reported so
|
|
// placement can require the mission's backend rather than assuming any
|
|
// KVM-capable node can boot any image — see microvm::available_backends.
|
|
let backends = microvm::available_backends();
|
|
capabilities_from(kvm, firecracker.as_deref(), &backends)
|
|
}
|
|
|
|
/// Shape the capability report from probe results.
|
|
///
|
|
/// Split from [`probe_capabilities`] so the rule can be tested without a
|
|
/// `/dev/kvm` to open — the machine running the tests is usually the one that
|
|
/// cannot host a microVM.
|
|
fn capabilities_from(kvm: bool, firecracker: Option<&str>, backends: &[String]) -> Value {
|
|
json!({
|
|
"kvm": kvm,
|
|
"firecracker": firecracker,
|
|
// The backends this node can boot. An ARRAY, and empty when there are
|
|
// none: `set_capabilities` REPLACES, so an image that was deleted stops
|
|
// being advertised on the next report instead of leaving a stale claim.
|
|
//
|
|
// Reported even when `microvm` is false, because it is a fact about the
|
|
// disk rather than a promise — placement requires both.
|
|
"rootfs": backends,
|
|
// BOTH must hold. A node with KVM but no firecracker binary looks
|
|
// capable by the obvious test and fails at launch; a node with the
|
|
// binary but no KVM is gw-04. Computed here rather than in the
|
|
// scheduler so the rule sits next to the probe that feeds it.
|
|
"microvm": kvm && firecracker.is_some(),
|
|
})
|
|
}
|
|
|
|
fn probe_tools() -> Value {
|
|
let home = std::env::var("HOME").unwrap_or_default();
|
|
let dirs = [
|
|
format!("{home}/.local/bin"),
|
|
"/opt/homebrew/bin".to_string(),
|
|
"/usr/local/bin".to_string(),
|
|
"/usr/bin".to_string(),
|
|
"/bin".to_string(),
|
|
format!("{home}/.local/share/uv/tools/kimi-cli/bin"),
|
|
format!("{home}/.cargo/bin"),
|
|
];
|
|
let mut out = serde_json::Map::new();
|
|
// (report key, binary name) — usually identical; Rust reports "rust" but its
|
|
// binary is `rustc`.
|
|
for (key, bin) in [
|
|
("docker", "docker"),
|
|
("claude", "claude"),
|
|
("kimi-cli", "kimi-cli"),
|
|
("ollama", "ollama"),
|
|
("rust", "rustc"),
|
|
] {
|
|
for d in &dirs {
|
|
let p = std::path::Path::new(d).join(bin);
|
|
if p.exists() {
|
|
if let Some(v) = tool_version(&p) {
|
|
out.insert(key.to_string(), Value::String(v));
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
Value::Object(out)
|
|
}
|
|
|
|
/// Run `<bin> --version` with a 2s cap (off-thread, so a hung binary can't wedge
|
|
/// the prober) and extract the first semver from its output.
|
|
fn tool_version(bin: &std::path::Path) -> Option<String> {
|
|
let bin = bin.to_path_buf();
|
|
let (tx, rx) = std::sync::mpsc::channel();
|
|
std::thread::spawn(move || {
|
|
let text = std::process::Command::new(&bin)
|
|
.arg("--version")
|
|
.output()
|
|
.ok()
|
|
.map(|o| {
|
|
let mut s = String::from_utf8_lossy(&o.stdout).into_owned();
|
|
s.push_str(&String::from_utf8_lossy(&o.stderr));
|
|
s
|
|
});
|
|
let _ = tx.send(text);
|
|
});
|
|
// 8s, not 2s: right after an update the binary is freshly replaced, and macOS
|
|
// Gatekeeper re-verifies an unsigned binary on first exec (several seconds) — a
|
|
// tight cap would miss the new version on the post-update re-probe.
|
|
let text = rx.recv_timeout(Duration::from_secs(8)).ok().flatten()?;
|
|
extract_semver(&text)
|
|
}
|
|
|
|
/// First `\d+\.\d+(\.\d+)?` run in `s` (e.g. "29.1.3" from "Docker version
|
|
/// 29.1.3, build …"; stops before a trailing `-0ubuntu…`).
|
|
/// Run the FIXED update command for a tool (no arbitrary shell — the key maps to a
|
|
/// hardcoded command). Returns (success, combined output). Async so it never blocks
|
|
/// the read loop; ~170s cap.
|
|
async fn update_tool(tool: &str) -> (bool, String) {
|
|
let home = std::env::var("HOME").unwrap_or_default();
|
|
let dirs = [
|
|
format!("{home}/.local/bin"),
|
|
"/opt/homebrew/bin".to_string(),
|
|
"/usr/local/bin".to_string(),
|
|
"/usr/bin".to_string(),
|
|
"/bin".to_string(),
|
|
format!("{home}/.cargo/bin"),
|
|
];
|
|
let find = |name: &str| {
|
|
dirs.iter()
|
|
.map(|d| std::path::Path::new(d).join(name))
|
|
.find(|p| p.exists())
|
|
};
|
|
let mut cmd = match tool {
|
|
"claude" | "glm" => match find("claude") {
|
|
Some(p) => {
|
|
let mut c = tokio::process::Command::new(p);
|
|
c.arg("update");
|
|
c
|
|
}
|
|
None => return (false, "claude not found".into()),
|
|
},
|
|
"kimi" => match find("uv") {
|
|
Some(p) => {
|
|
let mut c = tokio::process::Command::new(p);
|
|
c.args(["tool", "upgrade", "kimi-cli"]);
|
|
c
|
|
}
|
|
None => return (false, "uv not found".into()),
|
|
},
|
|
"ollama" => {
|
|
if cfg!(target_os = "macos") {
|
|
match find("brew") {
|
|
Some(p) => {
|
|
let mut c = tokio::process::Command::new(p);
|
|
c.args(["upgrade", "ollama"]);
|
|
c
|
|
}
|
|
None => return (false, "brew not found".into()),
|
|
}
|
|
} else {
|
|
let mut c = tokio::process::Command::new("sh");
|
|
c.args(["-c", "curl -fsSL https://ollama.com/install.sh | sh"]);
|
|
c
|
|
}
|
|
}
|
|
"rust" => match find("rustup") {
|
|
Some(p) => {
|
|
let mut c = tokio::process::Command::new(p);
|
|
c.args(["update", "stable"]);
|
|
c
|
|
}
|
|
None => return (false, "rustup not found".into()),
|
|
},
|
|
_ => return (false, "unsupported tool".into()),
|
|
};
|
|
match tokio::time::timeout(Duration::from_secs(170), cmd.output()).await {
|
|
Ok(Ok(o)) => {
|
|
let mut s = String::from_utf8_lossy(&o.stdout).into_owned();
|
|
s.push_str(&String::from_utf8_lossy(&o.stderr));
|
|
if s.len() > 4096 {
|
|
s.truncate(4096);
|
|
}
|
|
(o.status.success(), s)
|
|
}
|
|
Ok(Err(e)) => (false, format!("spawn error: {e}")),
|
|
Err(_) => (false, "update timed out (170s)".into()),
|
|
}
|
|
}
|
|
|
|
fn extract_semver(s: &str) -> Option<String> {
|
|
let c: Vec<char> = s.chars().collect();
|
|
let mut i = 0;
|
|
while i < c.len() {
|
|
if c[i].is_ascii_digit() {
|
|
let start = i;
|
|
while i < c.len() && (c[i].is_ascii_digit() || c[i] == '.') {
|
|
i += 1;
|
|
}
|
|
let mut end = i;
|
|
while end > start && c[end - 1] == '.' {
|
|
end -= 1;
|
|
}
|
|
let cand: String = c[start..end].iter().collect();
|
|
let parts: Vec<&str> = cand.split('.').collect();
|
|
if parts.len() >= 2 && parts.iter().all(|p| !p.is_empty()) {
|
|
return Some(cand);
|
|
}
|
|
} else {
|
|
i += 1;
|
|
}
|
|
}
|
|
None
|
|
}
|
|
|
|
/// Total + available bytes of the filesystem backing `/` (largest disk as a
|
|
/// fallback).
|
|
fn root_disk() -> (i64, i64) {
|
|
let disks = Disks::new_with_refreshed_list();
|
|
let mut best: Option<(i64, i64)> = None;
|
|
for d in &disks {
|
|
let total = d.total_space() as i64;
|
|
let free = d.available_space() as i64;
|
|
if d.mount_point().as_os_str() == "/" {
|
|
return (total, free);
|
|
}
|
|
if best.map(|(t, _)| total > t).unwrap_or(true) {
|
|
best = Some((total, free));
|
|
}
|
|
}
|
|
best.unwrap_or((0, 0))
|
|
}
|
|
|
|
/// Number of running Docker containers (0 if Docker is absent).
|
|
fn docker_count() -> i32 {
|
|
std::process::Command::new("docker")
|
|
.args(["ps", "-q"])
|
|
.output()
|
|
.ok()
|
|
.filter(|o| o.status.success())
|
|
.map(|o| String::from_utf8_lossy(&o.stdout).lines().count() as i32)
|
|
.unwrap_or(0)
|
|
}
|
|
|
|
/// The node's primary outbound IPv4 (the interface a default route would use).
|
|
/// Uses a connectionless UDP socket — no packet is actually sent.
|
|
fn local_ip() -> Option<String> {
|
|
let sock = std::net::UdpSocket::bind("0.0.0.0:0").ok()?;
|
|
sock.connect("8.8.8.8:80").ok()?;
|
|
Some(sock.local_addr().ok()?.ip().to_string())
|
|
}
|
|
|
|
/// This node's Tailscale IP, if Tailscale is up (BYO tailnet).
|
|
fn tailscale_ip() -> Option<String> {
|
|
let out = std::process::Command::new("tailscale")
|
|
.args(["ip", "-4"])
|
|
.output()
|
|
.ok()?;
|
|
if !out.status.success() {
|
|
return None;
|
|
}
|
|
let ip = String::from_utf8_lossy(&out.stdout).trim().to_owned();
|
|
(!ip.is_empty()).then_some(ip)
|
|
}
|
|
|
|
/// Handle a typed frame from the gateway. The gateway never sends arbitrary
|
|
/// shell — only vetted ops (verify, and an interactive host terminal the user
|
|
/// explicitly opened), so the host attack surface stays minimal.
|
|
async fn handle_frame(
|
|
text: &str,
|
|
out: &mpsc::UnboundedSender<String>,
|
|
ptys: &Ptys,
|
|
peers: &rtc::RtcPeers,
|
|
vms: µvm::Vms,
|
|
) {
|
|
let Ok(v) = serde_json::from_str::<Value>(text) else {
|
|
return;
|
|
};
|
|
match v.get("t").and_then(Value::as_str).unwrap_or_default() {
|
|
"verify" => {
|
|
if let Some(id) = v.get("id").and_then(Value::as_u64) {
|
|
let (ok, output) = verify().await;
|
|
let _ = out.send(
|
|
json!({ "t": "result", "id": id, "ok": ok, "output": output }).to_string(),
|
|
);
|
|
}
|
|
}
|
|
"sb_check" => {
|
|
if let Some(id) = v.get("id").and_then(Value::as_u64) {
|
|
let (ok, output) = sandbox_check().await;
|
|
let _ = out.send(
|
|
json!({ "t": "result", "id": id, "ok": ok, "output": output }).to_string(),
|
|
);
|
|
}
|
|
}
|
|
// One-click dev-tool update. Runs a FIXED per-tool command (no arbitrary
|
|
// shell), then re-probes so the new version reports. Spawned so the ~170s
|
|
// command never stalls the read loop (heartbeats keep flowing).
|
|
"tool_update" => {
|
|
if let Some(id) = v.get("id").and_then(Value::as_u64) {
|
|
let tool = v
|
|
.get("tool")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or_default()
|
|
.to_owned();
|
|
let out = out.clone();
|
|
tokio::spawn(async move {
|
|
let (ok, output) = update_tool(&tool).await;
|
|
let _ = out.send(
|
|
json!({ "t": "result", "id": id, "ok": ok, "output": output }).to_string(),
|
|
);
|
|
let tools = tokio::task::spawn_blocking(probe_tools)
|
|
.await
|
|
.unwrap_or_else(|_| json!({}));
|
|
let _ = out.send(json!({ "t": "node_tools", "tools": tools }).to_string());
|
|
});
|
|
}
|
|
}
|
|
// Herdr dispatch ops. Server sends `herdr_dispatch` to open a
|
|
// sibling pane on the node's Herdr session and start the requested
|
|
// CLI (claude / codex / kimi / etc.) with a prompt. `herdr_status`
|
|
// polls that pane's agent_status; `herdr_read` scrapes its recent
|
|
// transcript. Node just shells out to the `herdr` binary — the
|
|
// Herdr background daemon is expected to already be running.
|
|
op @ ("herdr_dispatch" | "herdr_status" | "herdr_read" | "herdr_workspaces"
|
|
| "herdr_snapshot") => {
|
|
if let Some(id) = v.get("id").and_then(Value::as_u64) {
|
|
let (ok, output) = herdr_op(op, &v).await;
|
|
let _ = out.send(
|
|
json!({ "t": "result", "id": id, "ok": ok, "output": output }).to_string(),
|
|
);
|
|
}
|
|
}
|
|
// Agent-sandbox container ops: drive the REAL DockerDriver so the
|
|
// hardening (cap-drop ALL, seccomp, no-net, read-only, non-root) is
|
|
// byte-identical to the gateway's local sandboxes.
|
|
// microVM ops. Same envelope as every other op, so adding them needed
|
|
// no protocol change. `vm_create` blocks until the guest agent answers:
|
|
// a VM that booted but serves nothing is worse than one that failed.
|
|
op @ ("vm_create" | "vm_inject" | "vm_exec" | "vm_collect" | "vm_destroy" | "vm_list") => {
|
|
if let Some(id) = v.get("id").and_then(Value::as_u64) {
|
|
let (op, v, out, vms) = (op.to_string(), v.clone(), out.clone(), vms.clone());
|
|
// Spawned: a VM boot takes ~1s and an exec can take an hour.
|
|
// Running it inline would stall heartbeats and the daemon would
|
|
// be declared offline mid-mission.
|
|
tokio::spawn(async move {
|
|
// While an `exec` runs, follow the turn's log and push each
|
|
// chunk to the server as it appears. The guest agent accepts
|
|
// concurrent connections (proved against a live VM: a tail
|
|
// returned data second-by-second while an 8s exec was still
|
|
// running), so this does not wait for, or delay, the turn.
|
|
//
|
|
// Only for `vm_exec`, and only when the caller named a run to
|
|
// attribute the output to — a probe exec has nothing to
|
|
// stream and no subscriber.
|
|
// Set when the turn returns, so the tail can DRAIN before it
|
|
// stops rather than being cut off mid-flush.
|
|
let turn_done = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
|
let tail = (op == "vm_exec")
|
|
.then(|| {
|
|
let run_id = v.get("run_id").and_then(Value::as_str)?.to_string();
|
|
let log_path = v
|
|
.get("log_path")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("/root/agent.log")
|
|
.to_string();
|
|
let vm_id = v.get("vm_id").and_then(Value::as_str)?.to_string();
|
|
Some(tokio::spawn(stream_vm_log(
|
|
vms.clone(),
|
|
vm_id,
|
|
run_id,
|
|
log_path,
|
|
out.clone(),
|
|
turn_done.clone(),
|
|
)))
|
|
})
|
|
.flatten();
|
|
let (ok, output) = microvm::handle_op(&op, &v, &vms).await;
|
|
// Let the tail DRAIN, then stop. Aborting here was wrong:
|
|
// `claude -p | tee` makes stdout a pipe, so the CLI block-
|
|
// buffers and flushes at EXIT — the most valuable output
|
|
// arrives in the instant the turn ends. Aborting raced that
|
|
// flush and lost it. Measured: a solo turn (minutes long) won
|
|
// the race and streamed 337 bytes; every node of a composed
|
|
// run (~20s each) lost it and streamed nothing at all.
|
|
//
|
|
// Bounded, because a VM that stopped answering must not hold
|
|
// this task open — the abort remains, as a backstop rather
|
|
// than the mechanism.
|
|
if let Some(t) = tail {
|
|
turn_done.store(true, std::sync::atomic::Ordering::Relaxed);
|
|
let drained =
|
|
tokio::time::timeout(std::time::Duration::from_secs(20), t).await;
|
|
if drained.is_err() {
|
|
eprintln!("clawmates-node: tail drain timed out for {op}");
|
|
}
|
|
}
|
|
let _ = out.send(
|
|
json!({ "t": "result", "id": id, "ok": ok, "output": output }).to_string(),
|
|
);
|
|
});
|
|
}
|
|
}
|
|
op @ ("sb_provision" | "sb_exec" | "sb_destroy" | "sb_health" | "sb_list") => {
|
|
if let Some(id) = v.get("id").and_then(Value::as_u64) {
|
|
let (ok, output) = sb_op(op, &v).await;
|
|
let _ = out.send(
|
|
json!({ "t": "result", "id": id, "ok": ok, "output": output }).to_string(),
|
|
);
|
|
}
|
|
}
|
|
"pty_open" => {
|
|
let sid = v.get("sid").and_then(Value::as_u64).unwrap_or(0);
|
|
let cols = v.get("cols").and_then(Value::as_u64).unwrap_or(80) as u16;
|
|
let rows = v.get("rows").and_then(Value::as_u64).unwrap_or(24) as u16;
|
|
eprintln!("[pty] received pty_open sid={sid}");
|
|
if let Err(e) = open_pty(
|
|
sid,
|
|
cols,
|
|
rows,
|
|
PtyTarget::from_frame(&v),
|
|
out.clone(),
|
|
ptys.clone(),
|
|
)
|
|
.await
|
|
{
|
|
eprintln!("[pty] open_pty FAILED sid={sid}: {e}");
|
|
let _ = out.send(json!({ "t": "pty_out", "sid": sid, "data": B64.encode(format!("\r\n\x1b[31m[clawmates] could not start shell: {e}\x1b[0m\r\n").as_bytes()) }).to_string());
|
|
let _ = out.send(json!({ "t": "pty_exit", "sid": sid, "error": e }).to_string());
|
|
}
|
|
}
|
|
"pty_in" => {
|
|
let sid = v.get("sid").and_then(Value::as_u64).unwrap_or(0);
|
|
if let Some(bytes) = v
|
|
.get("data")
|
|
.and_then(Value::as_str)
|
|
.and_then(|d| B64.decode(d).ok())
|
|
{
|
|
if let Some(p) = ptys.lock().await.get_mut(&sid) {
|
|
let _ = p.writer.write_all(&bytes);
|
|
let _ = p.writer.flush();
|
|
}
|
|
}
|
|
}
|
|
"pty_resize" => {
|
|
let sid = v.get("sid").and_then(Value::as_u64).unwrap_or(0);
|
|
let cols = v.get("cols").and_then(Value::as_u64).unwrap_or(80) as u16;
|
|
let rows = v.get("rows").and_then(Value::as_u64).unwrap_or(24) as u16;
|
|
if let Some(p) = ptys.lock().await.get(&sid) {
|
|
let _ = p.master.resize(PtySize {
|
|
rows,
|
|
cols,
|
|
pixel_width: 0,
|
|
pixel_height: 0,
|
|
});
|
|
}
|
|
}
|
|
"pty_close" => {
|
|
let sid = v.get("sid").and_then(Value::as_u64).unwrap_or(0);
|
|
if let Some(mut p) = ptys.lock().await.remove(&sid) {
|
|
let _ = p.child.kill();
|
|
}
|
|
}
|
|
// WebRTC signaling (browser is offerer). The DataChannel, once open,
|
|
// carries terminal I/O directly peer-to-peer; resize/close still arrive
|
|
// as pty_resize/pty_close keyed by the same sid.
|
|
"webrtc_offer" => {
|
|
let sid = v.get("sid").and_then(Value::as_u64).unwrap_or(0);
|
|
let sdp = v
|
|
.get("sdp")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("")
|
|
.to_owned();
|
|
rtc::handle_offer(
|
|
sid,
|
|
sdp,
|
|
PtyTarget::from_frame(&v),
|
|
out.clone(),
|
|
ptys.clone(),
|
|
peers.clone(),
|
|
)
|
|
.await;
|
|
}
|
|
"webrtc_ice" => {
|
|
let sid = v.get("sid").and_then(Value::as_u64).unwrap_or(0);
|
|
rtc::handle_ice(sid, &v, peers).await;
|
|
}
|
|
"webrtc_close" => {
|
|
let sid = v.get("sid").and_then(Value::as_u64).unwrap_or(0);
|
|
rtc::handle_close(sid, peers, ptys).await;
|
|
}
|
|
_ => {}
|
|
}
|
|
}
|
|
|
|
/// (master for resize, cloned reader, writer, child) — one PTY spawn reused by
|
|
/// both the WS-relay path and the WebRTC DataChannel path.
|
|
type PtyParts = (
|
|
Box<dyn MasterPty + Send>,
|
|
Box<dyn Read + Send>,
|
|
Box<dyn Write + Send>,
|
|
Box<dyn portable_pty::Child + Send + Sync>,
|
|
);
|
|
|
|
/// Spawn `cmd` in a fresh PTY, returning handles for resize/read/write/child.
|
|
fn spawn_pty(cmd: CommandBuilder, cols: u16, rows: u16) -> Result<PtyParts, String> {
|
|
let pair = native_pty_system()
|
|
.openpty(PtySize {
|
|
rows,
|
|
cols,
|
|
pixel_width: 0,
|
|
pixel_height: 0,
|
|
})
|
|
.map_err(|e| e.to_string())?;
|
|
let child = pair.slave.spawn_command(cmd).map_err(|e| e.to_string())?;
|
|
drop(pair.slave);
|
|
let reader = pair.master.try_clone_reader().map_err(|e| e.to_string())?;
|
|
let writer = pair.master.take_writer().map_err(|e| e.to_string())?;
|
|
Ok((pair.master, reader, writer, child))
|
|
}
|
|
|
|
/// Spawn a host login shell (tmux `-L clawmates`) in a fresh PTY.
|
|
fn spawn_terminal_pty(cols: u16, rows: u16) -> Result<PtyParts, String> {
|
|
spawn_pty(terminal_command(), cols, rows)
|
|
}
|
|
|
|
/// `docker exec -it` a tmux session inside an agent's container — the node-placed
|
|
/// agent terminal, which shares the container's node-local `~/drives`.
|
|
fn spawn_container_pty(
|
|
container: &str,
|
|
session: &str,
|
|
cols: u16,
|
|
rows: u16,
|
|
) -> Result<PtyParts, String> {
|
|
let mut c = CommandBuilder::new("docker");
|
|
for a in [
|
|
"exec",
|
|
"-it",
|
|
container,
|
|
"tmux",
|
|
"new-session",
|
|
"-A",
|
|
"-s",
|
|
session,
|
|
] {
|
|
c.arg(a);
|
|
}
|
|
spawn_pty(c, cols, rows)
|
|
}
|
|
|
|
/// Where a session's PTY runs: the node's host shell, or `docker exec` into a
|
|
/// specific agent container on this node.
|
|
pub(crate) enum PtyTarget {
|
|
Host,
|
|
Container {
|
|
container: String,
|
|
session: String,
|
|
},
|
|
/// Custom argv (Herdr Live Pane uses this to spawn `herdr` directly
|
|
/// so the browser xterm attaches straight into the node's Herdr TUI
|
|
/// instead of a login shell).
|
|
Command {
|
|
argv: Vec<String>,
|
|
},
|
|
}
|
|
|
|
impl PtyTarget {
|
|
/// Parse from a control frame. Precedence: explicit `command` (non-
|
|
/// empty array) → Command; else `container` → Container; else Host.
|
|
pub(crate) fn from_frame(v: &Value) -> Self {
|
|
if let Some(argv) = v.get("command").and_then(Value::as_array) {
|
|
let parts: Vec<String> = argv
|
|
.iter()
|
|
.filter_map(|x| x.as_str().map(str::to_owned))
|
|
.collect();
|
|
if !parts.is_empty() {
|
|
return PtyTarget::Command { argv: parts };
|
|
}
|
|
}
|
|
match v.get("container").and_then(Value::as_str) {
|
|
Some(c) if !c.is_empty() => PtyTarget::Container {
|
|
container: c.to_owned(),
|
|
session: v
|
|
.get("session")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("main")
|
|
.to_owned(),
|
|
},
|
|
_ => PtyTarget::Host,
|
|
}
|
|
}
|
|
|
|
pub(crate) fn spawn(&self, cols: u16, rows: u16) -> Result<PtyParts, String> {
|
|
match self {
|
|
PtyTarget::Host => spawn_terminal_pty(cols, rows),
|
|
PtyTarget::Container { container, session } => {
|
|
spawn_container_pty(container, session, cols, rows)
|
|
}
|
|
PtyTarget::Command { argv } => spawn_command_pty(argv, cols, rows),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Spawn an arbitrary command in a PTY. Argv[0] must be the program;
|
|
/// if it's a bare name (no slash), it's resolved via the process PATH.
|
|
/// Missing binary returns a clean error the client sees as a banner.
|
|
fn spawn_command_pty(argv: &[String], cols: u16, rows: u16) -> Result<PtyParts, String> {
|
|
let program = argv
|
|
.first()
|
|
.ok_or_else(|| "command argv is empty".to_string())?;
|
|
// Resolve bare names against common bin dirs so a headless daemon
|
|
// (no login shell / no PATH set for herdr install dir) still finds it.
|
|
let resolved = if program.contains('/') {
|
|
program.clone()
|
|
} else {
|
|
let home = std::env::var("HOME").unwrap_or_default();
|
|
let candidates = [
|
|
format!("{home}/.local/bin/{program}"),
|
|
format!("/opt/homebrew/bin/{program}"),
|
|
format!("/usr/local/bin/{program}"),
|
|
format!("/usr/bin/{program}"),
|
|
];
|
|
candidates
|
|
.into_iter()
|
|
.find(|p| std::path::Path::new(p).exists())
|
|
.unwrap_or_else(|| program.clone())
|
|
};
|
|
let mut c = CommandBuilder::new(&resolved);
|
|
for a in argv.iter().skip(1) {
|
|
c.arg(a);
|
|
}
|
|
spawn_pty(c, cols, rows)
|
|
}
|
|
|
|
/// Follow a running turn's log inside a VM and push each chunk to the server.
|
|
///
|
|
/// The other half of the observability path: the guest tails the file, this
|
|
/// forwards what it reads over the WebSocket the daemon already holds, and the
|
|
/// server appends it to the run so the live pane and the Output tab both have it.
|
|
///
|
|
/// Reconnects on a dropped tail, resuming from the last offset — following by
|
|
/// OFFSET rather than holding one socket open forever is what makes that cheap.
|
|
/// It gives up after a few consecutive failures rather than spinning: by then
|
|
/// the VM is gone and the turn's own result is the record.
|
|
async fn stream_vm_log(
|
|
vms: microvm::Vms,
|
|
vm_id: String,
|
|
run_id: String,
|
|
log_path: String,
|
|
out: tokio::sync::mpsc::UnboundedSender<String>,
|
|
turn_done: std::sync::Arc<std::sync::atomic::AtomicBool>,
|
|
) {
|
|
// Said out loud at the start, because the failure this replaced was
|
|
// invisible: the tail gave up during VM boot and logged nothing, so an empty
|
|
// Live tab looked identical to a feature that was never wired.
|
|
eprintln!("clawmates-node: following {log_path} in {vm_id} for run {run_id}");
|
|
let mut at: u64 = 0;
|
|
let mut failures = 0;
|
|
while failures < 3 {
|
|
let at_before = at;
|
|
let sent = out.clone();
|
|
let rid = run_id.clone();
|
|
match microvm::tail_into(&vms, &vm_id, &log_path, at, move |offset, data| {
|
|
let _ = sent.send(
|
|
json!({ "t": "vm_out", "run_id": rid, "at": offset, "data": data }).to_string(),
|
|
);
|
|
})
|
|
.await
|
|
{
|
|
Ok(reached) => {
|
|
// NO PROGRESS IS NOT THE END. The guest reports EOF whenever the
|
|
// file has been idle, and the first idle window is always the one
|
|
// before the turn writes anything — the VM is still booting and
|
|
// the CLI still starting. Returning here meant the tail gave up
|
|
// seconds into every run, before a single byte existed. Measured:
|
|
// a turn that streamed nothing at all.
|
|
//
|
|
// The caller aborts this task when the exec returns, so "keep
|
|
// waiting" cannot outlive the turn; the abort is the terminator,
|
|
// not a guess about idleness.
|
|
at = reached;
|
|
failures = 0;
|
|
// The turn has returned AND this pass read nothing new: the
|
|
// final flush is already in hand, so stop. Checked after a read,
|
|
// never before one — exiting on the flag alone would drop
|
|
// exactly the bytes this exists to capture.
|
|
if turn_done.load(std::sync::atomic::Ordering::Relaxed) && reached == at_before {
|
|
return;
|
|
}
|
|
tokio::time::sleep(std::time::Duration::from_millis(300)).await;
|
|
}
|
|
Err(e) => {
|
|
failures += 1;
|
|
eprintln!("clawmates-node: tail of {vm_id} for run {run_id} failed: {e}");
|
|
tokio::time::sleep(std::time::Duration::from_secs(2)).await;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Spawn a host login shell in a PTY; stream its output back as pty_out frames.
|
|
async fn open_pty(
|
|
sid: u64,
|
|
cols: u16,
|
|
rows: u16,
|
|
target: PtyTarget,
|
|
out: mpsc::UnboundedSender<String>,
|
|
ptys: Ptys,
|
|
) -> Result<(), String> {
|
|
let (master, mut reader, writer, child) = target.spawn(cols, rows)?;
|
|
ptys.lock().await.insert(
|
|
sid,
|
|
Pty {
|
|
master,
|
|
writer,
|
|
child,
|
|
},
|
|
);
|
|
let label = match &target {
|
|
PtyTarget::Host => format!(
|
|
"host shell ({})",
|
|
if has_tmux() { "tmux" } else { "login shell" }
|
|
),
|
|
PtyTarget::Container { container, .. } => format!("container {container}"),
|
|
PtyTarget::Command { argv } => format!("command {}", argv.join(" ")),
|
|
};
|
|
eprintln!("[pty] open sid={sid} cols={cols} rows={rows} target={label}");
|
|
// Immediate banner over the channel: if the browser shows this but no shell,
|
|
// the relay works and the shell is the problem (vs. a dead relay → nothing).
|
|
let host = System::host_name().unwrap_or_else(|| "node".into());
|
|
let banner = format!("\r\n\x1b[2m[clawmates] {label} on {host} — starting…\x1b[0m\r\n");
|
|
let _ = out.send(
|
|
json!({ "t": "pty_out", "sid": sid, "data": B64.encode(banner.as_bytes()) }).to_string(),
|
|
);
|
|
// Blocking PTY reads on a thread → base64 pty_out frames into the out channel.
|
|
std::thread::spawn(move || {
|
|
let mut buf = [0u8; 8192];
|
|
let mut total = 0usize;
|
|
loop {
|
|
match reader.read(&mut buf) {
|
|
Ok(0) => {
|
|
eprintln!("[pty] sid={sid} EOF after {total} bytes (shell exited)");
|
|
break;
|
|
}
|
|
Err(e) => {
|
|
eprintln!("[pty] sid={sid} read error after {total} bytes: {e}");
|
|
break;
|
|
}
|
|
Ok(n) => {
|
|
if total == 0 {
|
|
eprintln!("[pty] sid={sid} first read: {n} bytes");
|
|
}
|
|
total += n;
|
|
let data = B64.encode(&buf[..n]);
|
|
if out
|
|
.send(json!({ "t": "pty_out", "sid": sid, "data": data }).to_string())
|
|
.is_err()
|
|
{
|
|
eprintln!("[pty] sid={sid} out channel closed");
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
eprintln!("[pty] sid={sid} reader done, {total} bytes total");
|
|
let _ = out.send(json!({ "t": "pty_exit", "sid": sid }).to_string());
|
|
});
|
|
Ok(())
|
|
}
|
|
|
|
/// The daemon's built-in host check (fixed command — nothing caller-supplied
|
|
/// executes): kernel info + Docker presence.
|
|
async fn verify() -> (bool, String) {
|
|
let out = tokio::process::Command::new("sh")
|
|
.arg("-lc")
|
|
.arg("uname -a; echo '---'; docker version --format 'docker {{.Server.Version}}' 2>/dev/null || echo 'docker: not found'")
|
|
.output()
|
|
.await;
|
|
match out {
|
|
Ok(o) => {
|
|
let mut s = String::from_utf8_lossy(&o.stdout).into_owned();
|
|
if !o.stderr.is_empty() {
|
|
s.push_str(&String::from_utf8_lossy(&o.stderr));
|
|
}
|
|
(true, s.trim().to_owned())
|
|
}
|
|
Err(e) => (false, format!("verify error: {e}")),
|
|
}
|
|
}
|
|
|
|
/// Readiness check: provision a fully locked-down throwaway container (the same
|
|
/// hardening agent sandboxes use — cap-drop ALL, no-new-privileges, no network,
|
|
/// read-only rootfs, non-root, resource caps), run it, and tear it down. Proves
|
|
/// the node can host hardened agent workloads. Fixed command — nothing
|
|
/// caller-supplied runs.
|
|
async fn sandbox_check() -> (bool, String) {
|
|
let _ = tokio::process::Command::new("docker")
|
|
.args(["pull", "-q", "alpine:latest"])
|
|
.output()
|
|
.await;
|
|
let out = tokio::process::Command::new("docker")
|
|
.args([
|
|
"run",
|
|
"--rm",
|
|
"--cap-drop=ALL",
|
|
"--security-opt",
|
|
"no-new-privileges",
|
|
"--network",
|
|
"none",
|
|
"--read-only",
|
|
"--tmpfs",
|
|
"/tmp",
|
|
"--user",
|
|
"10001:10001",
|
|
"--memory",
|
|
"256m",
|
|
"--pids-limit",
|
|
"128",
|
|
"alpine:latest",
|
|
"sh",
|
|
"-c",
|
|
"echo sandbox-ok; id; uname -sm",
|
|
])
|
|
.output()
|
|
.await;
|
|
match out {
|
|
Ok(o) if o.status.success() => (true, String::from_utf8_lossy(&o.stdout).trim().to_owned()),
|
|
Ok(o) => {
|
|
let mut s = String::from_utf8_lossy(&o.stdout).into_owned();
|
|
s.push_str(&String::from_utf8_lossy(&o.stderr));
|
|
(false, s.trim().to_owned())
|
|
}
|
|
Err(e) => (false, format!("docker error: {e} — is Docker installed?")),
|
|
}
|
|
}
|
|
|
|
fn handle_of(v: &Value) -> SandboxHandle {
|
|
SandboxHandle {
|
|
id: v
|
|
.get("cid")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or_default()
|
|
.to_owned(),
|
|
name: v
|
|
.get("cname")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or_default()
|
|
.to_owned(),
|
|
}
|
|
}
|
|
|
|
/// Run an agent-sandbox container op via the local DockerDriver (full hardening),
|
|
/// returning the result payload as JSON or an error message.
|
|
/// Herdr control ops. Requires `herdr` in PATH and a running background
|
|
/// session (Phase 0 install). Returns raw JSON strings from the herdr
|
|
/// CLI so the server can parse pane_id / agent_status without a
|
|
/// second RPC hop.
|
|
///
|
|
/// Ops:
|
|
/// herdr_dispatch { mission_id, cli, prompt, direction? } →
|
|
/// runs `herdr pane split ... && herdr pane run ... "prompt"`
|
|
/// output = the split's JSON response so the server can extract
|
|
/// result.pane.pane_id
|
|
/// herdr_status { pane_id } → `herdr pane get <pane_id>` JSON
|
|
/// herdr_read { pane_id, lines? } → recent-unwrapped scrollback
|
|
async fn herdr_op(op: &str, v: &Value) -> (bool, String) {
|
|
let herdr = match std::env::var("HOME").ok().and_then(|h| {
|
|
[
|
|
format!("{h}/.local/bin/herdr"),
|
|
"/opt/homebrew/bin/herdr".to_string(),
|
|
"/usr/local/bin/herdr".to_string(),
|
|
]
|
|
.into_iter()
|
|
.find(|p| std::path::Path::new(p).exists())
|
|
}) {
|
|
Some(p) => p,
|
|
None => return (false, "herdr binary not found on PATH".into()),
|
|
};
|
|
let herdr = std::sync::Arc::new(herdr);
|
|
let run = move |args: Vec<String>| {
|
|
let herdr = herdr.clone();
|
|
async move {
|
|
let fut = tokio::process::Command::new(herdr.as_str())
|
|
.args(&args)
|
|
.output();
|
|
match tokio::time::timeout(Duration::from_secs(30), fut).await {
|
|
Ok(Ok(o)) => {
|
|
let mut s = String::from_utf8_lossy(&o.stdout).into_owned();
|
|
if !o.status.success() {
|
|
s.push_str(&String::from_utf8_lossy(&o.stderr));
|
|
}
|
|
(o.status.success(), s)
|
|
}
|
|
Ok(Err(e)) => (false, format!("spawn error: {e}")),
|
|
Err(_) => (false, "herdr op timed out".into()),
|
|
}
|
|
}
|
|
};
|
|
match op {
|
|
"herdr_dispatch" => {
|
|
let mission_id = v
|
|
.get("mission_id")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("unknown");
|
|
let cli = v.get("cli").and_then(Value::as_str).unwrap_or("claude");
|
|
let prompt = v.get("prompt").and_then(Value::as_str).unwrap_or("");
|
|
let direction = v
|
|
.get("direction")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or("right");
|
|
// 1. Ensure a mission workspace exists (idempotent — label collision falls through).
|
|
let _ = run(vec![
|
|
"workspace".into(),
|
|
"create".into(),
|
|
"--label".into(),
|
|
format!("mission-{mission_id}"),
|
|
])
|
|
.await;
|
|
// 2. Split off a fresh pane in that workspace and read its pane_id.
|
|
let (ok, split_out) = run(vec![
|
|
"pane".into(),
|
|
"split".into(),
|
|
"--direction".into(),
|
|
direction.into(),
|
|
"--no-focus".into(),
|
|
])
|
|
.await;
|
|
if !ok {
|
|
return (false, format!("split failed: {split_out}"));
|
|
}
|
|
let pane_id = match serde_json::from_str::<Value>(&split_out) {
|
|
Ok(j) => j
|
|
.pointer("/result/pane/pane_id")
|
|
.and_then(Value::as_str)
|
|
.unwrap_or_default()
|
|
.to_string(),
|
|
Err(_) => String::new(),
|
|
};
|
|
if pane_id.is_empty() {
|
|
return (false, format!("no pane_id in split response: {split_out}"));
|
|
}
|
|
// 3. Rename for operator readability.
|
|
let _ = run(vec![
|
|
"pane".into(),
|
|
"rename".into(),
|
|
pane_id.clone(),
|
|
format!("mission-{mission_id}"),
|
|
])
|
|
.await;
|
|
// 4. Launch the CLI with the prompt inline.
|
|
let launch = if prompt.is_empty() {
|
|
cli.to_string()
|
|
} else {
|
|
// Single-quoted so shell metacharacters in the prompt don't
|
|
// reinterpret. Herdr's pane.run sends this verbatim to the shell.
|
|
let escaped = prompt.replace('\'', "'\\''");
|
|
format!("{cli} '{escaped}'")
|
|
};
|
|
let (rok, rout) = run(vec!["pane".into(), "run".into(), pane_id.clone(), launch]).await;
|
|
let payload = serde_json::json!({
|
|
"pane_id": pane_id,
|
|
"split": split_out,
|
|
"run_output": rout,
|
|
"run_ok": rok,
|
|
});
|
|
(rok, payload.to_string())
|
|
}
|
|
"herdr_status" => {
|
|
let pane = v.get("pane_id").and_then(Value::as_str).unwrap_or_default();
|
|
if pane.is_empty() {
|
|
return (false, "pane_id required".into());
|
|
}
|
|
run(vec!["pane".into(), "get".into(), pane.to_string()]).await
|
|
}
|
|
"herdr_read" => {
|
|
let pane = v.get("pane_id").and_then(Value::as_str).unwrap_or_default();
|
|
let lines = v.get("lines").and_then(Value::as_u64).unwrap_or(200);
|
|
if pane.is_empty() {
|
|
return (false, "pane_id required".into());
|
|
}
|
|
run(vec![
|
|
"pane".into(),
|
|
"read".into(),
|
|
pane.to_string(),
|
|
"--source".into(),
|
|
"recent-unwrapped".into(),
|
|
"--lines".into(),
|
|
lines.to_string(),
|
|
])
|
|
.await
|
|
}
|
|
"herdr_workspaces" => run(vec!["workspace".into(), "list".into()]).await,
|
|
"herdr_snapshot" => run(vec!["api".into(), "snapshot".into()]).await,
|
|
_ => (false, format!("unknown herdr op {op}")),
|
|
}
|
|
}
|
|
|
|
async fn sb_op(op: &str, v: &Value) -> (bool, String) {
|
|
let driver = match DockerDriver::connect() {
|
|
Ok(d) => d,
|
|
Err(e) => return (false, format!("docker unavailable: {e}")),
|
|
};
|
|
match op {
|
|
"sb_provision" => {
|
|
let spec: SandboxSpec = match v
|
|
.get("spec")
|
|
.and_then(|s| serde_json::from_value(s.clone()).ok())
|
|
{
|
|
Some(s) => s,
|
|
None => return (false, "invalid spec".to_owned()),
|
|
};
|
|
// Pull the agent image if the node doesn't have it yet (best effort).
|
|
let _ = tokio::process::Command::new("docker")
|
|
.args(["pull", "-q", &spec.image])
|
|
.output()
|
|
.await;
|
|
match driver.provision(&spec).await {
|
|
Ok(h) => (true, json!({ "id": h.id, "name": h.name }).to_string()),
|
|
Err(e) => (false, format!("provision: {e}")),
|
|
}
|
|
}
|
|
"sb_exec" => {
|
|
let handle = handle_of(v);
|
|
let cmd: Vec<String> = v
|
|
.get("cmd")
|
|
.and_then(Value::as_array)
|
|
.map(|a| {
|
|
a.iter()
|
|
.filter_map(|x| x.as_str().map(str::to_owned))
|
|
.collect()
|
|
})
|
|
.unwrap_or_default();
|
|
let refs: Vec<&str> = cmd.iter().map(String::as_str).collect();
|
|
match driver.exec(&handle, &refs).await {
|
|
Ok(r) => (
|
|
r.exit_code == 0,
|
|
json!({ "exit": r.exit_code, "stdout": r.stdout, "stderr": r.stderr })
|
|
.to_string(),
|
|
),
|
|
Err(e) => (false, format!("exec: {e}")),
|
|
}
|
|
}
|
|
"sb_destroy" => match driver.destroy(&handle_of(v)).await {
|
|
Ok(()) => (true, "ok".to_owned()),
|
|
Err(e) => (false, format!("destroy: {e}")),
|
|
},
|
|
"sb_health" => match driver.health(&handle_of(v)).await {
|
|
Ok(alive) => (true, json!({ "alive": alive }).to_string()),
|
|
Err(e) => (false, format!("health: {e}")),
|
|
},
|
|
"sb_list" => {
|
|
let kind = v.get("kind").and_then(Value::as_str).unwrap_or("agent");
|
|
match driver.list_managed(kind).await {
|
|
Ok(list) => (
|
|
true,
|
|
json!(list
|
|
.iter()
|
|
.map(|m| json!({ "id": m.id, "created_unix": m.created_unix }))
|
|
.collect::<Vec<_>>())
|
|
.to_string(),
|
|
),
|
|
Err(e) => (false, format!("list: {e}")),
|
|
}
|
|
}
|
|
_ => (false, "unknown op".to_owned()),
|
|
}
|
|
}
|
|
|
|
/// Best-effort: join the user's tailnet (BYO Tailscale) and enable Tailscale SSH
|
|
/// so they can reach this node keylessly. ONLY when an auth key is explicitly
|
|
/// passed — enabling SSH unprompted can drop the user's current SSH session.
|
|
/// `--accept-risk` avoids the interactive abort when they're on Tailscale.
|
|
/// Non-fatal: the WSS control channel works regardless of Tailscale.
|
|
fn tailscale_up(authkey: &str) {
|
|
if authkey.is_empty() {
|
|
return;
|
|
}
|
|
let _ = std::process::Command::new("tailscale")
|
|
.args([
|
|
"up",
|
|
"--authkey",
|
|
authkey,
|
|
"--ssh",
|
|
"--accept-routes",
|
|
"--accept-risk=lose-ssh",
|
|
])
|
|
.status();
|
|
}
|
|
|
|
/// The command a host terminal runs: a resumable host tmux session (like the
|
|
/// agent terminal — `-A` attaches-or-creates "clawmates" and tmux redraws on
|
|
/// attach so the view is never blank), falling back to a login shell.
|
|
fn terminal_command() -> CommandBuilder {
|
|
let mut cmd = if has_tmux() {
|
|
let mut c = CommandBuilder::new("tmux");
|
|
// A DEDICATED server socket (-L) so we never collide with — or get refused
|
|
// by — the operator's own tmux. Without this, running the daemon inside a
|
|
// tmux makes the spawned tmux refuse to nest and exit instantly (writing
|
|
// its warning to stderr, not the PTY → the browser sees nothing).
|
|
c.arg("-L");
|
|
c.arg("clawmates");
|
|
c.arg("new-session");
|
|
c.arg("-A");
|
|
c.arg("-s");
|
|
c.arg("main");
|
|
c
|
|
} else {
|
|
let shell = std::env::var("SHELL").unwrap_or_else(|_| "/bin/bash".to_owned());
|
|
let mut c = CommandBuilder::new(shell);
|
|
c.arg("-l");
|
|
c
|
|
};
|
|
cmd.env("TERM", "xterm-256color");
|
|
// Clear any inherited $TMUX so the new tmux doesn't think it's nested.
|
|
cmd.env_remove("TMUX");
|
|
if let Ok(home) = std::env::var("HOME") {
|
|
cmd.cwd(home);
|
|
}
|
|
cmd
|
|
}
|
|
|
|
/// `--selftest`: open the host terminal PTY locally and print ~2.5s of its raw
|
|
/// output (diagnostic — confirms tmux/shell actually draws).
|
|
fn selftest() {
|
|
let pair = match native_pty_system().openpty(PtySize {
|
|
rows: 40,
|
|
cols: 120,
|
|
pixel_width: 0,
|
|
pixel_height: 0,
|
|
}) {
|
|
Ok(p) => p,
|
|
Err(e) => {
|
|
eprintln!("openpty failed: {e}");
|
|
return;
|
|
}
|
|
};
|
|
let mut child = match pair.slave.spawn_command(terminal_command()) {
|
|
Ok(c) => c,
|
|
Err(e) => {
|
|
eprintln!("spawn failed: {e}");
|
|
return;
|
|
}
|
|
};
|
|
drop(pair.slave);
|
|
let mut reader = pair.master.try_clone_reader().expect("reader");
|
|
let (tx, rx) = std::sync::mpsc::channel::<Vec<u8>>();
|
|
std::thread::spawn(move || {
|
|
let mut buf = [0u8; 8192];
|
|
while let Ok(n) = reader.read(&mut buf) {
|
|
if n == 0 || tx.send(buf[..n].to_vec()).is_err() {
|
|
break;
|
|
}
|
|
}
|
|
});
|
|
let mut total = 0usize;
|
|
let start = std::time::Instant::now();
|
|
while start.elapsed() < std::time::Duration::from_millis(2500) {
|
|
if let Ok(bytes) = rx.recv_timeout(std::time::Duration::from_millis(300)) {
|
|
total += bytes.len();
|
|
print!("{}", String::from_utf8_lossy(&bytes));
|
|
}
|
|
}
|
|
let _ = child.kill();
|
|
eprintln!(
|
|
"\n--- selftest: tmux={}, {} bytes of output ---",
|
|
has_tmux(),
|
|
total
|
|
);
|
|
}
|
|
|
|
/// Is tmux on PATH?
|
|
fn has_tmux() -> bool {
|
|
std::process::Command::new("tmux")
|
|
.arg("-V")
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false)
|
|
}
|
|
|
|
/// Best-effort: install tmux via the host package manager (works when the daemon
|
|
/// runs as root, e.g. systemd). Non-interactive; if it can't, host terminals fall
|
|
/// back to a plain login shell.
|
|
fn ensure_tmux() {
|
|
if has_tmux() {
|
|
return;
|
|
}
|
|
let mgrs: &[(&str, &[&str])] = &[
|
|
("apt-get", &["install", "-y", "tmux"]),
|
|
("dnf", &["install", "-y", "tmux"]),
|
|
("yum", &["install", "-y", "tmux"]),
|
|
("apk", &["add", "--no-cache", "tmux"]),
|
|
("pacman", &["-S", "--noconfirm", "tmux"]),
|
|
("brew", &["install", "tmux"]),
|
|
];
|
|
for (mgr, args) in mgrs {
|
|
let present = std::process::Command::new("sh")
|
|
.arg("-c")
|
|
.arg(format!("command -v {mgr}"))
|
|
.output()
|
|
.map(|o| o.status.success())
|
|
.unwrap_or(false);
|
|
if !present {
|
|
continue;
|
|
}
|
|
if *mgr == "apt-get" {
|
|
let _ = std::process::Command::new("apt-get")
|
|
.arg("update")
|
|
.env("DEBIAN_FRONTEND", "noninteractive")
|
|
.output();
|
|
}
|
|
let _ = std::process::Command::new(mgr)
|
|
.args(*args)
|
|
.env("DEBIAN_FRONTEND", "noninteractive")
|
|
.output();
|
|
break;
|
|
}
|
|
if has_tmux() {
|
|
println!("tmux ready for host terminals");
|
|
} else {
|
|
eprintln!("tmux not found (auto-install unavailable) — host terminal will use a plain shell; `apt install tmux` for resumable sessions");
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod capability_tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn microvm_needs_both_kvm_and_firecracker() {
|
|
assert_eq!(
|
|
capabilities_from(true, Some("Firecracker v1.16.1"), &[])["microvm"],
|
|
json!(true)
|
|
);
|
|
assert_eq!(
|
|
capabilities_from(true, None, &[])["microvm"],
|
|
json!(false),
|
|
"KVM without firecracker cannot host a microVM"
|
|
);
|
|
assert_eq!(
|
|
capabilities_from(false, Some("Firecracker v1.16.1"), &[])["microvm"],
|
|
json!(false),
|
|
"firecracker without KVM is gw-04 — it can never host one"
|
|
);
|
|
assert_eq!(capabilities_from(false, None, &[])["microvm"], json!(false));
|
|
}
|
|
|
|
/// The report replaces rather than merges server-side, so a node that has
|
|
/// LOST a capability must say so rather than omitting the key — an absent
|
|
/// key and a false one must not be distinguishable to the predicate.
|
|
#[test]
|
|
fn a_lost_capability_is_reported_false_not_omitted() {
|
|
let caps = capabilities_from(false, None, &[]);
|
|
assert!(caps.get("kvm").is_some(), "kvm must always be present");
|
|
assert!(
|
|
caps.get("microvm").is_some(),
|
|
"microvm must always be present"
|
|
);
|
|
// Same reasoning for the image list: a node that deleted its last rootfs
|
|
// must report an empty ARRAY, not omit the key. Placement asks "does this
|
|
// node have backend X"; against a missing key that question has no
|
|
// answer, and a scheduler with no answer picks something.
|
|
assert_eq!(
|
|
caps.get("rootfs"),
|
|
Some(&json!([])),
|
|
"rootfs must always be present, empty when there are no images"
|
|
);
|
|
}
|
|
|
|
/// The list is what placement matches a mission's `backend` against, so it
|
|
/// must carry the names verbatim.
|
|
#[test]
|
|
fn reported_backends_are_the_names_placement_will_ask_for() {
|
|
let caps = capabilities_from(
|
|
true,
|
|
Some("Firecracker v1.16.1"),
|
|
&["claude".to_string(), "default".to_string()],
|
|
);
|
|
assert_eq!(caps["rootfs"], json!(["claude", "default"]));
|
|
}
|
|
}
|