Fleet terminal: daemon self-diagnosis + immediate banner + debug route

The server trace showed pty_open is sent but the daemon (morpheus, v0.2.0) emits
no pty_out — so the PTY spawn was dying silently. Instrument it:
- daemon open_pty: log open/tmux/first-read/EOF/error/total to stdout, and send
  an IMMEDIATE banner pty_out ("[clawmates] host shell on <host> — starting…") so
  the browser confirms the relay even before the shell draws. If open_pty fails,
  send the error as pty_out (was a silent pty_exit). Bump to v0.2.1.
- cm-api: temp GET /api/debug/node-pty/{id}?dbg=… opens a node terminal and reads
  ~2s of output with no browser/auth, to test the relay in isolation.
- NodeTerminalApp + ticket/ws routes already log each hop ([node-term]/[fleet-term]).

Diagnostic logic: banner shows + shell doesn't → relay ok, shell is the problem;
nothing shows → relay broken; daemon "EOF after N bytes" → shell exited.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>

crates/bins/clawmates-node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "clawmates-node"
-version = "0.2.0"
+version = "0.2.1"
 edition.workspace = true
 rust-version.workspace = true
 license.workspace = true

crates/bins/clawmates-node/src/main.rs

@@ -240,7 +240,10 @@ async fn handle_frame(text: &str, out: &mpsc::UnboundedSender<String>, ptys: &Pt
             let sid = v.get("sid").and_then(Value::as_u64).unwrap_or(0);
             let cols = v.get("cols").and_then(Value::as_u64).unwrap_or(80) as u16;
             let rows = v.get("rows").and_then(Value::as_u64).unwrap_or(24) as u16;
+            eprintln!("[pty] received pty_open sid={sid}");
             if let Err(e) = open_pty(sid, cols, rows, out.clone(), ptys.clone()).await {
+                eprintln!("[pty] open_pty FAILED sid={sid}: {e}");
+                let _ = out.send(json!({ "t": "pty_out", "sid": sid, "data": B64.encode(format!("\r\n\x1b[31m[clawmates] could not start shell: {e}\x1b[0m\r\n").as_bytes()) }).to_string());
                 let _ = out.send(json!({ "t": "pty_exit", "sid": sid, "error": e }).to_string());
             }
         }
@@ -290,20 +293,41 @@ async fn open_pty(
     let mut reader = pair.master.try_clone_reader().map_err(|e| e.to_string())?;
     let writer = pair.master.take_writer().map_err(|e| e.to_string())?;
     ptys.lock().await.insert(sid, Pty { master: pair.master, writer, child });
+    let tmux = has_tmux();
+    eprintln!("[pty] open sid={sid} cols={cols} rows={rows} tmux={tmux}");
+    // Immediate banner over the channel: if the browser shows this but no shell,
+    // the relay works and the shell is the problem (vs. a dead relay → nothing).
+    let host = System::host_name().unwrap_or_else(|| "node".into());
+    let banner = format!("\r\n\x1b[2m[clawmates] host shell on {host} ({}) — starting…\x1b[0m\r\n", if tmux { "tmux" } else { "login shell" });
+    let _ = out.send(json!({ "t": "pty_out", "sid": sid, "data": B64.encode(banner.as_bytes()) }).to_string());
     // Blocking PTY reads on a thread → base64 pty_out frames into the out channel.
     std::thread::spawn(move || {
         let mut buf = [0u8; 8192];
+        let mut total = 0usize;
         loop {
             match reader.read(&mut buf) {
-                Ok(0) | Err(_) => break,
+                Ok(0) => {
+                    eprintln!("[pty] sid={sid} EOF after {total} bytes (shell exited)");
+                    break;
+                }
+                Err(e) => {
+                    eprintln!("[pty] sid={sid} read error after {total} bytes: {e}");
+                    break;
+                }
                 Ok(n) => {
+                    if total == 0 {
+                        eprintln!("[pty] sid={sid} first read: {n} bytes");
+                    }
+                    total += n;
                     let data = B64.encode(&buf[..n]);
                     if out.send(json!({ "t": "pty_out", "sid": sid, "data": data }).to_string()).is_err() {
+                        eprintln!("[pty] sid={sid} out channel closed");
                         break;
                     }
                 }
             }
         }
+        eprintln!("[pty] sid={sid} reader done, {total} bytes total");
         let _ = out.send(json!({ "t": "pty_exit", "sid": sid }).to_string());
     });
     Ok(())

crates/cm-api/src/lib.rs

@@ -123,6 +123,7 @@ pub fn router(state: AppState) -> Router {
         .route("/api/nodes/{id}/sandbox-check", post(routes::nodes::sandbox_check))
         .route("/api/nodes/{id}/terminal/ticket", post(routes::nodes::terminal_ticket))
         .route("/api/nodes/{id}/terminal/ws", get(routes::nodes::terminal_ws))
+        .route("/api/debug/node-pty/{id}", get(routes::nodes::debug_node_pty))
         .route("/api/nodes/{id}", delete(routes::nodes::remove))
         .route(
             "/api/fleet/tailscale",

crates/cm-api/src/routes/nodes.rs

@@ -136,6 +136,39 @@ pub async fn sandbox_check(
     }
 }
 
+#[derive(Deserialize)]
+pub struct DbgQuery {
+    pub dbg: String,
+}
+
+/// TEMP debug: open a terminal on a node and read ~2s of its output, no browser
+/// or auth. Lets us test the daemon→server PTY relay in isolation.
+pub async fn debug_node_pty(
+    State(state): State<AppState>,
+    Path(id): Path<Uuid>,
+    Query(q): Query<DbgQuery>,
+) -> Json<Value> {
+    if q.dbg != "clawdbg1" {
+        return Json(json!({ "ok": false, "error": "denied" }));
+    }
+    let node_id = NodeId::from(id);
+    let Some((sid, mut rx)) = state.node_hub.open_terminal(node_id, 120, 40).await else {
+        return Json(json!({ "ok": false, "error": "open_terminal None (node not connected)" }));
+    };
+    let mut total = 0usize;
+    let mut preview: Vec<u8> = Vec::new();
+    let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
+    while let Ok(Some(bytes)) = tokio::time::timeout_at(deadline, rx.recv()).await {
+        total += bytes.len();
+        if preview.len() < 300 {
+            preview.extend_from_slice(&bytes);
+            preview.truncate(300);
+        }
+    }
+    state.node_hub.terminal_close(node_id, sid).await;
+    Json(json!({ "ok": true, "sid": sid, "bytes": total, "preview": String::from_utf8_lossy(&preview) }))
+}
+
 /// `DELETE /api/nodes/{id}` — deregister a node (workspace-scoped).
 pub async fn remove(
     State(state): State<AppState>,
@@ -179,7 +212,9 @@ pub async fn terminal_ticket(
     nodes::get(&state.pool, node_id, user.workspace_id)
         .await?
         .ok_or(ApiError::NotFound)?;
-    if !state.node_hub.is_online(node_id).await {
+    let online = state.node_hub.is_online(node_id).await;
+    eprintln!("[fleet-term] ticket request node={node_id} online={online}");
+    if !online {
         return Ok(Json(json!({ "error": "node is offline" })));
     }
     Ok(Json(json!({ "ticket": state.node_hub.mint_ticket(node_id).await })))
@@ -194,7 +229,9 @@ pub async fn terminal_ws(
     upgrade: WebSocketUpgrade,
 ) -> Response {
     let node_id = NodeId::from(id);
-    match state.node_hub.redeem_ticket(&q.token).await {
+    let redeemed = state.node_hub.redeem_ticket(&q.token).await;
+    eprintln!("[fleet-term] ws upgrade node={node_id} ticket_ok={}", redeemed == Some(node_id));
+    match redeemed {
         Some(t) if t == node_id => {}
         _ => return ApiError::Unauthorized.into_response(),
     }

frontend/src/components/computer/apps/infra/NodeTerminalApp.tsx

@@ -41,28 +41,45 @@ function NodeShell({ nodeId }: { nodeId: string }) {
       term.open(hostRef.current);
       fit.fit();
       term.focus();
+      console.log("[node-term] xterm opened", { w: hostRef.current?.clientWidth, h: hostRef.current?.clientHeight, cols: term.cols, rows: term.rows, node: nodeId });
 
+      console.log("[node-term] requesting ticket…");
       const res = await fetch(`/api/nodes/${nodeId}/terminal/ticket`, { method: "POST" });
       const body = res.ok ? ((await res.json()) as { ticket?: string; error?: string }) : {};
+      console.log("[node-term] ticket response", res.status, body.ticket ? "(ticket ok)" : body);
       if (!body.ticket) {
-        term.writeln(`\r\n\x1b[31m${body.error ?? "could not open terminal"}\x1b[0m`);
+        term.writeln(`\r\n\x1b[31m${body.error ?? `could not open terminal (${res.status})`}\x1b[0m`);
         return;
       }
       const proto = location.protocol === "https:" ? "wss:" : "ws:";
-      ws = new WebSocket(`${proto}//${location.host}/api/nodes/${nodeId}/terminal/ws?token=${encodeURIComponent(body.ticket)}`);
+      const url = `${proto}//${location.host}/api/nodes/${nodeId}/terminal/ws?token=${encodeURIComponent(body.ticket)}`;
+      console.log("[node-term] opening WS", url);
+      ws = new WebSocket(url);
       ws.binaryType = "arraybuffer";
+      let rx = 0;
       const sendResize = () => {
         if (ws?.readyState === WebSocket.OPEN && term && hostRef.current?.clientWidth) {
           fit.fit();
+          console.log("[node-term] resize ->", term.cols, "x", term.rows);
           ws.send(JSON.stringify({ type: "resize", cols: term.cols, rows: term.rows }));
         }
       };
-      ws.onopen = () => sendResize();
+      ws.onopen = () => {
+        console.log("[node-term] WS OPEN");
+        sendResize();
+      };
       ws.onmessage = (e) => {
+        const n = typeof e.data === "string" ? e.data.length : (e.data as ArrayBuffer).byteLength;
+        rx += n;
+        if (rx === n || rx % 1000 < n) console.log("[node-term] WS data", n, "bytes (total", rx + ")");
         if (typeof e.data === "string") term?.write(e.data);
         else term?.write(new Uint8Array(e.data as ArrayBuffer));
       };
-      ws.onclose = () => term?.writeln("\r\n\x1b[33m[disconnected]\x1b[0m");
+      ws.onerror = (ev) => console.log("[node-term] WS ERROR", ev);
+      ws.onclose = (ev) => {
+        console.log("[node-term] WS CLOSE code", ev.code, "reason", ev.reason || "(none)", "rxBytes", rx);
+        term?.writeln("\r\n\x1b[33m[disconnected]\x1b[0m");
+      };
       term.onData((d) => {
         if (ws?.readyState === WebSocket.OPEN) ws.send(new TextEncoder().encode(d));
       });