clawmates-node --selftest + server terminal tracing (diagnose blank terminal)
ci / gates (push) Failing after 5s
ci / rust (push) Has been skipped
ci / sandbox-k8s (push) Has been skipped
ci / frontend (push) Has been skipped
ci / e2e (push) Has been skipped

- Daemon: factor the terminal command into terminal_command(); add `--selftest`
  which opens the host terminal PTY locally and prints ~2.5s of raw output, so
  you can confirm tmux/zsh actually draws on a given node without the browser.
  (Verified locally: tmux spawns zsh + draws its status bar.)
- cm-api: temporary [fleet-term] eprintln tracing in open_terminal, the pty_out
  router, and the browser bridge (byte counts + sink presence) to locate where
  output stops between daemon→server→browser. To be removed once diagnosed.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
This commit is contained in:
Omar Sobh
2026-06-24 21:28:26 -07:00
co-authored by Claude Opus 4.8
parent 6e81eaa52b
commit c2a0309ad7
3 changed files with 89 additions and 21 deletions
+77 -21
View File
@@ -36,6 +36,10 @@ async fn main() {
// The dep graph enables both rustls crypto providers (tungstenite + bollard), // The dep graph enables both rustls crypto providers (tungstenite + bollard),
// so rustls can't auto-pick — install one explicitly before any TLS. // so rustls can't auto-pick — install one explicitly before any TLS.
let _ = rustls::crypto::ring::default_provider().install_default(); let _ = rustls::crypto::ring::default_provider().install_default();
if std::env::args().any(|a| a == "--selftest") {
selftest();
return;
}
let (server, token, ts_authkey) = parse_args(); let (server, token, ts_authkey) = parse_args();
if server.is_empty() || token.is_empty() { if server.is_empty() || token.is_empty() {
eprintln!("usage: clawmates-node --server <https://gateway> --token <token> [--tailscale-authkey <key>]"); eprintln!("usage: clawmates-node --server <https://gateway> --token <token> [--tailscale-authkey <key>]");
@@ -278,27 +282,10 @@ async fn open_pty(
let pair = native_pty_system() let pair = native_pty_system()
.openpty(PtySize { rows, cols, pixel_width: 0, pixel_height: 0 }) .openpty(PtySize { rows, cols, pixel_width: 0, pixel_height: 0 })
.map_err(|e| e.to_string())?; .map_err(|e| e.to_string())?;
// Prefer a resumable host tmux session (like the agent terminal): `-A` attaches let child = pair
// to the existing "clawmates" session or creates it, and tmux redraws the whole .slave
// screen on attach (so the view is never blank). Fall back to a login shell. .spawn_command(terminal_command())
let mut cmd = if has_tmux() { .map_err(|e| e.to_string())?;
let mut c = CommandBuilder::new("tmux");
c.arg("new-session");
c.arg("-A");
c.arg("-s");
c.arg("clawmates");
c
} else {
let shell = std::env::var("SHELL").unwrap_or_else(|_| "/bin/bash".to_owned());
let mut c = CommandBuilder::new(shell);
c.arg("-l");
c
};
cmd.env("TERM", "xterm-256color");
if let Ok(home) = std::env::var("HOME") {
cmd.cwd(home);
}
let child = pair.slave.spawn_command(cmd).map_err(|e| e.to_string())?;
drop(pair.slave); drop(pair.slave);
let mut reader = pair.master.try_clone_reader().map_err(|e| e.to_string())?; let mut reader = pair.master.try_clone_reader().map_err(|e| e.to_string())?;
let writer = pair.master.take_writer().map_err(|e| e.to_string())?; let writer = pair.master.take_writer().map_err(|e| e.to_string())?;
@@ -458,6 +445,75 @@ fn tailscale_up(authkey: &str) {
.status(); .status();
} }
/// The command a host terminal runs: a resumable host tmux session (like the
/// agent terminal — `-A` attaches-or-creates "clawmates" and tmux redraws on
/// attach so the view is never blank), falling back to a login shell.
fn terminal_command() -> CommandBuilder {
let mut cmd = if has_tmux() {
let mut c = CommandBuilder::new("tmux");
c.arg("new-session");
c.arg("-A");
c.arg("-s");
c.arg("clawmates");
c
} else {
let shell = std::env::var("SHELL").unwrap_or_else(|_| "/bin/bash".to_owned());
let mut c = CommandBuilder::new(shell);
c.arg("-l");
c
};
cmd.env("TERM", "xterm-256color");
if let Ok(home) = std::env::var("HOME") {
cmd.cwd(home);
}
cmd
}
/// `--selftest`: open the host terminal PTY locally and print ~2.5s of its raw
/// output (diagnostic — confirms tmux/shell actually draws).
fn selftest() {
let pair = match native_pty_system().openpty(PtySize {
rows: 40,
cols: 120,
pixel_width: 0,
pixel_height: 0,
}) {
Ok(p) => p,
Err(e) => {
eprintln!("openpty failed: {e}");
return;
}
};
let mut child = match pair.slave.spawn_command(terminal_command()) {
Ok(c) => c,
Err(e) => {
eprintln!("spawn failed: {e}");
return;
}
};
drop(pair.slave);
let mut reader = pair.master.try_clone_reader().expect("reader");
let (tx, rx) = std::sync::mpsc::channel::<Vec<u8>>();
std::thread::spawn(move || {
let mut buf = [0u8; 8192];
while let Ok(n) = reader.read(&mut buf) {
if n == 0 || tx.send(buf[..n].to_vec()).is_err() {
break;
}
}
});
let mut total = 0usize;
let start = std::time::Instant::now();
while start.elapsed() < std::time::Duration::from_millis(2500) {
if let Ok(bytes) = rx.recv_timeout(std::time::Duration::from_millis(300)) {
total += bytes.len();
print!("{}", String::from_utf8_lossy(&bytes));
}
}
let _ = child.kill();
eprintln!("\n--- selftest: tmux={}, {} bytes of output ---", has_tmux(), total);
}
/// Is tmux on PATH? /// Is tmux on PATH?
fn has_tmux() -> bool { fn has_tmux() -> bool {
std::process::Command::new("tmux") std::process::Command::new("tmux")
+6
View File
@@ -129,6 +129,7 @@ impl NodeHub {
let (tx, rx) = mpsc::unbounded_channel(); let (tx, rx) = mpsc::unbounded_channel();
conn.pty_sinks.lock().await.insert(sid, tx); conn.pty_sinks.lock().await.insert(sid, tx);
let frame = json!({ "t": "pty_open", "sid": sid, "cols": cols, "rows": rows }).to_string(); let frame = json!({ "t": "pty_open", "sid": sid, "cols": cols, "rows": rows }).to_string();
eprintln!("[fleet-term] open_terminal sid={sid} node={id} cols={cols} rows={rows}");
if conn.tx.send(frame).is_err() { if conn.tx.send(frame).is_err() {
conn.pty_sinks.lock().await.remove(&sid); conn.pty_sinks.lock().await.remove(&sid);
return None; return None;
@@ -281,6 +282,11 @@ pub async fn run_channel(pool: PgPool, hub: Arc<NodeHub>, node_id: NodeId, socke
Ok(Uplink::PtyOut { sid, data }) => { Ok(Uplink::PtyOut { sid, data }) => {
if let Ok(bytes) = B64.decode(&data) { if let Ok(bytes) = B64.decode(&data) {
let sink = conn.pty_sinks.lock().await.get(&sid).cloned(); let sink = conn.pty_sinks.lock().await.get(&sid).cloned();
eprintln!(
"[fleet-term] pty_out sid={sid} bytes={} sink={}",
bytes.len(),
sink.is_some()
);
if let Some(s) = sink { if let Some(s) = sink {
let _ = s.send(bytes); let _ = s.send(bytes);
} }
+6
View File
@@ -212,15 +212,21 @@ struct TermCtrl {
async fn bridge_terminal(hub: Arc<NodeHub>, node_id: NodeId, socket: WebSocket) { async fn bridge_terminal(hub: Arc<NodeHub>, node_id: NodeId, socket: WebSocket) {
let Some((sid, mut rx)) = hub.open_terminal(node_id, 80, 24).await else { let Some((sid, mut rx)) = hub.open_terminal(node_id, 80, 24).await else {
eprintln!("[fleet-term] bridge: open_terminal None (node offline?) node={node_id}");
return; return;
}; };
eprintln!("[fleet-term] bridge start node={node_id} sid={sid}");
let (mut ws_tx, mut ws_rx) = socket.split(); let (mut ws_tx, mut ws_rx) = socket.split();
let to_browser = async { let to_browser = async {
let mut sent = 0usize;
while let Some(bytes) = rx.recv().await { while let Some(bytes) = rx.recv().await {
sent += bytes.len();
if ws_tx.send(Message::Binary(bytes.into())).await.is_err() { if ws_tx.send(Message::Binary(bytes.into())).await.is_err() {
eprintln!("[fleet-term] bridge: browser send failed after {sent} bytes");
break; break;
} }
} }
eprintln!("[fleet-term] bridge to_browser ended, {sent} bytes total");
}; };
let to_node = async { let to_node = async {
while let Some(Ok(msg)) = ws_rx.next().await { while let Some(Ok(msg)) = ws_rx.next().await {