wedged-run fix: pre-approve claude perms + verbose step log + reaper
Root cause: claude CLI in the per-topic research container runs as
uid=0(root). ZeroClaw's claude_cli provider passes
--dangerously-skip-permissions which Claude CLI rejects under root
for security — so the CLI hangs waiting for interactive permission
approval that never arrives, hitting the 600s provider timeout with
zero step records journaled.
Three-part fix:
1. Claude settings bind-mount (research_container.rs):
Optional CLAWMATES_CLAUDE_SETTINGS_PATH env — when set, mount the
host file at /root/.claude/settings.json (read-only) in every
spawned team container. deploy/claude-settings.json ships the
canonical config (permissions.defaultMode = bypassPermissions +
hasCompletedOnboarding). CLI accepts requests immediately with no
--dangerously-skip-permissions flag needed.
2. Verbose per-step log line (topology_worker.rs):
Every checkpoint now writes to stderr:
topology_worker::step run_id=X step=N node=Y role=Z phase=W
output_bytes=B tokens=T gated=G
Visible in docker logs clawmates_server_1 — gives us live
'topology is flowing' signal without opening the canvas, and
makes it obvious when a topology_kind is skipping stages it
shouldn't.
3. Stuck-container reaper (topology_worker.rs):
New 60s-tick loop reap_stuck_runs: for any research topology_run
older than 15 min with zero checkpoint.records, docker-stop its
container and mark the run failed with a diagnostic error. Only
reaps research-bound runs (non-research runs don't own a
container). The existing 180s stale-checkpoint requeuer stays
in place for other failure modes.
Deploy: gw-04 needs
ln -sf /path/to/repo/deploy/claude-settings.json /opt/clawmates/claude-settings.json
CLAWMATES_CLAUDE_SETTINGS_PATH=/opt/clawmates/claude-settings.json
in the server env, plus the timeout lowered from 600 -> 120 in
compose. Both handled in the deploy step outside this commit.
This commit is contained in:
@@ -222,7 +222,7 @@ pub async fn spawn(
|
|||||||
// disabling pairing here is safe.
|
// disabling pairing here is safe.
|
||||||
prewrite_daemon_config(state_host_path)?;
|
prewrite_daemon_config(state_host_path)?;
|
||||||
|
|
||||||
let mounts = vec![
|
let mut mounts = vec![
|
||||||
Mount {
|
Mount {
|
||||||
target: Some("/workspace/repo".into()),
|
target: Some("/workspace/repo".into()),
|
||||||
source: Some(repo_host_path.to_string_lossy().into_owned()),
|
source: Some(repo_host_path.to_string_lossy().into_owned()),
|
||||||
@@ -239,6 +239,29 @@ pub async fn spawn(
|
|||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
|
// 2026-07-15 (post-diagnosis): claude CLI in-container runs as
|
||||||
|
// uid=0 and refuses --dangerously-skip-permissions under root for
|
||||||
|
// security. Without pre-approved permissions the CLI hangs waiting
|
||||||
|
// for interactive approval → the whole run stalls until the 600s
|
||||||
|
// provider timeout. Bind-mount a settings.json with
|
||||||
|
// permissions.defaultMode=bypassPermissions so the CLI accepts
|
||||||
|
// requests immediately.
|
||||||
|
//
|
||||||
|
// Path via env so deployments can swap in a different settings
|
||||||
|
// file (e.g. a workspace-specific one) without a code change. When
|
||||||
|
// unset the mount is skipped (dev backwards compat).
|
||||||
|
if let Ok(claude_settings_path) = std::env::var("CLAWMATES_CLAUDE_SETTINGS_PATH") {
|
||||||
|
if !claude_settings_path.is_empty() {
|
||||||
|
mounts.push(Mount {
|
||||||
|
target: Some("/root/.claude/settings.json".into()),
|
||||||
|
source: Some(claude_settings_path),
|
||||||
|
typ: Some(MountTypeEnum::BIND),
|
||||||
|
read_only: Some(true),
|
||||||
|
..Default::default()
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let host_config = HostConfig {
|
let host_config = HostConfig {
|
||||||
mounts: Some(mounts),
|
mounts: Some(mounts),
|
||||||
network_mode: Some(team_network()),
|
network_mode: Some(team_network()),
|
||||||
|
|||||||
@@ -25,9 +25,28 @@ use crate::topology_exec::ZeroClawDriveExecutor;
|
|||||||
/// Requeue a `running` job whose worker hasn't checkpointed within this window.
|
/// Requeue a `running` job whose worker hasn't checkpointed within this window.
|
||||||
const STALE_AFTER_SECS: f64 = 180.0;
|
const STALE_AFTER_SECS: f64 = 180.0;
|
||||||
|
|
||||||
|
/// Maximum age a `running` run may spend WITHOUT journaling any step
|
||||||
|
/// records before the reaper kills its container and fails it. 15 min
|
||||||
|
/// is generous: healthy first-step latency is typically 5–60s; anything
|
||||||
|
/// past this is a stuck container (usually a wedged provider CLI).
|
||||||
|
const REAP_STUCK_AFTER_SECS: i64 = 15 * 60;
|
||||||
|
|
||||||
/// Spawn the durable topology job worker. Polls for queued jobs every `poll`
|
/// Spawn the durable topology job worker. Polls for queued jobs every `poll`
|
||||||
/// interval; runs each to completion (or failure), checkpointing per step.
|
/// interval; runs each to completion (or failure), checkpointing per step.
|
||||||
pub fn spawn(pool: PgPool, runtime: cm_runtime::Runtime, poll: Duration) {
|
pub fn spawn(pool: PgPool, runtime: cm_runtime::Runtime, poll: Duration) {
|
||||||
|
// Fire the stuck-container reaper on its own cadence — checking
|
||||||
|
// once a minute is plenty and keeps this off the hot claim loop.
|
||||||
|
let reaper_pool = pool.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let mut ticker = tokio::time::interval(Duration::from_secs(60));
|
||||||
|
loop {
|
||||||
|
ticker.tick().await;
|
||||||
|
if let Err(e) = reap_stuck_runs(&reaper_pool).await {
|
||||||
|
eprintln!("topology_worker::reaper: reap failed: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
loop {
|
loop {
|
||||||
// Recover jobs orphaned by a dead worker before claiming new ones.
|
// Recover jobs orphaned by a dead worker before claiming new ones.
|
||||||
@@ -47,6 +66,65 @@ pub fn spawn(pool: PgPool, runtime: cm_runtime::Runtime, poll: Duration) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Kill research team containers whose `running` topology_run has been
|
||||||
|
/// alive past [`REAP_STUCK_AFTER_SECS`] without journaling a single
|
||||||
|
/// step record. Marks the run `failed` with a diagnostic error so the
|
||||||
|
/// user sees WHY instead of an infinitely-spinning pipeline.
|
||||||
|
///
|
||||||
|
/// Only reaps runs bound to a research topic — non-research runs (raw
|
||||||
|
/// API-driven topology runs) don't own a container so there's nothing
|
||||||
|
/// to kill; they're left to the existing stale-checkpoint requeuer.
|
||||||
|
async fn reap_stuck_runs(pool: &PgPool) -> Result<(), sqlx::Error> {
|
||||||
|
use sqlx::Row;
|
||||||
|
let rows: Vec<sqlx::postgres::PgRow> = sqlx::query(
|
||||||
|
"SELECT id, research_topic_id
|
||||||
|
FROM topology_runs
|
||||||
|
WHERE status = 'running'
|
||||||
|
AND research_topic_id IS NOT NULL
|
||||||
|
AND created_at < now() - make_interval(secs => $1::float)
|
||||||
|
AND coalesce(jsonb_array_length(coalesce(checkpoint->'records', '[]'::jsonb)), 0) = 0",
|
||||||
|
)
|
||||||
|
.bind(REAP_STUCK_AFTER_SECS as f64)
|
||||||
|
.fetch_all(pool)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
if rows.is_empty() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Best-effort docker cleanup; even if the container is already gone
|
||||||
|
// (crashed, manually killed), we still want to mark the run failed.
|
||||||
|
let docker = crate::research_container::connect().ok();
|
||||||
|
|
||||||
|
for row in rows {
|
||||||
|
let id: Uuid = row.get("id");
|
||||||
|
let topic_id: Uuid = row.get("research_topic_id");
|
||||||
|
let container = crate::research_container::container_name_for(topic_id);
|
||||||
|
eprintln!(
|
||||||
|
"topology_worker::reaper: reaping stuck run run_id={} topic_id={} container={} (no step records after {}s)",
|
||||||
|
id, topic_id, container, REAP_STUCK_AFTER_SECS,
|
||||||
|
);
|
||||||
|
if let Some(d) = &docker {
|
||||||
|
let _ = d
|
||||||
|
.stop_container(
|
||||||
|
&container,
|
||||||
|
None::<bollard::query_parameters::StopContainerOptions>,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
let _ = cm_db::repo::topology_runs::fail(
|
||||||
|
pool,
|
||||||
|
id,
|
||||||
|
&format!(
|
||||||
|
"reaped: no step records after {}s (container {} stopped)",
|
||||||
|
REAP_STUCK_AFTER_SECS, container
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
/// Drive one claimed job to a terminal state, persisting checkpoints as it goes.
|
/// Drive one claimed job to a terminal state, persisting checkpoints as it goes.
|
||||||
async fn run_job(
|
async fn run_job(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
@@ -473,6 +551,31 @@ async fn drive<E: TurnExecutor>(
|
|||||||
execute_resumable(graph, task, executor, progress, move |snap| {
|
execute_resumable(graph, task, executor, progress, move |snap| {
|
||||||
let pool = pool_cb.clone();
|
let pool = pool_cb.clone();
|
||||||
async move {
|
async move {
|
||||||
|
// 2026-07-15: verbose per-step trace so `docker logs
|
||||||
|
// clawmates_server_1` shows which topology node just fired,
|
||||||
|
// its role, output size, tokens, and gated-action count.
|
||||||
|
// Cheap (one info! per step) and gives us the missing
|
||||||
|
// "topology is flowing" signal without needing to open the
|
||||||
|
// canvas.
|
||||||
|
if let Some(last) = snap.records.last() {
|
||||||
|
let phase = match &last.phase {
|
||||||
|
cm_orchestrator::StepPhase::Plan => "plan",
|
||||||
|
cm_orchestrator::StepPhase::Work => "work",
|
||||||
|
cm_orchestrator::StepPhase::Synth => "synth",
|
||||||
|
cm_orchestrator::StepPhase::Aggregate => "aggregate",
|
||||||
|
};
|
||||||
|
eprintln!(
|
||||||
|
"topology_worker::step run_id={} step={} node={} role={} phase={} output_bytes={} tokens={} gated={}",
|
||||||
|
id,
|
||||||
|
snap.completed,
|
||||||
|
last.node_id,
|
||||||
|
last.role,
|
||||||
|
phase,
|
||||||
|
last.output.len(),
|
||||||
|
last.tokens,
|
||||||
|
last.gated.len(),
|
||||||
|
);
|
||||||
|
}
|
||||||
// Best-effort checkpoint: a failed write just means we re-run the
|
// Best-effort checkpoint: a failed write just means we re-run the
|
||||||
// step on resume (idempotent — topology turns are pure reads here).
|
// step on resume (idempotent — topology turns are pure reads here).
|
||||||
if let Ok(v) = serde_json::to_value(&snap) {
|
if let Ok(v) = serde_json::to_value(&snap) {
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
{
|
||||||
|
"hasCompletedOnboarding": true,
|
||||||
|
"permissions": {
|
||||||
|
"defaultMode": "bypassPermissions"
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user