wedged-run fix: pre-approve claude perms + verbose step log + reaper
Root cause: claude CLI in the per-topic research container runs as
uid=0(root). ZeroClaw's claude_cli provider passes
--dangerously-skip-permissions which Claude CLI rejects under root
for security — so the CLI hangs waiting for interactive permission
approval that never arrives, hitting the 600s provider timeout with
zero step records journaled.
Three-part fix:
1. Claude settings bind-mount (research_container.rs):
Optional CLAWMATES_CLAUDE_SETTINGS_PATH env — when set, mount the
host file at /root/.claude/settings.json (read-only) in every
spawned team container. deploy/claude-settings.json ships the
canonical config (permissions.defaultMode = bypassPermissions +
hasCompletedOnboarding). CLI accepts requests immediately with no
--dangerously-skip-permissions flag needed.
2. Verbose per-step log line (topology_worker.rs):
Every checkpoint now writes to stderr:
topology_worker::step run_id=X step=N node=Y role=Z phase=W
output_bytes=B tokens=T gated=G
Visible in docker logs clawmates_server_1 — gives us live
'topology is flowing' signal without opening the canvas, and
makes it obvious when a topology_kind is skipping stages it
shouldn't.
3. Stuck-container reaper (topology_worker.rs):
New 60s-tick loop reap_stuck_runs: for any research topology_run
older than 15 min with zero checkpoint.records, docker-stop its
container and mark the run failed with a diagnostic error. Only
reaps research-bound runs (non-research runs don't own a
container). The existing 180s stale-checkpoint requeuer stays
in place for other failure modes.
Deploy: gw-04 needs
ln -sf /path/to/repo/deploy/claude-settings.json /opt/clawmates/claude-settings.json
CLAWMATES_CLAUDE_SETTINGS_PATH=/opt/clawmates/claude-settings.json
in the server env, plus the timeout lowered from 600 -> 120 in
compose. Both handled in the deploy step outside this commit.
This commit is contained in:
@@ -25,9 +25,28 @@ use crate::topology_exec::ZeroClawDriveExecutor;
|
||||
/// Requeue a `running` job whose worker hasn't checkpointed within this window.
|
||||
const STALE_AFTER_SECS: f64 = 180.0;
|
||||
|
||||
/// Maximum age a `running` run may spend WITHOUT journaling any step
|
||||
/// records before the reaper kills its container and fails it. 15 min
|
||||
/// is generous: healthy first-step latency is typically 5–60s; anything
|
||||
/// past this is a stuck container (usually a wedged provider CLI).
|
||||
const REAP_STUCK_AFTER_SECS: i64 = 15 * 60;
|
||||
|
||||
/// Spawn the durable topology job worker. Polls for queued jobs every `poll`
|
||||
/// interval; runs each to completion (or failure), checkpointing per step.
|
||||
pub fn spawn(pool: PgPool, runtime: cm_runtime::Runtime, poll: Duration) {
|
||||
// Fire the stuck-container reaper on its own cadence — checking
|
||||
// once a minute is plenty and keeps this off the hot claim loop.
|
||||
let reaper_pool = pool.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut ticker = tokio::time::interval(Duration::from_secs(60));
|
||||
loop {
|
||||
ticker.tick().await;
|
||||
if let Err(e) = reap_stuck_runs(&reaper_pool).await {
|
||||
eprintln!("topology_worker::reaper: reap failed: {e}");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
// Recover jobs orphaned by a dead worker before claiming new ones.
|
||||
@@ -47,6 +66,65 @@ pub fn spawn(pool: PgPool, runtime: cm_runtime::Runtime, poll: Duration) {
|
||||
});
|
||||
}
|
||||
|
||||
/// Kill research team containers whose `running` topology_run has been
|
||||
/// alive past [`REAP_STUCK_AFTER_SECS`] without journaling a single
|
||||
/// step record. Marks the run `failed` with a diagnostic error so the
|
||||
/// user sees WHY instead of an infinitely-spinning pipeline.
|
||||
///
|
||||
/// Only reaps runs bound to a research topic — non-research runs (raw
|
||||
/// API-driven topology runs) don't own a container so there's nothing
|
||||
/// to kill; they're left to the existing stale-checkpoint requeuer.
|
||||
async fn reap_stuck_runs(pool: &PgPool) -> Result<(), sqlx::Error> {
|
||||
use sqlx::Row;
|
||||
let rows: Vec<sqlx::postgres::PgRow> = sqlx::query(
|
||||
"SELECT id, research_topic_id
|
||||
FROM topology_runs
|
||||
WHERE status = 'running'
|
||||
AND research_topic_id IS NOT NULL
|
||||
AND created_at < now() - make_interval(secs => $1::float)
|
||||
AND coalesce(jsonb_array_length(coalesce(checkpoint->'records', '[]'::jsonb)), 0) = 0",
|
||||
)
|
||||
.bind(REAP_STUCK_AFTER_SECS as f64)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
if rows.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Best-effort docker cleanup; even if the container is already gone
|
||||
// (crashed, manually killed), we still want to mark the run failed.
|
||||
let docker = crate::research_container::connect().ok();
|
||||
|
||||
for row in rows {
|
||||
let id: Uuid = row.get("id");
|
||||
let topic_id: Uuid = row.get("research_topic_id");
|
||||
let container = crate::research_container::container_name_for(topic_id);
|
||||
eprintln!(
|
||||
"topology_worker::reaper: reaping stuck run run_id={} topic_id={} container={} (no step records after {}s)",
|
||||
id, topic_id, container, REAP_STUCK_AFTER_SECS,
|
||||
);
|
||||
if let Some(d) = &docker {
|
||||
let _ = d
|
||||
.stop_container(
|
||||
&container,
|
||||
None::<bollard::query_parameters::StopContainerOptions>,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let _ = cm_db::repo::topology_runs::fail(
|
||||
pool,
|
||||
id,
|
||||
&format!(
|
||||
"reaped: no step records after {}s (container {} stopped)",
|
||||
REAP_STUCK_AFTER_SECS, container
|
||||
),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Drive one claimed job to a terminal state, persisting checkpoints as it goes.
|
||||
async fn run_job(
|
||||
pool: &PgPool,
|
||||
@@ -473,6 +551,31 @@ async fn drive<E: TurnExecutor>(
|
||||
execute_resumable(graph, task, executor, progress, move |snap| {
|
||||
let pool = pool_cb.clone();
|
||||
async move {
|
||||
// 2026-07-15: verbose per-step trace so `docker logs
|
||||
// clawmates_server_1` shows which topology node just fired,
|
||||
// its role, output size, tokens, and gated-action count.
|
||||
// Cheap (one info! per step) and gives us the missing
|
||||
// "topology is flowing" signal without needing to open the
|
||||
// canvas.
|
||||
if let Some(last) = snap.records.last() {
|
||||
let phase = match &last.phase {
|
||||
cm_orchestrator::StepPhase::Plan => "plan",
|
||||
cm_orchestrator::StepPhase::Work => "work",
|
||||
cm_orchestrator::StepPhase::Synth => "synth",
|
||||
cm_orchestrator::StepPhase::Aggregate => "aggregate",
|
||||
};
|
||||
eprintln!(
|
||||
"topology_worker::step run_id={} step={} node={} role={} phase={} output_bytes={} tokens={} gated={}",
|
||||
id,
|
||||
snap.completed,
|
||||
last.node_id,
|
||||
last.role,
|
||||
phase,
|
||||
last.output.len(),
|
||||
last.tokens,
|
||||
last.gated.len(),
|
||||
);
|
||||
}
|
||||
// Best-effort checkpoint: a failed write just means we re-run the
|
||||
// step on resume (idempotent — topology turns are pure reads here).
|
||||
if let Ok(v) = serde_json::to_value(&snap) {
|
||||
|
||||
Reference in New Issue
Block a user