feat(telemetry): record per-agent usage for mission turns
deploy / test (push) Successful in 4m31s
deploy / build (push) Successful in 5m23s

The command centre's SPEND, ACTIVITY and THROUGHPUT cards read `usage_events`,
and nothing on the mission path ever wrote a row: `cm_billing::charge` was
called only from the agent-run path. Measured mid-mission with 14 agents live,
`usage_events` was 0 while a crew had just burned 15k tokens — so an agent that
had done real work reported zero cost and zero activity.

The worker already knew everything needed: it logs node, role and token count
per step, and the node's `attrs.agent` carries the `claw_<uuid>` binding the
runtime dispatches on. This routes that to the ledger.

`charge`'s run_id is now Option. `usage_events.run_id` references `agent_runs`,
and a topology turn has no row there — passing its `topology_runs` id was a
foreign-key violation, which is exactly what the first attempt hit. NULL is the
honest value; the agent-run caller still passes its real id.

The executor reports one total rather than an in/out split, so the cost is right
(credits price the sum) and the columns record it as output rather than
inventing a split.

Verified end to end on a real mission: 4 agents, 1046-8670 tokens each, credits
attributed per agent, and the SPEND/ACTIVITY queries now return real numbers.

Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
Omar Sobh
2026-08-15 05:09:56 -07:00
co-authored by Claude Opus 5
parent 8ef7067467
commit 8be7b3c9b2
4 changed files with 99 additions and 21 deletions
+81 -7
View File
@@ -201,12 +201,14 @@ async fn run_job(
// to no mission — a bare topology run has no phase to hang tool calls on. // to no mission — a bare topology run has no phase to hang tool calls on.
let tap = mission_binding let tap = mission_binding
.as_ref() .as_ref()
.map(|(_, _, mission_id, phase_id)| crate::topology_exec::MissionTap { .map(
|(_, _, mission_id, phase_id)| crate::topology_exec::MissionTap {
pool: pool.clone(), pool: pool.clone(),
mission_id: *mission_id, mission_id: *mission_id,
phase_id: *phase_id, phase_id: *phase_id,
run_id: Some(id), run_id: Some(id),
}); },
);
let leaf_result = match mission_binding { let leaf_result = match mission_binding {
Some((Some(url), Some(code), _, _)) => { Some((Some(url), Some(code), _, _)) => {
ZeroClawDriveExecutor::from_env_for_gateway_with_code(url, code) ZeroClawDriveExecutor::from_env_for_gateway_with_code(url, code)
@@ -246,9 +248,29 @@ async fn run_job(
id, id,
Arc::new(leaf), Arc::new(leaf),
); );
drive(pool, id, &graph, &job.task, progress, &exec).await drive(
pool,
id,
job.workspace_id,
&graph,
&job.task,
progress,
&exec,
)
.await
}
_ => {
drive(
pool,
id,
job.workspace_id,
&graph,
&job.task,
progress,
&leaf,
)
.await
} }
_ => drive(pool, id, &graph, &job.task, progress, &leaf).await,
}; };
finish(pool, id, result).await; finish(pool, id, result).await;
@@ -303,8 +325,7 @@ async fn run_composed(
OrchestratorError::Executor("a composed run must belong to a mission phase".into()) OrchestratorError::Executor("a composed run must belong to a mission phase".into())
})?; })?;
let mission: (Option<Uuid>, Option<String>, Option<String>, bool) = let mission: (Option<Uuid>, Option<String>, Option<String>, bool) = sqlx::query_as(
sqlx::query_as(
"SELECT target_node_id, backend, team_engine, (repo_id IS NOT NULL) \ "SELECT target_node_id, backend, team_engine, (repo_id IS NOT NULL) \
FROM missions WHERE id = $1", FROM missions WHERE id = $1",
) )
@@ -347,7 +368,16 @@ async fn run_composed(
completed_steps: progress.completed as u32, completed_steps: progress.completed as u32,
}, },
); );
drive(pool, job.id, graph, &job.task, progress, &exec).await drive(
pool,
job.id,
job.workspace_id,
graph,
&job.task,
progress,
&exec,
)
.await
} }
/// Post-terminal hook: if this run's team is `ephemeral` and no siblings are /// Post-terminal hook: if this run's team is `ephemeral` and no siblings are
@@ -406,14 +436,30 @@ async fn maybe_teardown_ephemeral_team(pool: &PgPool, runtime: &cm_runtime::Runt
async fn drive<E: TurnExecutor>( async fn drive<E: TurnExecutor>(
pool: &PgPool, pool: &PgPool,
id: Uuid, id: Uuid,
workspace_id: Uuid,
graph: &TopologyGraph, graph: &TopologyGraph,
task: &str, task: &str,
progress: RunProgress, progress: RunProgress,
executor: &E, executor: &E,
) -> Result<RunRecord, OrchestratorError> { ) -> Result<RunRecord, OrchestratorError> {
let pool_cb = pool.clone(); let pool_cb = pool.clone();
// node_id -> agent id, resolved once. The binding lives in the node's
// attrs (`agent = claw_<uuid>`), which is also what the runtime dispatches
// on — so usage is attributed to exactly the claw that did the work.
let agent_of: std::sync::Arc<std::collections::HashMap<String, Uuid>> = std::sync::Arc::new(
graph
.nodes
.iter()
.filter_map(|n| {
let alias = n.attrs.get("agent")?;
let uuid = alias.strip_prefix("claw_")?;
Some((n.id.clone(), Uuid::parse_str(uuid).ok()?))
})
.collect(),
);
execute_resumable(graph, task, executor, progress, move |snap| { execute_resumable(graph, task, executor, progress, move |snap| {
let pool = pool_cb.clone(); let pool = pool_cb.clone();
let agent_of = agent_of.clone();
async move { async move {
// 2026-07-15: verbose per-step trace so `docker logs // 2026-07-15: verbose per-step trace so `docker logs
// clawmates_server_1` shows which topology node just fired, // clawmates_server_1` shows which topology node just fired,
@@ -439,6 +485,34 @@ async fn drive<E: TurnExecutor>(
last.tokens, last.tokens,
last.gated.len(), last.gated.len(),
); );
// Per-agent usage. Without this the command centre's SPEND,
// ACTIVITY and THROUGHPUT cards read `usage_events`, which
// nothing on the mission path ever wrote — so they showed 0 for
// an agent that had just burned 15k tokens.
//
// `charge` also decrements credit lots, which is the point: a
// mission turn costs what it costs. It clamps at the available
// balance and still records the full obligation, so an empty
// wallet cannot fail a turn.
if let Some(agent_id) = agent_of.get(&last.node_id).copied() {
if last.tokens > 0 {
// The executor reports ONE total, not an in/out split.
// Credits price the sum, so cost is right; the columns
// record it as output rather than inventing a split.
if let Err(e) = cm_billing::charge(
&pool,
cm_domain::WorkspaceId::from(workspace_id),
cm_domain::AgentId::from(agent_id),
None,
0,
last.tokens as u64,
)
.await
{
eprintln!("topology_worker: usage for {agent_id} failed: {e}");
}
}
}
} }
// Best-effort checkpoint: a failed write just means we re-run the // Best-effort checkpoint: a failed write just means we re-run the
// step on resume (idempotent — topology turns are pure reads here). // step on resume (idempotent — topology turns are pure reads here).
+5 -1
View File
@@ -29,7 +29,11 @@ pub async fn charge(
pool: &PgPool, pool: &PgPool,
workspace_id: WorkspaceId, workspace_id: WorkspaceId,
agent_id: AgentId, agent_id: AgentId,
run_id: Uuid, // Optional: `usage_events.run_id` references `agent_runs`, and a topology
// turn has no row there — its id lives in `topology_runs`. Passing that id
// was a foreign-key violation, so mission usage went unrecorded. NULL is
// the honest value for a charge that is not an agent_run.
run_id: Option<Uuid>,
input_tokens: u64, input_tokens: u64,
output_tokens: u64, output_tokens: u64,
) -> Result<i64, BillingError> { ) -> Result<i64, BillingError> {
+2 -2
View File
@@ -62,7 +62,7 @@ async fn charges_span_lots_oldest_first_and_record_usage() {
.unwrap(); .unwrap();
// 2500 tokens → 3 credits: drains the first lot (2) then one more. // 2500 tokens → 3 credits: drains the first lot (2) then one more.
let deducted = charge(&pool, ws.id, agent.id, run_id, 1500, 1000) let deducted = charge(&pool, ws.id, agent.id, Some(run_id), 1500, 1000)
.await .await
.unwrap(); .unwrap();
assert_eq!(deducted, 3); assert_eq!(deducted, 3);
@@ -96,7 +96,7 @@ async fn an_empty_workspace_records_usage_but_clamps_at_zero() {
.unwrap(); .unwrap();
// Owes 5, only 1 available: deducts 1, balance hits zero, never negative. // Owes 5, only 1 available: deducts 1, balance hits zero, never negative.
let deducted = charge(&pool, ws.id, agent.id, run_id, 4000, 500) let deducted = charge(&pool, ws.id, agent.id, Some(run_id), 4000, 500)
.await .await
.unwrap(); .unwrap();
assert_eq!(deducted, 1); assert_eq!(deducted, 1);
+1 -1
View File
@@ -879,7 +879,7 @@ impl Runtime {
&self.inner.pool, &self.inner.pool,
state.workspace_id, state.workspace_id,
state.agent_id, state.agent_id,
run_id, Some(run_id),
state.input_tokens, state.input_tokens,
state.output_tokens, state.output_tokens,
) )