feat(telemetry): record per-agent usage for mission turns
The command centre's SPEND, ACTIVITY and THROUGHPUT cards read `usage_events`, and nothing on the mission path ever wrote a row: `cm_billing::charge` was called only from the agent-run path. Measured mid-mission with 14 agents live, `usage_events` was 0 while a crew had just burned 15k tokens — so an agent that had done real work reported zero cost and zero activity. The worker already knew everything needed: it logs node, role and token count per step, and the node's `attrs.agent` carries the `claw_<uuid>` binding the runtime dispatches on. This routes that to the ledger. `charge`'s run_id is now Option. `usage_events.run_id` references `agent_runs`, and a topology turn has no row there — passing its `topology_runs` id was a foreign-key violation, which is exactly what the first attempt hit. NULL is the honest value; the agent-run caller still passes its real id. The executor reports one total rather than an in/out split, so the cost is right (credits price the sum) and the columns record it as output rather than inventing a split. Verified end to end on a real mission: 4 agents, 1046-8670 tokens each, credits attributed per agent, and the SPEND/ACTIVITY queries now return real numbers. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5
parent
8ef7067467
commit
8be7b3c9b2
@@ -201,12 +201,14 @@ async fn run_job(
|
|||||||
// to no mission — a bare topology run has no phase to hang tool calls on.
|
// to no mission — a bare topology run has no phase to hang tool calls on.
|
||||||
let tap = mission_binding
|
let tap = mission_binding
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|(_, _, mission_id, phase_id)| crate::topology_exec::MissionTap {
|
.map(
|
||||||
|
|(_, _, mission_id, phase_id)| crate::topology_exec::MissionTap {
|
||||||
pool: pool.clone(),
|
pool: pool.clone(),
|
||||||
mission_id: *mission_id,
|
mission_id: *mission_id,
|
||||||
phase_id: *phase_id,
|
phase_id: *phase_id,
|
||||||
run_id: Some(id),
|
run_id: Some(id),
|
||||||
});
|
},
|
||||||
|
);
|
||||||
let leaf_result = match mission_binding {
|
let leaf_result = match mission_binding {
|
||||||
Some((Some(url), Some(code), _, _)) => {
|
Some((Some(url), Some(code), _, _)) => {
|
||||||
ZeroClawDriveExecutor::from_env_for_gateway_with_code(url, code)
|
ZeroClawDriveExecutor::from_env_for_gateway_with_code(url, code)
|
||||||
@@ -246,9 +248,29 @@ async fn run_job(
|
|||||||
id,
|
id,
|
||||||
Arc::new(leaf),
|
Arc::new(leaf),
|
||||||
);
|
);
|
||||||
drive(pool, id, &graph, &job.task, progress, &exec).await
|
drive(
|
||||||
|
pool,
|
||||||
|
id,
|
||||||
|
job.workspace_id,
|
||||||
|
&graph,
|
||||||
|
&job.task,
|
||||||
|
progress,
|
||||||
|
&exec,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
drive(
|
||||||
|
pool,
|
||||||
|
id,
|
||||||
|
job.workspace_id,
|
||||||
|
&graph,
|
||||||
|
&job.task,
|
||||||
|
progress,
|
||||||
|
&leaf,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
_ => drive(pool, id, &graph, &job.task, progress, &leaf).await,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
finish(pool, id, result).await;
|
finish(pool, id, result).await;
|
||||||
@@ -303,8 +325,7 @@ async fn run_composed(
|
|||||||
OrchestratorError::Executor("a composed run must belong to a mission phase".into())
|
OrchestratorError::Executor("a composed run must belong to a mission phase".into())
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
let mission: (Option<Uuid>, Option<String>, Option<String>, bool) =
|
let mission: (Option<Uuid>, Option<String>, Option<String>, bool) = sqlx::query_as(
|
||||||
sqlx::query_as(
|
|
||||||
"SELECT target_node_id, backend, team_engine, (repo_id IS NOT NULL) \
|
"SELECT target_node_id, backend, team_engine, (repo_id IS NOT NULL) \
|
||||||
FROM missions WHERE id = $1",
|
FROM missions WHERE id = $1",
|
||||||
)
|
)
|
||||||
@@ -347,7 +368,16 @@ async fn run_composed(
|
|||||||
completed_steps: progress.completed as u32,
|
completed_steps: progress.completed as u32,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
drive(pool, job.id, graph, &job.task, progress, &exec).await
|
drive(
|
||||||
|
pool,
|
||||||
|
job.id,
|
||||||
|
job.workspace_id,
|
||||||
|
graph,
|
||||||
|
&job.task,
|
||||||
|
progress,
|
||||||
|
&exec,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Post-terminal hook: if this run's team is `ephemeral` and no siblings are
|
/// Post-terminal hook: if this run's team is `ephemeral` and no siblings are
|
||||||
@@ -406,14 +436,30 @@ async fn maybe_teardown_ephemeral_team(pool: &PgPool, runtime: &cm_runtime::Runt
|
|||||||
async fn drive<E: TurnExecutor>(
|
async fn drive<E: TurnExecutor>(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
id: Uuid,
|
id: Uuid,
|
||||||
|
workspace_id: Uuid,
|
||||||
graph: &TopologyGraph,
|
graph: &TopologyGraph,
|
||||||
task: &str,
|
task: &str,
|
||||||
progress: RunProgress,
|
progress: RunProgress,
|
||||||
executor: &E,
|
executor: &E,
|
||||||
) -> Result<RunRecord, OrchestratorError> {
|
) -> Result<RunRecord, OrchestratorError> {
|
||||||
let pool_cb = pool.clone();
|
let pool_cb = pool.clone();
|
||||||
|
// node_id -> agent id, resolved once. The binding lives in the node's
|
||||||
|
// attrs (`agent = claw_<uuid>`), which is also what the runtime dispatches
|
||||||
|
// on — so usage is attributed to exactly the claw that did the work.
|
||||||
|
let agent_of: std::sync::Arc<std::collections::HashMap<String, Uuid>> = std::sync::Arc::new(
|
||||||
|
graph
|
||||||
|
.nodes
|
||||||
|
.iter()
|
||||||
|
.filter_map(|n| {
|
||||||
|
let alias = n.attrs.get("agent")?;
|
||||||
|
let uuid = alias.strip_prefix("claw_")?;
|
||||||
|
Some((n.id.clone(), Uuid::parse_str(uuid).ok()?))
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
);
|
||||||
execute_resumable(graph, task, executor, progress, move |snap| {
|
execute_resumable(graph, task, executor, progress, move |snap| {
|
||||||
let pool = pool_cb.clone();
|
let pool = pool_cb.clone();
|
||||||
|
let agent_of = agent_of.clone();
|
||||||
async move {
|
async move {
|
||||||
// 2026-07-15: verbose per-step trace so `docker logs
|
// 2026-07-15: verbose per-step trace so `docker logs
|
||||||
// clawmates_server_1` shows which topology node just fired,
|
// clawmates_server_1` shows which topology node just fired,
|
||||||
@@ -439,6 +485,34 @@ async fn drive<E: TurnExecutor>(
|
|||||||
last.tokens,
|
last.tokens,
|
||||||
last.gated.len(),
|
last.gated.len(),
|
||||||
);
|
);
|
||||||
|
// Per-agent usage. Without this the command centre's SPEND,
|
||||||
|
// ACTIVITY and THROUGHPUT cards read `usage_events`, which
|
||||||
|
// nothing on the mission path ever wrote — so they showed 0 for
|
||||||
|
// an agent that had just burned 15k tokens.
|
||||||
|
//
|
||||||
|
// `charge` also decrements credit lots, which is the point: a
|
||||||
|
// mission turn costs what it costs. It clamps at the available
|
||||||
|
// balance and still records the full obligation, so an empty
|
||||||
|
// wallet cannot fail a turn.
|
||||||
|
if let Some(agent_id) = agent_of.get(&last.node_id).copied() {
|
||||||
|
if last.tokens > 0 {
|
||||||
|
// The executor reports ONE total, not an in/out split.
|
||||||
|
// Credits price the sum, so cost is right; the columns
|
||||||
|
// record it as output rather than inventing a split.
|
||||||
|
if let Err(e) = cm_billing::charge(
|
||||||
|
&pool,
|
||||||
|
cm_domain::WorkspaceId::from(workspace_id),
|
||||||
|
cm_domain::AgentId::from(agent_id),
|
||||||
|
None,
|
||||||
|
0,
|
||||||
|
last.tokens as u64,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
eprintln!("topology_worker: usage for {agent_id} failed: {e}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
// Best-effort checkpoint: a failed write just means we re-run the
|
// Best-effort checkpoint: a failed write just means we re-run the
|
||||||
// step on resume (idempotent — topology turns are pure reads here).
|
// step on resume (idempotent — topology turns are pure reads here).
|
||||||
|
|||||||
@@ -29,7 +29,11 @@ pub async fn charge(
|
|||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
workspace_id: WorkspaceId,
|
workspace_id: WorkspaceId,
|
||||||
agent_id: AgentId,
|
agent_id: AgentId,
|
||||||
run_id: Uuid,
|
// Optional: `usage_events.run_id` references `agent_runs`, and a topology
|
||||||
|
// turn has no row there — its id lives in `topology_runs`. Passing that id
|
||||||
|
// was a foreign-key violation, so mission usage went unrecorded. NULL is
|
||||||
|
// the honest value for a charge that is not an agent_run.
|
||||||
|
run_id: Option<Uuid>,
|
||||||
input_tokens: u64,
|
input_tokens: u64,
|
||||||
output_tokens: u64,
|
output_tokens: u64,
|
||||||
) -> Result<i64, BillingError> {
|
) -> Result<i64, BillingError> {
|
||||||
|
|||||||
@@ -62,7 +62,7 @@ async fn charges_span_lots_oldest_first_and_record_usage() {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
// 2500 tokens → 3 credits: drains the first lot (2) then one more.
|
// 2500 tokens → 3 credits: drains the first lot (2) then one more.
|
||||||
let deducted = charge(&pool, ws.id, agent.id, run_id, 1500, 1000)
|
let deducted = charge(&pool, ws.id, agent.id, Some(run_id), 1500, 1000)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(deducted, 3);
|
assert_eq!(deducted, 3);
|
||||||
@@ -96,7 +96,7 @@ async fn an_empty_workspace_records_usage_but_clamps_at_zero() {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
// Owes 5, only 1 available: deducts 1, balance hits zero, never negative.
|
// Owes 5, only 1 available: deducts 1, balance hits zero, never negative.
|
||||||
let deducted = charge(&pool, ws.id, agent.id, run_id, 4000, 500)
|
let deducted = charge(&pool, ws.id, agent.id, Some(run_id), 4000, 500)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(deducted, 1);
|
assert_eq!(deducted, 1);
|
||||||
|
|||||||
@@ -879,7 +879,7 @@ impl Runtime {
|
|||||||
&self.inner.pool,
|
&self.inner.pool,
|
||||||
state.workspace_id,
|
state.workspace_id,
|
||||||
state.agent_id,
|
state.agent_id,
|
||||||
run_id,
|
Some(run_id),
|
||||||
state.input_tokens,
|
state.input_tokens,
|
||||||
state.output_tokens,
|
state.output_tokens,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user