fix(llm): two modules were posting to Anthropic behind the providers' back
The research scenario passed 4/4 and the log underneath it said: phase_summarizer: ... failed: anthropic 400 Bad Request: "Your credit balance is too low to access the Anthropic API" `phase_summarizer` and `mission_refiner` each built their own reqwest POST to the Messages API with `x-api-key: $ANTHROPIC_API_KEY`. No audit of `.complete(` call sites could have found them — they never touched a provider — so every phase summary and every mission-brief refinement on this deployment had been failing against an empty account while the phases themselves ran fine. The summarizer even persisted an error row per phase, which is why nothing ever retried loudly enough to notice. Both now go through `subscription::complete_with_fallback`, so they inherit the subscription-first credential choice, the 429 backoff, and the opus -> haiku -> glm chain. The summarizer records the model that ANSWERED in mission_phase_summaries.model rather than the one it asked for. The guard is a source WALK, not a file list: any .rs under cm-api/src that mentions the Messages API host or `x-api-key` fails the test. A hand-listed set of files is exactly what let these two hide. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5
parent
52500a689c
commit
d48bdbc9a7
@@ -371,7 +371,7 @@ async fn run() -> Result<(), String> {
|
|||||||
// Phase completion summarizer: reads terminal-state phases and
|
// Phase completion summarizer: reads terminal-state phases and
|
||||||
// asks Claude Opus 4.8 to synthesize a "what got done" card that
|
// asks Claude Opus 4.8 to synthesize a "what got done" card that
|
||||||
// the UI renders under the phase.
|
// the UI renders under the phase.
|
||||||
cm_api::phase_summarizer::spawn(pool.clone());
|
cm_api::phase_summarizer::spawn(pool.clone(), runtime.clone());
|
||||||
// Outbound-email delivery: drains the §15-gated `outbox` over SMTP. Inert
|
// Outbound-email delivery: drains the §15-gated `outbox` over SMTP. Inert
|
||||||
// until CLAWMATES_SMTP_* is set, so it ships safely before credentials exist.
|
// until CLAWMATES_SMTP_* is set, so it ships safely before credentials exist.
|
||||||
cm_runtime::spawn_drainer(pool.clone(), std::time::Duration::from_secs(10));
|
cm_runtime::spawn_drainer(pool.clone(), std::time::Duration::from_secs(10));
|
||||||
|
|||||||
@@ -2,16 +2,18 @@
|
|||||||
//! mission and rewrite it into a coherent, sectioned Markdown brief
|
//! mission and rewrite it into a coherent, sectioned Markdown brief
|
||||||
//! that downstream research + coding agents can ingest cleanly.
|
//! that downstream research + coding agents can ingest cleanly.
|
||||||
//!
|
//!
|
||||||
//! Calls Anthropic Claude Opus 4.8 by default. Prod already carries
|
//! Asks for Claude Opus 4.8 by default, but goes through
|
||||||
//! ANTHROPIC_API_KEY for ZeroClaw's provider config, so no separate
|
//! `subscription::complete_with_fallback` like every other server-side model
|
||||||
//! env is needed.
|
//! call. It used to hand-roll its own HTTPS POST to the Messages API with the
|
||||||
|
//! metered key — a comment above this line still claimed prod "already carries
|
||||||
|
//! ANTHROPIC_API_KEY, so no separate env is needed", which stopped being true
|
||||||
|
//! the moment that account ran out of credit. See `subscription`, whose
|
||||||
|
//! source-walk test is what found this module.
|
||||||
|
|
||||||
use serde_json::json;
|
|
||||||
use sqlx::PgPool;
|
use sqlx::PgPool;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
const DEFAULT_MODEL: &str = "claude-opus-4-8";
|
const DEFAULT_MODEL: &str = "claude-opus-4-8";
|
||||||
const ANTHROPIC_API_VERSION: &str = "2023-06-01";
|
|
||||||
|
|
||||||
fn model_name() -> String {
|
fn model_name() -> String {
|
||||||
std::env::var("CLAWMATES_REFINER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
std::env::var("CLAWMATES_REFINER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
||||||
@@ -28,6 +30,7 @@ pub struct RefineResult {
|
|||||||
/// an audit table.
|
/// an audit table.
|
||||||
pub async fn refine(
|
pub async fn refine(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
workspace_id: cm_domain::WorkspaceId,
|
workspace_id: cm_domain::WorkspaceId,
|
||||||
mission_id: Uuid,
|
mission_id: Uuid,
|
||||||
) -> Result<RefineResult, String> {
|
) -> Result<RefineResult, String> {
|
||||||
@@ -54,7 +57,8 @@ pub async fn refine(
|
|||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let refined =
|
let refined =
|
||||||
call_anthropic(&mission.title, &mission.template_kind, &phase_kinds, &raw).await?;
|
call_anthropic(runtime, &mission.title, &mission.template_kind, &phase_kinds, &raw)
|
||||||
|
.await?;
|
||||||
|
|
||||||
Ok(RefineResult {
|
Ok(RefineResult {
|
||||||
original: raw,
|
original: raw,
|
||||||
@@ -63,13 +67,12 @@ pub async fn refine(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn call_anthropic(
|
async fn call_anthropic(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
title: &str,
|
title: &str,
|
||||||
template_kind: &str,
|
template_kind: &str,
|
||||||
phase_kinds: &[String],
|
phase_kinds: &[String],
|
||||||
raw: &str,
|
raw: &str,
|
||||||
) -> Result<String, String> {
|
) -> Result<String, String> {
|
||||||
let api_key =
|
|
||||||
std::env::var("ANTHROPIC_API_KEY").map_err(|_| "ANTHROPIC_API_KEY unset".to_string())?;
|
|
||||||
let model = model_name();
|
let model = model_name();
|
||||||
|
|
||||||
let system = "You are a technical brief editor for an autonomous software \
|
let system = "You are a technical brief editor for an autonomous software \
|
||||||
@@ -131,57 +134,14 @@ async fn call_anthropic(
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Opus 4.8 rejects the `temperature` parameter — the model runs at
|
// Opus 4.8 rejects the `temperature` parameter — the model runs at
|
||||||
// its own calibrated setting. Older Claude models accepted 0.0–1.0.
|
// its own calibrated setting. Older Claude models accepted 0.0–1.0, and
|
||||||
let body = json!({
|
// `ChatRequest` does not carry one, so nothing is lost by the move.
|
||||||
"model": model,
|
let (text, answered_by) =
|
||||||
"max_tokens": 4096,
|
crate::subscription::complete_with_fallback(runtime, system, &user, &model, 4096, false)
|
||||||
"system": system,
|
.await?;
|
||||||
"messages": [
|
let text = text.trim().to_string();
|
||||||
{ "role": "user", "content": user }
|
|
||||||
]
|
|
||||||
});
|
|
||||||
|
|
||||||
let client = reqwest::Client::builder()
|
|
||||||
.timeout(std::time::Duration::from_secs(90))
|
|
||||||
.build()
|
|
||||||
.map_err(|e| format!("http client: {e}"))?;
|
|
||||||
let resp = client
|
|
||||||
.post("https://api.anthropic.com/v1/messages")
|
|
||||||
.header("x-api-key", &api_key)
|
|
||||||
.header("anthropic-version", ANTHROPIC_API_VERSION)
|
|
||||||
.header("content-type", "application/json")
|
|
||||||
.json(&body)
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("anthropic call: {e}"))?;
|
|
||||||
if !resp.status().is_success() {
|
|
||||||
let code = resp.status();
|
|
||||||
let body = resp.text().await.unwrap_or_default();
|
|
||||||
return Err(format!(
|
|
||||||
"anthropic {code}: {}",
|
|
||||||
&body[..body.len().min(500)]
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let json: serde_json::Value = resp
|
|
||||||
.json()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("anthropic json: {e}"))?;
|
|
||||||
// Anthropic Messages API returns content as an array of blocks;
|
|
||||||
// the first text block holds the assistant's reply.
|
|
||||||
let text = json
|
|
||||||
.get("content")
|
|
||||||
.and_then(|c| c.as_array())
|
|
||||||
.and_then(|arr| {
|
|
||||||
arr.iter()
|
|
||||||
.find(|b| b.get("type").and_then(|t| t.as_str()) == Some("text"))
|
|
||||||
})
|
|
||||||
.and_then(|b| b.get("text"))
|
|
||||||
.and_then(|t| t.as_str())
|
|
||||||
.ok_or_else(|| "anthropic response missing text block".to_string())?
|
|
||||||
.trim()
|
|
||||||
.to_string();
|
|
||||||
if text.is_empty() {
|
if text.is_empty() {
|
||||||
return Err("anthropic returned empty text".into());
|
return Err(format!("{answered_by} returned empty text"));
|
||||||
}
|
}
|
||||||
Ok(text)
|
Ok(text)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,7 +23,6 @@ use std::time::Duration;
|
|||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
const DEFAULT_MODEL: &str = "claude-opus-4-8";
|
const DEFAULT_MODEL: &str = "claude-opus-4-8";
|
||||||
const ANTHROPIC_API_VERSION: &str = "2023-06-01";
|
|
||||||
const POLL_INTERVAL: Duration = Duration::from_secs(30);
|
const POLL_INTERVAL: Duration = Duration::from_secs(30);
|
||||||
/// Cap the raw material we send to the model. Missions can produce
|
/// Cap the raw material we send to the model. Missions can produce
|
||||||
/// hundreds of KB of agent output; we slice by turn and by phase
|
/// hundreds of KB of agent output; we slice by turn and by phase
|
||||||
@@ -34,21 +33,26 @@ fn model_name() -> String {
|
|||||||
std::env::var("CLAWMATES_SUMMARIZER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
std::env::var("CLAWMATES_SUMMARIZER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn spawn(pool: PgPool) {
|
/// The runtime is carried purely so the summarizer can reach the SAME
|
||||||
|
/// providers as everything else. It used to hand-roll its own HTTPS POST with
|
||||||
|
/// `x-api-key: $ANTHROPIC_API_KEY`, which is why no audit of `.complete(` call
|
||||||
|
/// sites ever found it — and why every phase summary on this deployment died
|
||||||
|
/// with "credit balance is too low" while the phases themselves ran fine.
|
||||||
|
pub fn spawn(pool: PgPool, runtime: cm_runtime::Runtime) {
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
tokio::time::sleep(Duration::from_secs(45)).await;
|
tokio::time::sleep(Duration::from_secs(45)).await;
|
||||||
let mut ticker = tokio::time::interval(POLL_INTERVAL);
|
let mut ticker = tokio::time::interval(POLL_INTERVAL);
|
||||||
ticker.tick().await;
|
ticker.tick().await;
|
||||||
loop {
|
loop {
|
||||||
ticker.tick().await;
|
ticker.tick().await;
|
||||||
if let Err(e) = sweep_once(&pool).await {
|
if let Err(e) = sweep_once(&pool, &runtime).await {
|
||||||
eprintln!("phase_summarizer: sweep failed: {e}");
|
eprintln!("phase_summarizer: sweep failed: {e}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn sweep_once(pool: &PgPool) -> Result<(), String> {
|
async fn sweep_once(pool: &PgPool, runtime: &cm_runtime::Runtime) -> Result<(), String> {
|
||||||
// Terminal phases with no summary yet.
|
// Terminal phases with no summary yet.
|
||||||
let rows = sqlx::query(
|
let rows = sqlx::query(
|
||||||
"SELECT mp.id, mp.mission_id, mp.kind
|
"SELECT mp.id, mp.mission_id, mp.kind
|
||||||
@@ -65,7 +69,7 @@ async fn sweep_once(pool: &PgPool) -> Result<(), String> {
|
|||||||
let phase_id: Uuid = row.get("id");
|
let phase_id: Uuid = row.get("id");
|
||||||
let mission_id: Uuid = row.get("mission_id");
|
let mission_id: Uuid = row.get("mission_id");
|
||||||
let kind: String = row.get("kind");
|
let kind: String = row.get("kind");
|
||||||
if let Err(e) = summarize_one(pool, mission_id, phase_id, &kind).await {
|
if let Err(e) = summarize_one(pool, runtime, mission_id, phase_id, &kind).await {
|
||||||
// Persist an error row so we don't infinite-retry a broken
|
// Persist an error row so we don't infinite-retry a broken
|
||||||
// phase — the UI can surface "summary unavailable: <e>".
|
// phase — the UI can surface "summary unavailable: <e>".
|
||||||
eprintln!("phase_summarizer: {phase_id} ({kind}) failed: {e}");
|
eprintln!("phase_summarizer: {phase_id} ({kind}) failed: {e}");
|
||||||
@@ -77,6 +81,7 @@ async fn sweep_once(pool: &PgPool) -> Result<(), String> {
|
|||||||
|
|
||||||
async fn summarize_one(
|
async fn summarize_one(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
mission_id: Uuid,
|
mission_id: Uuid,
|
||||||
phase_id: Uuid,
|
phase_id: Uuid,
|
||||||
kind: &str,
|
kind: &str,
|
||||||
@@ -105,7 +110,7 @@ async fn summarize_one(
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
let (narrative, structured) = call_anthropic(kind, &material).await?;
|
let (narrative, structured, answered_by) = call_anthropic(runtime, kind, &material).await?;
|
||||||
let metrics = structured
|
let metrics = structured
|
||||||
.get("metrics")
|
.get("metrics")
|
||||||
.cloned()
|
.cloned()
|
||||||
@@ -128,7 +133,7 @@ async fn summarize_one(
|
|||||||
mission_id,
|
mission_id,
|
||||||
phase_id,
|
phase_id,
|
||||||
kind,
|
kind,
|
||||||
&model_name(),
|
&answered_by,
|
||||||
&narrative,
|
&narrative,
|
||||||
&metrics,
|
&metrics,
|
||||||
&sources,
|
&sources,
|
||||||
@@ -323,58 +328,25 @@ async fn collect_material(
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn call_anthropic(kind: &str, material: &PhaseMaterial) -> Result<(String, Value), String> {
|
/// Returns the narrative, the parsed object, and **the model that answered** —
|
||||||
let api_key =
|
/// which may be a fallback link rather than `model_name()`, and is recorded as
|
||||||
std::env::var("ANTHROPIC_API_KEY").map_err(|_| "ANTHROPIC_API_KEY unset".to_string())?;
|
/// such.
|
||||||
|
async fn call_anthropic(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
|
kind: &str,
|
||||||
|
material: &PhaseMaterial,
|
||||||
|
) -> Result<(String, Value, String), String> {
|
||||||
let model = model_name();
|
let model = model_name();
|
||||||
let system = system_prompt(kind);
|
let system = system_prompt(kind);
|
||||||
let user = user_prompt(kind, material);
|
let user = user_prompt(kind, material);
|
||||||
|
|
||||||
let body = json!({
|
let (raw, answered_by) = crate::subscription::complete_with_fallback(
|
||||||
"model": model,
|
runtime, &system, &user, &model, 4096, false,
|
||||||
"max_tokens": 4096,
|
)
|
||||||
"system": system,
|
.await?;
|
||||||
"messages": [ { "role": "user", "content": user } ]
|
let raw = raw.trim().to_string();
|
||||||
});
|
|
||||||
let client = reqwest::Client::builder()
|
|
||||||
.timeout(std::time::Duration::from_secs(120))
|
|
||||||
.build()
|
|
||||||
.map_err(|e| format!("http client: {e}"))?;
|
|
||||||
let resp = client
|
|
||||||
.post("https://api.anthropic.com/v1/messages")
|
|
||||||
.header("x-api-key", &api_key)
|
|
||||||
.header("anthropic-version", ANTHROPIC_API_VERSION)
|
|
||||||
.header("content-type", "application/json")
|
|
||||||
.json(&body)
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("anthropic call: {e}"))?;
|
|
||||||
if !resp.status().is_success() {
|
|
||||||
let code = resp.status();
|
|
||||||
let body = resp.text().await.unwrap_or_default();
|
|
||||||
return Err(format!(
|
|
||||||
"anthropic {code}: {}",
|
|
||||||
&body[..body.len().min(500)]
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let json: Value = resp
|
|
||||||
.json()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("anthropic json: {e}"))?;
|
|
||||||
let raw = json
|
|
||||||
.get("content")
|
|
||||||
.and_then(|c| c.as_array())
|
|
||||||
.and_then(|arr| {
|
|
||||||
arr.iter()
|
|
||||||
.find(|b| b.get("type").and_then(|t| t.as_str()) == Some("text"))
|
|
||||||
})
|
|
||||||
.and_then(|b| b.get("text"))
|
|
||||||
.and_then(|t| t.as_str())
|
|
||||||
.ok_or_else(|| "anthropic response missing text block".to_string())?
|
|
||||||
.trim()
|
|
||||||
.to_string();
|
|
||||||
if raw.is_empty() {
|
if raw.is_empty() {
|
||||||
return Err("anthropic returned empty text".into());
|
return Err(format!("{answered_by} returned empty text"));
|
||||||
}
|
}
|
||||||
// Model returns a JSON object; extract narrative + rest.
|
// Model returns a JSON object; extract narrative + rest.
|
||||||
let parsed: Value = serde_json::from_str(&strip_code_fence(&raw)).map_err(|e| {
|
let parsed: Value = serde_json::from_str(&strip_code_fence(&raw)).map_err(|e| {
|
||||||
@@ -392,7 +364,7 @@ async fn call_anthropic(kind: &str, material: &PhaseMaterial) -> Result<(String,
|
|||||||
if narrative.is_empty() {
|
if narrative.is_empty() {
|
||||||
return Err("summarizer response missing narrative".into());
|
return Err("summarizer response missing narrative".into());
|
||||||
}
|
}
|
||||||
Ok((narrative, parsed))
|
Ok((narrative, parsed, answered_by))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Trim a leading/trailing ```json … ``` fence the model sometimes wraps
|
/// Trim a leading/trailing ```json … ``` fence the model sometimes wraps
|
||||||
|
|||||||
@@ -619,7 +619,7 @@ pub async fn refine(
|
|||||||
Authed(user): Authed,
|
Authed(user): Authed,
|
||||||
Path(id): Path<Uuid>,
|
Path(id): Path<Uuid>,
|
||||||
) -> Result<Json<RefineResponse>, ApiError> {
|
) -> Result<Json<RefineResponse>, ApiError> {
|
||||||
let result = crate::mission_refiner::refine(&state.pool, user.workspace_id, id)
|
let result = crate::mission_refiner::refine(&state.pool, &state.runtime, user.workspace_id, id)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
eprintln!("mission {id}: refine failed: {e}");
|
eprintln!("mission {id}: refine failed: {e}");
|
||||||
|
|||||||
@@ -343,6 +343,50 @@ mod tests {
|
|||||||
assert!(!is_transient(&LlmError::Wire("bad json".into())));
|
assert!(!is_transient(&LlmError::Wire("bad json".into())));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Nobody hand-rolls their own Anthropic HTTP call.
|
||||||
|
///
|
||||||
|
/// `phase_summarizer` did — its own `reqwest` POST to `api.anthropic.com`
|
||||||
|
/// with `x-api-key: $ANTHROPIC_API_KEY`. No audit of `.complete(` call
|
||||||
|
/// sites could ever have found it, and it was the last thing on this
|
||||||
|
/// deployment still billing an account with no credit: every phase summary
|
||||||
|
/// died with "credit balance is too low" while the phases themselves ran.
|
||||||
|
/// A call site is only routable if it goes through a provider, so walk the
|
||||||
|
/// whole crate rather than a hand-listed set of files.
|
||||||
|
#[test]
|
||||||
|
fn no_module_talks_to_anthropic_behind_the_providers_back() {
|
||||||
|
fn walk(dir: &std::path::Path, out: &mut Vec<std::path::PathBuf>) {
|
||||||
|
for entry in std::fs::read_dir(dir).expect("readable source dir") {
|
||||||
|
let path = entry.expect("readable entry").path();
|
||||||
|
if path.is_dir() {
|
||||||
|
walk(&path, out);
|
||||||
|
} else if path.extension().is_some_and(|e| e == "rs") {
|
||||||
|
out.push(path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("src");
|
||||||
|
let mut files = Vec::new();
|
||||||
|
walk(&root, &mut files);
|
||||||
|
assert!(files.len() > 20, "source walk found suspiciously few files");
|
||||||
|
|
||||||
|
for path in files {
|
||||||
|
// This module names the host in prose; it is the one that may.
|
||||||
|
if path.ends_with("subscription.rs") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let src = std::fs::read_to_string(&path).expect("readable source");
|
||||||
|
for needle in ["api.anthropic.com", "\"x-api-key\""] {
|
||||||
|
assert!(
|
||||||
|
!src.contains(needle),
|
||||||
|
"{} contains {needle} — build the request through cm_llm and \
|
||||||
|
route it via `subscription::complete_or`, so credential \
|
||||||
|
choice and the capacity fallback live in ONE place",
|
||||||
|
path.display()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// The chain never retries the capped model as its own fallback.
|
/// The chain never retries the capped model as its own fallback.
|
||||||
///
|
///
|
||||||
/// Without the filter, asking for haiku while haiku is capped would try
|
/// Without the filter, asking for haiku while haiku is capped would try
|
||||||
|
|||||||
Reference in New Issue
Block a user