feat(llm): the subscription is the default provider, with a recorded fallback chain
Two changes so an empty metered account stops being a platform outage. 1. `build_provider` prefers the subscription token over ANTHROPIC_API_KEY. A bare model name resolves to whatever this returns, so making it the subscription means no server-side call can reach the metered key by construction — rather than by a source-grep test that already missed four call sites once. The metered key remains a fallback and now warns loudly when it is the one in use; boot no longer requires it at all. 2. `complete_with_fallback` walks a declared chain when a model has no capacity: opus -> haiku -> glm:glm-4.7 by default, overridable via CLAWMATES_MODEL_FALLBACK, empty to disable. Measured on gw-04 today: opus and sonnet return 429 on the subscription while haiku, GLM and Kimi all return 200, so a capped window no longer means "the planner is gone". The chain returns the model that ANSWERED, and every caller persists it — mission_plan_proposals.author_model, mission_team_proposals.author_model, and the swarm's step role. A plan drafted by the third link and filed as an opus plan is a silent quality change, which is the failure shape this project keeps paying for. Two negative controls hold the design: the chain never retries the model that just failed as its own fallback, and it steps down ONLY for a capacity failure — walking it on a malformed prompt would ask three models the same bad question and report the third one's confusion. Co-Authored-By: Claude Opus 5 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 5
parent
ee5a939ce6
commit
9c9439a271
@@ -24,11 +24,32 @@ async fn main() -> ExitCode {
|
|||||||
|
|
||||||
/// Instantiates the configured LLM provider. The Anthropic key comes from
|
/// Instantiates the configured LLM provider. The Anthropic key comes from
|
||||||
/// the environment until the secret broker lands in P2.
|
/// the environment until the secret broker lands in P2.
|
||||||
|
///
|
||||||
|
/// The **subscription wins** when both credentials are present. This is the
|
||||||
|
/// structural half of the fix that `cm_api::subscription` does per-call: a bare
|
||||||
|
/// model name resolves to whatever this function returns, so making that the
|
||||||
|
/// subscription means no server-side call can reach the metered key by
|
||||||
|
/// accident — by construction, rather than by a source-grep test that has
|
||||||
|
/// already missed four call sites once. The metered key stays usable as a
|
||||||
|
/// fallback for deployments that have credit; ours does not, which is what
|
||||||
|
/// made the ordering matter.
|
||||||
fn build_provider(config: &AppConfig) -> Result<Arc<dyn LlmProvider>, String> {
|
fn build_provider(config: &AppConfig) -> Result<Arc<dyn LlmProvider>, String> {
|
||||||
match config.llm.provider {
|
match config.llm.provider {
|
||||||
LlmProviderKind::Anthropic => {
|
LlmProviderKind::Anthropic => {
|
||||||
let key = std::env::var("ANTHROPIC_API_KEY")
|
if let Some(provider) = cm_api::subscription::provider() {
|
||||||
.map_err(|_| "llm.provider = \"anthropic\" requires ANTHROPIC_API_KEY")?;
|
println!(
|
||||||
|
"clawmates-server: default LLM provider = Claude Code subscription \
|
||||||
|
(bare model names bill no metered key)"
|
||||||
|
);
|
||||||
|
return Ok(Arc::new(provider));
|
||||||
|
}
|
||||||
|
let key = std::env::var("ANTHROPIC_API_KEY").map_err(|_| {
|
||||||
|
"llm.provider = \"anthropic\" needs a credential: either \
|
||||||
|
ANTHROPIC_OAUTH_TOKEN / CLAUDE_CODE_OAUTH_TOKEN (sk-ant-oat…, \
|
||||||
|
the Claude Code subscription, preferred) or ANTHROPIC_API_KEY \
|
||||||
|
(sk-ant-api…, metered)"
|
||||||
|
.to_string()
|
||||||
|
})?;
|
||||||
// A subscription OAuth token pasted where an API key belongs
|
// A subscription OAuth token pasted where an API key belongs
|
||||||
// authenticates nothing here and fails on the first model call,
|
// authenticates nothing here and fails on the first model call,
|
||||||
// far from the mistake. Both start `sk-ant-`, so the confusion is
|
// far from the mistake. Both start `sk-ant-`, so the confusion is
|
||||||
@@ -40,6 +61,11 @@ fn build_provider(config: &AppConfig) -> Result<Arc<dyn LlmProvider>, String> {
|
|||||||
bearer auth and is what the phase evaluator reads."
|
bearer auth and is what the phase evaluator reads."
|
||||||
.to_string());
|
.to_string());
|
||||||
}
|
}
|
||||||
|
eprintln!(
|
||||||
|
"clawmates-server: WARNING — no subscription token; the default LLM \
|
||||||
|
provider is the METERED ANTHROPIC_API_KEY and every bare model name \
|
||||||
|
bills it"
|
||||||
|
);
|
||||||
Ok(Arc::new(AnthropicProvider::new(key)))
|
Ok(Arc::new(AnthropicProvider::new(key)))
|
||||||
}
|
}
|
||||||
LlmProviderKind::OpenAiCompat => {
|
LlmProviderKind::OpenAiCompat => {
|
||||||
|
|||||||
@@ -174,7 +174,9 @@ pub async fn suggest(
|
|||||||
PLANNABLE_KINDS.join(", "),
|
PLANNABLE_KINDS.join(", "),
|
||||||
);
|
);
|
||||||
|
|
||||||
let raw = crate::subscription::complete_or(
|
// The stored `author_model` is whichever link of the fallback chain
|
||||||
|
// actually answered — see `subscription::complete_with_fallback`.
|
||||||
|
let (raw, author_model) = crate::subscription::complete_with_fallback(
|
||||||
&state.runtime,
|
&state.runtime,
|
||||||
PLAN_SYSTEM,
|
PLAN_SYSTEM,
|
||||||
&prompt,
|
&prompt,
|
||||||
@@ -212,7 +214,7 @@ pub async fn suggest(
|
|||||||
id,
|
id,
|
||||||
ws.as_uuid().to_owned(),
|
ws.as_uuid().to_owned(),
|
||||||
&stored,
|
&stored,
|
||||||
PLANNER_MODEL,
|
&author_model,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
@@ -220,7 +222,7 @@ pub async fn suggest(
|
|||||||
ApiError::Internal
|
ApiError::Internal
|
||||||
})?;
|
})?;
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"mission_plan: mission {id} — {PLANNER_MODEL} proposed {} phase(s): {}",
|
"mission_plan: mission {id} — {author_model} proposed {} phase(s): {}",
|
||||||
plan.phases.len(),
|
plan.phases.len(),
|
||||||
plan.phases
|
plan.phases
|
||||||
.iter()
|
.iter()
|
||||||
@@ -232,7 +234,7 @@ pub async fn suggest(
|
|||||||
Ok(Json(PlanProposalResponse {
|
Ok(Json(PlanProposalResponse {
|
||||||
id: pid,
|
id: pid,
|
||||||
plan: stored,
|
plan: stored,
|
||||||
author_model: PLANNER_MODEL.to_string(),
|
author_model,
|
||||||
status: "proposed".into(),
|
status: "proposed".into(),
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -115,7 +115,10 @@ pub async fn suggest(
|
|||||||
// `Runtime::complete` with a bare model name resolves to the default
|
// `Runtime::complete` with a bare model name resolves to the default
|
||||||
// provider, which is the pay-as-you-go key; this planner died with
|
// provider, which is the pay-as-you-go key; this planner died with
|
||||||
// "credit balance is too low" while missions on the same box ran fine.
|
// "credit balance is too low" while missions on the same box ran fine.
|
||||||
let raw = crate::subscription::complete_or(
|
// `author_model` is what ANSWERED, not what was asked for. When opus is
|
||||||
|
// capped the chain steps down to haiku and then to GLM, and a plan drafted
|
||||||
|
// by the third link but filed as an opus plan is a silent quality change.
|
||||||
|
let (raw, author_model) = crate::subscription::complete_with_fallback(
|
||||||
&state.runtime,
|
&state.runtime,
|
||||||
ROSTER_SYSTEM,
|
ROSTER_SYSTEM,
|
||||||
&prompt,
|
&prompt,
|
||||||
@@ -158,7 +161,7 @@ pub async fn suggest(
|
|||||||
id,
|
id,
|
||||||
ws.as_uuid().to_owned(),
|
ws.as_uuid().to_owned(),
|
||||||
&stored,
|
&stored,
|
||||||
PLANNER_MODEL,
|
&author_model,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
@@ -167,7 +170,7 @@ pub async fn suggest(
|
|||||||
})?;
|
})?;
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"mission_roster: mission {id} — {} proposed {} member(s): {}",
|
"mission_roster: mission {id} — {} proposed {} member(s): {}",
|
||||||
PLANNER_MODEL,
|
author_model,
|
||||||
roster.members.len(),
|
roster.members.len(),
|
||||||
roster
|
roster
|
||||||
.members
|
.members
|
||||||
@@ -180,7 +183,7 @@ pub async fn suggest(
|
|||||||
Ok(Json(ProposalResponse {
|
Ok(Json(ProposalResponse {
|
||||||
id: pid,
|
id: pid,
|
||||||
roster: stored,
|
roster: stored,
|
||||||
author_model: PLANNER_MODEL.to_string(),
|
author_model,
|
||||||
status: "proposed".into(),
|
status: "proposed".into(),
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -124,6 +124,74 @@ fn is_transient(e: &cm_llm::LlmError) -> bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Models to try, in order, when the requested one is rate limited.
|
||||||
|
///
|
||||||
|
/// Measured on this deployment 2026-08-08: opus and sonnet returned 429 on the
|
||||||
|
/// subscription while haiku returned 200, and GLM — funded separately, and
|
||||||
|
/// already the project's measured-best independent judge — returned 200. So the
|
||||||
|
/// default chain steps down within the subscription first, then leaves Anthropic
|
||||||
|
/// entirely rather than making a capped window mean "the planner is gone".
|
||||||
|
///
|
||||||
|
/// Override with `CLAWMATES_MODEL_FALLBACK` (comma-separated). An empty value
|
||||||
|
/// disables fallback and restores plain "503 and wait".
|
||||||
|
const DEFAULT_FALLBACK: &str = "claude-haiku-4-5-20251001,glm:glm-4.7";
|
||||||
|
|
||||||
|
/// The chain to walk after `requested`, with `requested` itself removed so a
|
||||||
|
/// capped model is never retried as its own fallback.
|
||||||
|
pub fn fallback_chain(requested: &str) -> Vec<String> {
|
||||||
|
let raw =
|
||||||
|
std::env::var("CLAWMATES_MODEL_FALLBACK").unwrap_or_else(|_| DEFAULT_FALLBACK.to_string());
|
||||||
|
raw.split(',')
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|m| !m.is_empty() && *m != requested.trim())
|
||||||
|
.map(str::to_string)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Whether a failure means "this model has no capacity right now" as opposed
|
||||||
|
/// to "this request was wrong".
|
||||||
|
///
|
||||||
|
/// The distinction is the whole safety of the chain: walking it on a malformed
|
||||||
|
/// prompt would ask three models the same bad question and report the third
|
||||||
|
/// one's confusion, while walking it on a rate limit is exactly the point.
|
||||||
|
pub fn is_capacity_failure(err: &str) -> bool {
|
||||||
|
err.contains("rate_limit") || err.contains("429") || err.contains("credit balance")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One completion, stepping down `fallback_chain` when a model has no capacity.
|
||||||
|
///
|
||||||
|
/// Returns the text **and the model that actually produced it**. Callers must
|
||||||
|
/// persist that second value: a plan drafted by the third link in the chain and
|
||||||
|
/// filed as an opus plan is a silent quality change, which is the failure shape
|
||||||
|
/// this project keeps paying for. Every hop is logged.
|
||||||
|
pub async fn complete_with_fallback(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
|
system: &str,
|
||||||
|
user: &str,
|
||||||
|
model: &str,
|
||||||
|
max_tokens: u32,
|
||||||
|
web_search: bool,
|
||||||
|
) -> Result<(String, String), String> {
|
||||||
|
let mut last = match complete_or(runtime, system, user, model, max_tokens, web_search).await {
|
||||||
|
Ok(text) => return Ok((text, model.to_string())),
|
||||||
|
Err(e) if is_capacity_failure(&e) => e,
|
||||||
|
// A real error. Do not launder it through two more models.
|
||||||
|
Err(e) => return Err(e),
|
||||||
|
};
|
||||||
|
for next in fallback_chain(model) {
|
||||||
|
eprintln!("model fallback: {model} has no capacity ({last}) — trying {next}");
|
||||||
|
match complete_or(runtime, system, user, &next, max_tokens, web_search).await {
|
||||||
|
Ok(text) => {
|
||||||
|
eprintln!("model fallback: {next} answered in place of {model}");
|
||||||
|
return Ok((text, next));
|
||||||
|
}
|
||||||
|
Err(e) if is_capacity_failure(&e) => last = e,
|
||||||
|
Err(e) => return Err(format!("fallback {next}: {e}")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(last)
|
||||||
|
}
|
||||||
|
|
||||||
/// Turn a `complete_or` failure into the right API error.
|
/// Turn a `complete_or` failure into the right API error.
|
||||||
///
|
///
|
||||||
/// A rate limit that outlived the backoff is not a bug in this server, and
|
/// A rate limit that outlived the backoff is not a bug in this server, and
|
||||||
@@ -275,6 +343,42 @@ mod tests {
|
|||||||
assert!(!is_transient(&LlmError::Wire("bad json".into())));
|
assert!(!is_transient(&LlmError::Wire("bad json".into())));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The chain never retries the capped model as its own fallback.
|
||||||
|
///
|
||||||
|
/// Without the filter, asking for haiku while haiku is capped would try
|
||||||
|
/// haiku, fail, and try haiku again — a chain that looks like resilience
|
||||||
|
/// and delivers none.
|
||||||
|
#[test]
|
||||||
|
fn the_chain_excludes_the_model_that_just_failed() {
|
||||||
|
// No env override in scope: this asserts the SHIPPED default.
|
||||||
|
let chain = fallback_chain("claude-opus-4-8");
|
||||||
|
assert_eq!(chain, vec!["claude-haiku-4-5-20251001", "glm:glm-4.7"]);
|
||||||
|
assert!(!fallback_chain("claude-haiku-4-5-20251001")
|
||||||
|
.iter()
|
||||||
|
.any(|m| m == "claude-haiku-4-5-20251001"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The chain is walked for "no capacity" and NOT for "bad request".
|
||||||
|
///
|
||||||
|
/// Walking it on a malformed prompt would ask three models the same bad
|
||||||
|
/// question and report the third one's confusion as the answer, burning
|
||||||
|
/// the two credentials that still work in order to hide the real error.
|
||||||
|
#[test]
|
||||||
|
fn only_a_capacity_failure_steps_down_the_chain() {
|
||||||
|
assert!(is_capacity_failure(
|
||||||
|
"subscription call: provider returned an error: 429 Too Many Requests"
|
||||||
|
));
|
||||||
|
assert!(is_capacity_failure("rate_limit_error"));
|
||||||
|
// The metered key's wall counts too — same meaning, different wording.
|
||||||
|
assert!(is_capacity_failure(
|
||||||
|
"400: Your credit balance is too low to access the Anthropic API"
|
||||||
|
));
|
||||||
|
|
||||||
|
assert!(!is_capacity_failure("400: messages.0: text content is empty"));
|
||||||
|
assert!(!is_capacity_failure("401: invalid x-api-key"));
|
||||||
|
assert!(!is_capacity_failure("404: model not found"));
|
||||||
|
}
|
||||||
|
|
||||||
/// An operator's explicit provider choice is never hijacked.
|
/// An operator's explicit provider choice is never hijacked.
|
||||||
///
|
///
|
||||||
/// The swarm worker model is a configured `name:model` spec. Routing that
|
/// The swarm worker model is a configured `name:model` spec. Routing that
|
||||||
|
|||||||
@@ -122,9 +122,11 @@ pub async fn run_swarm_job(
|
|||||||
let worker_model = resolve_worker_model(&job.worker_model);
|
let worker_model = resolve_worker_model(&job.worker_model);
|
||||||
|
|
||||||
// 1) PLAN — Opus decomposes the goal into worker tasks.
|
// 1) PLAN — Opus decomposes the goal into worker tasks.
|
||||||
|
// This record is written BEFORE the call, so it cannot name the model that
|
||||||
|
// answers. The record after the call can, and does.
|
||||||
records.push(step(
|
records.push(step(
|
||||||
"planner",
|
"planner",
|
||||||
"planner:opus",
|
"planner",
|
||||||
StepPhase::Plan,
|
StepPhase::Plan,
|
||||||
format!("Planning tasks for: {goal}"),
|
format!("Planning tasks for: {goal}"),
|
||||||
));
|
));
|
||||||
@@ -137,7 +139,10 @@ pub async fn run_swarm_job(
|
|||||||
"GOAL:\n{goal}\n\nCHECKLIST each task's output must satisfy:\n{}{want}",
|
"GOAL:\n{goal}\n\nCHECKLIST each task's output must satisfy:\n{}{want}",
|
||||||
checklist_lines(&checklist)
|
checklist_lines(&checklist)
|
||||||
);
|
);
|
||||||
let plan_raw = crate::subscription::complete_or(
|
// The recorded role says which model ANSWERED. When opus is capped the
|
||||||
|
// chain steps down, and a step labelled "planner:opus" that GLM wrote is a
|
||||||
|
// lie in the one place an operator looks to explain a bad decomposition.
|
||||||
|
let (plan_raw, plan_model) = crate::subscription::complete_with_fallback(
|
||||||
runtime,
|
runtime,
|
||||||
PLAN_SYSTEM,
|
PLAN_SYSTEM,
|
||||||
&plan_user,
|
&plan_user,
|
||||||
@@ -160,7 +165,7 @@ pub async fn run_swarm_job(
|
|||||||
}
|
}
|
||||||
records.push(step(
|
records.push(step(
|
||||||
"planner",
|
"planner",
|
||||||
"planner:opus",
|
format!("planner:{plan_model}"),
|
||||||
StepPhase::Plan,
|
StepPhase::Plan,
|
||||||
format!(
|
format!(
|
||||||
"Decomposed into {} tasks. Workers: {worker_model}. Verifier: claude-opus-4-8.",
|
"Decomposed into {} tasks. Workers: {worker_model}. Verifier: claude-opus-4-8.",
|
||||||
@@ -198,7 +203,7 @@ pub async fn run_swarm_job(
|
|||||||
ckpt(pool, id, &records, &totals).await;
|
ckpt(pool, id, &records, &totals).await;
|
||||||
|
|
||||||
let vuser = format!("TASK:\n{task}\n\nWORKER OUTPUT:\n{out}");
|
let vuser = format!("TASK:\n{task}\n\nWORKER OUTPUT:\n{out}");
|
||||||
let v_raw = crate::subscription::complete_or(
|
let v_raw = crate::subscription::complete_with_fallback(
|
||||||
runtime,
|
runtime,
|
||||||
&vsys,
|
&vsys,
|
||||||
&vuser,
|
&vuser,
|
||||||
@@ -207,6 +212,7 @@ pub async fn run_swarm_job(
|
|||||||
true,
|
true,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
.map(|(text, _)| text)
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
let v = extract_json(&v_raw);
|
let v = extract_json(&v_raw);
|
||||||
let passed = v
|
let passed = v
|
||||||
|
|||||||
Reference in New Issue
Block a user