fix(planner): wait out a rate limit instead of failing the whole proposal

Moving the roster and planner onto the subscription removed the credit wall
and revealed the next one: the harness went from
`400 credit balance too low` to `429 rate_limit_error`. A one-shot proposal
call had no retry — there is no retry convention anywhere in cm-llm — so a
limit that clears in seconds killed the "propose a team" button outright.

Four attempts, 2/8/20s backoff, and only for errors that can actually clear:
429/5xx/transport. A 400, 401 or 404 returns immediately, because retrying
those is a 30s hang ending in the identical message, which reads to an
operator as a stall rather than a bad request.

Co-Authored-By: Claude Opus 5 <[email protected]>

crates/cm-api/src/subscription.rs

@@ -76,7 +76,42 @@ pub async fn complete_or(
     complete_with(&provider, system, user, model, max_tokens, web_search).await
 }
 
-/// Stream one request and collect its text.
+/// How long to wait before each retry. Four attempts, ~30s of patience total.
+///
+/// The subscription has no credit wall, but it does have a rate limit, and a
+/// roster proposal is a single one-shot call: a 429 that a browser would shrug
+/// off used to fail the whole "propose a team" button. Measured on this
+/// deployment — moving the roster onto the subscription turned
+/// `400 credit balance too low` into `429 rate_limit_error`, i.e. a wall that
+/// clears on its own became the failure mode, so waiting is the right answer.
+const BACKOFF_SECS: &[u64] = &[2, 8, 20];
+
+/// Whether an error is worth waiting out rather than reporting.
+///
+/// Deliberately narrow. A 400 (bad request), 401 (wrong token) or 404 (unknown
+/// model) will never succeed on a retry, and retrying them turns a legible
+/// error into a 30-second hang followed by the same error.
+fn is_transient(e: &cm_llm::LlmError) -> bool {
+    use cm_llm::LlmError;
+    match e {
+        // The transport never reached Anthropic — a dropped connection or a
+        // DNS blip, not a rejected request.
+        LlmError::Transport(_) => true,
+        LlmError::Api(detail) => {
+            // `anthropic.rs` formats these as `"{status}: {body}"`.
+            detail.starts_with("429")
+                || detail.starts_with("500")
+                || detail.starts_with("502")
+                || detail.starts_with("503")
+                || detail.starts_with("529")
+                || detail.contains("rate_limit")
+                || detail.contains("overloaded")
+        }
+        LlmError::Scenario(_) | LlmError::Wire(_) => false,
+    }
+}
+
+/// Stream one request and collect its text, waiting out transient failures.
 async fn complete_with(
     provider: &cm_llm::AnthropicProvider,
     system: &str,
@@ -85,6 +120,38 @@ async fn complete_with(
     max_tokens: u32,
     web_search: bool,
 ) -> Result<String, String> {
+    let mut attempt = 0usize;
+    loop {
+        match attempt_once(provider, system, user, model, max_tokens, web_search).await {
+            Ok(text) => return Ok(text),
+            Err((stage, e)) => {
+                let Some(delay) = BACKOFF_SECS.get(attempt).copied().filter(|_| is_transient(&e))
+                else {
+                    return Err(format!("subscription {stage}: {e}"));
+                };
+                eprintln!(
+                    "subscription {stage}: {e} — retrying in {delay}s \
+                     (attempt {} of {})",
+                    attempt + 2,
+                    BACKOFF_SECS.len() + 1
+                );
+                tokio::time::sleep(std::time::Duration::from_secs(delay)).await;
+                attempt += 1;
+            }
+        }
+    }
+}
+
+/// One attempt. The collected text is discarded on failure, so a retry never
+/// concatenates a partial answer onto a whole one.
+async fn attempt_once(
+    provider: &cm_llm::AnthropicProvider,
+    system: &str,
+    user: &str,
+    model: &str,
+    max_tokens: u32,
+    web_search: bool,
+) -> Result<String, (&'static str, cm_llm::LlmError)> {
     use cm_llm::{ChatMessage, ChatRequest, ChatRole, ContentPart, LlmEvent, LlmProvider};
     use futures::StreamExt as _;
 
@@ -99,16 +166,13 @@ async fn complete_with(
         max_tokens,
         web_search,
     };
-    let mut stream = provider
+    let mut stream = provider.stream(request).await.map_err(|e| ("call", e))?;
-        .stream(request)
-        .await
-        .map_err(|e| format!("subscription call: {e}"))?;
     let mut text = String::new();
     while let Some(event) = stream.next().await {
         match event {
             Ok(LlmEvent::TextDelta(t)) => text.push_str(&t),
             Ok(_) => {}
-            Err(e) => return Err(format!("subscription stream: {e}")),
+            Err(e) => return Err(("stream", e)),
         }
     }
     Ok(text)
@@ -146,6 +210,33 @@ mod tests {
         );
     }
 
+    /// Only errors that can clear on their own are waited out.
+    ///
+    /// The negative half is the point: a 400 or a 401 retried three times is a
+    /// 30-second hang ending in the identical message, which reads as a stall
+    /// rather than a bad request — the failure mode this project keeps hitting.
+    #[test]
+    fn a_wall_that_clears_is_waited_out_and_one_that_does_not_is_not() {
+        use cm_llm::LlmError;
+        let api = |s: &str| LlmError::Api(s.to_string());
+
+        assert!(is_transient(&api(
+            "429 Too Many Requests: {\"type\":\"rate_limit_error\"}"
+        )));
+        assert!(is_transient(&api("529: overloaded_error")));
+        assert!(is_transient(&api("503 Service Unavailable")));
+        assert!(is_transient(&LlmError::Transport("connection reset".into())));
+
+        // The exact error that started this: it never clears by waiting, it
+        // clears by moving to the other credential — which is now done.
+        assert!(!is_transient(&api(
+            "400 Bad Request: Your credit balance is too low"
+        )));
+        assert!(!is_transient(&api("401 Unauthorized: invalid x-api-key")));
+        assert!(!is_transient(&api("404 Not Found: model not found")));
+        assert!(!is_transient(&LlmError::Wire("bad json".into())));
+    }
+
     /// A metered key in the OAuth slot must be REFUSED, not used.
     ///
     /// Accepting it would authenticate, work, and bill the pay-as-you-go account