judges use claude-opus-4-8; everything else stays on sonnet-4-6

Both LLM-as-judge sites — the door governor (Runtime::judge) and the topology
comparison scorer (JudgeScorer) — now use the judge model (default
claude-opus-4-8, override CLAWMATES_JUDGE_MODEL). Execution/reasoning turns keep
using the configured default model (claude-sonnet-4-6).

Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
Omar Sobh
2026-06-17 03:36:43 -07:00
co-authored by Claude Opus 4.8
parent e0e150084d
commit 325038e806
2 changed files with 19 additions and 5 deletions
+6 -2
View File
@@ -91,10 +91,14 @@ pub async fn compare_topologies(
Json(req): Json<CompareRequest>, Json(req): Json<CompareRequest>,
) -> Result<Json<Comparison>, ApiError> { ) -> Result<Json<Comparison>, ApiError> {
let provider = state.runtime.provider(); let provider = state.runtime.provider();
// Execution turns run on the configured default model (sonnet); the judge
// uses the stronger judge model (default claude-opus-4-8).
let model = state.runtime.model().to_string(); let model = state.runtime.model().to_string();
let judge_model =
std::env::var("CLAWMATES_JUDGE_MODEL").unwrap_or_else(|_| "claude-opus-4-8".to_string());
let executor = let executor =
ProviderExecutor::new(provider.clone(), model.clone(), state.runtime.max_tokens()); ProviderExecutor::new(provider.clone(), model, state.runtime.max_tokens());
let scorer = JudgeScorer::new(provider, model, 16); let scorer = JudgeScorer::new(provider, judge_model, 16);
let cmp = compare(&req.graphs, &req.task, &executor, &scorer) let cmp = compare(&req.graphs, &req.task, &executor, &scorer)
.await .await
.map_err(|_| ApiError::Internal)?; .map_err(|_| ApiError::Internal)?;
+13 -3
View File
@@ -26,6 +26,14 @@ use crate::tools::{ToolContext, ToolRegistry};
/// How long a pending approval stays decidable before it expires. /// How long a pending approval stays decidable before it expires.
const APPROVAL_TTL: time::Duration = time::Duration::hours(24); const APPROVAL_TTL: time::Duration = time::Duration::hours(24);
/// Model used for LLM-as-judge roles (the door governor and the topology
/// comparison scorer). Judges use the strongest model — default
/// `claude-opus-4-8` — while everything else runs on the configured default
/// model (`claude-sonnet-4-6`). Override with `CLAWMATES_JUDGE_MODEL`.
pub fn judge_model() -> String {
std::env::var("CLAWMATES_JUDGE_MODEL").unwrap_or_else(|_| "claude-opus-4-8".to_string())
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct RuntimeConfig { pub struct RuntimeConfig {
pub model: String, pub model: String,
@@ -235,21 +243,23 @@ impl Runtime {
} }
} }
/// Governor agent: ask the configured model to judge an action. Returns /// Governor agent: ask the **judge model** to judge an action. Returns
/// `(allow, reason)`. The verdict is the first token (`ALLOW`/`DENY`) of the /// `(allow, reason)`. The verdict is the first token (`ALLOW`/`DENY`) of the
/// model's reply. Best-effort and **fail-open** — if the model is /// model's reply. Best-effort and **fail-open** — if the model is
/// unreachable it returns `(true, …)` so a governor outage doesn't halt /// unreachable it returns `(true, …)` so a governor outage doesn't halt
/// autonomous agents (the governor is an extra soft check atop deterministic /// autonomous agents (the governor is an extra soft check atop deterministic
/// policy, not the only gate). /// policy, not the only gate). Judges use the strongest model
/// ([`judge_model`], default `claude-opus-4-8`); everything else runs on the
/// configured default model.
pub async fn judge(&self, system: &str, user: &str) -> (bool, String) { pub async fn judge(&self, system: &str, user: &str) -> (bool, String) {
let request = ChatRequest { let request = ChatRequest {
system: system.to_string(), system: system.to_string(),
model: judge_model(),
messages: vec![ChatMessage { messages: vec![ChatMessage {
role: ChatRole::User, role: ChatRole::User,
parts: vec![ContentPart::text(user)], parts: vec![ContentPart::text(user)],
}], }],
tools: vec![], tools: vec![],
model: self.inner.config.model.clone(),
max_tokens: 256, max_tokens: 256,
}; };
let mut text = String::new(); let mut text = String::new();