refactor: no feature depends on Gemini any more
Depleted Gemini prepayment credits took out PDF rendering. The same key was the
only thing standing between level-up proposals and the same fate, so both are
off it.
- `pdf_renderer` is DELETED, not disabled. Nothing sets `render_pdf: true` since
markdown became the deliverable (821cbb8), so the worker polled forever for
rows that can no longer exist. It was also the only caller of the Gemini
MD->HTML conversion. A worker that cannot do anything is worse than absent: it
reads as a feature.
- `level_up` now resolves its proposer through the provider REGISTRY
(`Runtime::resolve_provider`), the same path the evaluator uses, defaulting to
`glm:glm-4.7` — the validator this project measured and chose in
scripts/judge-eval.sh. `CLAWMATES_LEVEL_UP_MODEL` takes a registry spec
(`glm:glm-4.7`, `kimi:k2`, `claude-sonnet-5`), so every provider the platform
can already reach works and no single vendor's billing can take it down.
The non-obvious part of that swap: Gemini was asked for
`response_mime_type: application/json` and obliged, so the old code parsed the
raw reply. Anthropic-format models are under no such obligation and wrap objects
in prose or a ```json fence. `extract_json_object` brace-counts to the matching
close — string-aware, so a `}` inside a value does not end it, and nested (these
proposals nest by design). Tested against bare, fenced, nested, brace-in-string
and absent. Parsing raw text would have worked in review and failed on the first
real proposal.
What deliberately still MENTIONS Gemini: `mission_runtime` forwards
GEMINI_API_KEY to agent containers alongside GROQ/OPENAI/ZAI/KIMI, and the claw
model selector offers it. Those are user options, not platform requirements —
the ask was to remove the NEED.
Also corrected a comment in mission_delivery that cited `pdf_renderer` as the
authority on artifact path resolution. It never was: it joined the mission id
first and produced a doubled path that never resolved.
238 lib tests, 20 test binaries.
This commit is contained in:
@@ -346,12 +346,6 @@ async fn run() -> Result<(), String> {
|
|||||||
// asks Claude Opus 4.8 to synthesize a "what got done" card that
|
// asks Claude Opus 4.8 to synthesize a "what got done" card that
|
||||||
// the UI renders under the phase.
|
// the UI renders under the phase.
|
||||||
cm_api::phase_summarizer::spawn(pool.clone());
|
cm_api::phase_summarizer::spawn(pool.clone());
|
||||||
// PDF renderer worker (Slice 6): watches mission_artifacts for
|
|
||||||
// MD entries with render_pdf_status='pending', calls the
|
|
||||||
// configured LLM (default Gemini 2.5 Flash) for styled HTML,
|
|
||||||
// prints to PDF via chromium --headless. No-op-friendly when
|
|
||||||
// GEMINI_API_KEY / chromium binary aren't configured.
|
|
||||||
cm_api::pdf_renderer::spawn(pool.clone());
|
|
||||||
// Outbound-email delivery: drains the §15-gated `outbox` over SMTP. Inert
|
// Outbound-email delivery: drains the §15-gated `outbox` over SMTP. Inert
|
||||||
// until CLAWMATES_SMTP_* is set, so it ships safely before credentials exist.
|
// until CLAWMATES_SMTP_* is set, so it ships safely before credentials exist.
|
||||||
cm_runtime::spawn_drainer(pool.clone(), std::time::Duration::from_secs(10));
|
cm_runtime::spawn_drainer(pool.clone(), std::time::Duration::from_secs(10));
|
||||||
|
|||||||
+147
-44
@@ -13,16 +13,28 @@
|
|||||||
//! reviewer picked. Rejected proposals move to status='rejected';
|
//! reviewer picked. Rejected proposals move to status='rejected';
|
||||||
//! partial approvals move to status='partial'.
|
//! partial approvals move to status='partial'.
|
||||||
//!
|
//!
|
||||||
//! Uses Gemini 2.5 Flash as the default proposer model — cheap,
|
//! The proposer model resolves through the provider REGISTRY
|
||||||
//! JSON-mode-native, plenty of room for structured output. Configurable
|
//! (`Runtime::resolve_provider`), the same path the evaluator uses, and defaults
|
||||||
//! via CLAWMATES_LEVEL_UP_MODEL.
|
//! to `glm:glm-4.7`. Configurable via `CLAWMATES_LEVEL_UP_MODEL` as a registry
|
||||||
|
//! spec (`glm:glm-4.7`, `kimi:k2`, `claude-sonnet-5`, …).
|
||||||
|
//!
|
||||||
|
//! It used to call Gemini directly over bespoke HTTP with `GEMINI_API_KEY`. Two
|
||||||
|
//! problems with that, one fatal: it was the only thing standing between this
|
||||||
|
//! feature and a dead prepayment balance, and it duplicated a provider client
|
||||||
|
//! the codebase already has. Going through the registry means every provider the
|
||||||
|
//! platform can already reach works here, and no single vendor's billing can
|
||||||
|
//! take the feature down.
|
||||||
|
|
||||||
use serde_json::{json, Value};
|
use serde_json::{json, Value};
|
||||||
use sqlx::PgPool;
|
use sqlx::PgPool;
|
||||||
use sqlx::Row;
|
use sqlx::Row;
|
||||||
use uuid::Uuid;
|
use uuid::Uuid;
|
||||||
|
|
||||||
const DEFAULT_MODEL: &str = "gemini-2.5-flash";
|
/// Registry spec, not a bare model name — the registry needs the provider.
|
||||||
|
///
|
||||||
|
/// GLM: cheap, reliable at structured output, and already the validator this
|
||||||
|
/// project measured and chose (see `scripts/judge-eval.sh`).
|
||||||
|
const DEFAULT_MODEL: &str = "glm:glm-4.7";
|
||||||
|
|
||||||
fn model_name() -> String {
|
fn model_name() -> String {
|
||||||
std::env::var("CLAWMATES_LEVEL_UP_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
std::env::var("CLAWMATES_LEVEL_UP_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
||||||
@@ -31,6 +43,7 @@ fn model_name() -> String {
|
|||||||
/// Analyze an agent + insert a pending proposal. Returns the proposal id.
|
/// Analyze an agent + insert a pending proposal. Returns the proposal id.
|
||||||
pub async fn propose_agent(
|
pub async fn propose_agent(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
workspace_id: cm_domain::WorkspaceId,
|
workspace_id: cm_domain::WorkspaceId,
|
||||||
created_by: cm_domain::UserId,
|
created_by: cm_domain::UserId,
|
||||||
agent_id: Uuid,
|
agent_id: Uuid,
|
||||||
@@ -50,6 +63,7 @@ pub async fn propose_agent(
|
|||||||
.flatten();
|
.flatten();
|
||||||
|
|
||||||
let payload = call_llm_for_agent(
|
let payload = call_llm_for_agent(
|
||||||
|
runtime,
|
||||||
&agent.name,
|
&agent.name,
|
||||||
&agent.job_title,
|
&agent.job_title,
|
||||||
&agent.system_prompt,
|
&agent.system_prompt,
|
||||||
@@ -79,6 +93,7 @@ pub async fn propose_agent(
|
|||||||
/// Analyze a team + insert a pending proposal. Returns the proposal id.
|
/// Analyze a team + insert a pending proposal. Returns the proposal id.
|
||||||
pub async fn propose_team(
|
pub async fn propose_team(
|
||||||
pool: &PgPool,
|
pool: &PgPool,
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
workspace_id: cm_domain::WorkspaceId,
|
workspace_id: cm_domain::WorkspaceId,
|
||||||
created_by: cm_domain::UserId,
|
created_by: cm_domain::UserId,
|
||||||
team_id: Uuid,
|
team_id: Uuid,
|
||||||
@@ -115,7 +130,7 @@ pub async fn propose_team(
|
|||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
let payload = call_llm_for_team(&member_summaries).await?;
|
let payload = call_llm_for_team(runtime, &member_summaries).await?;
|
||||||
|
|
||||||
let model = model_name();
|
let model = model_name();
|
||||||
let id = cm_db::repo::level_up::insert(
|
let id = cm_db::repo::level_up::insert(
|
||||||
@@ -418,6 +433,7 @@ async fn recent_run_summary(pool: &PgPool, agent_id: Uuid, limit: i64) -> Result
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn call_llm_for_agent(
|
async fn call_llm_for_agent(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
name: &str,
|
name: &str,
|
||||||
role: &str,
|
role: &str,
|
||||||
system_prompt: &str,
|
system_prompt: &str,
|
||||||
@@ -462,10 +478,13 @@ the sake of proposing."#;
|
|||||||
})
|
})
|
||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
call_gemini_json(system, &user).await
|
call_llm_json(runtime, system, &user).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn call_llm_for_team(members: &[Value]) -> Result<Value, String> {
|
async fn call_llm_for_team(
|
||||||
|
runtime: &cm_runtime::Runtime,
|
||||||
|
members: &[Value],
|
||||||
|
) -> Result<Value, String> {
|
||||||
let system = r#"You review an AI team's roster + recent history and propose
|
let system = r#"You review an AI team's roster + recent history and propose
|
||||||
targeted improvements. Return ONLY JSON:
|
targeted improvements. Return ONLY JSON:
|
||||||
{
|
{
|
||||||
@@ -482,47 +501,90 @@ prompts over adding skills. Only add skills when a clear
|
|||||||
"the team keeps getting stuck on <X>" pattern appears."#;
|
"the team keeps getting stuck on <X>" pattern appears."#;
|
||||||
|
|
||||||
let user = json!({ "members": members }).to_string();
|
let user = json!({ "members": members }).to_string();
|
||||||
call_gemini_json(system, &user).await
|
call_llm_json(runtime, system, &user).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn call_gemini_json(system: &str, user: &str) -> Result<Value, String> {
|
/// Ask the configured proposer model for one JSON object.
|
||||||
let api_key =
|
///
|
||||||
std::env::var("GEMINI_API_KEY").map_err(|_| "GEMINI_API_KEY unset".to_string())?;
|
/// Goes through the provider registry rather than a vendor's HTTP API, so any
|
||||||
let model = model_name();
|
/// model the platform can already reach works and no single vendor's billing can
|
||||||
let url = format!(
|
/// take level-up down.
|
||||||
"https://generativelanguage.googleapis.com/v1beta/models/{}:generateContent?key={}",
|
///
|
||||||
model, api_key
|
/// The JSON is extracted rather than assumed: an anthropic-format model is not
|
||||||
);
|
/// bound by Gemini's `response_mime_type: application/json`, and will happily
|
||||||
let body = json!({
|
/// wrap an object in prose or a ```json fence. Parsing the raw reply worked
|
||||||
"system_instruction": { "parts": [{ "text": system }] },
|
/// against Gemini and would fail on everything else.
|
||||||
"contents": [{ "role": "user", "parts": [{ "text": user }] }],
|
async fn call_llm_json(
|
||||||
"generationConfig": {
|
runtime: &cm_runtime::Runtime,
|
||||||
"temperature": 0.2,
|
system: &str,
|
||||||
"response_mime_type": "application/json",
|
user: &str,
|
||||||
"maxOutputTokens": 8192,
|
) -> Result<Value, String> {
|
||||||
}
|
use cm_llm::{ChatMessage, ChatRequest, ChatRole, ContentPart, LlmEvent};
|
||||||
});
|
use futures::StreamExt as _;
|
||||||
let client = reqwest::Client::builder()
|
|
||||||
.timeout(std::time::Duration::from_secs(60))
|
let spec = model_name();
|
||||||
.build()
|
let (provider, model) = runtime.resolve_provider(&spec);
|
||||||
.map_err(|e| format!("http client: {e}"))?;
|
let request = ChatRequest {
|
||||||
let resp = client
|
system: system.to_string(),
|
||||||
.post(&url)
|
model: model.to_string(),
|
||||||
.json(&body)
|
messages: vec![ChatMessage {
|
||||||
.send()
|
role: ChatRole::User,
|
||||||
|
parts: vec![ContentPart::text(user)],
|
||||||
|
}],
|
||||||
|
tools: vec![],
|
||||||
|
max_tokens: 8192,
|
||||||
|
web_search: false,
|
||||||
|
};
|
||||||
|
let mut stream = provider
|
||||||
|
.stream(request)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("gemini call: {e}"))?;
|
.map_err(|e| format!("level-up call ({spec}): {e}"))?;
|
||||||
if !resp.status().is_success() {
|
let mut text = String::new();
|
||||||
let code = resp.status();
|
while let Some(event) = stream.next().await {
|
||||||
let body = resp.text().await.unwrap_or_default();
|
match event {
|
||||||
return Err(format!("gemini {code}: {}", &body[..body.len().min(500)]));
|
Ok(LlmEvent::TextDelta(t)) => text.push_str(&t),
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(e) => return Err(format!("level-up stream ({spec}): {e}")),
|
||||||
}
|
}
|
||||||
let json: Value = resp.json().await.map_err(|e| format!("gemini json: {e}"))?;
|
}
|
||||||
let text = json
|
let body = extract_json_object(&text)
|
||||||
.pointer("/candidates/0/content/parts/0/text")
|
.ok_or_else(|| format!("no JSON object in {spec} reply: {}", excerpt(&text, 300)))?;
|
||||||
.and_then(|v| v.as_str())
|
serde_json::from_str(body).map_err(|e| format!("parse suggestion json: {e}"))
|
||||||
.ok_or_else(|| "gemini response missing text".to_string())?;
|
}
|
||||||
serde_json::from_str(text).map_err(|e| format!("parse suggestion json: {e}"))
|
|
||||||
|
/// The outermost `{...}` in a reply, so a fenced or prose-wrapped object parses.
|
||||||
|
///
|
||||||
|
/// Brace-counting rather than a regex: a nested object would end a lazy match at
|
||||||
|
/// the first inner `}`, and these proposals are nested by design (items carry
|
||||||
|
/// per-role objects).
|
||||||
|
fn extract_json_object(text: &str) -> Option<&str> {
|
||||||
|
let start = text.find('{')?;
|
||||||
|
let mut depth = 0usize;
|
||||||
|
let mut in_string = false;
|
||||||
|
let mut escaped = false;
|
||||||
|
for (i, c) in text[start..].char_indices() {
|
||||||
|
if in_string {
|
||||||
|
match c {
|
||||||
|
_ if escaped => escaped = false,
|
||||||
|
'\\' => escaped = true,
|
||||||
|
'"' => in_string = false,
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
match c {
|
||||||
|
'"' => in_string = true,
|
||||||
|
'{' => depth += 1,
|
||||||
|
'}' => {
|
||||||
|
depth -= 1;
|
||||||
|
if depth == 0 {
|
||||||
|
return Some(&text[start..start + i + 1]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
fn excerpt(s: &str, max: usize) -> String {
|
fn excerpt(s: &str, max: usize) -> String {
|
||||||
@@ -546,3 +608,44 @@ fn workspace_skill_id(workspace_id: Uuid, name: &str) -> Uuid {
|
|||||||
bytes[8] = (bytes[8] & 0x3f) | 0x80;
|
bytes[8] = (bytes[8] & 0x3f) | 0x80;
|
||||||
Uuid::from_bytes(bytes)
|
Uuid::from_bytes(bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
/// Gemini was asked for `response_mime_type: application/json` and obliged.
|
||||||
|
/// Anthropic-format models are under no such obligation and routinely wrap
|
||||||
|
/// the object in prose or a fenced block, so the reply is EXTRACTED, not
|
||||||
|
/// assumed. Parsing the raw text worked against Gemini and would fail
|
||||||
|
/// everywhere else — exactly the shape of bug a provider swap hides until
|
||||||
|
/// the first real proposal.
|
||||||
|
#[test]
|
||||||
|
fn a_json_object_is_extracted_from_however_the_model_wrapped_it() {
|
||||||
|
let bare = r#"{"items":[]}"#;
|
||||||
|
assert_eq!(super::extract_json_object(bare), Some(bare));
|
||||||
|
|
||||||
|
let fenced = "Here is my proposal:\n```json\n{\"items\":[1]}\n```\nDone.";
|
||||||
|
assert_eq!(super::extract_json_object(fenced), Some(r#"{"items":[1]}"#));
|
||||||
|
|
||||||
|
// Nested objects: a lazy match would stop at the first inner brace and
|
||||||
|
// hand back invalid JSON. These proposals are nested by design.
|
||||||
|
let nested = r#"prose {"a":{"b":{"c":1}},"d":2} trailing"#;
|
||||||
|
assert_eq!(
|
||||||
|
super::extract_json_object(nested),
|
||||||
|
Some(r#"{"a":{"b":{"c":1}},"d":2}"#)
|
||||||
|
);
|
||||||
|
|
||||||
|
// A brace inside a string must not close the object.
|
||||||
|
let stringy = r#"{"note":"an unmatched } here","ok":true}"#;
|
||||||
|
assert_eq!(super::extract_json_object(stringy), Some(stringy));
|
||||||
|
|
||||||
|
assert_eq!(super::extract_json_object("no object here"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The default must not be a vendor whose billing already took a feature
|
||||||
|
/// down. It is a REGISTRY SPEC (`provider:model`), not a bare model name —
|
||||||
|
/// `resolve_provider` needs the provider half.
|
||||||
|
#[test]
|
||||||
|
fn the_default_proposer_is_a_registry_spec_and_not_gemini() {
|
||||||
|
assert!(super::DEFAULT_MODEL.contains(':'), "{}", super::DEFAULT_MODEL);
|
||||||
|
assert!(!super::DEFAULT_MODEL.contains("gemini"), "{}", super::DEFAULT_MODEL);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -38,7 +38,6 @@ pub mod mission_roster;
|
|||||||
pub mod mission_runtime;
|
pub mod mission_runtime;
|
||||||
pub mod mission_workspace;
|
pub mod mission_workspace;
|
||||||
pub mod node_rules;
|
pub mod node_rules;
|
||||||
pub mod pdf_renderer;
|
|
||||||
pub mod phase_runner;
|
pub mod phase_runner;
|
||||||
pub mod phase_summarizer;
|
pub mod phase_summarizer;
|
||||||
pub mod quota;
|
pub mod quota;
|
||||||
|
|||||||
@@ -443,8 +443,11 @@ pub async fn capture_phase_diff_at(
|
|||||||
)
|
)
|
||||||
.map_err(|e| format!("write delivery.json: {e}"))?;
|
.map_err(|e| format!("write delivery.json: {e}"))?;
|
||||||
|
|
||||||
// Path is stored relative to the missions root, matching how
|
// Path is stored relative to the MISSIONS ROOT — the convention every
|
||||||
// `pdf_renderer` resolves artifact paths.
|
// artifact uses, and what `routes::missions::artifact_content` resolves
|
||||||
|
// against. (The old `pdf_renderer` claimed to match this and did not: it
|
||||||
|
// joined the mission id first, producing a doubled id and ENOENT. It is
|
||||||
|
// gone; this comment named it as the authority, which it never was.)
|
||||||
let rel = format!("_outputs/{mission_id}/{phase_id}/diff.patch");
|
let rel = format!("_outputs/{mission_id}/{phase_id}/diff.patch");
|
||||||
cm_db::repo::missions::register_artifact(
|
cm_db::repo::missions::register_artifact(
|
||||||
pool,
|
pool,
|
||||||
|
|||||||
@@ -1,290 +0,0 @@
|
|||||||
//! LLM + Chromium PDF renderer worker — Slice 6.
|
|
||||||
//!
|
|
||||||
//! Watches `mission_artifacts` for rows with `render_pdf_status =
|
|
||||||
//! 'pending'`. For each:
|
|
||||||
//! 1. Read the source MD from `<mission_root>/<path>` on disk
|
|
||||||
//! 2. Call the configured LLM (default: Gemini 2.5 Flash) with a
|
|
||||||
//! "produce styled HTML" prompt anchored to a design-system
|
|
||||||
//! example. LLM writes HTML with inline CSS.
|
|
||||||
//! 3. Print that HTML to PDF via `chromium --headless
|
|
||||||
//! --print-to-pdf`
|
|
||||||
//! 4. Save the PDF alongside the MD, update `rendered_pdf_path` +
|
|
||||||
//! status = 'done'
|
|
||||||
//!
|
|
||||||
//! Graceful degradation: if `GEMINI_API_KEY` is unset or the
|
|
||||||
//! chromium binary isn't on PATH, the worker marks the row `failed`
|
|
||||||
//! with a descriptive error rather than blocking boot. Ops enables
|
|
||||||
//! rendering by wiring both.
|
|
||||||
//!
|
|
||||||
//! The frontend already renders `rendered_pdf_path` as an "Open PDF"
|
|
||||||
//! button on artifact cards (Slice 2).
|
|
||||||
|
|
||||||
use serde_json::json;
|
|
||||||
use sqlx::PgPool;
|
|
||||||
use std::path::{Path, PathBuf};
|
|
||||||
use std::time::Duration;
|
|
||||||
|
|
||||||
const POLL_INTERVAL: Duration = Duration::from_secs(30);
|
|
||||||
const MAX_PARALLEL: usize = 2;
|
|
||||||
const DEFAULT_MODEL: &str = "gemini-2.5-flash";
|
|
||||||
|
|
||||||
/// Where per-mission artifacts land on disk. Overridable so dev vs.
|
|
||||||
/// prod can move the tree; matches the pattern in
|
|
||||||
/// `research_container::research_workspace_root`.
|
|
||||||
fn missions_root() -> PathBuf {
|
|
||||||
std::env::var("CLAWMATES_MISSIONS_ROOT")
|
|
||||||
.map(PathBuf::from)
|
|
||||||
.unwrap_or_else(|_| PathBuf::from("/var/lib/clawmates-missions"))
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Resolve `mission_artifacts.path` to a file on disk.
|
|
||||||
///
|
|
||||||
/// Artifact paths are relative to the MISSIONS ROOT, not to the per-mission
|
|
||||||
/// directory: every registration site writes `_outputs/<mission>/<phase>/...`,
|
|
||||||
/// and `_outputs` is deliberately a sibling of the per-mission dirs so it
|
|
||||||
/// survives their reaping.
|
|
||||||
///
|
|
||||||
/// This used to join `missions_root()/<mission_id>/` first, producing
|
|
||||||
/// `<root>/<mission>/_outputs/<mission>/<phase>/...` — the mission id twice and
|
|
||||||
/// no such file. It went unnoticed because the only artifacts that existed were
|
|
||||||
/// `code_diff` rows registered with `render_pdf: false`, which this worker never
|
|
||||||
/// reads. The first artifacts to ask for rendering were the first to find it.
|
|
||||||
fn artifact_abs(rel: &str) -> PathBuf {
|
|
||||||
missions_root().join(rel)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn chromium_bin() -> String {
|
|
||||||
std::env::var("CHROMIUM_BIN").unwrap_or_else(|_| "chromium".to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn renderer_model() -> String {
|
|
||||||
std::env::var("CLAWMATES_PDF_RENDERER_MODEL").unwrap_or_else(|_| DEFAULT_MODEL.to_string())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Spawn the poller. No-op-friendly: if there's nothing pending or
|
|
||||||
/// no rendering pipeline configured, we still tick + observe.
|
|
||||||
pub fn spawn(pool: PgPool) {
|
|
||||||
tokio::spawn(async move {
|
|
||||||
// Small startup delay so migrations + loaders finish first.
|
|
||||||
tokio::time::sleep(Duration::from_secs(8)).await;
|
|
||||||
let mut ticker = tokio::time::interval(POLL_INTERVAL);
|
|
||||||
ticker.tick().await;
|
|
||||||
loop {
|
|
||||||
ticker.tick().await;
|
|
||||||
if let Err(e) = sweep_once(&pool).await {
|
|
||||||
eprintln!("pdf_renderer: sweep failed: {e}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn sweep_once(pool: &PgPool) -> Result<(), String> {
|
|
||||||
let pending = cm_db::repo::missions::next_pdf_pending(pool, MAX_PARALLEL as i64)
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("next_pdf_pending: {e}"))?;
|
|
||||||
for artifact in pending {
|
|
||||||
let pool = pool.clone();
|
|
||||||
let id = artifact.id;
|
|
||||||
tokio::spawn(async move {
|
|
||||||
match render_one(&pool, &artifact).await {
|
|
||||||
Ok(pdf_path) => {
|
|
||||||
let _ = cm_db::repo::missions::set_pdf_result(&pool, id, Some(&pdf_path), None)
|
|
||||||
.await;
|
|
||||||
eprintln!("pdf_renderer: rendered {id} → {pdf_path}");
|
|
||||||
}
|
|
||||||
Err(e) => {
|
|
||||||
let _ = cm_db::repo::missions::set_pdf_result(&pool, id, None, Some(&e)).await;
|
|
||||||
eprintln!("pdf_renderer: {id} failed: {e}");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn render_one(
|
|
||||||
_pool: &PgPool,
|
|
||||||
artifact: &cm_db::repo::missions::MissionArtifact,
|
|
||||||
) -> Result<String, String> {
|
|
||||||
// 1. Locate the source MD on disk.
|
|
||||||
let src_path = artifact_abs(&artifact.path);
|
|
||||||
let md = tokio::fs::read_to_string(&src_path)
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("read {}: {e}", src_path.display()))?;
|
|
||||||
|
|
||||||
// 2. LLM → styled HTML.
|
|
||||||
let html = md_to_html_via_llm(&md, artifact.title.as_deref())
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("llm render: {e}"))?;
|
|
||||||
|
|
||||||
// 3. Chromium → PDF.
|
|
||||||
let tmp = tempdir_for(artifact.id)?;
|
|
||||||
let html_path = tmp.join("in.html");
|
|
||||||
let pdf_path = tmp.join("out.pdf");
|
|
||||||
tokio::fs::write(&html_path, html)
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("write {}: {e}", html_path.display()))?;
|
|
||||||
|
|
||||||
let status = tokio::process::Command::new(chromium_bin())
|
|
||||||
.args([
|
|
||||||
"--headless=new",
|
|
||||||
"--disable-gpu",
|
|
||||||
"--no-sandbox",
|
|
||||||
"--hide-scrollbars",
|
|
||||||
&format!("--print-to-pdf={}", pdf_path.display()),
|
|
||||||
"--print-to-pdf-no-header",
|
|
||||||
"--virtual-time-budget=10000",
|
|
||||||
&format!("file://{}", html_path.display()),
|
|
||||||
])
|
|
||||||
.stderr(std::process::Stdio::piped())
|
|
||||||
.stdout(std::process::Stdio::piped())
|
|
||||||
.status()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("spawn chromium: {e}"))?;
|
|
||||||
if !status.success() {
|
|
||||||
return Err(format!("chromium exited {status}"));
|
|
||||||
}
|
|
||||||
|
|
||||||
// 4. Move next to the source MD so the artifact tree stays self-
|
|
||||||
// contained. Filename derived from the MD path (foo.md → foo.pdf).
|
|
||||||
let out_rel = pdf_sibling(&artifact.path);
|
|
||||||
let out_abs = artifact_abs(&out_rel);
|
|
||||||
if let Some(parent) = out_abs.parent() {
|
|
||||||
tokio::fs::create_dir_all(parent)
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("mkdir {}: {e}", parent.display()))?;
|
|
||||||
}
|
|
||||||
tokio::fs::copy(&pdf_path, &out_abs)
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("copy pdf: {e}"))?;
|
|
||||||
// Best-effort tmp cleanup — the temp dir lives under /tmp so the
|
|
||||||
// OS will reap it anyway.
|
|
||||||
let _ = tokio::fs::remove_dir_all(&tmp).await;
|
|
||||||
Ok(out_rel)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Ask the configured LLM to turn `md` into a fully self-contained
|
|
||||||
/// styled HTML doc. Uses whichever provider `CLAWMATES_PDF_RENDERER_MODEL`
|
|
||||||
/// resolves to. Defaults to Gemini 2.5 Flash + GEMINI_API_KEY.
|
|
||||||
async fn md_to_html_via_llm(md: &str, title: Option<&str>) -> Result<String, String> {
|
|
||||||
let model = renderer_model();
|
|
||||||
// For now we hardcode the Gemini path — anthropic + openai
|
|
||||||
// variants land when the design-system template stabilizes.
|
|
||||||
if !model.starts_with("gemini") {
|
|
||||||
return Err(format!(
|
|
||||||
"renderer model {model} not yet wired (only gemini-* supported in Slice 6)"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let api_key =
|
|
||||||
std::env::var("GEMINI_API_KEY").map_err(|_| "GEMINI_API_KEY unset".to_string())?;
|
|
||||||
|
|
||||||
let system = r#"You are a document typesetter. Given a Markdown source,
|
|
||||||
produce ONE self-contained HTML document that:
|
|
||||||
- Has ALL styles inline in a single <style> block in <head>. No external
|
|
||||||
fonts, no external CSS. System font stack only.
|
|
||||||
- Uses a clean, modern, readable serif for body copy (Georgia / "Iowan Old
|
|
||||||
Style" / "Charter" / serif) and a sans for headings.
|
|
||||||
- Uses ONLY these accent colors: #ff8a7a (heading), #5ec8d8 (link),
|
|
||||||
#101014 (body text), #f7f7f8 (page bg).
|
|
||||||
- Renders code blocks with a monospace stack and a subtle background.
|
|
||||||
- Uses page-break-inside: avoid on headings and images.
|
|
||||||
- Puts a document title in an <h1> at the top if provided.
|
|
||||||
- Includes NOTHING outside the HTML — no ```html fence, no commentary."#;
|
|
||||||
|
|
||||||
let prompt = match title {
|
|
||||||
Some(t) => format!("Document title: {t}\n\nMarkdown:\n\n{md}"),
|
|
||||||
None => md.to_string(),
|
|
||||||
};
|
|
||||||
|
|
||||||
let url = format!(
|
|
||||||
"https://generativelanguage.googleapis.com/v1beta/models/{}:generateContent?key={}",
|
|
||||||
model, api_key
|
|
||||||
);
|
|
||||||
let body = json!({
|
|
||||||
"system_instruction": { "parts": [{ "text": system }] },
|
|
||||||
"contents": [{ "role": "user", "parts": [{ "text": prompt }] }],
|
|
||||||
"generationConfig": {
|
|
||||||
"temperature": 0.2,
|
|
||||||
"maxOutputTokens": 32000,
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
let client = reqwest::Client::builder()
|
|
||||||
.timeout(Duration::from_secs(120))
|
|
||||||
.build()
|
|
||||||
.map_err(|e| format!("http client: {e}"))?;
|
|
||||||
let resp = client
|
|
||||||
.post(&url)
|
|
||||||
.json(&body)
|
|
||||||
.send()
|
|
||||||
.await
|
|
||||||
.map_err(|e| format!("gemini call: {e}"))?;
|
|
||||||
if !resp.status().is_success() {
|
|
||||||
let code = resp.status();
|
|
||||||
let body = resp.text().await.unwrap_or_default();
|
|
||||||
return Err(format!("gemini {code}: {}", &body[..body.len().min(500)]));
|
|
||||||
}
|
|
||||||
let json: serde_json::Value = resp.json().await.map_err(|e| format!("gemini json: {e}"))?;
|
|
||||||
let text = json
|
|
||||||
.pointer("/candidates/0/content/parts/0/text")
|
|
||||||
.and_then(|v| v.as_str())
|
|
||||||
.ok_or_else(|| "gemini response missing text".to_string())?;
|
|
||||||
// Strip a stray ```html fence if the model added one despite the
|
|
||||||
// system prompt — cheap belt to the suspenders.
|
|
||||||
let cleaned = text
|
|
||||||
.trim()
|
|
||||||
.strip_prefix("```html")
|
|
||||||
.and_then(|s| s.strip_suffix("```"))
|
|
||||||
.map(|s| s.trim())
|
|
||||||
.unwrap_or(text.trim())
|
|
||||||
.to_string();
|
|
||||||
Ok(cleaned)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn tempdir_for(id: uuid::Uuid) -> Result<PathBuf, String> {
|
|
||||||
let dir = std::env::temp_dir().join(format!("clawmates-pdf-{id}"));
|
|
||||||
std::fs::create_dir_all(&dir).map_err(|e| format!("mkdir tmp: {e}"))?;
|
|
||||||
Ok(dir)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// `research/v3/spec.md` → `research/v3/spec.pdf`.
|
|
||||||
/// `foo/bar/without_ext` → `foo/bar/without_ext.pdf` (rare — parser
|
|
||||||
/// never emits an extension-less MD, but we're defensive).
|
|
||||||
fn pdf_sibling(md_path: &str) -> String {
|
|
||||||
let p = Path::new(md_path);
|
|
||||||
let stem = p.file_stem().and_then(|s| s.to_str()).unwrap_or("output");
|
|
||||||
let parent = p.parent().map(|x| x.to_string_lossy().to_string());
|
|
||||||
let base = format!("{stem}.pdf");
|
|
||||||
match parent {
|
|
||||||
Some(pp) if !pp.is_empty() => format!("{pp}/{base}"),
|
|
||||||
_ => base,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
mod tests {
|
|
||||||
use super::*;
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn an_artifact_path_resolves_against_the_missions_root() {
|
|
||||||
// Exactly what `mission_delivery` and `mission_outputs` register.
|
|
||||||
let rel = "_outputs/11111111-1111-1111-1111-111111111111/22222222-2222-2222-2222-222222222222/repo/research/01_summary.md";
|
|
||||||
let abs = artifact_abs(rel);
|
|
||||||
let root = missions_root();
|
|
||||||
assert_eq!(abs, root.join(rel), "{abs:?}");
|
|
||||||
// The regression this exists for: the mission id must appear ONCE.
|
|
||||||
let s = abs.to_string_lossy();
|
|
||||||
assert_eq!(
|
|
||||||
s.matches("11111111-1111-1111-1111-111111111111").count(),
|
|
||||||
1,
|
|
||||||
"the mission id must not be doubled: {s}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn pdf_sibling_paths() {
|
|
||||||
assert_eq!(pdf_sibling("research/v3/spec.md"), "research/v3/spec.pdf");
|
|
||||||
assert_eq!(pdf_sibling("spec.md"), "spec.pdf");
|
|
||||||
assert_eq!(pdf_sibling("no_ext"), "no_ext.pdf");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -36,7 +36,7 @@ pub async fn propose_for_agent(
|
|||||||
Authed(user): Authed,
|
Authed(user): Authed,
|
||||||
Path(agent_id): Path<Uuid>,
|
Path(agent_id): Path<Uuid>,
|
||||||
) -> Result<Json<serde_json::Value>, ApiError> {
|
) -> Result<Json<serde_json::Value>, ApiError> {
|
||||||
let id = crate::level_up::propose_agent(&state.pool, user.workspace_id, user.user_id, agent_id)
|
let id = crate::level_up::propose_agent(&state.pool, &state.runtime, user.workspace_id, user.user_id, agent_id)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
eprintln!("level_up: propose_agent {agent_id} failed: {e}");
|
eprintln!("level_up: propose_agent {agent_id} failed: {e}");
|
||||||
@@ -51,7 +51,7 @@ pub async fn propose_for_team(
|
|||||||
Authed(user): Authed,
|
Authed(user): Authed,
|
||||||
Path(team_id): Path<Uuid>,
|
Path(team_id): Path<Uuid>,
|
||||||
) -> Result<Json<serde_json::Value>, ApiError> {
|
) -> Result<Json<serde_json::Value>, ApiError> {
|
||||||
let id = crate::level_up::propose_team(&state.pool, user.workspace_id, user.user_id, team_id)
|
let id = crate::level_up::propose_team(&state.pool, &state.runtime, user.workspace_id, user.user_id, team_id)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| {
|
.map_err(|e| {
|
||||||
eprintln!("level_up: propose_team {team_id} failed: {e}");
|
eprintln!("level_up: propose_team {team_id} failed: {e}");
|
||||||
|
|||||||
Reference in New Issue
Block a user