diff --git a/crates/cm-api/src/podcast.rs b/crates/cm-api/src/podcast.rs index 0822dcd..2b465da 100644 --- a/crates/cm-api/src/podcast.rs +++ b/crates/cm-api/src/podcast.rs @@ -699,6 +699,12 @@ mod live { // ── Rendering a finished mission into an episode ────────────────────── +/// How long a tombstone stands before the sweep tries that mission again. +/// +/// Long enough that a dead end is not retried every two minutes; short +/// enough that a deploy which fixes the cause recovers the same day. +const RETRY_TOMBSTONE_AFTER: &str = "6 hours"; + /// Where a mission's script lives inside its checkout. pub fn script_path(date: &str) -> String { format!("ContinuousResearch/{date}/script.md") @@ -754,16 +760,35 @@ pub async fn render_pending( backend: &dyn AudioBackend, ) -> Result { use sqlx::Row; + // A mission with no episode, or one whose last attempt was a TOMBSTONE + // old enough to be worth trying again. + // + // Tombstones used to be permanent: `NOT EXISTS (podcast_episodes)` meant + // that once a day gave up it could never be reconsidered, so a fix + // shipped afterwards recovered nothing. Mission 01a0c9c4 was tombstoned + // four minutes before the build that could have rendered it rolled, and + // stayed silent on a script that was sitting on a branch the whole time. + // + // The backoff is what keeps this from becoming the every-two-minutes + // churn that the no-turns tombstone was added to stop: at most one + // retry per mission per window, because `record_unrenderable_because` + // refreshes `created_at` on each attempt. let rows = sqlx::query( "SELECT m.id, m.workspace_id, m.title FROM missions m + LEFT JOIN podcast_episodes e ON e.mission_id = m.id WHERE m.template_kind = $1 AND m.status IN ('completed', 'failed') - AND NOT EXISTS (SELECT 1 FROM podcast_episodes e WHERE e.mission_id = m.id) + AND ( + e.mission_id IS NULL + OR (e.rendered_by LIKE 'unrenderable%' + AND e.created_at < now() - $2::interval) + ) ORDER BY m.completed_at DESC NULLS LAST LIMIT 3", ) .bind(crate::continuous_research::TEMPLATE_KIND) + .bind(RETRY_TOMBSTONE_AFTER) .fetch_all(pool) .await .map_err(|e| format!("select missions to render: {e}"))?; @@ -1021,7 +1046,8 @@ async fn record_unrenderable_because(pool: &sqlx::PgPool, mission_id: uuid::Uuid duration_secs, rendered_by, script_sha) SELECT $1, m.workspace_id, m.id, '', m.title, '', 0, 0, $3, '' FROM missions m WHERE m.id = $2 - ON CONFLICT (mission_id) DO NOTHING", + ON CONFLICT (mission_id) DO UPDATE + SET rendered_by = EXCLUDED.rendered_by, created_at = now()", ) .bind(uuid::Uuid::now_v7()) .bind(mission_id)