fix(podcast): a tombstone must not be permanent
Proven by the thing it broke. Mission 01a0c9c4 was tombstoned at
16:17:39 by the old code, in the four-minute gap after the reaper deleted
its checkout and before the build that could render it started at
16:21:58. Because the sweep selected on NOT EXISTS (podcast_episodes),
that row made the mission invisible forever: the fix shipped, and
recovered nothing, on a script that was sitting on a branch the whole
time.
The sweep now reconsiders a tombstone after RETRY_TOMBSTONE_AFTER, and
record_unrenderable_because refreshes created_at on each attempt, so the
backoff restarts rather than compounding. At most ~4 attempts a day per
mission: enough to recover the same day a cause is fixed, not enough to
become the every-two-minutes churn the no-turns tombstone was added to
stop.
Verified live by clearing the stale row and letting the deployed build
re-attempt:
podcast: mission 01a0c9c4 — checkout is gone; rendering from the vault
(clawmates/mission-01a0c9c4-cf8ba78d:.../script.md)
podcast: episode ... 31 turns, 419s, 6708231 bytes
audio: HTTP 200, 6708231 bytes, audio/mpeg; feed.xml carries the item
which is the first complete pass this pipeline has made: vault fallback,
the bold-speaker parser fix, ElevenLabs render, blob, feed.
Co-Authored-By: Claude Opus 5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01WZb5A2kfVfjpdwSochkuHz
This commit is contained in:
co-authored by
Claude Opus 5
parent
ad6ce261d5
commit
9d427900a5
@@ -699,6 +699,12 @@ mod live {
|
|||||||
|
|
||||||
// ── Rendering a finished mission into an episode ──────────────────────
|
// ── Rendering a finished mission into an episode ──────────────────────
|
||||||
|
|
||||||
|
/// How long a tombstone stands before the sweep tries that mission again.
|
||||||
|
///
|
||||||
|
/// Long enough that a dead end is not retried every two minutes; short
|
||||||
|
/// enough that a deploy which fixes the cause recovers the same day.
|
||||||
|
const RETRY_TOMBSTONE_AFTER: &str = "6 hours";
|
||||||
|
|
||||||
/// Where a mission's script lives inside its checkout.
|
/// Where a mission's script lives inside its checkout.
|
||||||
pub fn script_path(date: &str) -> String {
|
pub fn script_path(date: &str) -> String {
|
||||||
format!("ContinuousResearch/{date}/script.md")
|
format!("ContinuousResearch/{date}/script.md")
|
||||||
@@ -754,16 +760,35 @@ pub async fn render_pending(
|
|||||||
backend: &dyn AudioBackend,
|
backend: &dyn AudioBackend,
|
||||||
) -> Result<usize, String> {
|
) -> Result<usize, String> {
|
||||||
use sqlx::Row;
|
use sqlx::Row;
|
||||||
|
// A mission with no episode, or one whose last attempt was a TOMBSTONE
|
||||||
|
// old enough to be worth trying again.
|
||||||
|
//
|
||||||
|
// Tombstones used to be permanent: `NOT EXISTS (podcast_episodes)` meant
|
||||||
|
// that once a day gave up it could never be reconsidered, so a fix
|
||||||
|
// shipped afterwards recovered nothing. Mission 01a0c9c4 was tombstoned
|
||||||
|
// four minutes before the build that could have rendered it rolled, and
|
||||||
|
// stayed silent on a script that was sitting on a branch the whole time.
|
||||||
|
//
|
||||||
|
// The backoff is what keeps this from becoming the every-two-minutes
|
||||||
|
// churn that the no-turns tombstone was added to stop: at most one
|
||||||
|
// retry per mission per window, because `record_unrenderable_because`
|
||||||
|
// refreshes `created_at` on each attempt.
|
||||||
let rows = sqlx::query(
|
let rows = sqlx::query(
|
||||||
"SELECT m.id, m.workspace_id, m.title
|
"SELECT m.id, m.workspace_id, m.title
|
||||||
FROM missions m
|
FROM missions m
|
||||||
|
LEFT JOIN podcast_episodes e ON e.mission_id = m.id
|
||||||
WHERE m.template_kind = $1
|
WHERE m.template_kind = $1
|
||||||
AND m.status IN ('completed', 'failed')
|
AND m.status IN ('completed', 'failed')
|
||||||
AND NOT EXISTS (SELECT 1 FROM podcast_episodes e WHERE e.mission_id = m.id)
|
AND (
|
||||||
|
e.mission_id IS NULL
|
||||||
|
OR (e.rendered_by LIKE 'unrenderable%'
|
||||||
|
AND e.created_at < now() - $2::interval)
|
||||||
|
)
|
||||||
ORDER BY m.completed_at DESC NULLS LAST
|
ORDER BY m.completed_at DESC NULLS LAST
|
||||||
LIMIT 3",
|
LIMIT 3",
|
||||||
)
|
)
|
||||||
.bind(crate::continuous_research::TEMPLATE_KIND)
|
.bind(crate::continuous_research::TEMPLATE_KIND)
|
||||||
|
.bind(RETRY_TOMBSTONE_AFTER)
|
||||||
.fetch_all(pool)
|
.fetch_all(pool)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("select missions to render: {e}"))?;
|
.map_err(|e| format!("select missions to render: {e}"))?;
|
||||||
@@ -1021,7 +1046,8 @@ async fn record_unrenderable_because(pool: &sqlx::PgPool, mission_id: uuid::Uuid
|
|||||||
duration_secs, rendered_by, script_sha)
|
duration_secs, rendered_by, script_sha)
|
||||||
SELECT $1, m.workspace_id, m.id, '', m.title, '', 0, 0, $3, ''
|
SELECT $1, m.workspace_id, m.id, '', m.title, '', 0, 0, $3, ''
|
||||||
FROM missions m WHERE m.id = $2
|
FROM missions m WHERE m.id = $2
|
||||||
ON CONFLICT (mission_id) DO NOTHING",
|
ON CONFLICT (mission_id) DO UPDATE
|
||||||
|
SET rendered_by = EXCLUDED.rendered_by, created_at = now()",
|
||||||
)
|
)
|
||||||
.bind(uuid::Uuid::now_v7())
|
.bind(uuid::Uuid::now_v7())
|
||||||
.bind(mission_id)
|
.bind(mission_id)
|
||||||
|
|||||||
Reference in New Issue
Block a user