URGENT FIX: stale-gc must not treat last_active=None as stale

Previous commit 643ba17 unmasked a latent bug in the None-handling
branch of both gc_stale_targets AND the new stale_project_names:
they treated 'no last_active timestamp' as 'stale, evict'. This
combined with the new proactive sweep (fires every tick, not just
under space pressure) meant daemon restart wiped runtime state,
saw every project as None-timestamped, and mass-deactivated
everything active on the next tick.

Real damage in this session on live daemons:
  * architect: clawverse/omni-cortex — 134 GB hot artifacts freed
  * tank: clawverse/omni-cortex (44 GB), rustyverse/rustytorch (4 GB),
    plus ~10 more with empty hot targets

Warm clones under /slab/projects are intact (per deactivate flow) —
impact is only rebuild cost on next activation.

Fix: None is treated as NOT stale in both places. Absence of a
timestamp is normal: update_active_projects only stamps projects
whose cargo/rustc it catches mid-run. A freshly-activated project
with nothing built yet, or a project whose builds all finished
between poll ticks, will have None. That's not stale — that's
'we haven't seen it hit the threshold'. Staleness must always be
a positive assertion.

Added test test_gc_skips_none_last_active to lock the invariant.
This commit is contained in:
Omar Sobh
2026-07-02 17:00:58 -07:00
parent 643ba170b6
commit aefa1cce58
2 changed files with 33 additions and 10 deletions
+13 -9
View File
@@ -175,21 +175,25 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
} }
/// Names of projects that qualify for the proactive stale sweep: /// Names of projects that qualify for the proactive stale sweep:
/// unpinned, still have a hot target on disk, and haven't seen cargo/rustc /// unpinned, still have a hot target on disk, and last_active is a real
/// activity in `stale_hours`. A project with `last_active = None` is /// timestamp older than `stale_hours`.
/// treated as stale (it never registered activity — must be an old ///
/// activation the daemon didn't see finish). /// `last_active = None` is treated as NOT stale. That case happens
/// constantly — daemon restart resets the runtime timestamp state, and
/// `update_active_projects` only stamps projects that have cargo/rustc
/// running RIGHT NOW during a poll tick. A freshly-activated project
/// with no activity yet must not be nuked; the operator explicitly asked
/// for it to be active. Staleness is a positive assertion ("we've seen
/// this project sit idle past the threshold"), never an absence of data.
fn stale_project_names(manifest: &Manifest, stale_hours: u64) -> Vec<String> { fn stale_project_names(manifest: &Manifest, stale_hours: u64) -> Vec<String> {
let now = Utc::now(); let now = Utc::now();
let threshold = chrono::Duration::hours(stale_hours as i64); let threshold = chrono::Duration::hours(stale_hours as i64);
manifest.projects.iter() manifest.projects.iter()
.filter(|p| !p.pinned) .filter(|p| !p.pinned)
.filter(|p| p.hot_target_path.exists()) .filter(|p| p.hot_target_path.exists())
.filter(|p| match p.last_active { .filter_map(|p| p.last_active.map(|t| (p, t)))
None => true, .filter(|(_, t)| (now - *t) > threshold)
Some(t) => (now - t) > threshold, .map(|(p, _)| p.name.clone())
})
.map(|p| p.name.clone())
.collect() .collect()
} }
+20 -1
View File
@@ -38,8 +38,12 @@ pub fn gc_stale_targets(manifest: &Manifest, stale_hours: u64) -> Result<Vec<Str
if p.pinned { if p.pinned {
continue; continue;
} }
// `last_active = None` is NOT stale. Absence of a timestamp is
// normal — it happens on daemon restart (runtime state resets)
// and for projects whose cargo/rustc hasn't been caught mid-run
// by a poll yet. Staleness must be a positive assertion.
let is_stale = match p.last_active { let is_stale = match p.last_active {
None => true, None => false,
Some(t) => (now - t) > threshold, Some(t) => (now - t) > threshold,
}; };
if is_stale && p.hot_target_path.exists() { if is_stale && p.hot_target_path.exists() {
@@ -134,6 +138,21 @@ mod tests {
assert!(!dir.path().join("stale-proj").exists()); assert!(!dir.path().join("stale-proj").exists());
} }
#[test]
fn test_gc_skips_none_last_active() {
// last_active = None must be treated as "unknown, don't touch".
// This is the safe default that prevents daemon-restart-driven
// mass evictions of freshly-activated projects.
let dir = TempDir::new().unwrap();
let mut manifest = Manifest::default();
manifest.projects.push(make_project(&dir, "fresh-no-timestamp"));
// last_active stays None (make_project default)
let evicted = gc_stale_targets(&manifest, 48).unwrap();
assert!(evicted.is_empty(), "None-timestamped project was evicted");
assert!(dir.path().join("fresh-no-timestamp").exists());
}
#[test] #[test]
fn test_gc_skips_active_project() { fn test_gc_skips_active_project() {
let dir = TempDir::new().unwrap(); let dir = TempDir::new().unwrap();