Previous commit 643ba17 unmasked a latent bug in the None-handling
branch of both gc_stale_targets AND the new stale_project_names:
they treated 'no last_active timestamp' as 'stale, evict'. This
combined with the new proactive sweep (fires every tick, not just
under space pressure) meant daemon restart wiped runtime state,
saw every project as None-timestamped, and mass-deactivated
everything active on the next tick.
Real damage in this session on live daemons:
* architect: clawverse/omni-cortex — 134 GB hot artifacts freed
* tank: clawverse/omni-cortex (44 GB), rustyverse/rustytorch (4 GB),
plus ~10 more with empty hot targets
Warm clones under /slab/projects are intact (per deactivate flow) —
impact is only rebuild cost on next activation.
Fix: None is treated as NOT stale in both places. Absence of a
timestamp is normal: update_active_projects only stamps projects
whose cargo/rustc it catches mid-run. A freshly-activated project
with nothing built yet, or a project whose builds all finished
between poll ticks, will have None. That's not stale — that's
'we haven't seen it hit the threshold'. Staleness must always be
a positive assertion.
Added test test_gc_skips_none_last_active to lock the invariant.
205 lines
7.4 KiB
Rust
205 lines
7.4 KiB
Rust
use crate::manifest::Manifest;
|
|
use anyhow::{Context, Result};
|
|
use chrono::Utc;
|
|
use std::path::Path;
|
|
|
|
pub fn target_size_bytes(path: &Path) -> Result<u64> {
|
|
if !path.exists() { return Ok(0); }
|
|
let path_str = path.to_str()
|
|
.with_context(|| format!("non-UTF-8 path: {}", path.display()))?;
|
|
let out = std::process::Command::new("du")
|
|
.args(["-sb", path_str])
|
|
.output()
|
|
.context("running du")?;
|
|
let line = String::from_utf8_lossy(&out.stdout);
|
|
let bytes = line.split_whitespace()
|
|
.next()
|
|
.and_then(|s| s.parse::<u64>().ok())
|
|
.unwrap_or(0);
|
|
Ok(bytes)
|
|
}
|
|
|
|
pub fn total_used_gb(manifest: &Manifest) -> Result<f64> {
|
|
let mut total = 0u64;
|
|
for p in &manifest.projects {
|
|
total += target_size_bytes(&p.hot_target_path)?;
|
|
}
|
|
Ok(total as f64 / 1_073_741_824.0)
|
|
}
|
|
|
|
pub fn gc_stale_targets(manifest: &Manifest, stale_hours: u64) -> Result<Vec<String>> {
|
|
let now = Utc::now();
|
|
let threshold = chrono::Duration::hours(stale_hours as i64);
|
|
let mut evicted = Vec::new();
|
|
for p in &manifest.projects {
|
|
// v0.2.0 — operator-pinned projects survive every GC pass, both
|
|
// the stale sweep here and the space-pressure LRU below. The
|
|
// manifest is authoritative for intent; activation alone isn't.
|
|
if p.pinned {
|
|
continue;
|
|
}
|
|
// `last_active = None` is NOT stale. Absence of a timestamp is
|
|
// normal — it happens on daemon restart (runtime state resets)
|
|
// and for projects whose cargo/rustc hasn't been caught mid-run
|
|
// by a poll yet. Staleness must be a positive assertion.
|
|
let is_stale = match p.last_active {
|
|
None => false,
|
|
Some(t) => (now - t) > threshold,
|
|
};
|
|
if is_stale && p.hot_target_path.exists() {
|
|
std::fs::remove_dir_all(&p.hot_target_path)
|
|
.with_context(|| format!("removing hot target for {}", p.name))?;
|
|
evicted.push(p.name.clone());
|
|
}
|
|
}
|
|
Ok(evicted)
|
|
}
|
|
|
|
pub fn gc_by_space(manifest: &mut Manifest, max_gb: f64) -> Result<Vec<String>> {
|
|
let mut evicted = Vec::new();
|
|
loop {
|
|
let used = total_used_gb(manifest)?;
|
|
if used <= max_gb {
|
|
break;
|
|
}
|
|
// LRU eviction skips pinned projects — they may NEVER be evicted
|
|
// for space pressure. The trade-off: if every non-pinned project
|
|
// is gone and we're still over `max_gb`, we stop and log; better
|
|
// to over-allocate hot than to violate operator intent.
|
|
let lru_name = manifest
|
|
.projects
|
|
.iter()
|
|
.filter(|p| !p.pinned)
|
|
.filter(|p| p.hot_target_path.exists())
|
|
.min_by_key(|p| p.last_active)
|
|
.map(|p| p.name.clone());
|
|
match lru_name {
|
|
None => {
|
|
tracing::warn!(
|
|
used_gb = used,
|
|
max_gb = max_gb,
|
|
"hot tier over budget but every remaining project is pinned"
|
|
);
|
|
break;
|
|
}
|
|
Some(name) => {
|
|
if let Some(p) = manifest.projects.iter().find(|p| p.name == name) {
|
|
if p.hot_target_path.exists() {
|
|
std::fs::remove_dir_all(&p.hot_target_path)?;
|
|
evicted.push(name);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Ok(evicted)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::manifest::{Manifest, Project};
|
|
use tempfile::TempDir;
|
|
|
|
fn make_project(dir: &TempDir, name: &str) -> Project {
|
|
let path = dir.path().join(name);
|
|
std::fs::create_dir_all(&path).unwrap();
|
|
std::fs::write(path.join("dummy"), vec![0u8; 1024 * 1024]).unwrap();
|
|
Project {
|
|
name: name.into(),
|
|
warm_path: format!("/slab/projects/{}", name).into(),
|
|
hot_target_path: path.clone(),
|
|
last_build: None,
|
|
last_active: None,
|
|
last_sync: None,
|
|
pinned: false,
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_project_target_size_bytes() {
|
|
let dir = TempDir::new().unwrap();
|
|
let p = make_project(&dir, "test-proj");
|
|
let size = target_size_bytes(&p.hot_target_path).unwrap();
|
|
assert!(size >= 1024 * 1024, "expected at least 1MB, got {}", size);
|
|
}
|
|
|
|
#[test]
|
|
fn test_gc_evicts_stale_project() {
|
|
let dir = TempDir::new().unwrap();
|
|
let mut manifest = Manifest::default();
|
|
manifest.projects.push(make_project(&dir, "stale-proj"));
|
|
manifest.projects[0].last_active =
|
|
Some(chrono::Utc::now() - chrono::Duration::hours(100));
|
|
|
|
let evicted = gc_stale_targets(&manifest, 48).unwrap();
|
|
assert_eq!(evicted.len(), 1);
|
|
assert_eq!(evicted[0], "stale-proj");
|
|
assert!(!dir.path().join("stale-proj").exists());
|
|
}
|
|
|
|
#[test]
|
|
fn test_gc_skips_none_last_active() {
|
|
// last_active = None must be treated as "unknown, don't touch".
|
|
// This is the safe default that prevents daemon-restart-driven
|
|
// mass evictions of freshly-activated projects.
|
|
let dir = TempDir::new().unwrap();
|
|
let mut manifest = Manifest::default();
|
|
manifest.projects.push(make_project(&dir, "fresh-no-timestamp"));
|
|
// last_active stays None (make_project default)
|
|
|
|
let evicted = gc_stale_targets(&manifest, 48).unwrap();
|
|
assert!(evicted.is_empty(), "None-timestamped project was evicted");
|
|
assert!(dir.path().join("fresh-no-timestamp").exists());
|
|
}
|
|
|
|
#[test]
|
|
fn test_gc_skips_active_project() {
|
|
let dir = TempDir::new().unwrap();
|
|
let mut manifest = Manifest::default();
|
|
manifest.projects.push(make_project(&dir, "active-proj"));
|
|
manifest.projects[0].last_active =
|
|
Some(chrono::Utc::now() - chrono::Duration::hours(2));
|
|
|
|
let evicted = gc_stale_targets(&manifest, 48).unwrap();
|
|
assert!(evicted.is_empty());
|
|
assert!(dir.path().join("active-proj").exists());
|
|
}
|
|
|
|
#[test]
|
|
fn pinned_projects_survive_stale_gc() {
|
|
let dir = TempDir::new().unwrap();
|
|
let mut manifest = Manifest::default();
|
|
let mut p = make_project(&dir, "pinned-proj");
|
|
p.last_active = Some(chrono::Utc::now() - chrono::Duration::hours(500));
|
|
p.pinned = true;
|
|
manifest.projects.push(p);
|
|
|
|
let evicted = gc_stale_targets(&manifest, 48).unwrap();
|
|
assert!(evicted.is_empty(), "pinned project was evicted: {evicted:?}");
|
|
assert!(dir.path().join("pinned-proj").exists());
|
|
}
|
|
|
|
#[test]
|
|
fn pinned_projects_survive_space_gc_even_when_lru() {
|
|
// Pinned + ancient → would be the natural LRU eviction target.
|
|
// Confirm it stays put even when we set max_gb tiny.
|
|
let dir = TempDir::new().unwrap();
|
|
let mut manifest = Manifest::default();
|
|
let mut pinned = make_project(&dir, "pinned");
|
|
pinned.last_active = Some(chrono::Utc::now() - chrono::Duration::hours(500));
|
|
pinned.pinned = true;
|
|
manifest.projects.push(pinned);
|
|
let mut hot = make_project(&dir, "hot-but-unpinned");
|
|
hot.last_active = Some(chrono::Utc::now());
|
|
manifest.projects.push(hot);
|
|
|
|
// Force eviction: cap below total size.
|
|
let evicted = gc_by_space(&mut manifest, 0.0).unwrap();
|
|
assert!(
|
|
!evicted.iter().any(|n| n == "pinned"),
|
|
"pinned was evicted under space pressure"
|
|
);
|
|
}
|
|
}
|