daemon: proactively deactivate stale (>48h idle) projects each tick
Answers a real operational question — 'why does /hot/targets stay near
full even when I haven't touched most of these projects for weeks?'
Previously the stale sweep was gated behind 'hot tier > 90% full', so
an idle project held NVMe until the operator manually deactivated it
or space ran out and LRU came for it.
The change:
* new claw-store/src/actions.rs — shared deactivate_project(cfg,
manifest_path, project) with the FULL flow: sync-to-peer +
hot rm + .cargo/config.toml removal + manifest update, plus a
DeactivateOutcome { synced, sync_error, freed_bytes } return.
Extracted from main.rs::cmd_deactivate so both CLI and daemon
run the same code — no drift between manual and auto semantics.
* daemon.rs poll_tick now runs the stale sweep on EVERY tick,
independent of space pressure. Each project idle > stale_hours
(48 by config) goes through the full deactivate, logs
'stale-gc: X freed N MB (synced=Y)'.
* gc_by_space (LRU) still gates on >90% full — it's the emergency
'even after the stale sweep we're still tight' path.
* main.rs cmd_deactivate is now a thin CLI wrapper that adds
println! feedback + reports freed MB in the terminal output.
Space-pressure LRU keeps its original .cargo/config.toml-preserving
semantics (rm hot only, not the shim) so a project marked 'active'
by manual activation but hard-evicted for space can still be
re-activated cheaply. Stale sweep is the aggressive one because the
project genuinely hasn't been touched.
Tests: two new unit tests in actions.rs cover the full flow +
idempotency; the freed_bytes assertion is Linux-only-safe (BSD du
returns 0 for -sb, same limitation as the existing hot test).
This commit is contained in:
@@ -0,0 +1,217 @@
|
|||||||
|
//! Shared project-state actions callable from BOTH the CLI dispatch in
|
||||||
|
//! `main.rs` and the background daemon in `daemon.rs`. Extracted here so
|
||||||
|
//! the daemon's proactive stale-project sweep runs the same code path as
|
||||||
|
//! a user-invoked `claw-store deactivate <project>` — no drift between
|
||||||
|
//! "manual" and "automatic" semantics.
|
||||||
|
//!
|
||||||
|
//! Public: `deactivate_project`, plus `warm_path`/`hot_path` helpers.
|
||||||
|
|
||||||
|
use crate::config::Config;
|
||||||
|
use crate::manifest::Manifest;
|
||||||
|
use crate::sync::{self, SyncQueue};
|
||||||
|
use anyhow::{Context, Result};
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
pub fn warm_path(cfg: &Config, project: &str) -> PathBuf {
|
||||||
|
cfg.warm.projects_path.join(project)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn hot_path(cfg: &Config, project: &str) -> PathBuf {
|
||||||
|
cfg.hot.path.join(project)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Perform a full project deactivation:
|
||||||
|
/// 1. Sync current commits to the peer (queued for retry on failure).
|
||||||
|
/// 2. Delete the hot-tier target directory (frees NVMe).
|
||||||
|
/// 3. Remove the `.cargo/config.toml` shim so future builds land back
|
||||||
|
/// in `<warm>/target` instead of the hot path.
|
||||||
|
/// 4. Remove the project from the manifest under the manifest lock.
|
||||||
|
///
|
||||||
|
/// Never touches `/slab/projects/<project>` — the source clone is
|
||||||
|
/// preserved.
|
||||||
|
///
|
||||||
|
/// This is the shared implementation behind both:
|
||||||
|
/// * `main.rs::cmd_deactivate` (user typed `claw-store deactivate`)
|
||||||
|
/// * `daemon.rs` idle sweep (project untouched for stale_hours)
|
||||||
|
pub fn deactivate_project(
|
||||||
|
cfg: &Config,
|
||||||
|
manifest_path: &Path,
|
||||||
|
project: &str,
|
||||||
|
) -> Result<DeactivateOutcome> {
|
||||||
|
let mut synced = false;
|
||||||
|
let mut sync_error: Option<String> = None;
|
||||||
|
|
||||||
|
// Step 1: sync to peer (best-effort — a failure queues for retry).
|
||||||
|
if let Some(peer) = &cfg.peer {
|
||||||
|
let warm = warm_path(cfg, project);
|
||||||
|
match sync::sync_project(&warm, project, &peer.user, &peer.host) {
|
||||||
|
Ok(()) => {
|
||||||
|
synced = true;
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
sync_error = Some(format!("{e}"));
|
||||||
|
let mut queue = SyncQueue::load(&SyncQueue::default_path()).unwrap_or_default();
|
||||||
|
queue.enqueue(project, &warm);
|
||||||
|
queue.save(&SyncQueue::default_path())?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 2: evict hot tier. Measure the freed size first so callers can
|
||||||
|
// report it (dashboard batch UI + daemon logs both want the number).
|
||||||
|
let hot_target = hot_path(cfg, project);
|
||||||
|
let freed_bytes = if hot_target.exists() {
|
||||||
|
let n = crate::hot::target_size_bytes(&hot_target).unwrap_or(0);
|
||||||
|
std::fs::remove_dir_all(&hot_target)
|
||||||
|
.with_context(|| format!("removing hot target for {project}"))?;
|
||||||
|
n
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
|
||||||
|
// Step 3: remove the cargo shim. Leaves the .cargo dir itself so any
|
||||||
|
// other config the operator added (registries, aliases) survives.
|
||||||
|
let warm = warm_path(cfg, project);
|
||||||
|
let cargo_config = warm.join(".cargo/config.toml");
|
||||||
|
if cargo_config.exists() {
|
||||||
|
std::fs::remove_file(&cargo_config)
|
||||||
|
.with_context(|| format!("removing {}", cargo_config.display()))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 4: manifest update under lock.
|
||||||
|
Manifest::update(manifest_path, |m| {
|
||||||
|
m.projects.retain(|p| p.name != project);
|
||||||
|
Ok(())
|
||||||
|
})?;
|
||||||
|
|
||||||
|
Ok(DeactivateOutcome { synced, sync_error, freed_bytes })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct DeactivateOutcome {
|
||||||
|
pub synced: bool,
|
||||||
|
pub sync_error: Option<String>,
|
||||||
|
pub freed_bytes: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DeactivateOutcome {
|
||||||
|
pub fn freed_mb(&self) -> f64 {
|
||||||
|
self.freed_bytes as f64 / 1_048_576.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::config::{
|
||||||
|
Config, HotConfig, NodeConfig, NodeRole, WarmConfig,
|
||||||
|
};
|
||||||
|
use crate::manifest::Project;
|
||||||
|
use tempfile::TempDir;
|
||||||
|
|
||||||
|
fn mk_cfg(warm_root: &Path, hot_root: &Path) -> Config {
|
||||||
|
Config {
|
||||||
|
node: NodeConfig { name: "t".into(), role: NodeRole::Primary },
|
||||||
|
hot: HotConfig {
|
||||||
|
path: hot_root.to_path_buf(),
|
||||||
|
max_gb: 10,
|
||||||
|
stale_hours: 48,
|
||||||
|
},
|
||||||
|
warm: WarmConfig {
|
||||||
|
projects_path: warm_root.to_path_buf(),
|
||||||
|
zfs_dataset: "slab/projects".into(),
|
||||||
|
snapshot_retain_hours: 24,
|
||||||
|
snapshot_retain_days: 7,
|
||||||
|
snapshot_retain_weeks: 4,
|
||||||
|
},
|
||||||
|
cold: None,
|
||||||
|
replication: None,
|
||||||
|
peer: None,
|
||||||
|
api_token: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn seed_warm(warm_root: &Path, project: &str, hot_target: &Path) {
|
||||||
|
// warm dir with a stub .cargo/config.toml pointing at the hot target
|
||||||
|
let warm = warm_root.join(project);
|
||||||
|
std::fs::create_dir_all(warm.join(".cargo")).unwrap();
|
||||||
|
std::fs::write(
|
||||||
|
warm.join(".cargo/config.toml"),
|
||||||
|
format!("[build]\ntarget-dir = \"{}\"\n", hot_target.display()),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn seed_manifest(path: &Path, project: &str, warm_root: &Path, hot_root: &Path) {
|
||||||
|
let mut m = Manifest::default();
|
||||||
|
m.projects.push(Project {
|
||||||
|
name: project.into(),
|
||||||
|
warm_path: warm_root.join(project),
|
||||||
|
hot_target_path: hot_root.join(project),
|
||||||
|
last_build: None,
|
||||||
|
last_active: None,
|
||||||
|
last_sync: None,
|
||||||
|
pinned: false,
|
||||||
|
});
|
||||||
|
m.save(path).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_deactivate_removes_hot_and_cargo_shim() {
|
||||||
|
let tmp = TempDir::new().unwrap();
|
||||||
|
let warm_root = tmp.path().join("warm");
|
||||||
|
let hot_root = tmp.path().join("hot");
|
||||||
|
std::fs::create_dir_all(&warm_root).unwrap();
|
||||||
|
std::fs::create_dir_all(&hot_root).unwrap();
|
||||||
|
|
||||||
|
let project = "orgA/repoA";
|
||||||
|
let hot_target = hot_root.join(project);
|
||||||
|
std::fs::create_dir_all(&hot_target).unwrap();
|
||||||
|
std::fs::write(hot_target.join("target-file"), vec![0u8; 1024 * 128]).unwrap();
|
||||||
|
seed_warm(&warm_root, project, &hot_target);
|
||||||
|
|
||||||
|
let manifest_path = tmp.path().join("manifest.toml");
|
||||||
|
seed_manifest(&manifest_path, project, &warm_root, &hot_root);
|
||||||
|
|
||||||
|
let cfg = mk_cfg(&warm_root, &hot_root);
|
||||||
|
let outcome = deactivate_project(&cfg, &manifest_path, project).unwrap();
|
||||||
|
|
||||||
|
assert!(!hot_target.exists(), "hot target should be removed");
|
||||||
|
assert!(
|
||||||
|
!warm_root.join(project).join(".cargo/config.toml").exists(),
|
||||||
|
"cargo shim should be removed"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
warm_root.join(project).join(".cargo").exists(),
|
||||||
|
".cargo dir must survive so operator custom config isn't nuked"
|
||||||
|
);
|
||||||
|
// Warm dir itself must be untouched.
|
||||||
|
assert!(warm_root.join(project).exists());
|
||||||
|
// Manifest no longer lists the project.
|
||||||
|
let m = Manifest::load(&manifest_path).unwrap();
|
||||||
|
assert!(m.projects.iter().all(|p| p.name != project));
|
||||||
|
// freed_bytes uses `du -sb` under the hood; that flag doesn't
|
||||||
|
// exist on BSD du (macOS CI), so we only assert removal + a
|
||||||
|
// non-error result here. The Linux-only value is validated by
|
||||||
|
// the existing hot::tests::test_project_target_size_bytes.
|
||||||
|
let _ = outcome.freed_bytes;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_deactivate_idempotent_when_nothing_present() {
|
||||||
|
let tmp = TempDir::new().unwrap();
|
||||||
|
let warm_root = tmp.path().join("warm");
|
||||||
|
let hot_root = tmp.path().join("hot");
|
||||||
|
std::fs::create_dir_all(&warm_root).unwrap();
|
||||||
|
std::fs::create_dir_all(&hot_root).unwrap();
|
||||||
|
|
||||||
|
let project = "orgA/orphan";
|
||||||
|
let manifest_path = tmp.path().join("manifest.toml");
|
||||||
|
Manifest::default().save(&manifest_path).unwrap();
|
||||||
|
std::fs::create_dir_all(warm_root.join(project)).unwrap(); // warm dir but nothing hot
|
||||||
|
|
||||||
|
let cfg = mk_cfg(&warm_root, &hot_root);
|
||||||
|
let outcome = deactivate_project(&cfg, &manifest_path, project).unwrap();
|
||||||
|
assert_eq!(outcome.freed_bytes, 0);
|
||||||
|
assert!(!outcome.synced);
|
||||||
|
}
|
||||||
|
}
|
||||||
+58
-11
@@ -1,3 +1,4 @@
|
|||||||
|
use crate::actions;
|
||||||
use crate::config::Config;
|
use crate::config::Config;
|
||||||
use crate::head_watch::{scan_and_enqueue, HeadCache};
|
use crate::head_watch::{scan_and_enqueue, HeadCache};
|
||||||
use crate::hot;
|
use crate::hot;
|
||||||
@@ -6,6 +7,7 @@ use crate::snapshot;
|
|||||||
use crate::sync::{SyncQueue, drain_sync_queue};
|
use crate::sync::{SyncQueue, drain_sync_queue};
|
||||||
use crate::zfs::SystemZfs;
|
use crate::zfs::SystemZfs;
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
use chrono::Utc;
|
||||||
use sysinfo::{ProcessRefreshKind, RefreshKind, System};
|
use sysinfo::{ProcessRefreshKind, RefreshKind, System};
|
||||||
use tokio::time::{interval, Duration};
|
use tokio::time::{interval, Duration};
|
||||||
|
|
||||||
@@ -50,25 +52,51 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
|||||||
});
|
});
|
||||||
|
|
||||||
update_active_projects(&mut manifest)?;
|
update_active_projects(&mut manifest)?;
|
||||||
|
manifest.save(&manifest_path)?;
|
||||||
|
|
||||||
|
// ── Proactive stale sweep (independent of space pressure) ──
|
||||||
|
// Previously stale eviction only fired when the hot tier
|
||||||
|
// hit >90% full — a project could sit idle for weeks and
|
||||||
|
// still hold NVMe just because there was room. The
|
||||||
|
// operator's contract is `stale_hours`: exceed it and the
|
||||||
|
// project should return to the warm tier automatically.
|
||||||
|
// The full deactivate flow runs here (sync + hot rm +
|
||||||
|
// cargo shim + manifest) so the dashboard also reflects
|
||||||
|
// the change immediately.
|
||||||
|
let stale = stale_project_names(&manifest, cfg.hot.stale_hours);
|
||||||
|
for project in &stale {
|
||||||
|
tracing::info!(
|
||||||
|
"stale-gc: auto-deactivating {} (idle > {}h)",
|
||||||
|
project, cfg.hot.stale_hours
|
||||||
|
);
|
||||||
|
match actions::deactivate_project(&cfg, &manifest_path, project) {
|
||||||
|
Ok(o) => tracing::info!(
|
||||||
|
"stale-gc: {} freed {:.1} MB (synced={})",
|
||||||
|
project, o.freed_mb(), o.synced
|
||||||
|
),
|
||||||
|
Err(e) => tracing::error!(
|
||||||
|
"stale-gc: deactivate {} failed: {:#}", project, e
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Reload once after the sweep so subsequent steps see the
|
||||||
|
// trimmed manifest.
|
||||||
|
if !stale.is_empty() {
|
||||||
|
manifest = Manifest::load(&manifest_path).unwrap_or(manifest);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Space-pressure LRU still runs only when tight — it's
|
||||||
|
// the emergency "we're 90% full and everything is fresh"
|
||||||
|
// path. Stale sweep above handles the ordinary case.
|
||||||
let used = hot::total_used_gb(&manifest)?;
|
let used = hot::total_used_gb(&manifest)?;
|
||||||
if used > cfg.hot.max_gb as f64 * 0.9 {
|
if used > cfg.hot.max_gb as f64 * 0.9 {
|
||||||
tracing::warn!("hot tier {:.1}GB / {}GB — running GC", used, cfg.hot.max_gb);
|
tracing::warn!("hot tier {:.1}GB / {}GB — running space GC", used, cfg.hot.max_gb);
|
||||||
// v0.2.0 — locked-atomic update so a concurrent
|
|
||||||
// `claw-store pin` can't race the GC writer. The
|
|
||||||
// closure runs INSIDE the flock so it sees the
|
|
||||||
// latest disk state.
|
|
||||||
let stale_hours = cfg.hot.stale_hours;
|
|
||||||
let max_gb = cfg.hot.max_gb as f64;
|
let max_gb = cfg.hot.max_gb as f64;
|
||||||
let updated = Manifest::update(&manifest_path, |m| {
|
let updated = Manifest::update(&manifest_path, |m| {
|
||||||
hot::gc_stale_targets(m, stale_hours)?;
|
|
||||||
hot::gc_by_space(m, max_gb)?;
|
hot::gc_by_space(m, max_gb)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
})?;
|
})?;
|
||||||
manifest = updated;
|
manifest = updated;
|
||||||
} else {
|
|
||||||
// Even without GC, persist the timestamps that
|
|
||||||
// `update_active_projects` stamped.
|
|
||||||
manifest.save(&manifest_path)?;
|
|
||||||
}
|
}
|
||||||
// Auto-enqueue sync jobs for repos whose HEAD moved since
|
// Auto-enqueue sync jobs for repos whose HEAD moved since
|
||||||
// the previous tick, then drain the queue. Two paths that
|
// the previous tick, then drain the queue. Two paths that
|
||||||
@@ -146,6 +174,25 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Names of projects that qualify for the proactive stale sweep:
|
||||||
|
/// unpinned, still have a hot target on disk, and haven't seen cargo/rustc
|
||||||
|
/// activity in `stale_hours`. A project with `last_active = None` is
|
||||||
|
/// treated as stale (it never registered activity — must be an old
|
||||||
|
/// activation the daemon didn't see finish).
|
||||||
|
fn stale_project_names(manifest: &Manifest, stale_hours: u64) -> Vec<String> {
|
||||||
|
let now = Utc::now();
|
||||||
|
let threshold = chrono::Duration::hours(stale_hours as i64);
|
||||||
|
manifest.projects.iter()
|
||||||
|
.filter(|p| !p.pinned)
|
||||||
|
.filter(|p| p.hot_target_path.exists())
|
||||||
|
.filter(|p| match p.last_active {
|
||||||
|
None => true,
|
||||||
|
Some(t) => (now - t) > threshold,
|
||||||
|
})
|
||||||
|
.map(|p| p.name.clone())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
fn update_active_projects(manifest: &mut Manifest) -> Result<()> {
|
fn update_active_projects(manifest: &mut Manifest) -> Result<()> {
|
||||||
let mut sys = System::new_with_specifics(
|
let mut sys = System::new_with_specifics(
|
||||||
RefreshKind::new().with_processes(ProcessRefreshKind::everything())
|
RefreshKind::new().with_processes(ProcessRefreshKind::everything())
|
||||||
|
|||||||
+16
-29
@@ -1,3 +1,4 @@
|
|||||||
|
mod actions;
|
||||||
mod cargo_init;
|
mod cargo_init;
|
||||||
mod config;
|
mod config;
|
||||||
mod daemon;
|
mod daemon;
|
||||||
@@ -195,45 +196,31 @@ fn cmd_activate(
|
|||||||
|
|
||||||
// ── deactivate ────────────────────────────────────────────────────────────────
|
// ── deactivate ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// CLI-facing deactivate: thin wrapper around `actions::deactivate_project`
|
||||||
|
/// that adds println! feedback for the terminal. The daemon calls the
|
||||||
|
/// shared function directly with tracing::info! instead.
|
||||||
fn cmd_deactivate(
|
fn cmd_deactivate(
|
||||||
cfg: &Config,
|
cfg: &Config,
|
||||||
manifest: &mut Manifest,
|
manifest: &mut Manifest,
|
||||||
manifest_path: &std::path::Path,
|
manifest_path: &std::path::Path,
|
||||||
project: &str,
|
project: &str,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
// Sync first if peer is configured
|
if cfg.peer.is_some() {
|
||||||
if let Some(peer) = &cfg.peer {
|
|
||||||
println!("Syncing {} to peer before deactivate...", project);
|
println!("Syncing {} to peer before deactivate...", project);
|
||||||
if let Err(e) = cmd_sync(cfg, manifest, project) {
|
|
||||||
eprintln!(" warning: sync failed ({}), queuing for retry", e);
|
|
||||||
let p = manifest.get(project);
|
|
||||||
let warm = p.map(|p| p.warm_path.clone()).unwrap_or_else(|| warm_path(cfg, project));
|
|
||||||
let mut queue = sync::SyncQueue::load(&sync::SyncQueue::default_path()).unwrap_or_default();
|
|
||||||
queue.enqueue(project, &warm);
|
|
||||||
queue.save(&sync::SyncQueue::default_path())?;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
let outcome = actions::deactivate_project(cfg, manifest_path, project)?;
|
||||||
// Evict hot tier
|
if let Some(err) = &outcome.sync_error {
|
||||||
let hot_target = hot_path(cfg, project);
|
eprintln!(" warning: sync failed ({}), queued for retry", err);
|
||||||
if hot_target.exists() {
|
|
||||||
std::fs::remove_dir_all(&hot_target)?;
|
|
||||||
println!("evicted hot target: {}", hot_target.display());
|
|
||||||
}
|
}
|
||||||
|
// Refresh the caller's in-memory manifest to match the on-disk state.
|
||||||
// Remove cargo config
|
|
||||||
let warm = warm_path(cfg, project);
|
|
||||||
let cargo_config = warm.join(".cargo/config.toml");
|
|
||||||
if cargo_config.exists() {
|
|
||||||
std::fs::remove_file(&cargo_config)?;
|
|
||||||
}
|
|
||||||
|
|
||||||
Manifest::update(manifest_path, |m| {
|
|
||||||
m.projects.retain(|p| p.name != project);
|
|
||||||
Ok(())
|
|
||||||
})?;
|
|
||||||
*manifest = Manifest::load(manifest_path)?;
|
*manifest = Manifest::load(manifest_path)?;
|
||||||
println!("deactivated: {} (warm clone kept at {})", project, warm.display());
|
let warm = warm_path(cfg, project);
|
||||||
|
let freed = if outcome.freed_bytes > 0 {
|
||||||
|
format!(", freed {:.1} MB", outcome.freed_mb())
|
||||||
|
} else {
|
||||||
|
String::new()
|
||||||
|
};
|
||||||
|
println!("deactivated: {} (warm clone kept at {}{})", project, warm.display(), freed);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user