daemon: proactively deactivate stale (>48h idle) projects each tick
Answers a real operational question — 'why does /hot/targets stay near
full even when I haven't touched most of these projects for weeks?'
Previously the stale sweep was gated behind 'hot tier > 90% full', so
an idle project held NVMe until the operator manually deactivated it
or space ran out and LRU came for it.
The change:
* new claw-store/src/actions.rs — shared deactivate_project(cfg,
manifest_path, project) with the FULL flow: sync-to-peer +
hot rm + .cargo/config.toml removal + manifest update, plus a
DeactivateOutcome { synced, sync_error, freed_bytes } return.
Extracted from main.rs::cmd_deactivate so both CLI and daemon
run the same code — no drift between manual and auto semantics.
* daemon.rs poll_tick now runs the stale sweep on EVERY tick,
independent of space pressure. Each project idle > stale_hours
(48 by config) goes through the full deactivate, logs
'stale-gc: X freed N MB (synced=Y)'.
* gc_by_space (LRU) still gates on >90% full — it's the emergency
'even after the stale sweep we're still tight' path.
* main.rs cmd_deactivate is now a thin CLI wrapper that adds
println! feedback + reports freed MB in the terminal output.
Space-pressure LRU keeps its original .cargo/config.toml-preserving
semantics (rm hot only, not the shim) so a project marked 'active'
by manual activation but hard-evicted for space can still be
re-activated cheaply. Stale sweep is the aggressive one because the
project genuinely hasn't been touched.
Tests: two new unit tests in actions.rs cover the full flow +
idempotency; the freed_bytes assertion is Linux-only-safe (BSD du
returns 0 for -sb, same limitation as the existing hot test).
This commit is contained in:
@@ -0,0 +1,217 @@
|
||||
//! Shared project-state actions callable from BOTH the CLI dispatch in
|
||||
//! `main.rs` and the background daemon in `daemon.rs`. Extracted here so
|
||||
//! the daemon's proactive stale-project sweep runs the same code path as
|
||||
//! a user-invoked `claw-store deactivate <project>` — no drift between
|
||||
//! "manual" and "automatic" semantics.
|
||||
//!
|
||||
//! Public: `deactivate_project`, plus `warm_path`/`hot_path` helpers.
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::manifest::Manifest;
|
||||
use crate::sync::{self, SyncQueue};
|
||||
use anyhow::{Context, Result};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub fn warm_path(cfg: &Config, project: &str) -> PathBuf {
|
||||
cfg.warm.projects_path.join(project)
|
||||
}
|
||||
|
||||
pub fn hot_path(cfg: &Config, project: &str) -> PathBuf {
|
||||
cfg.hot.path.join(project)
|
||||
}
|
||||
|
||||
/// Perform a full project deactivation:
|
||||
/// 1. Sync current commits to the peer (queued for retry on failure).
|
||||
/// 2. Delete the hot-tier target directory (frees NVMe).
|
||||
/// 3. Remove the `.cargo/config.toml` shim so future builds land back
|
||||
/// in `<warm>/target` instead of the hot path.
|
||||
/// 4. Remove the project from the manifest under the manifest lock.
|
||||
///
|
||||
/// Never touches `/slab/projects/<project>` — the source clone is
|
||||
/// preserved.
|
||||
///
|
||||
/// This is the shared implementation behind both:
|
||||
/// * `main.rs::cmd_deactivate` (user typed `claw-store deactivate`)
|
||||
/// * `daemon.rs` idle sweep (project untouched for stale_hours)
|
||||
pub fn deactivate_project(
|
||||
cfg: &Config,
|
||||
manifest_path: &Path,
|
||||
project: &str,
|
||||
) -> Result<DeactivateOutcome> {
|
||||
let mut synced = false;
|
||||
let mut sync_error: Option<String> = None;
|
||||
|
||||
// Step 1: sync to peer (best-effort — a failure queues for retry).
|
||||
if let Some(peer) = &cfg.peer {
|
||||
let warm = warm_path(cfg, project);
|
||||
match sync::sync_project(&warm, project, &peer.user, &peer.host) {
|
||||
Ok(()) => {
|
||||
synced = true;
|
||||
}
|
||||
Err(e) => {
|
||||
sync_error = Some(format!("{e}"));
|
||||
let mut queue = SyncQueue::load(&SyncQueue::default_path()).unwrap_or_default();
|
||||
queue.enqueue(project, &warm);
|
||||
queue.save(&SyncQueue::default_path())?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 2: evict hot tier. Measure the freed size first so callers can
|
||||
// report it (dashboard batch UI + daemon logs both want the number).
|
||||
let hot_target = hot_path(cfg, project);
|
||||
let freed_bytes = if hot_target.exists() {
|
||||
let n = crate::hot::target_size_bytes(&hot_target).unwrap_or(0);
|
||||
std::fs::remove_dir_all(&hot_target)
|
||||
.with_context(|| format!("removing hot target for {project}"))?;
|
||||
n
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
// Step 3: remove the cargo shim. Leaves the .cargo dir itself so any
|
||||
// other config the operator added (registries, aliases) survives.
|
||||
let warm = warm_path(cfg, project);
|
||||
let cargo_config = warm.join(".cargo/config.toml");
|
||||
if cargo_config.exists() {
|
||||
std::fs::remove_file(&cargo_config)
|
||||
.with_context(|| format!("removing {}", cargo_config.display()))?;
|
||||
}
|
||||
|
||||
// Step 4: manifest update under lock.
|
||||
Manifest::update(manifest_path, |m| {
|
||||
m.projects.retain(|p| p.name != project);
|
||||
Ok(())
|
||||
})?;
|
||||
|
||||
Ok(DeactivateOutcome { synced, sync_error, freed_bytes })
|
||||
}
|
||||
|
||||
pub struct DeactivateOutcome {
|
||||
pub synced: bool,
|
||||
pub sync_error: Option<String>,
|
||||
pub freed_bytes: u64,
|
||||
}
|
||||
|
||||
impl DeactivateOutcome {
|
||||
pub fn freed_mb(&self) -> f64 {
|
||||
self.freed_bytes as f64 / 1_048_576.0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::{
|
||||
Config, HotConfig, NodeConfig, NodeRole, WarmConfig,
|
||||
};
|
||||
use crate::manifest::Project;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn mk_cfg(warm_root: &Path, hot_root: &Path) -> Config {
|
||||
Config {
|
||||
node: NodeConfig { name: "t".into(), role: NodeRole::Primary },
|
||||
hot: HotConfig {
|
||||
path: hot_root.to_path_buf(),
|
||||
max_gb: 10,
|
||||
stale_hours: 48,
|
||||
},
|
||||
warm: WarmConfig {
|
||||
projects_path: warm_root.to_path_buf(),
|
||||
zfs_dataset: "slab/projects".into(),
|
||||
snapshot_retain_hours: 24,
|
||||
snapshot_retain_days: 7,
|
||||
snapshot_retain_weeks: 4,
|
||||
},
|
||||
cold: None,
|
||||
replication: None,
|
||||
peer: None,
|
||||
api_token: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn seed_warm(warm_root: &Path, project: &str, hot_target: &Path) {
|
||||
// warm dir with a stub .cargo/config.toml pointing at the hot target
|
||||
let warm = warm_root.join(project);
|
||||
std::fs::create_dir_all(warm.join(".cargo")).unwrap();
|
||||
std::fs::write(
|
||||
warm.join(".cargo/config.toml"),
|
||||
format!("[build]\ntarget-dir = \"{}\"\n", hot_target.display()),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn seed_manifest(path: &Path, project: &str, warm_root: &Path, hot_root: &Path) {
|
||||
let mut m = Manifest::default();
|
||||
m.projects.push(Project {
|
||||
name: project.into(),
|
||||
warm_path: warm_root.join(project),
|
||||
hot_target_path: hot_root.join(project),
|
||||
last_build: None,
|
||||
last_active: None,
|
||||
last_sync: None,
|
||||
pinned: false,
|
||||
});
|
||||
m.save(path).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deactivate_removes_hot_and_cargo_shim() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let warm_root = tmp.path().join("warm");
|
||||
let hot_root = tmp.path().join("hot");
|
||||
std::fs::create_dir_all(&warm_root).unwrap();
|
||||
std::fs::create_dir_all(&hot_root).unwrap();
|
||||
|
||||
let project = "orgA/repoA";
|
||||
let hot_target = hot_root.join(project);
|
||||
std::fs::create_dir_all(&hot_target).unwrap();
|
||||
std::fs::write(hot_target.join("target-file"), vec![0u8; 1024 * 128]).unwrap();
|
||||
seed_warm(&warm_root, project, &hot_target);
|
||||
|
||||
let manifest_path = tmp.path().join("manifest.toml");
|
||||
seed_manifest(&manifest_path, project, &warm_root, &hot_root);
|
||||
|
||||
let cfg = mk_cfg(&warm_root, &hot_root);
|
||||
let outcome = deactivate_project(&cfg, &manifest_path, project).unwrap();
|
||||
|
||||
assert!(!hot_target.exists(), "hot target should be removed");
|
||||
assert!(
|
||||
!warm_root.join(project).join(".cargo/config.toml").exists(),
|
||||
"cargo shim should be removed"
|
||||
);
|
||||
assert!(
|
||||
warm_root.join(project).join(".cargo").exists(),
|
||||
".cargo dir must survive so operator custom config isn't nuked"
|
||||
);
|
||||
// Warm dir itself must be untouched.
|
||||
assert!(warm_root.join(project).exists());
|
||||
// Manifest no longer lists the project.
|
||||
let m = Manifest::load(&manifest_path).unwrap();
|
||||
assert!(m.projects.iter().all(|p| p.name != project));
|
||||
// freed_bytes uses `du -sb` under the hood; that flag doesn't
|
||||
// exist on BSD du (macOS CI), so we only assert removal + a
|
||||
// non-error result here. The Linux-only value is validated by
|
||||
// the existing hot::tests::test_project_target_size_bytes.
|
||||
let _ = outcome.freed_bytes;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_deactivate_idempotent_when_nothing_present() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let warm_root = tmp.path().join("warm");
|
||||
let hot_root = tmp.path().join("hot");
|
||||
std::fs::create_dir_all(&warm_root).unwrap();
|
||||
std::fs::create_dir_all(&hot_root).unwrap();
|
||||
|
||||
let project = "orgA/orphan";
|
||||
let manifest_path = tmp.path().join("manifest.toml");
|
||||
Manifest::default().save(&manifest_path).unwrap();
|
||||
std::fs::create_dir_all(warm_root.join(project)).unwrap(); // warm dir but nothing hot
|
||||
|
||||
let cfg = mk_cfg(&warm_root, &hot_root);
|
||||
let outcome = deactivate_project(&cfg, &manifest_path, project).unwrap();
|
||||
assert_eq!(outcome.freed_bytes, 0);
|
||||
assert!(!outcome.synced);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user