use crate::actions; use crate::cluster::services::ClusterServices; use crate::config::Config; use crate::head_watch::{scan_and_enqueue, HeadCache}; use crate::hot; use crate::manifest::Manifest; use crate::snapshot; use crate::sync::{SyncQueue, drain_sync_queue}; use crate::zfs::SystemZfs; use anyhow::{Context, Result}; use chrono::Utc; use sysinfo::{ProcessRefreshKind, RefreshKind, System}; use tokio::time::{interval, Duration}; pub const DAEMON_STARTED_PATH: &str = "/var/lib/claw-store/daemon-started"; pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> { tracing::info!("claw-store daemon starting on node '{}'", cfg.node.name); // Write start time for uptime reporting by the serve process. if let Some(parent) = std::path::Path::new(DAEMON_STARTED_PATH).parent() { let _ = std::fs::create_dir_all(parent); } let _ = std::fs::write(DAEMON_STARTED_PATH, chrono::Utc::now().timestamp().to_string()); let zfs = SystemZfs; let manifest_path = Manifest::default_path(); // Cluster services — gossip + optional QUIC RPC + hot-tier metric // publisher. Held for the daemon's lifetime; dropped on shutdown so // peers observe us going away within the failure detector's grace // window. `None` when `[cluster]` is absent from config — pre-v2 // deployments still work unchanged. let cluster_services: Option = match cfg.cluster.as_ref() { Some(cluster_cfg) => { let hot_dir = cfg.hot.path.clone(); let hot_max_bytes = cfg.hot.max_gb.saturating_mul(1024 * 1024 * 1024); let blob_root = cluster_cfg.blob_store_root.clone(); // Fail fast rather than degrade silently: a bind failure here is // almost always a boot-time race against DHCP/network-online // (the bind address isn't assigned to the interface yet). The // systemd unit has `Restart=on-failure`; exiting lets it retry // a few seconds later once the network is actually up, instead // of leaving the daemon running indefinitely with no gossip, // RPC, or Prometheus endpoint and no visible failure state. let svc = ClusterServices::start( cluster_cfg, cfg.node.name.clone(), hot_dir, hot_max_bytes, blob_root, ) .await .context("starting cluster services")?; tracing::info!( rpc_enabled = svc.rpc_enabled(), blob_store_enabled = svc.blob_store_enabled(), zone = %cluster_cfg.zone, "cluster services online" ); Some(svc) } None => { tracing::info!("no [cluster] section in config; running standalone"); None } }; let mut shutdown = tokio::signal::unix::signal( tokio::signal::unix::SignalKind::terminate() )?; let mut poll_tick = interval(Duration::from_secs(300)); let mut snap_tick = interval(Duration::from_secs(3600)); let mut repl_tick = interval(Duration::from_secs(86400)); loop { tokio::select! { _ = shutdown.recv() => { tracing::info!("received SIGTERM — shutting down cleanly"); if let Some(svc) = cluster_services { svc.shutdown(); } let _ = std::fs::remove_file(DAEMON_STARTED_PATH); break; } _ = poll_tick.tick() => { // v0.2.0 — reload from disk at the start of each tick // so any out-of-band CLI activations / dashboard pokes // since the previous tick are visible. Without this, // the daemon's in-memory `manifest` drifts from disk // and its next `save` clobbers anything written meanwhile. manifest = Manifest::load(&manifest_path).unwrap_or_else(|e| { tracing::warn!(error = %e, "manifest reload failed; using in-memory copy"); manifest.clone() }); update_active_projects(&mut manifest)?; manifest.save(&manifest_path)?; // ── Proactive stale sweep (independent of space pressure) ── // Previously stale eviction only fired when the hot tier // hit >90% full — a project could sit idle for weeks and // still hold NVMe just because there was room. The // operator's contract is `stale_hours`: exceed it and the // project should return to the warm tier automatically. // The full deactivate flow runs here (sync + hot rm + // cargo shim + manifest) so the dashboard also reflects // the change immediately. let stale = stale_project_names(&manifest, cfg.hot.stale_hours); for project in &stale { tracing::info!( "stale-gc: auto-deactivating {} (idle > {}h)", project, cfg.hot.stale_hours ); match actions::deactivate_project(&cfg, &manifest_path, project) { Ok(o) => tracing::info!( "stale-gc: {} freed {:.1} MB (synced={})", project, o.freed_mb(), o.synced ), Err(e) => tracing::error!( "stale-gc: deactivate {} failed: {:#}", project, e ), } } // Reload once after the sweep so subsequent steps see the // trimmed manifest. if !stale.is_empty() { manifest = Manifest::load(&manifest_path).unwrap_or(manifest); } // Space-pressure LRU still runs only when tight — it's // the emergency "we're 90% full and everything is fresh" // path. Stale sweep above handles the ordinary case. let used = hot::total_used_gb(&manifest)?; if used > cfg.hot.max_gb as f64 * 0.9 { tracing::warn!("hot tier {:.1}GB / {}GB — running space GC", used, cfg.hot.max_gb); let max_gb = cfg.hot.max_gb as f64; let updated = Manifest::update(&manifest_path, |m| { hot::gc_by_space(m, max_gb)?; Ok(()) })?; manifest = updated; } // Auto-enqueue sync jobs for repos whose HEAD moved since // the previous tick, then drain the queue. Two paths that // used to be manual (or wait for `claw-store deactivate`) // are now folded into the ordinary poll cycle: // 1. head_watch::scan_and_enqueue catches `git commit` // done directly in /slab/projects/*/* and adds a // sync job — no need for the operator to remember to // run `claw-store sync`. // 2. drain_sync_queue then notifies the peer for every // job (both freshly-enqueued and previously-stuck). if let Some(peer) = &cfg.peer { let queue_path = SyncQueue::default_path(); let cache_path = HeadCache::default_path(); let mut queue = SyncQueue::load(&queue_path).unwrap_or_default(); let mut cache = HeadCache::load(&cache_path); match scan_and_enqueue( &cfg.warm.projects_path, &mut queue, &mut cache, ) { Ok((scanned, enqueued)) if enqueued > 0 => { tracing::info!( "head-watch: scanned {}, {} new sync(s) enqueued", scanned, enqueued ); let _ = queue.save(&queue_path); } Ok(_) => { /* nothing moved — quiet */ } Err(e) => { tracing::warn!("head-watch scan failed: {:#}", e); } } // Persist the cache even when nothing was enqueued — // covers the first-scan populate case. if let Err(e) = cache.save(&cache_path) { tracing::warn!("head cache save failed: {:#}", e); } if !queue.jobs.is_empty() { tracing::info!("draining {} queued sync job(s)", queue.jobs.len()); if let Err(e) = drain_sync_queue(&mut queue, &peer.user, &peer.host, &queue_path) { tracing::error!("sync queue drain failed: {:#}", e); } } } } _ = snap_tick.tick() => { if !cfg.warm.zfs_enabled() { tracing::debug!("skipping snapshot tick — zfs_dataset = \"none\" on this node"); } else { let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string(); tracing::info!("taking snapshot {}", ts); if let Err(e) = snapshot::run_snapshot_cycle( &zfs, &cfg.warm.zfs_dataset, &ts, cfg.warm.snapshot_retain_hours as usize, cfg.warm.snapshot_retain_days as usize, cfg.warm.snapshot_retain_weeks as usize, ) { tracing::error!("snapshot failed: {:#}", e); } } } _ = repl_tick.tick() => { if !cfg.warm.zfs_enabled() { tracing::debug!("skipping replication tick — zfs_dataset = \"none\" on this node"); } else if let Some(rep) = &cfg.replication { if let (Some(host), Some(user), Some(dest)) = ( &rep.send_to_host, &rep.send_to_user, &rep.cold_dataset_on_peer ) { tracing::info!("replicating warm → cold on {}", host); if let Err(e) = snapshot::replicate_to_cold( &zfs, &cfg.warm.zfs_dataset, user, host, dest ) { tracing::error!("replication failed: {:#}", e); } } } } } } Ok(()) } /// Names of projects that qualify for the proactive stale sweep: /// unpinned, still have a hot target on disk, and last_active is a real /// timestamp older than `stale_hours`. /// /// `last_active = None` is treated as NOT stale. That case happens /// constantly — daemon restart resets the runtime timestamp state, and /// `update_active_projects` only stamps projects that have cargo/rustc /// running RIGHT NOW during a poll tick. A freshly-activated project /// with no activity yet must not be nuked; the operator explicitly asked /// for it to be active. Staleness is a positive assertion ("we've seen /// this project sit idle past the threshold"), never an absence of data. fn stale_project_names(manifest: &Manifest, stale_hours: u64) -> Vec { let now = Utc::now(); let threshold = chrono::Duration::hours(stale_hours as i64); manifest.projects.iter() .filter(|p| !p.pinned) .filter(|p| p.hot_target_path.exists()) .filter_map(|p| p.last_active.map(|t| (p, t))) .filter(|(_, t)| (now - *t) > threshold) .map(|(p, _)| p.name.clone()) .collect() } fn update_active_projects(manifest: &mut Manifest) -> Result<()> { let mut sys = System::new_with_specifics( RefreshKind::new().with_processes(ProcessRefreshKind::everything()) ); sys.refresh_processes(); let now = chrono::Utc::now(); for process in sys.processes().values() { let cmd = process.exe().map(|p| p.to_string_lossy().to_string()).unwrap_or_default(); if cmd.contains("cargo") || cmd.contains("rustc") { let cwd = process.root().map(|p| p.to_path_buf()); if let Some(cwd) = cwd { for project in &mut manifest.projects { if cwd.starts_with(&project.warm_path) { project.last_active = Some(now); project.last_build = Some(now); } } } } } Ok(()) }