clawstor/claw-store/src/daemon.rs

use crate::actions;
use crate::cluster::services::ClusterServices;
use crate::config::Config;
use crate::head_watch::{scan_and_enqueue, HeadCache};
use crate::hot;
use crate::manifest::Manifest;
use crate::snapshot;
use crate::sync::{SyncQueue, drain_sync_queue};
use crate::zfs::SystemZfs;
use anyhow::{Context, Result};
use chrono::Utc;
use sysinfo::{ProcessRefreshKind, RefreshKind, System};
use tokio::time::{interval, Duration};

pub const DAEMON_STARTED_PATH: &str = "/var/lib/claw-store/daemon-started";

pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
    tracing::info!("claw-store daemon starting on node '{}'", cfg.node.name);

    // Write start time for uptime reporting by the serve process.
    if let Some(parent) = std::path::Path::new(DAEMON_STARTED_PATH).parent() {
        let _ = std::fs::create_dir_all(parent);
    }
    let _ = std::fs::write(DAEMON_STARTED_PATH, chrono::Utc::now().timestamp().to_string());

    let zfs = SystemZfs;
    let manifest_path = Manifest::default_path();

    // Cluster services — gossip + optional QUIC RPC + hot-tier metric
    // publisher. Held for the daemon's lifetime; dropped on shutdown so
    // peers observe us going away within the failure detector's grace
    // window. `None` when `[cluster]` is absent from config — pre-v2
    // deployments still work unchanged.
    let cluster_services: Option<ClusterServices> = match cfg.cluster.as_ref() {
        Some(cluster_cfg) => {
            let hot_dir = cfg.hot.path.clone();
            let hot_max_bytes = cfg.hot.max_gb.saturating_mul(1024 * 1024 * 1024);
            let blob_root = cluster_cfg.blob_store_root.clone();
            // Fail fast rather than degrade silently: a bind failure here is
            // almost always a boot-time race against DHCP/network-online
            // (the bind address isn't assigned to the interface yet). The
            // systemd unit has `Restart=on-failure`; exiting lets it retry
            // a few seconds later once the network is actually up, instead
            // of leaving the daemon running indefinitely with no gossip,
            // RPC, or Prometheus endpoint and no visible failure state.
            let svc = ClusterServices::start(
                cluster_cfg,
                cfg.node.name.clone(),
                hot_dir,
                hot_max_bytes,
                blob_root,
            )
            .await
            .context("starting cluster services")?;
            tracing::info!(
                rpc_enabled = svc.rpc_enabled(),
                blob_store_enabled = svc.blob_store_enabled(),
                zone = %cluster_cfg.zone,
                "cluster services online"
            );
            Some(svc)
        }
        None => {
            tracing::info!("no [cluster] section in config; running standalone");
            None
        }
    };

    let mut shutdown = tokio::signal::unix::signal(
        tokio::signal::unix::SignalKind::terminate()
    )?;

    let mut poll_tick = interval(Duration::from_secs(300));
    let mut snap_tick = interval(Duration::from_secs(3600));
    let mut repl_tick = interval(Duration::from_secs(86400));

    loop {
        tokio::select! {
            _ = shutdown.recv() => {
                tracing::info!("received SIGTERM — shutting down cleanly");
                if let Some(svc) = cluster_services {
                    svc.shutdown();
                }
                let _ = std::fs::remove_file(DAEMON_STARTED_PATH);
                break;
            }
            _ = poll_tick.tick() => {
                // v0.2.0 — reload from disk at the start of each tick
                // so any out-of-band CLI activations / dashboard pokes
                // since the previous tick are visible. Without this,
                // the daemon's in-memory `manifest` drifts from disk
                // and its next `save` clobbers anything written meanwhile.
                manifest = Manifest::load(&manifest_path).unwrap_or_else(|e| {
                    tracing::warn!(error = %e, "manifest reload failed; using in-memory copy");
                    manifest.clone()
                });

                update_active_projects(&mut manifest)?;
                manifest.save(&manifest_path)?;

                // ── Proactive stale sweep (independent of space pressure) ──
                // Previously stale eviction only fired when the hot tier
                // hit >90% full — a project could sit idle for weeks and
                // still hold NVMe just because there was room. The
                // operator's contract is `stale_hours`: exceed it and the
                // project should return to the warm tier automatically.
                // The full deactivate flow runs here (sync + hot rm +
                // cargo shim + manifest) so the dashboard also reflects
                // the change immediately.
                let stale = stale_project_names(&manifest, cfg.hot.stale_hours);
                for project in &stale {
                    tracing::info!(
                        "stale-gc: auto-deactivating {} (idle > {}h)",
                        project, cfg.hot.stale_hours
                    );
                    match actions::deactivate_project(&cfg, &manifest_path, project) {
                        Ok(o) => tracing::info!(
                            "stale-gc: {} freed {:.1} MB (synced={})",
                            project, o.freed_mb(), o.synced
                        ),
                        Err(e) => tracing::error!(
                            "stale-gc: deactivate {} failed: {:#}", project, e
                        ),
                    }
                }
                // Reload once after the sweep so subsequent steps see the
                // trimmed manifest.
                if !stale.is_empty() {
                    manifest = Manifest::load(&manifest_path).unwrap_or(manifest);
                }

                // Space-pressure LRU still runs only when tight — it's
                // the emergency "we're 90% full and everything is fresh"
                // path. Stale sweep above handles the ordinary case.
                let used = hot::total_used_gb(&manifest)?;
                if used > cfg.hot.max_gb as f64 * 0.9 {
                    tracing::warn!("hot tier {:.1}GB / {}GB — running space GC", used, cfg.hot.max_gb);
                    let max_gb = cfg.hot.max_gb as f64;
                    let updated = Manifest::update(&manifest_path, |m| {
                        hot::gc_by_space(m, max_gb)?;
                        Ok(())
                    })?;
                    manifest = updated;
                }
                // Auto-enqueue sync jobs for repos whose HEAD moved since
                // the previous tick, then drain the queue. Two paths that
                // used to be manual (or wait for `claw-store deactivate`)
                // are now folded into the ordinary poll cycle:
                //   1. head_watch::scan_and_enqueue catches `git commit`
                //      done directly in /slab/projects/*/* and adds a
                //      sync job — no need for the operator to remember to
                //      run `claw-store sync`.
                //   2. drain_sync_queue then notifies the peer for every
                //      job (both freshly-enqueued and previously-stuck).
                if let Some(peer) = &cfg.peer {
                    let queue_path = SyncQueue::default_path();
                    let cache_path = HeadCache::default_path();
                    let mut queue = SyncQueue::load(&queue_path).unwrap_or_default();
                    let mut cache = HeadCache::load(&cache_path);

                    match scan_and_enqueue(
                        &cfg.warm.projects_path, &mut queue, &mut cache,
                    ) {
                        Ok((scanned, enqueued)) if enqueued > 0 => {
                            tracing::info!(
                                "head-watch: scanned {}, {} new sync(s) enqueued",
                                scanned, enqueued
                            );
                            let _ = queue.save(&queue_path);
                        }
                        Ok(_) => { /* nothing moved — quiet */ }
                        Err(e) => {
                            tracing::warn!("head-watch scan failed: {:#}", e);
                        }
                    }
                    // Persist the cache even when nothing was enqueued —
                    // covers the first-scan populate case.
                    if let Err(e) = cache.save(&cache_path) {
                        tracing::warn!("head cache save failed: {:#}", e);
                    }

                    if !queue.jobs.is_empty() {
                        tracing::info!("draining {} queued sync job(s)", queue.jobs.len());
                        if let Err(e) = drain_sync_queue(&mut queue, &peer.user, &peer.host, &queue_path) {
                            tracing::error!("sync queue drain failed: {:#}", e);
                        }
                    }
                }
            }
            _ = snap_tick.tick() => {
                if !cfg.warm.zfs_enabled() {
                    tracing::debug!("skipping snapshot tick — zfs_dataset = \"none\" on this node");
                } else {
                    let ts = chrono::Utc::now().format("%Y-%m-%d-%H%M").to_string();
                    tracing::info!("taking snapshot {}", ts);
                    if let Err(e) = snapshot::run_snapshot_cycle(
                        &zfs, &cfg.warm.zfs_dataset, &ts,
                        cfg.warm.snapshot_retain_hours as usize,
                        cfg.warm.snapshot_retain_days as usize,
                        cfg.warm.snapshot_retain_weeks as usize,
                    ) {
                        tracing::error!("snapshot failed: {:#}", e);
                    }
                }
            }
            _ = repl_tick.tick() => {
                if !cfg.warm.zfs_enabled() {
                    tracing::debug!("skipping replication tick — zfs_dataset = \"none\" on this node");
                } else if let Some(rep) = &cfg.replication {
                    if let (Some(host), Some(user), Some(dest)) = (
                        &rep.send_to_host, &rep.send_to_user, &rep.cold_dataset_on_peer
                    ) {
                        tracing::info!("replicating warm → cold on {}", host);
                        if let Err(e) = snapshot::replicate_to_cold(
                            &zfs, &cfg.warm.zfs_dataset, user, host, dest
                        ) {
                            tracing::error!("replication failed: {:#}", e);
                        }
                    }
                }
            }
        }
    }
    Ok(())
}

/// Names of projects that qualify for the proactive stale sweep:
/// unpinned, still have a hot target on disk, and last_active is a real
/// timestamp older than `stale_hours`.
///
/// `last_active = None` is treated as NOT stale. That case happens
/// constantly — daemon restart resets the runtime timestamp state, and
/// `update_active_projects` only stamps projects that have cargo/rustc
/// running RIGHT NOW during a poll tick. A freshly-activated project
/// with no activity yet must not be nuked; the operator explicitly asked
/// for it to be active. Staleness is a positive assertion ("we've seen
/// this project sit idle past the threshold"), never an absence of data.
fn stale_project_names(manifest: &Manifest, stale_hours: u64) -> Vec<String> {
    let now = Utc::now();
    let threshold = chrono::Duration::hours(stale_hours as i64);
    manifest.projects.iter()
        .filter(|p| !p.pinned)
        .filter(|p| p.hot_target_path.exists())
        .filter_map(|p| p.last_active.map(|t| (p, t)))
        .filter(|(_, t)| (now - *t) > threshold)
        .map(|(p, _)| p.name.clone())
        .collect()
}

fn update_active_projects(manifest: &mut Manifest) -> Result<()> {
    let mut sys = System::new_with_specifics(
        RefreshKind::new().with_processes(ProcessRefreshKind::everything())
    );
    sys.refresh_processes();
    let now = chrono::Utc::now();
    for process in sys.processes().values() {
        let cmd = process.exe().map(|p| p.to_string_lossy().to_string()).unwrap_or_default();
        if cmd.contains("cargo") || cmd.contains("rustc") {
            let cwd = process.root().map(|p| p.to_path_buf());
            if let Some(cwd) = cwd {
                for project in &mut manifest.projects {
                    if cwd.starts_with(&project.warm_path) {
                        project.last_active = Some(now);
                        project.last_build = Some(now);
                    }
                }
            }
        }
    }
    Ok(())
}