clawstor/claw-store/src/cluster.rs

//! Cluster peer routing and gossip.
//!
//! Two submodules of the v2 distributed FS:
//! * this file — LAN-first peer probe (Phase 1a).
//! * [`gossip`] — chitchat SWIM membership + KV state broadcast (Phase 1b).
//!
//! Every fleet node has two potential endpoints: a LAN socket (fast, direct,
//! no WireGuard framing) and a Tailscale socket (fallback, always reachable
//! when the tailnet is up). This file races LAN first and falls through to
//! Tailscale, returning the winning route so the caller can cache it for
//! the session.
//!
//! Cache the returned [`RouteWinner`] and re-probe on transport error or
//! after ~5 min so route changes (node moved networks, LAN NIC came back)
//! propagate without a daemon restart.

pub mod blob;
pub mod build_cache;
pub mod client_config;
pub mod gossip;
pub mod metrics;
pub mod prom;
pub mod refs;
pub mod rpc;
pub mod services;
pub mod tags;
pub mod transport;

use crate::config::PeerEntry;
use anyhow::{bail, Result};
use std::net::SocketAddr;
use std::time::{Duration, Instant};
use tokio::net::TcpStream;
use tokio::time::timeout;

/// Which network route reached a peer.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RouteKind {
    /// Direct LAN socket. No WireGuard overhead.
    Lan,
    /// Tailscale-provided socket. WireGuard-encapsulated.
    Tailscale,
}

impl RouteKind {
    /// Short lowercase label suitable for logs, metrics, config.
    pub fn as_str(self) -> &'static str {
        match self {
            RouteKind::Lan => "lan",
            RouteKind::Tailscale => "tailscale",
        }
    }
}

/// A successful probe result: which address answered, which route kind it was,
/// and how long the probe took from start to first-success (useful for logs).
#[derive(Debug, Clone)]
pub struct RouteWinner {
    pub addr: SocketAddr,
    pub kind: RouteKind,
    pub elapsed: Duration,
}

/// LAN-first probe.
///
/// Attempts the peer's LAN address with a tight timeout; on any error or
/// timeout, falls through to the Tailscale address with a longer timeout
/// (Tailscale may need to set up a direct WireGuard tunnel on first probe).
/// Errors only when neither route succeeds.
pub struct LanFirstProbe {
    lan_timeout: Duration,
    tailscale_timeout: Duration,
}

impl Default for LanFirstProbe {
    /// LAN 200ms, Tailscale 500ms. Rationale: on a healthy LAN the TCP SYN
    /// round-trip is sub-millisecond; anything slower than 200ms means the
    /// LAN route is not usable and we should fall through immediately.
    /// Tailscale needs headroom for WireGuard handshakes and DERP fallback.
    fn default() -> Self {
        Self {
            lan_timeout: Duration::from_millis(200),
            tailscale_timeout: Duration::from_millis(500),
        }
    }
}

impl LanFirstProbe {
    /// Probe using default timeouts (200ms LAN, 500ms Tailscale).
    pub fn new() -> Self {
        Self::default()
    }

    /// Probe with custom timeouts. Useful for tests and for tuning per
    /// deployment shape.
    pub fn with_timeouts(lan: Duration, tailscale: Duration) -> Self {
        Self {
            lan_timeout: lan,
            tailscale_timeout: tailscale,
        }
    }

    /// Probe a peer. Returns the first route (LAN preferred) that accepts a
    /// TCP connection within its per-route timeout. Errors when neither
    /// route succeeds, or when the peer has no addresses configured.
    pub async fn probe(&self, peer: &PeerEntry) -> Result<RouteWinner> {
        let start = Instant::now();
        if let Some(lan) = peer.lan_addr {
            if try_connect(lan, self.lan_timeout).await.is_ok() {
                return Ok(RouteWinner {
                    addr: lan,
                    kind: RouteKind::Lan,
                    elapsed: start.elapsed(),
                });
            }
        }
        if let Some(ts) = peer.tailscale_addr {
            if try_connect(ts, self.tailscale_timeout).await.is_ok() {
                return Ok(RouteWinner {
                    addr: ts,
                    kind: RouteKind::Tailscale,
                    elapsed: start.elapsed(),
                });
            }
        }
        bail!(
            "peer {} has no reachable address (lan={:?}, tailscale={:?})",
            peer.name,
            peer.lan_addr,
            peer.tailscale_addr,
        );
    }
}

/// Attempt a TCP connect with a deadline. Returns Ok on successful handshake,
/// Err on connection error or timeout.
async fn try_connect(addr: SocketAddr, deadline: Duration) -> Result<()> {
    match timeout(deadline, TcpStream::connect(addr)).await {
        Ok(Ok(_stream)) => Ok(()),
        Ok(Err(e)) => Err(anyhow::Error::from(e)),
        Err(_) => bail!("connect to {} timed out after {:?}", addr, deadline),
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use tokio::net::TcpListener;

    /// Bind a TCP listener on an OS-assigned port, then drop it. The returned
    /// address points at a port that's very likely unbound. Tiny race window
    /// between drop and the caller's probe (microseconds); in practice tests
    /// pass reliably. If ever flaky, we upgrade to a strict SO_LINGER=0 trick.
    async fn free_but_unbound_addr() -> SocketAddr {
        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let addr = listener.local_addr().unwrap();
        drop(listener);
        addr
    }

    /// Bind a listener and spawn an accept loop so it stays alive across probes.
    /// Returns the listener's local address.
    async fn spawn_accepting_listener() -> SocketAddr {
        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let addr = listener.local_addr().unwrap();
        tokio::spawn(async move {
            loop {
                if listener.accept().await.is_err() {
                    return;
                }
            }
        });
        addr
    }

    fn peer_with(lan: Option<SocketAddr>, ts: Option<SocketAddr>) -> PeerEntry {
        PeerEntry {
            name: "test-peer".into(),
            zone: "test-zone".into(),
            lan_addr: lan,
            tailscale_addr: ts,
        }
    }

    #[tokio::test]
    async fn probe_prefers_lan_when_available() {
        let lan = spawn_accepting_listener().await;
        // Tailscale unreachable — must not be attempted when LAN succeeds.
        let ts = free_but_unbound_addr().await;
        let probe = LanFirstProbe::new();
        let win = probe.probe(&peer_with(Some(lan), Some(ts))).await.unwrap();
        assert_eq!(win.kind, RouteKind::Lan);
        assert_eq!(win.addr, lan);
    }

    #[tokio::test]
    async fn probe_falls_through_to_tailscale_when_lan_fails() {
        let lan_dead = free_but_unbound_addr().await;
        let ts = spawn_accepting_listener().await;
        // Short LAN timeout so the test runs fast even if the OS holds the
        // failed connect in a queue rather than refusing immediately.
        let probe =
            LanFirstProbe::with_timeouts(Duration::from_millis(50), Duration::from_millis(500));
        let win = probe
            .probe(&peer_with(Some(lan_dead), Some(ts)))
            .await
            .unwrap();
        assert_eq!(win.kind, RouteKind::Tailscale);
        assert_eq!(win.addr, ts);
    }

    #[tokio::test]
    async fn probe_errors_when_both_routes_fail() {
        let lan_dead = free_but_unbound_addr().await;
        let ts_dead = free_but_unbound_addr().await;
        let probe =
            LanFirstProbe::with_timeouts(Duration::from_millis(50), Duration::from_millis(50));
        let err = probe
            .probe(&peer_with(Some(lan_dead), Some(ts_dead)))
            .await
            .unwrap_err()
            .to_string();
        assert!(
            err.contains("no reachable address"),
            "unexpected error: {err}"
        );
    }

    #[tokio::test]
    async fn probe_uses_only_tailscale_when_lan_absent() {
        let ts = spawn_accepting_listener().await;
        let probe = LanFirstProbe::new();
        let win = probe.probe(&peer_with(None, Some(ts))).await.unwrap();
        assert_eq!(win.kind, RouteKind::Tailscale);
        assert_eq!(win.addr, ts);
    }

    #[tokio::test]
    async fn probe_uses_only_lan_when_tailscale_absent() {
        let lan = spawn_accepting_listener().await;
        let probe = LanFirstProbe::new();
        let win = probe.probe(&peer_with(Some(lan), None)).await.unwrap();
        assert_eq!(win.kind, RouteKind::Lan);
        assert_eq!(win.addr, lan);
    }

    #[tokio::test]
    async fn probe_errors_when_no_addresses_configured() {
        // PeerEntry::validate would reject this at config-load time, but the
        // probe must still handle it gracefully (defense in depth).
        let probe = LanFirstProbe::new();
        let err = probe
            .probe(&peer_with(None, None))
            .await
            .unwrap_err()
            .to_string();
        assert!(
            err.contains("no reachable address"),
            "unexpected error: {err}"
        );
    }

    #[tokio::test]
    async fn probe_records_elapsed_time_under_timeout() {
        let lan = spawn_accepting_listener().await;
        let probe = LanFirstProbe::new();
        let win = probe.probe(&peer_with(Some(lan), None)).await.unwrap();
        assert!(
            win.elapsed < Duration::from_millis(500),
            "probe should complete quickly on a live local listener; took {:?}",
            win.elapsed
        );
    }

    #[test]
    fn route_kind_as_str_matches_variants() {
        assert_eq!(RouteKind::Lan.as_str(), "lan");
        assert_eq!(RouteKind::Tailscale.as_str(), "tailscale");
    }
}