clawstor/claw-store/src/cluster/wal_queue.rs

//! Phase 4d (2026-07-13): `WalQueue` — the caller-facing wrapper
//! around `WriteAheadLog` + `wal_mutation` + `wal_replay`.
//!
//! Callers that want to add client-mode / offline durability to a
//! previously-synchronous RPC path shouldn't have to reason about
//! three modules. This wrapper collapses the flow to two calls:
//!
//! ```ignore
//! let mut q = WalQueue::open(state_dir.join("wal")).await?;
//! q.enqueue(&WalMutation::PutTagVersioned { .. }).await?;
//! // ...later, on reconnect:
//! let report = q.drain(&conn).await?;
//! ```
//!
//! `drain` runs `drive_replay` and, when it walked the whole log
//! cleanly, `truncate_up_to(last_applied)`. On partial progress
//! (hard error mid-stream), `truncate_up_to` still runs up to the
//! last successfully-applied seq — nothing is truncated past the
//! failure point, so the failing record and everything after it
//! are retried on the next `drain`.
//!
//! Introspection surface (`pending_count`, `oldest_pending_seq`,
//! `newest_pending_seq`, `snapshot`) is what a metrics endpoint /
//! CLI status view wants.

use crate::cluster::wal::WriteAheadLog;
use crate::cluster::wal_mutation::{
    append_mutation, replay_mutations, WalMutation, WalMutationError,
};
use crate::cluster::wal_replay::{drive_replay, DriveReport};
use anyhow::Result;
use quinn::Connection;
use std::path::PathBuf;

/// Wrapper around a WAL that speaks in `WalMutation`s.
pub struct WalQueue {
    wal: WriteAheadLog,
}

impl WalQueue {
    /// Open (or create) a queue at `root`. Delegates to
    /// [`WriteAheadLog::open`] with the default segment cap.
    pub async fn open(root: impl Into<PathBuf>) -> Result<Self> {
        Ok(Self {
            wal: WriteAheadLog::open(root).await?,
        })
    }

    /// Open with a custom per-segment cap. Useful for tests that
    /// want rotation without writing 8 MiB.
    pub async fn open_with_options(
        root: impl Into<PathBuf>,
        max_segment_bytes: u64,
    ) -> Result<Self> {
        Ok(Self {
            wal: WriteAheadLog::open_with_options(root, max_segment_bytes).await?,
        })
    }

    /// Encode + append a mutation. Returns the assigned seq.
    /// Durable after this returns (fsync'd inside the WAL layer).
    pub async fn enqueue(&mut self, mutation: &WalMutation) -> Result<u64> {
        append_mutation(&mut self.wal, mutation).await
    }

    /// How many records are still on disk. This is O(#pending) via
    /// a full log scan; callers who need this on a hot path should
    /// cache the result instead of calling every request.
    pub async fn pending_count(&self) -> Result<usize> {
        Ok(self.wal.iter_from(0).await?.len())
    }

    /// Lowest seq still on disk. `None` when empty.
    pub fn oldest_pending_seq(&self) -> Option<u64> {
        if self.wal.is_empty() {
            None
        } else {
            Some(self.wal.head_seq())
        }
    }

    /// Highest seq assigned. `None` when the log has never had a
    /// record (or was fully truncated back to empty).
    pub fn newest_pending_seq(&self) -> Option<u64> {
        if self.wal.is_empty() {
            None
        } else {
            Some(self.wal.tail_seq())
        }
    }

    /// Whether the queue has any pending mutations.
    pub fn is_empty(&self) -> bool {
        self.wal.is_empty()
    }

    /// Decode every pending record. Kept out of the drain path so
    /// callers can inspect what's about to be replayed (status
    /// views, tests, log dumps).
    pub async fn snapshot(&self) -> Result<Vec<(u64, Result<WalMutation, WalMutationError>)>> {
        replay_mutations(&self.wal, 0).await
    }

    /// Replay pending mutations against a peer. Advances the
    /// watermark to the last successfully-applied (or Superseded)
    /// seq, whether or not the drive stopped on a hard error mid-
    /// stream. The `DriveReport` returned from `drive_replay` is
    /// forwarded verbatim so the caller can log / retry / alert.
    pub async fn drain(&mut self, conn: &Connection) -> Result<DriveReport> {
        let report = drive_replay(conn, &self.wal, 0).await?;
        if report.last_applied > 0 {
            self.wal.truncate_up_to(report.last_applied).await?;
        }
        Ok(report)
    }

    /// Expose the backing WAL for advanced callers (metrics, low-
    /// level ops). Discouraged for normal use — go through
    /// `enqueue` / `drain` / `snapshot` where possible.
    pub fn wal(&self) -> &WriteAheadLog {
        &self.wal
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::cluster::gossip::ClusterGossip;
    use crate::cluster::refs::{node_stamp_for, RefStore, StampedRef};
    use crate::cluster::rpc::{
        call_get_ref, call_get_tag_versioned, call_put_ref_versioned, serve_connection,
        RpcRouter,
    };
    use crate::cluster::tags::{StampedTagValue, TagStore};
    use crate::cluster::transport::{NodeIdentity, QuicClient, QuicServer};
    use crate::cluster::wal_mutation::Kind;
    use crate::config::ClusterConfig;
    use std::net::SocketAddr;
    use std::sync::atomic::{AtomicU16, Ordering};
    use std::sync::Arc;
    use std::time::Duration;

    static NEXT_PORT: AtomicU16 = AtomicU16::new(49001);
    fn next_port() -> u16 {
        NEXT_PORT.fetch_add(1, Ordering::Relaxed)
    }
    fn loopback(port: u16) -> SocketAddr {
        format!("127.0.0.1:{port}").parse().unwrap()
    }

    async fn bootstrap_router(name: &str, port: u16) -> (tempfile::TempDir, Arc<RpcRouter>) {
        let cfg = ClusterConfig {
            zone: "fabric-10g".into(),
            bind_lan: Some(loopback(port)),
            ..Default::default()
        };
        let gossip = Arc::new(ClusterGossip::bootstrap(&cfg, name).await.unwrap());
        let tmp = tempfile::TempDir::new().unwrap();
        let blob_store = Arc::new(
            crate::cluster::blob::BlobStore::open(tmp.path().join("blobs")).unwrap(),
        );
        let ref_store = Arc::new(RefStore::open(tmp.path().join("refs")).unwrap());
        let tag_store = Arc::new(TagStore::open(tmp.path().join("tags")).unwrap());
        let router = Arc::new(
            RpcRouter::new(gossip, name.into(), "fabric-10g".into())
                .with_blob_store(blob_store)
                .with_ref_store(ref_store)
                .with_tag_store(tag_store),
        );
        (tmp, router)
    }

    async fn start_peer(
        id_a: NodeIdentity,
    ) -> (tempfile::TempDir, Arc<RpcRouter>, SocketAddr, tokio::task::JoinHandle<()>) {
        let (tmp, router) = bootstrap_router("a", next_port()).await;
        let server = QuicServer::bind(loopback(0), id_a).unwrap();
        let addr = server.local_addr().unwrap();
        let r = router.clone();
        let acc = tokio::spawn(async move {
            while let Some(Ok(conn)) = server.accept().await {
                let r = r.clone();
                tokio::spawn(async move {
                    let _ = serve_connection(conn, r).await;
                });
            }
        });
        (tmp, router, addr, acc)
    }

    #[tokio::test]
    async fn empty_queue_reports_empty() {
        let tmp = tempfile::TempDir::new().unwrap();
        let q = WalQueue::open(tmp.path().join("wal")).await.unwrap();
        assert!(q.is_empty());
        assert_eq!(q.pending_count().await.unwrap(), 0);
        assert_eq!(q.oldest_pending_seq(), None);
        assert_eq!(q.newest_pending_seq(), None);
    }

    #[tokio::test]
    async fn enqueue_updates_bounds() {
        let tmp = tempfile::TempDir::new().unwrap();
        let mut q = WalQueue::open(tmp.path().join("wal")).await.unwrap();
        q.enqueue(&WalMutation::DeleteTag { key: "a".into() })
            .await
            .unwrap();
        q.enqueue(&WalMutation::DeleteTag { key: "b".into() })
            .await
            .unwrap();
        q.enqueue(&WalMutation::DeleteTag { key: "c".into() })
            .await
            .unwrap();
        assert_eq!(q.pending_count().await.unwrap(), 3);
        assert_eq!(q.oldest_pending_seq(), Some(1));
        assert_eq!(q.newest_pending_seq(), Some(3));
    }

    #[tokio::test]
    async fn snapshot_returns_decoded_mutations_in_order() {
        let tmp = tempfile::TempDir::new().unwrap();
        let mut q = WalQueue::open(tmp.path().join("wal")).await.unwrap();
        let m1 = WalMutation::DeleteTag { key: "a".into() };
        let m2 = WalMutation::SetTagExpiry {
            key: "b".into(),
            expires_at_unix: 999,
        };
        q.enqueue(&m1).await.unwrap();
        q.enqueue(&m2).await.unwrap();
        let snap = q.snapshot().await.unwrap();
        assert_eq!(snap.len(), 2);
        assert_eq!(snap[0].0, 1);
        assert_eq!(snap[0].1.as_ref().unwrap().kind(), Kind::DeleteTag);
        assert_eq!(snap[1].1.as_ref().unwrap(), &m2);
    }

    #[tokio::test]
    async fn drain_clears_queue_and_applies_to_peer() {
        let (id_a, id_b) = NodeIdentity::generate_test_pair("a", "b").unwrap();
        let (_tmp_peer, _router, addr, acc) = start_peer(id_a).await;
        let client = QuicClient::new(loopback(0), id_b).unwrap();
        let conn = client.connect(addr, "a").await.unwrap();

        let tmp = tempfile::TempDir::new().unwrap();
        let mut q = WalQueue::open(tmp.path().join("wal")).await.unwrap();
        let stamped = StampedTagValue {
            value: [0x42; 32],
            clock: 10,
            node: node_stamp_for("client"),
        };
        q.enqueue(&WalMutation::PutTagVersioned {
            key: "clawverse:main".into(),
            stamped,
        })
        .await
        .unwrap();
        q.enqueue(&WalMutation::PutRef {
            key: [0xAB; 32],
            value: [0xCD; 32],
        })
        .await
        .unwrap();

        let report = q.drain(&conn).await.unwrap();
        assert!(report.is_clean(), "unexpected stop: {:?}", report.stopped_at);
        assert_eq!(report.applied, 2);
        assert!(q.is_empty());
        assert_eq!(q.pending_count().await.unwrap(), 0);

        // Peer state is populated.
        assert_eq!(
            call_get_tag_versioned(&conn, "clawverse:main").await.unwrap(),
            Some(stamped)
        );
        assert_eq!(
            call_get_ref(&conn, &[0xAB; 32]).await.unwrap(),
            Some([0xCD; 32])
        );

        conn.close(quinn::VarInt::from_u32(0), b"done");
        client.shutdown().await;
        tokio::time::sleep(Duration::from_millis(50)).await;
        acc.abort();
    }

    #[tokio::test]
    async fn drain_survives_peer_side_supersession() {
        let (id_a, id_b) = NodeIdentity::generate_test_pair("a", "b").unwrap();
        let (_tmp_peer, _router, addr, acc) = start_peer(id_a).await;
        let client = QuicClient::new(loopback(0), id_b).unwrap();
        let conn = client.connect(addr, "a").await.unwrap();

        // Peer already has a dominant version.
        let key = [0x77u8; 32];
        let winner = StampedRef {
            value: [0xFF; 32],
            clock: 100,
            node: node_stamp_for("winner"),
        };
        assert!(call_put_ref_versioned(&conn, &key, &winner).await.unwrap());

        let tmp = tempfile::TempDir::new().unwrap();
        let mut q = WalQueue::open(tmp.path().join("wal")).await.unwrap();
        q.enqueue(&WalMutation::PutRefVersioned {
            key,
            stamped: StampedRef {
                value: [0xAA; 32],
                clock: 1,
                node: node_stamp_for("loser"),
            },
        })
        .await
        .unwrap();

        let report = q.drain(&conn).await.unwrap();
        assert!(report.is_clean());
        assert_eq!(report.applied, 0);
        assert_eq!(report.superseded, 1);
        // Superseded still advances the watermark — queue is drained.
        assert!(q.is_empty());

        conn.close(quinn::VarInt::from_u32(0), b"done");
        client.shutdown().await;
        tokio::time::sleep(Duration::from_millis(50)).await;
        acc.abort();
    }

    #[tokio::test]
    async fn enqueue_survives_reopen() {
        let tmp = tempfile::TempDir::new().unwrap();
        let path = tmp.path().join("wal");
        {
            let mut q = WalQueue::open(&path).await.unwrap();
            q.enqueue(&WalMutation::DeleteTag { key: "a".into() })
                .await
                .unwrap();
            q.enqueue(&WalMutation::DeleteTag { key: "b".into() })
                .await
                .unwrap();
        }
        let q = WalQueue::open(&path).await.unwrap();
        assert_eq!(q.pending_count().await.unwrap(), 2);
        assert_eq!(q.oldest_pending_seq(), Some(1));
        assert_eq!(q.newest_pending_seq(), Some(2));
    }
}