//! Phase 4d (2026-07-13): `WalQueue` — the caller-facing wrapper //! around `WriteAheadLog` + `wal_mutation` + `wal_replay`. //! //! Callers that want to add client-mode / offline durability to a //! previously-synchronous RPC path shouldn't have to reason about //! three modules. This wrapper collapses the flow to two calls: //! //! ```ignore //! let mut q = WalQueue::open(state_dir.join("wal")).await?; //! q.enqueue(&WalMutation::PutTagVersioned { .. }).await?; //! // ...later, on reconnect: //! let report = q.drain(&conn).await?; //! ``` //! //! `drain` runs `drive_replay` and, when it walked the whole log //! cleanly, `truncate_up_to(last_applied)`. On partial progress //! (hard error mid-stream), `truncate_up_to` still runs up to the //! last successfully-applied seq — nothing is truncated past the //! failure point, so the failing record and everything after it //! are retried on the next `drain`. //! //! Introspection surface (`pending_count`, `oldest_pending_seq`, //! `newest_pending_seq`, `snapshot`) is what a metrics endpoint / //! CLI status view wants. use crate::cluster::wal::WriteAheadLog; use crate::cluster::wal_mutation::{ append_mutation, replay_mutations, WalMutation, WalMutationError, }; use crate::cluster::wal_replay::{drive_replay, DriveReport}; use anyhow::Result; use quinn::Connection; use std::path::PathBuf; /// Wrapper around a WAL that speaks in `WalMutation`s. pub struct WalQueue { wal: WriteAheadLog, } impl WalQueue { /// Open (or create) a queue at `root`. Delegates to /// [`WriteAheadLog::open`] with the default segment cap. pub async fn open(root: impl Into) -> Result { Ok(Self { wal: WriteAheadLog::open(root).await?, }) } /// Open with a custom per-segment cap. Useful for tests that /// want rotation without writing 8 MiB. pub async fn open_with_options( root: impl Into, max_segment_bytes: u64, ) -> Result { Ok(Self { wal: WriteAheadLog::open_with_options(root, max_segment_bytes).await?, }) } /// Encode + append a mutation. Returns the assigned seq. /// Durable after this returns (fsync'd inside the WAL layer). pub async fn enqueue(&mut self, mutation: &WalMutation) -> Result { append_mutation(&mut self.wal, mutation).await } /// How many records are still on disk. This is O(#pending) via /// a full log scan; callers who need this on a hot path should /// cache the result instead of calling every request. pub async fn pending_count(&self) -> Result { Ok(self.wal.iter_from(0).await?.len()) } /// Lowest seq still on disk. `None` when empty. pub fn oldest_pending_seq(&self) -> Option { if self.wal.is_empty() { None } else { Some(self.wal.head_seq()) } } /// Highest seq assigned. `None` when the log has never had a /// record (or was fully truncated back to empty). pub fn newest_pending_seq(&self) -> Option { if self.wal.is_empty() { None } else { Some(self.wal.tail_seq()) } } /// Whether the queue has any pending mutations. pub fn is_empty(&self) -> bool { self.wal.is_empty() } /// Decode every pending record. Kept out of the drain path so /// callers can inspect what's about to be replayed (status /// views, tests, log dumps). pub async fn snapshot(&self) -> Result)>> { replay_mutations(&self.wal, 0).await } /// Replay pending mutations against a peer. Advances the /// watermark to the last successfully-applied (or Superseded) /// seq, whether or not the drive stopped on a hard error mid- /// stream. The `DriveReport` returned from `drive_replay` is /// forwarded verbatim so the caller can log / retry / alert. pub async fn drain(&mut self, conn: &Connection) -> Result { let report = drive_replay(conn, &self.wal, 0).await?; if report.last_applied > 0 { self.wal.truncate_up_to(report.last_applied).await?; } Ok(report) } /// Expose the backing WAL for advanced callers (metrics, low- /// level ops). Discouraged for normal use — go through /// `enqueue` / `drain` / `snapshot` where possible. pub fn wal(&self) -> &WriteAheadLog { &self.wal } } #[cfg(test)] mod tests { use super::*; use crate::cluster::gossip::ClusterGossip; use crate::cluster::refs::{node_stamp_for, RefStore, StampedRef}; use crate::cluster::rpc::{ call_get_ref, call_get_tag_versioned, call_put_ref_versioned, serve_connection, RpcRouter, }; use crate::cluster::tags::{StampedTagValue, TagStore}; use crate::cluster::transport::{NodeIdentity, QuicClient, QuicServer}; use crate::cluster::wal_mutation::Kind; use crate::config::ClusterConfig; use std::net::SocketAddr; use std::sync::atomic::{AtomicU16, Ordering}; use std::sync::Arc; use std::time::Duration; static NEXT_PORT: AtomicU16 = AtomicU16::new(49001); fn next_port() -> u16 { NEXT_PORT.fetch_add(1, Ordering::Relaxed) } fn loopback(port: u16) -> SocketAddr { format!("127.0.0.1:{port}").parse().unwrap() } async fn bootstrap_router(name: &str, port: u16) -> (tempfile::TempDir, Arc) { let cfg = ClusterConfig { zone: "fabric-10g".into(), bind_lan: Some(loopback(port)), ..Default::default() }; let gossip = Arc::new(ClusterGossip::bootstrap(&cfg, name).await.unwrap()); let tmp = tempfile::TempDir::new().unwrap(); let blob_store = Arc::new( crate::cluster::blob::BlobStore::open(tmp.path().join("blobs")).unwrap(), ); let ref_store = Arc::new(RefStore::open(tmp.path().join("refs")).unwrap()); let tag_store = Arc::new(TagStore::open(tmp.path().join("tags")).unwrap()); let router = Arc::new( RpcRouter::new(gossip, name.into(), "fabric-10g".into()) .with_blob_store(blob_store) .with_ref_store(ref_store) .with_tag_store(tag_store), ); (tmp, router) } async fn start_peer( id_a: NodeIdentity, ) -> (tempfile::TempDir, Arc, SocketAddr, tokio::task::JoinHandle<()>) { let (tmp, router) = bootstrap_router("a", next_port()).await; let server = QuicServer::bind(loopback(0), id_a).unwrap(); let addr = server.local_addr().unwrap(); let r = router.clone(); let acc = tokio::spawn(async move { while let Some(Ok(conn)) = server.accept().await { let r = r.clone(); tokio::spawn(async move { let _ = serve_connection(conn, r).await; }); } }); (tmp, router, addr, acc) } #[tokio::test] async fn empty_queue_reports_empty() { let tmp = tempfile::TempDir::new().unwrap(); let q = WalQueue::open(tmp.path().join("wal")).await.unwrap(); assert!(q.is_empty()); assert_eq!(q.pending_count().await.unwrap(), 0); assert_eq!(q.oldest_pending_seq(), None); assert_eq!(q.newest_pending_seq(), None); } #[tokio::test] async fn enqueue_updates_bounds() { let tmp = tempfile::TempDir::new().unwrap(); let mut q = WalQueue::open(tmp.path().join("wal")).await.unwrap(); q.enqueue(&WalMutation::DeleteTag { key: "a".into() }) .await .unwrap(); q.enqueue(&WalMutation::DeleteTag { key: "b".into() }) .await .unwrap(); q.enqueue(&WalMutation::DeleteTag { key: "c".into() }) .await .unwrap(); assert_eq!(q.pending_count().await.unwrap(), 3); assert_eq!(q.oldest_pending_seq(), Some(1)); assert_eq!(q.newest_pending_seq(), Some(3)); } #[tokio::test] async fn snapshot_returns_decoded_mutations_in_order() { let tmp = tempfile::TempDir::new().unwrap(); let mut q = WalQueue::open(tmp.path().join("wal")).await.unwrap(); let m1 = WalMutation::DeleteTag { key: "a".into() }; let m2 = WalMutation::SetTagExpiry { key: "b".into(), expires_at_unix: 999, }; q.enqueue(&m1).await.unwrap(); q.enqueue(&m2).await.unwrap(); let snap = q.snapshot().await.unwrap(); assert_eq!(snap.len(), 2); assert_eq!(snap[0].0, 1); assert_eq!(snap[0].1.as_ref().unwrap().kind(), Kind::DeleteTag); assert_eq!(snap[1].1.as_ref().unwrap(), &m2); } #[tokio::test] async fn drain_clears_queue_and_applies_to_peer() { let (id_a, id_b) = NodeIdentity::generate_test_pair("a", "b").unwrap(); let (_tmp_peer, _router, addr, acc) = start_peer(id_a).await; let client = QuicClient::new(loopback(0), id_b).unwrap(); let conn = client.connect(addr, "a").await.unwrap(); let tmp = tempfile::TempDir::new().unwrap(); let mut q = WalQueue::open(tmp.path().join("wal")).await.unwrap(); let stamped = StampedTagValue { value: [0x42; 32], clock: 10, node: node_stamp_for("client"), }; q.enqueue(&WalMutation::PutTagVersioned { key: "clawverse:main".into(), stamped, }) .await .unwrap(); q.enqueue(&WalMutation::PutRef { key: [0xAB; 32], value: [0xCD; 32], }) .await .unwrap(); let report = q.drain(&conn).await.unwrap(); assert!(report.is_clean(), "unexpected stop: {:?}", report.stopped_at); assert_eq!(report.applied, 2); assert!(q.is_empty()); assert_eq!(q.pending_count().await.unwrap(), 0); // Peer state is populated. assert_eq!( call_get_tag_versioned(&conn, "clawverse:main").await.unwrap(), Some(stamped) ); assert_eq!( call_get_ref(&conn, &[0xAB; 32]).await.unwrap(), Some([0xCD; 32]) ); conn.close(quinn::VarInt::from_u32(0), b"done"); client.shutdown().await; tokio::time::sleep(Duration::from_millis(50)).await; acc.abort(); } #[tokio::test] async fn drain_survives_peer_side_supersession() { let (id_a, id_b) = NodeIdentity::generate_test_pair("a", "b").unwrap(); let (_tmp_peer, _router, addr, acc) = start_peer(id_a).await; let client = QuicClient::new(loopback(0), id_b).unwrap(); let conn = client.connect(addr, "a").await.unwrap(); // Peer already has a dominant version. let key = [0x77u8; 32]; let winner = StampedRef { value: [0xFF; 32], clock: 100, node: node_stamp_for("winner"), }; assert!(call_put_ref_versioned(&conn, &key, &winner).await.unwrap()); let tmp = tempfile::TempDir::new().unwrap(); let mut q = WalQueue::open(tmp.path().join("wal")).await.unwrap(); q.enqueue(&WalMutation::PutRefVersioned { key, stamped: StampedRef { value: [0xAA; 32], clock: 1, node: node_stamp_for("loser"), }, }) .await .unwrap(); let report = q.drain(&conn).await.unwrap(); assert!(report.is_clean()); assert_eq!(report.applied, 0); assert_eq!(report.superseded, 1); // Superseded still advances the watermark — queue is drained. assert!(q.is_empty()); conn.close(quinn::VarInt::from_u32(0), b"done"); client.shutdown().await; tokio::time::sleep(Duration::from_millis(50)).await; acc.abort(); } #[tokio::test] async fn enqueue_survives_reopen() { let tmp = tempfile::TempDir::new().unwrap(); let path = tmp.path().join("wal"); { let mut q = WalQueue::open(&path).await.unwrap(); q.enqueue(&WalMutation::DeleteTag { key: "a".into() }) .await .unwrap(); q.enqueue(&WalMutation::DeleteTag { key: "b".into() }) .await .unwrap(); } let q = WalQueue::open(&path).await.unwrap(); assert_eq!(q.pending_count().await.unwrap(), 2); assert_eq!(q.oldest_pending_seq(), Some(1)); assert_eq!(q.newest_pending_seq(), Some(2)); } }