Phase 5g: cache metrics + GetMetrics RPC + peer-metrics CLI #16

Merged
osobh merged 1 commits from phase-5g-cache-metrics into main 2026-07-12 11:16:04 +00:00
6 changed files with 589 additions and 7 deletions
Showing only changes of commit d36cec11a6 - Show all commits
+78 -2
View File
@@ -46,8 +46,8 @@ use crate::cluster::build_cache::{
}; };
use crate::cluster::client_config::{ClientConfig, ResolvedClientConfig}; use crate::cluster::client_config::{ClientConfig, ResolvedClientConfig};
use crate::cluster::rpc::{ use crate::cluster::rpc::{
call_blob_get_stream, call_blob_put_stream, call_blob_stat, call_delete_tag, call_get_ref, call_blob_get_stream, call_blob_put_stream, call_blob_stat, call_delete_tag, call_get_metrics,
call_get_tag, call_list_tags, call_put_ref, call_put_tag, call_get_ref, call_get_tag, call_list_tags, call_put_ref, call_put_tag,
}; };
use crate::cluster::transport::{NodeIdentity, QuicClient}; use crate::cluster::transport::{NodeIdentity, QuicClient};
@@ -93,6 +93,9 @@ enum Cmd {
/// downstream side. Useful as a Gitea-webhook target so the runner's /// downstream side. Useful as a Gitea-webhook target so the runner's
/// local daemon has the cache warm before the build starts. /// local daemon has the cache warm before the build starts.
Prewarm(PrewarmArgs), Prewarm(PrewarmArgs),
/// Fetch and print the peer's cache-metrics snapshot (Phase 5g).
/// Shows hit/miss counts, hit rates, byte volumes served/ingested.
PeerMetrics(PeerArgs),
} }
#[derive(clap::Args, Debug, Clone)] #[derive(clap::Args, Debug, Clone)]
@@ -218,6 +221,7 @@ async fn main() -> Result<()> {
Cmd::Unpin(args) => cmd_unpin(args).await, Cmd::Unpin(args) => cmd_unpin(args).await,
Cmd::ListTags(args) => cmd_list_tags(args).await, Cmd::ListTags(args) => cmd_list_tags(args).await,
Cmd::Prewarm(args) => cmd_prewarm(args).await, Cmd::Prewarm(args) => cmd_prewarm(args).await,
Cmd::PeerMetrics(args) => cmd_peer_metrics(args).await,
} }
} }
@@ -684,6 +688,78 @@ async fn _reserved_call_get_tag(conn: &quinn::Connection, key: &str) -> Result<O
call_get_tag(conn, key).await call_get_tag(conn, key).await
} }
// ── peer-metrics ─────────────────────────────────────────────────────
async fn cmd_peer_metrics(args: PeerArgs) -> Result<()> {
let workspace = args
.workspace
.clone()
.unwrap_or_else(|| std::env::current_dir().expect("cwd"));
let layered = ClientConfig::load_layered(&workspace)?;
let resolved = ResolvedClientConfig::resolve(
layered,
args.peer.clone(),
args.peer_addr,
args.tls_dir.clone(),
args.profile.clone(),
args.features.clone(),
"dev",
);
let (client, conn) = connect_peer(&resolved).await?;
let m = call_get_metrics(&conn).await?;
conn.close(quinn::VarInt::from_u32(0), b"done");
client.shutdown().await;
let uptime = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs().saturating_sub(m.started_unix))
.unwrap_or(0);
println!("── peer cache metrics ──────────────────────────────");
println!("counter uptime: {}s", uptime);
println!();
println!("GetRef hits/misses: {} / {}", m.get_ref_hits, m.get_ref_misses);
match m.get_ref_hit_rate() {
Some(r) => println!(" hit rate: {:.2}%", r * 100.0),
None => println!(" hit rate: n/a (no lookups)"),
}
println!();
println!("GetTag hits/misses: {} / {}", m.get_tag_hits, m.get_tag_misses);
match m.get_tag_hit_rate() {
Some(r) => println!(" hit rate: {:.2}%", r * 100.0),
None => println!(" hit rate: n/a"),
}
println!();
println!("HasChunk hits/miss: {} / {}", m.has_chunk_hits, m.has_chunk_misses);
match m.has_chunk_hit_rate() {
Some(r) => println!(" hit rate: {:.2}%", r * 100.0),
None => println!(" hit rate: n/a"),
}
println!();
println!("GetChunk hits/miss: {} / {}", m.get_chunk_hits, m.get_chunk_misses);
println!();
println!("Blob GET bytes: {}", human_bytes(m.blob_get_bytes));
println!("Blob PUT bytes: {}", human_bytes(m.blob_put_bytes));
println!("────────────────────────────────────────────────────");
Ok(())
}
/// Human-readable byte count (KiB / MiB / GiB). Test helper too.
fn human_bytes(n: u64) -> String {
const KIB: u64 = 1024;
const MIB: u64 = KIB * 1024;
const GIB: u64 = MIB * 1024;
if n >= GIB {
format!("{:.2} GiB", n as f64 / GIB as f64)
} else if n >= MIB {
format!("{:.2} MiB", n as f64 / MIB as f64)
} else if n >= KIB {
format!("{:.2} KiB", n as f64 / KIB as f64)
} else {
format!("{} B", n)
}
}
// ── prewarm ────────────────────────────────────────────────────────── // ── prewarm ──────────────────────────────────────────────────────────
async fn cmd_prewarm(args: PrewarmArgs) -> Result<()> { async fn cmd_prewarm(args: PrewarmArgs) -> Result<()> {
+1
View File
@@ -18,6 +18,7 @@ pub mod blob;
pub mod build_cache; pub mod build_cache;
pub mod client_config; pub mod client_config;
pub mod gossip; pub mod gossip;
pub mod metrics;
pub mod refs; pub mod refs;
pub mod rpc; pub mod rpc;
pub mod services; pub mod services;
+268
View File
@@ -0,0 +1,268 @@
//! Cache metrics (Phase 5g).
//!
//! Lock-free per-router counters that record whether a lookup found
//! what the client asked for, plus cumulative bytes served / stored.
//! Read via the `GetMetrics` RPC — placement engines use the hit rate
//! and byte-volume to decide which peer a runner should point at for
//! a given repo.
//!
//! All fields are `AtomicU64` so handlers can increment without
//! locking, and the snapshot read is a simple `Ordering::Relaxed`
//! load per field (metrics are advisory, not consistency-critical).
use serde::{Deserialize, Serialize};
use std::sync::atomic::{AtomicU64, Ordering};
/// The atomic counters. Held in an `Arc<CacheMetrics>` on the router
/// and read by [`CacheMetrics::snapshot`] whenever `GetMetrics` fires.
#[derive(Debug, Default)]
pub struct CacheMetrics {
/// Timestamp (unix seconds) the counters were reset. Set on
/// [`CacheMetrics::new`]; not touched afterward.
started_unix: AtomicU64,
// GetRef — fingerprint-keyed cache lookups.
get_ref_hits: AtomicU64,
get_ref_misses: AtomicU64,
// GetTag — human-readable pin lookups.
get_tag_hits: AtomicU64,
get_tag_misses: AtomicU64,
// Byte volumes for the blob transfer paths. Both bounded and
// streaming variants roll up into the same counter — the caller
// just sees "how much did we serve/ingest".
blob_get_bytes: AtomicU64,
blob_put_bytes: AtomicU64,
// Chunk-level activity for partial-sync visibility (Phase 2d).
get_chunk_hits: AtomicU64,
get_chunk_misses: AtomicU64,
has_chunk_hits: AtomicU64,
has_chunk_misses: AtomicU64,
}
impl CacheMetrics {
/// Fresh counters. Every field starts at 0 except `started_unix`.
/// Falls back to 0 pre-1970 (never happens; keeps this pure).
pub fn new() -> Self {
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
Self {
started_unix: AtomicU64::new(now),
..Default::default()
}
}
// ── recorders ────────────────────────────────────────────────────
pub fn record_get_ref_hit(&self) {
self.get_ref_hits.fetch_add(1, Ordering::Relaxed);
}
pub fn record_get_ref_miss(&self) {
self.get_ref_misses.fetch_add(1, Ordering::Relaxed);
}
pub fn record_get_tag_hit(&self) {
self.get_tag_hits.fetch_add(1, Ordering::Relaxed);
}
pub fn record_get_tag_miss(&self) {
self.get_tag_misses.fetch_add(1, Ordering::Relaxed);
}
pub fn record_blob_get_bytes(&self, n: u64) {
self.blob_get_bytes.fetch_add(n, Ordering::Relaxed);
}
pub fn record_blob_put_bytes(&self, n: u64) {
self.blob_put_bytes.fetch_add(n, Ordering::Relaxed);
}
pub fn record_get_chunk_hit(&self) {
self.get_chunk_hits.fetch_add(1, Ordering::Relaxed);
}
pub fn record_get_chunk_miss(&self) {
self.get_chunk_misses.fetch_add(1, Ordering::Relaxed);
}
pub fn record_has_chunk_hit(&self) {
self.has_chunk_hits.fetch_add(1, Ordering::Relaxed);
}
pub fn record_has_chunk_miss(&self) {
self.has_chunk_misses.fetch_add(1, Ordering::Relaxed);
}
/// Cheap read-side capture of every counter. Uses `Relaxed`
/// ordering — the snapshot may not be strictly consistent across
/// fields, but for a metric API that's fine (each field is
/// individually up-to-date within a nanosecond).
pub fn snapshot(&self) -> MetricsReply {
MetricsReply {
started_unix: self.started_unix.load(Ordering::Relaxed),
get_ref_hits: self.get_ref_hits.load(Ordering::Relaxed),
get_ref_misses: self.get_ref_misses.load(Ordering::Relaxed),
get_tag_hits: self.get_tag_hits.load(Ordering::Relaxed),
get_tag_misses: self.get_tag_misses.load(Ordering::Relaxed),
blob_get_bytes: self.blob_get_bytes.load(Ordering::Relaxed),
blob_put_bytes: self.blob_put_bytes.load(Ordering::Relaxed),
get_chunk_hits: self.get_chunk_hits.load(Ordering::Relaxed),
get_chunk_misses: self.get_chunk_misses.load(Ordering::Relaxed),
has_chunk_hits: self.has_chunk_hits.load(Ordering::Relaxed),
has_chunk_misses: self.has_chunk_misses.load(Ordering::Relaxed),
}
}
}
/// JSON-shaped snapshot returned by `GetMetrics`. Fields are all
/// counters — hit rate + byte volume computations happen client-side
/// so we don't lock the router into a specific set of derived metrics.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct MetricsReply {
/// Unix timestamp (seconds) when this counter set started.
pub started_unix: u64,
pub get_ref_hits: u64,
pub get_ref_misses: u64,
pub get_tag_hits: u64,
pub get_tag_misses: u64,
pub blob_get_bytes: u64,
pub blob_put_bytes: u64,
pub get_chunk_hits: u64,
pub get_chunk_misses: u64,
pub has_chunk_hits: u64,
pub has_chunk_misses: u64,
}
impl MetricsReply {
/// Hit rate for `GetRef` lookups, as a float in `[0, 1]`. Returns
/// `None` when zero lookups have been recorded (0/0 is undefined).
pub fn get_ref_hit_rate(&self) -> Option<f64> {
let total = self.get_ref_hits.saturating_add(self.get_ref_misses);
if total == 0 {
None
} else {
Some(self.get_ref_hits as f64 / total as f64)
}
}
/// Hit rate for `GetTag` lookups.
pub fn get_tag_hit_rate(&self) -> Option<f64> {
let total = self.get_tag_hits.saturating_add(self.get_tag_misses);
if total == 0 {
None
} else {
Some(self.get_tag_hits as f64 / total as f64)
}
}
/// Hit rate for `HasChunk` probes — the dominant signal for how
/// much dedup we're saving in partial-sync operations.
pub fn has_chunk_hit_rate(&self) -> Option<f64> {
let total = self.has_chunk_hits.saturating_add(self.has_chunk_misses);
if total == 0 {
None
} else {
Some(self.has_chunk_hits as f64 / total as f64)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn new_starts_all_counters_at_zero_except_timestamp() {
let m = CacheMetrics::new();
let s = m.snapshot();
assert!(s.started_unix > 0);
assert_eq!(s.get_ref_hits, 0);
assert_eq!(s.get_ref_misses, 0);
assert_eq!(s.get_tag_hits, 0);
assert_eq!(s.get_tag_misses, 0);
assert_eq!(s.blob_get_bytes, 0);
assert_eq!(s.blob_put_bytes, 0);
assert_eq!(s.get_chunk_hits, 0);
assert_eq!(s.get_chunk_misses, 0);
assert_eq!(s.has_chunk_hits, 0);
assert_eq!(s.has_chunk_misses, 0);
}
#[test]
fn recorders_increment_the_right_field() {
let m = CacheMetrics::new();
m.record_get_ref_hit();
m.record_get_ref_hit();
m.record_get_ref_miss();
m.record_get_tag_hit();
m.record_get_tag_miss();
m.record_get_tag_miss();
m.record_blob_get_bytes(1024);
m.record_blob_get_bytes(512);
m.record_blob_put_bytes(2048);
m.record_get_chunk_hit();
m.record_get_chunk_miss();
m.record_has_chunk_hit();
m.record_has_chunk_miss();
m.record_has_chunk_miss();
let s = m.snapshot();
assert_eq!(s.get_ref_hits, 2);
assert_eq!(s.get_ref_misses, 1);
assert_eq!(s.get_tag_hits, 1);
assert_eq!(s.get_tag_misses, 2);
assert_eq!(s.blob_get_bytes, 1024 + 512);
assert_eq!(s.blob_put_bytes, 2048);
assert_eq!(s.get_chunk_hits, 1);
assert_eq!(s.get_chunk_misses, 1);
assert_eq!(s.has_chunk_hits, 1);
assert_eq!(s.has_chunk_misses, 2);
}
#[test]
fn hit_rates_none_when_zero_events() {
let s = CacheMetrics::new().snapshot();
assert_eq!(s.get_ref_hit_rate(), None);
assert_eq!(s.get_tag_hit_rate(), None);
assert_eq!(s.has_chunk_hit_rate(), None);
}
#[test]
fn hit_rates_compute_correctly() {
let m = CacheMetrics::new();
for _ in 0..3 {
m.record_get_ref_hit();
}
m.record_get_ref_miss();
let rate = m.snapshot().get_ref_hit_rate().unwrap();
assert!(
(rate - 0.75).abs() < 1e-9,
"expected 0.75, got {rate}"
);
}
#[test]
fn snapshot_round_trips_through_json() {
let m = CacheMetrics::new();
m.record_get_ref_hit();
m.record_blob_get_bytes(999);
let original = m.snapshot();
let json = serde_json::to_string(&original).unwrap();
let round: MetricsReply = serde_json::from_str(&json).unwrap();
assert_eq!(round, original);
}
#[test]
fn snapshots_across_threads_are_consistent_up_to_relaxed_ordering() {
// Fire many increments from N threads and verify the final
// snapshot totals match. Relaxed ordering + fetch_add is
// atomic → the total is exact; individual field ordering
// across threads may not be, but that's fine.
use std::sync::Arc;
let m = Arc::new(CacheMetrics::new());
let mut handles = Vec::new();
for _ in 0..8 {
let m = m.clone();
handles.push(std::thread::spawn(move || {
for _ in 0..1000 {
m.record_get_ref_hit();
}
}));
}
for h in handles {
h.join().unwrap();
}
assert_eq!(m.snapshot().get_ref_hits, 8_000);
}
}
+46 -5
View File
@@ -27,6 +27,7 @@
use crate::cluster::blob::{BlobId, BlobManifest, BlobStat, BlobStore, ChunkHash}; use crate::cluster::blob::{BlobId, BlobManifest, BlobStat, BlobStore, ChunkHash};
use crate::cluster::gossip::{ClusterGossip, PeerView}; use crate::cluster::gossip::{ClusterGossip, PeerView};
use crate::cluster::metrics::{CacheMetrics, MetricsReply};
use crate::cluster::refs::{RefKey, RefStore, RefValue}; use crate::cluster::refs::{RefKey, RefStore, RefValue};
use crate::cluster::tags::{TagEntry, TagStore}; use crate::cluster::tags::{TagEntry, TagStore};
use anyhow::{bail, Context, Result}; use anyhow::{bail, Context, Result};
@@ -126,6 +127,9 @@ pub enum Method {
/// Phase 5d: list all tags on the peer. `payload`: empty. /// Phase 5d: list all tags on the peer. `payload`: empty.
/// Reply: JSON `Vec<TagEntry>` sorted by key. /// Reply: JSON `Vec<TagEntry>` sorted by key.
ListTags = 0x12, ListTags = 0x12,
/// Phase 5g: fetch a snapshot of this peer's cache metrics.
/// `payload`: empty. Reply: JSON [`MetricsReply`].
GetMetrics = 0x13,
} }
impl Method { impl Method {
@@ -151,6 +155,7 @@ impl Method {
0x10 => Some(Method::GetTag), 0x10 => Some(Method::GetTag),
0x11 => Some(Method::DeleteTag), 0x11 => Some(Method::DeleteTag),
0x12 => Some(Method::ListTags), 0x12 => Some(Method::ListTags),
0x13 => Some(Method::GetMetrics),
_ => None, _ => None,
} }
} }
@@ -238,6 +243,7 @@ pub struct RpcRouter {
blob_store: Option<Arc<BlobStore>>, blob_store: Option<Arc<BlobStore>>,
ref_store: Option<Arc<RefStore>>, ref_store: Option<Arc<RefStore>>,
tag_store: Option<Arc<TagStore>>, tag_store: Option<Arc<TagStore>>,
metrics: Arc<CacheMetrics>,
local_name: String, local_name: String,
local_zone: String, local_zone: String,
} }
@@ -257,11 +263,19 @@ impl RpcRouter {
blob_store: None, blob_store: None,
ref_store: None, ref_store: None,
tag_store: None, tag_store: None,
metrics: Arc::new(CacheMetrics::new()),
local_name, local_name,
local_zone, local_zone,
} }
} }
/// Read-only handle to the router's metrics. Used by
/// `ClusterServices` (or tests) to sample counts without going
/// through the RPC layer.
pub fn metrics(&self) -> &Arc<CacheMetrics> {
&self.metrics
}
/// Attach a local blob store. Enables the `Blob*` methods; nodes /// Attach a local blob store. Enables the `Blob*` methods; nodes
/// without a store return [`ErrorCode::NotConfigured`] for those. /// without a store return [`ErrorCode::NotConfigured`] for those.
pub fn with_blob_store(mut self, store: Arc<BlobStore>) -> Self { pub fn with_blob_store(mut self, store: Arc<BlobStore>) -> Self {
@@ -368,6 +382,7 @@ impl RpcRouter {
MAX_MESSAGE_BYTES MAX_MESSAGE_BYTES
); );
} }
self.metrics.record_blob_get_bytes(bytes.len() as u64);
Ok(HandlerOutcome::Reply(bytes)) Ok(HandlerOutcome::Reply(bytes))
} }
None => Ok(HandlerOutcome::Error(ErrorCode::NotFound)), None => Ok(HandlerOutcome::Error(ErrorCode::NotFound)),
@@ -382,6 +397,7 @@ impl RpcRouter {
// through to store.put_bytes(&[]) which yields the // through to store.put_bytes(&[]) which yields the
// hash of the empty byte sequence. // hash of the empty byte sequence.
let id = store.put_bytes(payload).await?; let id = store.put_bytes(payload).await?;
self.metrics.record_blob_put_bytes(payload.len() as u64);
Ok(HandlerOutcome::Reply(id.as_bytes().to_vec())) Ok(HandlerOutcome::Reply(id.as_bytes().to_vec()))
} }
Method::BlobLoadManifest => { Method::BlobLoadManifest => {
@@ -422,8 +438,10 @@ impl RpcRouter {
None => return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest)), None => return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest)),
}; };
if store.has_chunk(&hash).await? { if store.has_chunk(&hash).await? {
self.metrics.record_has_chunk_hit();
Ok(HandlerOutcome::Reply(vec![STREAM_STATUS_OK])) Ok(HandlerOutcome::Reply(vec![STREAM_STATUS_OK]))
} else { } else {
self.metrics.record_has_chunk_miss();
Ok(HandlerOutcome::Error(ErrorCode::NotFound)) Ok(HandlerOutcome::Error(ErrorCode::NotFound))
} }
} }
@@ -461,12 +479,17 @@ impl RpcRouter {
// STREAM_STATUS_OK prefix so a legitimate first // STREAM_STATUS_OK prefix so a legitimate first
// content byte of 0xf3 isn't confused with // content byte of 0xf3 isn't confused with
// NotFound. Fixed 1-byte overhead. // NotFound. Fixed 1-byte overhead.
self.metrics.record_get_chunk_hit();
self.metrics.record_blob_get_bytes(bytes.len() as u64);
let mut reply = Vec::with_capacity(1 + bytes.len()); let mut reply = Vec::with_capacity(1 + bytes.len());
reply.push(STREAM_STATUS_OK); reply.push(STREAM_STATUS_OK);
reply.extend_from_slice(&bytes); reply.extend_from_slice(&bytes);
Ok(HandlerOutcome::Reply(reply)) Ok(HandlerOutcome::Reply(reply))
} }
None => Ok(HandlerOutcome::Error(ErrorCode::NotFound)), None => {
self.metrics.record_get_chunk_miss();
Ok(HandlerOutcome::Error(ErrorCode::NotFound))
}
} }
} }
Method::PutManifest => { Method::PutManifest => {
@@ -497,8 +520,14 @@ impl RpcRouter {
None => return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest)), None => return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest)),
}; };
match store.get(&key).await? { match store.get(&key).await? {
Some(value) => Ok(HandlerOutcome::Reply(value.to_vec())), Some(value) => {
None => Ok(HandlerOutcome::Error(ErrorCode::NotFound)), self.metrics.record_get_ref_hit();
Ok(HandlerOutcome::Reply(value.to_vec()))
}
None => {
self.metrics.record_get_ref_miss();
Ok(HandlerOutcome::Error(ErrorCode::NotFound))
}
} }
} }
Method::PutRef => { Method::PutRef => {
@@ -543,8 +572,14 @@ impl RpcRouter {
_ => return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest)), _ => return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest)),
}; };
match store.get(key).await? { match store.get(key).await? {
Some(value) => Ok(HandlerOutcome::Reply(value.to_vec())), Some(value) => {
None => Ok(HandlerOutcome::Error(ErrorCode::NotFound)), self.metrics.record_get_tag_hit();
Ok(HandlerOutcome::Reply(value.to_vec()))
}
None => {
self.metrics.record_get_tag_miss();
Ok(HandlerOutcome::Error(ErrorCode::NotFound))
}
} }
} }
Method::DeleteTag => { Method::DeleteTag => {
@@ -572,6 +607,12 @@ impl RpcRouter {
.context("encoding TagEntry list as JSON")?; .context("encoding TagEntry list as JSON")?;
Ok(HandlerOutcome::Reply(json)) Ok(HandlerOutcome::Reply(json))
} }
Method::GetMetrics => {
let snapshot = self.metrics.snapshot();
let json = serde_json::to_vec(&snapshot)
.context("encoding MetricsReply as JSON")?;
Ok(HandlerOutcome::Reply(json))
}
} }
} }
} }
+13
View File
@@ -454,6 +454,19 @@ pub async fn call_put_ref(
} }
} }
// ── Phase 5g: cache metrics client helper ────────────────────────────
/// Fetch the peer's current cache-metrics snapshot.
pub async fn call_get_metrics(conn: &Connection) -> Result<MetricsReply> {
let reply = rpc_call(conn, Method::GetMetrics, &[]).await?;
if reply.len() == 1 {
if let Some(err) = decode_error(reply[0]) {
bail!("peer replied with error: {}", err.describe());
}
}
serde_json::from_slice(&reply).context("decoding MetricsReply JSON")
}
// ── Phase 5d: tag-store client helpers ─────────────────────────────── // ── Phase 5d: tag-store client helpers ───────────────────────────────
/// Publish (or overwrite) a named tag pointing at a 32-byte value. /// Publish (or overwrite) a named tag pointing at a 32-byte value.
+183
View File
@@ -290,6 +290,189 @@ async fn list_tags_returns_json_sorted() {
assert_eq!(decoded[0].decode_value().unwrap(), [1u8; 32]); assert_eq!(decoded[0].decode_value().unwrap(), [1u8; 32]);
} }
// ── Phase 5g: cache metrics RPC ──────────────────────────────────────
#[test]
fn phase_5g_method_byte_encoding() {
assert_eq!(Method::GetMetrics.as_byte(), 0x13);
assert_eq!(Method::from_byte(0x13), Some(Method::GetMetrics));
}
#[tokio::test]
async fn get_metrics_returns_empty_snapshot_before_any_activity() {
let (_tmp, router) = router_with_full_stack("solo", next_port()).await;
let reply = dispatch(&router, &[Method::GetMetrics.as_byte()]).await;
let snapshot: crate::cluster::metrics::MetricsReply =
serde_json::from_slice(&reply).unwrap();
assert!(snapshot.started_unix > 0);
assert_eq!(snapshot.get_ref_hits, 0);
assert_eq!(snapshot.get_ref_misses, 0);
assert_eq!(snapshot.blob_get_bytes, 0);
}
#[tokio::test]
async fn get_ref_records_hit_and_miss_counters() {
let (_tmp, router) = router_with_full_stack("solo", next_port()).await;
let store = router.ref_store().unwrap().clone();
let key = [0x11u8; 32];
store.put(&key, &[0x22u8; 32]).await.unwrap();
// Two hits.
for _ in 0..2 {
let mut req = vec![Method::GetRef.as_byte()];
req.extend_from_slice(&key);
let reply = dispatch(&router, &req).await;
assert_eq!(reply.len(), 32);
}
// One miss.
let mut req = vec![Method::GetRef.as_byte()];
req.extend_from_slice(&[0xffu8; 32]);
let reply = dispatch(&router, &req).await;
assert_eq!(reply, vec![ErrorCode::NotFound.as_byte()]);
let snapshot = router.metrics().snapshot();
assert_eq!(snapshot.get_ref_hits, 2);
assert_eq!(snapshot.get_ref_misses, 1);
}
#[tokio::test]
async fn get_tag_records_hit_and_miss_counters() {
let (_tmp, router) = router_with_full_stack("solo", next_port()).await;
let store = router.tag_store().unwrap().clone();
store.put("clawverse:main", &[0u8; 32]).await.unwrap();
let mut hit = vec![Method::GetTag.as_byte()];
hit.extend_from_slice(b"clawverse:main");
dispatch(&router, &hit).await;
let mut miss = vec![Method::GetTag.as_byte()];
miss.extend_from_slice(b"never-set");
dispatch(&router, &miss).await;
let snapshot = router.metrics().snapshot();
assert_eq!(snapshot.get_tag_hits, 1);
assert_eq!(snapshot.get_tag_misses, 1);
}
#[tokio::test]
async fn blob_get_and_blob_put_record_byte_counts() {
let (_tmp, router) = router_with_full_stack("solo", next_port()).await;
let payload = b"metrics witness";
// Put — should record blob_put_bytes.
let mut put_req = vec![Method::BlobPut.as_byte()];
put_req.extend_from_slice(payload);
let put_reply = dispatch(&router, &put_req).await;
assert_eq!(put_reply.len(), 32);
let mut id_bytes = [0u8; 32];
id_bytes.copy_from_slice(&put_reply);
// Get — should record blob_get_bytes.
let mut get_req = vec![Method::BlobGet.as_byte()];
get_req.extend_from_slice(&id_bytes);
let get_reply = dispatch(&router, &get_req).await;
assert_eq!(get_reply, payload);
let snapshot = router.metrics().snapshot();
assert_eq!(snapshot.blob_put_bytes, payload.len() as u64);
assert_eq!(snapshot.blob_get_bytes, payload.len() as u64);
}
#[tokio::test]
async fn has_chunk_and_get_chunk_record_hit_miss_counters() {
let (_tmp, router) = router_with_full_stack("solo", next_port()).await;
let store = router.blob_store().unwrap().clone();
let bytes = b"chunk-in-store";
let hash = crate::cluster::blob::ChunkHash::from_bytes(blake3::hash(bytes).into());
store.put_chunk(&hash, bytes).await.unwrap();
// HasChunk hit + miss.
let mut hit = vec![Method::HasChunk.as_byte()];
hit.extend_from_slice(hash.as_bytes());
dispatch(&router, &hit).await;
let mut miss = vec![Method::HasChunk.as_byte()];
miss.extend_from_slice(&[0u8; 32]);
dispatch(&router, &miss).await;
// GetChunk hit + miss.
let mut get_hit = vec![Method::GetChunk.as_byte()];
get_hit.extend_from_slice(hash.as_bytes());
dispatch(&router, &get_hit).await;
let mut get_miss = vec![Method::GetChunk.as_byte()];
get_miss.extend_from_slice(&[0u8; 32]);
dispatch(&router, &get_miss).await;
let snapshot = router.metrics().snapshot();
assert_eq!(snapshot.has_chunk_hits, 1);
assert_eq!(snapshot.has_chunk_misses, 1);
assert_eq!(snapshot.get_chunk_hits, 1);
assert_eq!(snapshot.get_chunk_misses, 1);
// Get_chunk hit also records blob_get_bytes.
assert_eq!(snapshot.blob_get_bytes, bytes.len() as u64);
}
#[tokio::test]
async fn end_to_end_get_metrics_over_real_quic() {
// Exercise every counter, then fetch the metrics reply through
// real QUIC and verify each field.
let (id_a, id_b) = NodeIdentity::generate_test_pair("a", "b").unwrap();
let (_tmp, router) = router_with_full_stack("a", next_port()).await;
// Seed some activity locally so the counters have real values.
let store = router.ref_store().unwrap().clone();
store.put(&[0x11u8; 32], &[0x22u8; 32]).await.unwrap();
let tag_store = router.tag_store().unwrap().clone();
tag_store.put("hit-me", &[0u8; 32]).await.unwrap();
let blob_store = router.blob_store().unwrap().clone();
let payload = vec![0x5au8; 4096];
let blob_id = blob_store.put_bytes(&payload).await.unwrap();
// Fire dispatches to move the counters.
let mut ref_hit = vec![Method::GetRef.as_byte()];
ref_hit.extend_from_slice(&[0x11u8; 32]);
dispatch(&router, &ref_hit).await;
let mut ref_miss = vec![Method::GetRef.as_byte()];
ref_miss.extend_from_slice(&[0x99u8; 32]);
dispatch(&router, &ref_miss).await;
let mut tag_hit = vec![Method::GetTag.as_byte()];
tag_hit.extend_from_slice(b"hit-me");
dispatch(&router, &tag_hit).await;
let mut blob_get = vec![Method::BlobGet.as_byte()];
blob_get.extend_from_slice(blob_id.as_bytes());
dispatch(&router, &blob_get).await;
// Now start the server + fetch metrics over the wire.
let server = QuicServer::bind(loopback(0), id_a).unwrap();
let server_addr = server.local_addr().unwrap();
let router_srv = router.clone();
let accept_task = tokio::spawn(async move {
if let Some(Ok(conn)) = server.accept().await {
let _ = serve_connection(conn, router_srv).await;
}
});
let client = QuicClient::new(loopback(0), id_b).unwrap();
let conn = client.connect(server_addr, "a").await.unwrap();
let m = call_get_metrics(&conn).await.unwrap();
assert_eq!(m.get_ref_hits, 1);
assert_eq!(m.get_ref_misses, 1);
assert_eq!(m.get_tag_hits, 1);
assert_eq!(m.get_tag_misses, 0);
assert_eq!(m.blob_get_bytes, payload.len() as u64);
assert_eq!(m.get_ref_hit_rate(), Some(0.5));
conn.close(quinn::VarInt::from_u32(0), b"done");
client.shutdown().await;
tokio::time::sleep(Duration::from_millis(50)).await;
accept_task.abort();
}
#[tokio::test] #[tokio::test]
async fn end_to_end_prewarm_copies_tagged_blob_between_two_peers() { async fn end_to_end_prewarm_copies_tagged_blob_between_two_peers() {
// Phase 5f: `claw-cargo prewarm --from A --to C --pin tag`. // Phase 5f: `claw-cargo prewarm --from A --to C --pin tag`.