Phase 5g: cache metrics + GetMetrics RPC + peer-metrics CLI #16
@@ -46,8 +46,8 @@ use crate::cluster::build_cache::{
|
|||||||
};
|
};
|
||||||
use crate::cluster::client_config::{ClientConfig, ResolvedClientConfig};
|
use crate::cluster::client_config::{ClientConfig, ResolvedClientConfig};
|
||||||
use crate::cluster::rpc::{
|
use crate::cluster::rpc::{
|
||||||
call_blob_get_stream, call_blob_put_stream, call_blob_stat, call_delete_tag, call_get_ref,
|
call_blob_get_stream, call_blob_put_stream, call_blob_stat, call_delete_tag, call_get_metrics,
|
||||||
call_get_tag, call_list_tags, call_put_ref, call_put_tag,
|
call_get_ref, call_get_tag, call_list_tags, call_put_ref, call_put_tag,
|
||||||
};
|
};
|
||||||
use crate::cluster::transport::{NodeIdentity, QuicClient};
|
use crate::cluster::transport::{NodeIdentity, QuicClient};
|
||||||
|
|
||||||
@@ -93,6 +93,9 @@ enum Cmd {
|
|||||||
/// downstream side. Useful as a Gitea-webhook target so the runner's
|
/// downstream side. Useful as a Gitea-webhook target so the runner's
|
||||||
/// local daemon has the cache warm before the build starts.
|
/// local daemon has the cache warm before the build starts.
|
||||||
Prewarm(PrewarmArgs),
|
Prewarm(PrewarmArgs),
|
||||||
|
/// Fetch and print the peer's cache-metrics snapshot (Phase 5g).
|
||||||
|
/// Shows hit/miss counts, hit rates, byte volumes served/ingested.
|
||||||
|
PeerMetrics(PeerArgs),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(clap::Args, Debug, Clone)]
|
#[derive(clap::Args, Debug, Clone)]
|
||||||
@@ -218,6 +221,7 @@ async fn main() -> Result<()> {
|
|||||||
Cmd::Unpin(args) => cmd_unpin(args).await,
|
Cmd::Unpin(args) => cmd_unpin(args).await,
|
||||||
Cmd::ListTags(args) => cmd_list_tags(args).await,
|
Cmd::ListTags(args) => cmd_list_tags(args).await,
|
||||||
Cmd::Prewarm(args) => cmd_prewarm(args).await,
|
Cmd::Prewarm(args) => cmd_prewarm(args).await,
|
||||||
|
Cmd::PeerMetrics(args) => cmd_peer_metrics(args).await,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -684,6 +688,78 @@ async fn _reserved_call_get_tag(conn: &quinn::Connection, key: &str) -> Result<O
|
|||||||
call_get_tag(conn, key).await
|
call_get_tag(conn, key).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── peer-metrics ─────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
async fn cmd_peer_metrics(args: PeerArgs) -> Result<()> {
|
||||||
|
let workspace = args
|
||||||
|
.workspace
|
||||||
|
.clone()
|
||||||
|
.unwrap_or_else(|| std::env::current_dir().expect("cwd"));
|
||||||
|
let layered = ClientConfig::load_layered(&workspace)?;
|
||||||
|
let resolved = ResolvedClientConfig::resolve(
|
||||||
|
layered,
|
||||||
|
args.peer.clone(),
|
||||||
|
args.peer_addr,
|
||||||
|
args.tls_dir.clone(),
|
||||||
|
args.profile.clone(),
|
||||||
|
args.features.clone(),
|
||||||
|
"dev",
|
||||||
|
);
|
||||||
|
let (client, conn) = connect_peer(&resolved).await?;
|
||||||
|
let m = call_get_metrics(&conn).await?;
|
||||||
|
conn.close(quinn::VarInt::from_u32(0), b"done");
|
||||||
|
client.shutdown().await;
|
||||||
|
|
||||||
|
let uptime = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs().saturating_sub(m.started_unix))
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
|
println!("── peer cache metrics ──────────────────────────────");
|
||||||
|
println!("counter uptime: {}s", uptime);
|
||||||
|
println!();
|
||||||
|
println!("GetRef hits/misses: {} / {}", m.get_ref_hits, m.get_ref_misses);
|
||||||
|
match m.get_ref_hit_rate() {
|
||||||
|
Some(r) => println!(" hit rate: {:.2}%", r * 100.0),
|
||||||
|
None => println!(" hit rate: n/a (no lookups)"),
|
||||||
|
}
|
||||||
|
println!();
|
||||||
|
println!("GetTag hits/misses: {} / {}", m.get_tag_hits, m.get_tag_misses);
|
||||||
|
match m.get_tag_hit_rate() {
|
||||||
|
Some(r) => println!(" hit rate: {:.2}%", r * 100.0),
|
||||||
|
None => println!(" hit rate: n/a"),
|
||||||
|
}
|
||||||
|
println!();
|
||||||
|
println!("HasChunk hits/miss: {} / {}", m.has_chunk_hits, m.has_chunk_misses);
|
||||||
|
match m.has_chunk_hit_rate() {
|
||||||
|
Some(r) => println!(" hit rate: {:.2}%", r * 100.0),
|
||||||
|
None => println!(" hit rate: n/a"),
|
||||||
|
}
|
||||||
|
println!();
|
||||||
|
println!("GetChunk hits/miss: {} / {}", m.get_chunk_hits, m.get_chunk_misses);
|
||||||
|
println!();
|
||||||
|
println!("Blob GET bytes: {}", human_bytes(m.blob_get_bytes));
|
||||||
|
println!("Blob PUT bytes: {}", human_bytes(m.blob_put_bytes));
|
||||||
|
println!("────────────────────────────────────────────────────");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Human-readable byte count (KiB / MiB / GiB). Test helper too.
|
||||||
|
fn human_bytes(n: u64) -> String {
|
||||||
|
const KIB: u64 = 1024;
|
||||||
|
const MIB: u64 = KIB * 1024;
|
||||||
|
const GIB: u64 = MIB * 1024;
|
||||||
|
if n >= GIB {
|
||||||
|
format!("{:.2} GiB", n as f64 / GIB as f64)
|
||||||
|
} else if n >= MIB {
|
||||||
|
format!("{:.2} MiB", n as f64 / MIB as f64)
|
||||||
|
} else if n >= KIB {
|
||||||
|
format!("{:.2} KiB", n as f64 / KIB as f64)
|
||||||
|
} else {
|
||||||
|
format!("{} B", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ── prewarm ──────────────────────────────────────────────────────────
|
// ── prewarm ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
async fn cmd_prewarm(args: PrewarmArgs) -> Result<()> {
|
async fn cmd_prewarm(args: PrewarmArgs) -> Result<()> {
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ pub mod blob;
|
|||||||
pub mod build_cache;
|
pub mod build_cache;
|
||||||
pub mod client_config;
|
pub mod client_config;
|
||||||
pub mod gossip;
|
pub mod gossip;
|
||||||
|
pub mod metrics;
|
||||||
pub mod refs;
|
pub mod refs;
|
||||||
pub mod rpc;
|
pub mod rpc;
|
||||||
pub mod services;
|
pub mod services;
|
||||||
|
|||||||
@@ -0,0 +1,268 @@
|
|||||||
|
//! Cache metrics (Phase 5g).
|
||||||
|
//!
|
||||||
|
//! Lock-free per-router counters that record whether a lookup found
|
||||||
|
//! what the client asked for, plus cumulative bytes served / stored.
|
||||||
|
//! Read via the `GetMetrics` RPC — placement engines use the hit rate
|
||||||
|
//! and byte-volume to decide which peer a runner should point at for
|
||||||
|
//! a given repo.
|
||||||
|
//!
|
||||||
|
//! All fields are `AtomicU64` so handlers can increment without
|
||||||
|
//! locking, and the snapshot read is a simple `Ordering::Relaxed`
|
||||||
|
//! load per field (metrics are advisory, not consistency-critical).
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
|
||||||
|
/// The atomic counters. Held in an `Arc<CacheMetrics>` on the router
|
||||||
|
/// and read by [`CacheMetrics::snapshot`] whenever `GetMetrics` fires.
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub struct CacheMetrics {
|
||||||
|
/// Timestamp (unix seconds) the counters were reset. Set on
|
||||||
|
/// [`CacheMetrics::new`]; not touched afterward.
|
||||||
|
started_unix: AtomicU64,
|
||||||
|
// GetRef — fingerprint-keyed cache lookups.
|
||||||
|
get_ref_hits: AtomicU64,
|
||||||
|
get_ref_misses: AtomicU64,
|
||||||
|
// GetTag — human-readable pin lookups.
|
||||||
|
get_tag_hits: AtomicU64,
|
||||||
|
get_tag_misses: AtomicU64,
|
||||||
|
// Byte volumes for the blob transfer paths. Both bounded and
|
||||||
|
// streaming variants roll up into the same counter — the caller
|
||||||
|
// just sees "how much did we serve/ingest".
|
||||||
|
blob_get_bytes: AtomicU64,
|
||||||
|
blob_put_bytes: AtomicU64,
|
||||||
|
// Chunk-level activity for partial-sync visibility (Phase 2d).
|
||||||
|
get_chunk_hits: AtomicU64,
|
||||||
|
get_chunk_misses: AtomicU64,
|
||||||
|
has_chunk_hits: AtomicU64,
|
||||||
|
has_chunk_misses: AtomicU64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CacheMetrics {
|
||||||
|
/// Fresh counters. Every field starts at 0 except `started_unix`.
|
||||||
|
/// Falls back to 0 pre-1970 (never happens; keeps this pure).
|
||||||
|
pub fn new() -> Self {
|
||||||
|
let now = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs())
|
||||||
|
.unwrap_or(0);
|
||||||
|
Self {
|
||||||
|
started_unix: AtomicU64::new(now),
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── recorders ────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
pub fn record_get_ref_hit(&self) {
|
||||||
|
self.get_ref_hits.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
pub fn record_get_ref_miss(&self) {
|
||||||
|
self.get_ref_misses.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
pub fn record_get_tag_hit(&self) {
|
||||||
|
self.get_tag_hits.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
pub fn record_get_tag_miss(&self) {
|
||||||
|
self.get_tag_misses.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
pub fn record_blob_get_bytes(&self, n: u64) {
|
||||||
|
self.blob_get_bytes.fetch_add(n, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
pub fn record_blob_put_bytes(&self, n: u64) {
|
||||||
|
self.blob_put_bytes.fetch_add(n, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
pub fn record_get_chunk_hit(&self) {
|
||||||
|
self.get_chunk_hits.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
pub fn record_get_chunk_miss(&self) {
|
||||||
|
self.get_chunk_misses.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
pub fn record_has_chunk_hit(&self) {
|
||||||
|
self.has_chunk_hits.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
pub fn record_has_chunk_miss(&self) {
|
||||||
|
self.has_chunk_misses.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Cheap read-side capture of every counter. Uses `Relaxed`
|
||||||
|
/// ordering — the snapshot may not be strictly consistent across
|
||||||
|
/// fields, but for a metric API that's fine (each field is
|
||||||
|
/// individually up-to-date within a nanosecond).
|
||||||
|
pub fn snapshot(&self) -> MetricsReply {
|
||||||
|
MetricsReply {
|
||||||
|
started_unix: self.started_unix.load(Ordering::Relaxed),
|
||||||
|
get_ref_hits: self.get_ref_hits.load(Ordering::Relaxed),
|
||||||
|
get_ref_misses: self.get_ref_misses.load(Ordering::Relaxed),
|
||||||
|
get_tag_hits: self.get_tag_hits.load(Ordering::Relaxed),
|
||||||
|
get_tag_misses: self.get_tag_misses.load(Ordering::Relaxed),
|
||||||
|
blob_get_bytes: self.blob_get_bytes.load(Ordering::Relaxed),
|
||||||
|
blob_put_bytes: self.blob_put_bytes.load(Ordering::Relaxed),
|
||||||
|
get_chunk_hits: self.get_chunk_hits.load(Ordering::Relaxed),
|
||||||
|
get_chunk_misses: self.get_chunk_misses.load(Ordering::Relaxed),
|
||||||
|
has_chunk_hits: self.has_chunk_hits.load(Ordering::Relaxed),
|
||||||
|
has_chunk_misses: self.has_chunk_misses.load(Ordering::Relaxed),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// JSON-shaped snapshot returned by `GetMetrics`. Fields are all
|
||||||
|
/// counters — hit rate + byte volume computations happen client-side
|
||||||
|
/// so we don't lock the router into a specific set of derived metrics.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct MetricsReply {
|
||||||
|
/// Unix timestamp (seconds) when this counter set started.
|
||||||
|
pub started_unix: u64,
|
||||||
|
pub get_ref_hits: u64,
|
||||||
|
pub get_ref_misses: u64,
|
||||||
|
pub get_tag_hits: u64,
|
||||||
|
pub get_tag_misses: u64,
|
||||||
|
pub blob_get_bytes: u64,
|
||||||
|
pub blob_put_bytes: u64,
|
||||||
|
pub get_chunk_hits: u64,
|
||||||
|
pub get_chunk_misses: u64,
|
||||||
|
pub has_chunk_hits: u64,
|
||||||
|
pub has_chunk_misses: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl MetricsReply {
|
||||||
|
/// Hit rate for `GetRef` lookups, as a float in `[0, 1]`. Returns
|
||||||
|
/// `None` when zero lookups have been recorded (0/0 is undefined).
|
||||||
|
pub fn get_ref_hit_rate(&self) -> Option<f64> {
|
||||||
|
let total = self.get_ref_hits.saturating_add(self.get_ref_misses);
|
||||||
|
if total == 0 {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(self.get_ref_hits as f64 / total as f64)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Hit rate for `GetTag` lookups.
|
||||||
|
pub fn get_tag_hit_rate(&self) -> Option<f64> {
|
||||||
|
let total = self.get_tag_hits.saturating_add(self.get_tag_misses);
|
||||||
|
if total == 0 {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(self.get_tag_hits as f64 / total as f64)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Hit rate for `HasChunk` probes — the dominant signal for how
|
||||||
|
/// much dedup we're saving in partial-sync operations.
|
||||||
|
pub fn has_chunk_hit_rate(&self) -> Option<f64> {
|
||||||
|
let total = self.has_chunk_hits.saturating_add(self.has_chunk_misses);
|
||||||
|
if total == 0 {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(self.has_chunk_hits as f64 / total as f64)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn new_starts_all_counters_at_zero_except_timestamp() {
|
||||||
|
let m = CacheMetrics::new();
|
||||||
|
let s = m.snapshot();
|
||||||
|
assert!(s.started_unix > 0);
|
||||||
|
assert_eq!(s.get_ref_hits, 0);
|
||||||
|
assert_eq!(s.get_ref_misses, 0);
|
||||||
|
assert_eq!(s.get_tag_hits, 0);
|
||||||
|
assert_eq!(s.get_tag_misses, 0);
|
||||||
|
assert_eq!(s.blob_get_bytes, 0);
|
||||||
|
assert_eq!(s.blob_put_bytes, 0);
|
||||||
|
assert_eq!(s.get_chunk_hits, 0);
|
||||||
|
assert_eq!(s.get_chunk_misses, 0);
|
||||||
|
assert_eq!(s.has_chunk_hits, 0);
|
||||||
|
assert_eq!(s.has_chunk_misses, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn recorders_increment_the_right_field() {
|
||||||
|
let m = CacheMetrics::new();
|
||||||
|
m.record_get_ref_hit();
|
||||||
|
m.record_get_ref_hit();
|
||||||
|
m.record_get_ref_miss();
|
||||||
|
m.record_get_tag_hit();
|
||||||
|
m.record_get_tag_miss();
|
||||||
|
m.record_get_tag_miss();
|
||||||
|
m.record_blob_get_bytes(1024);
|
||||||
|
m.record_blob_get_bytes(512);
|
||||||
|
m.record_blob_put_bytes(2048);
|
||||||
|
m.record_get_chunk_hit();
|
||||||
|
m.record_get_chunk_miss();
|
||||||
|
m.record_has_chunk_hit();
|
||||||
|
m.record_has_chunk_miss();
|
||||||
|
m.record_has_chunk_miss();
|
||||||
|
let s = m.snapshot();
|
||||||
|
assert_eq!(s.get_ref_hits, 2);
|
||||||
|
assert_eq!(s.get_ref_misses, 1);
|
||||||
|
assert_eq!(s.get_tag_hits, 1);
|
||||||
|
assert_eq!(s.get_tag_misses, 2);
|
||||||
|
assert_eq!(s.blob_get_bytes, 1024 + 512);
|
||||||
|
assert_eq!(s.blob_put_bytes, 2048);
|
||||||
|
assert_eq!(s.get_chunk_hits, 1);
|
||||||
|
assert_eq!(s.get_chunk_misses, 1);
|
||||||
|
assert_eq!(s.has_chunk_hits, 1);
|
||||||
|
assert_eq!(s.has_chunk_misses, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn hit_rates_none_when_zero_events() {
|
||||||
|
let s = CacheMetrics::new().snapshot();
|
||||||
|
assert_eq!(s.get_ref_hit_rate(), None);
|
||||||
|
assert_eq!(s.get_tag_hit_rate(), None);
|
||||||
|
assert_eq!(s.has_chunk_hit_rate(), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn hit_rates_compute_correctly() {
|
||||||
|
let m = CacheMetrics::new();
|
||||||
|
for _ in 0..3 {
|
||||||
|
m.record_get_ref_hit();
|
||||||
|
}
|
||||||
|
m.record_get_ref_miss();
|
||||||
|
let rate = m.snapshot().get_ref_hit_rate().unwrap();
|
||||||
|
assert!(
|
||||||
|
(rate - 0.75).abs() < 1e-9,
|
||||||
|
"expected 0.75, got {rate}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn snapshot_round_trips_through_json() {
|
||||||
|
let m = CacheMetrics::new();
|
||||||
|
m.record_get_ref_hit();
|
||||||
|
m.record_blob_get_bytes(999);
|
||||||
|
let original = m.snapshot();
|
||||||
|
let json = serde_json::to_string(&original).unwrap();
|
||||||
|
let round: MetricsReply = serde_json::from_str(&json).unwrap();
|
||||||
|
assert_eq!(round, original);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn snapshots_across_threads_are_consistent_up_to_relaxed_ordering() {
|
||||||
|
// Fire many increments from N threads and verify the final
|
||||||
|
// snapshot totals match. Relaxed ordering + fetch_add is
|
||||||
|
// atomic → the total is exact; individual field ordering
|
||||||
|
// across threads may not be, but that's fine.
|
||||||
|
use std::sync::Arc;
|
||||||
|
let m = Arc::new(CacheMetrics::new());
|
||||||
|
let mut handles = Vec::new();
|
||||||
|
for _ in 0..8 {
|
||||||
|
let m = m.clone();
|
||||||
|
handles.push(std::thread::spawn(move || {
|
||||||
|
for _ in 0..1000 {
|
||||||
|
m.record_get_ref_hit();
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
for h in handles {
|
||||||
|
h.join().unwrap();
|
||||||
|
}
|
||||||
|
assert_eq!(m.snapshot().get_ref_hits, 8_000);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -27,6 +27,7 @@
|
|||||||
|
|
||||||
use crate::cluster::blob::{BlobId, BlobManifest, BlobStat, BlobStore, ChunkHash};
|
use crate::cluster::blob::{BlobId, BlobManifest, BlobStat, BlobStore, ChunkHash};
|
||||||
use crate::cluster::gossip::{ClusterGossip, PeerView};
|
use crate::cluster::gossip::{ClusterGossip, PeerView};
|
||||||
|
use crate::cluster::metrics::{CacheMetrics, MetricsReply};
|
||||||
use crate::cluster::refs::{RefKey, RefStore, RefValue};
|
use crate::cluster::refs::{RefKey, RefStore, RefValue};
|
||||||
use crate::cluster::tags::{TagEntry, TagStore};
|
use crate::cluster::tags::{TagEntry, TagStore};
|
||||||
use anyhow::{bail, Context, Result};
|
use anyhow::{bail, Context, Result};
|
||||||
@@ -126,6 +127,9 @@ pub enum Method {
|
|||||||
/// Phase 5d: list all tags on the peer. `payload`: empty.
|
/// Phase 5d: list all tags on the peer. `payload`: empty.
|
||||||
/// Reply: JSON `Vec<TagEntry>` sorted by key.
|
/// Reply: JSON `Vec<TagEntry>` sorted by key.
|
||||||
ListTags = 0x12,
|
ListTags = 0x12,
|
||||||
|
/// Phase 5g: fetch a snapshot of this peer's cache metrics.
|
||||||
|
/// `payload`: empty. Reply: JSON [`MetricsReply`].
|
||||||
|
GetMetrics = 0x13,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Method {
|
impl Method {
|
||||||
@@ -151,6 +155,7 @@ impl Method {
|
|||||||
0x10 => Some(Method::GetTag),
|
0x10 => Some(Method::GetTag),
|
||||||
0x11 => Some(Method::DeleteTag),
|
0x11 => Some(Method::DeleteTag),
|
||||||
0x12 => Some(Method::ListTags),
|
0x12 => Some(Method::ListTags),
|
||||||
|
0x13 => Some(Method::GetMetrics),
|
||||||
_ => None,
|
_ => None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -238,6 +243,7 @@ pub struct RpcRouter {
|
|||||||
blob_store: Option<Arc<BlobStore>>,
|
blob_store: Option<Arc<BlobStore>>,
|
||||||
ref_store: Option<Arc<RefStore>>,
|
ref_store: Option<Arc<RefStore>>,
|
||||||
tag_store: Option<Arc<TagStore>>,
|
tag_store: Option<Arc<TagStore>>,
|
||||||
|
metrics: Arc<CacheMetrics>,
|
||||||
local_name: String,
|
local_name: String,
|
||||||
local_zone: String,
|
local_zone: String,
|
||||||
}
|
}
|
||||||
@@ -257,11 +263,19 @@ impl RpcRouter {
|
|||||||
blob_store: None,
|
blob_store: None,
|
||||||
ref_store: None,
|
ref_store: None,
|
||||||
tag_store: None,
|
tag_store: None,
|
||||||
|
metrics: Arc::new(CacheMetrics::new()),
|
||||||
local_name,
|
local_name,
|
||||||
local_zone,
|
local_zone,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Read-only handle to the router's metrics. Used by
|
||||||
|
/// `ClusterServices` (or tests) to sample counts without going
|
||||||
|
/// through the RPC layer.
|
||||||
|
pub fn metrics(&self) -> &Arc<CacheMetrics> {
|
||||||
|
&self.metrics
|
||||||
|
}
|
||||||
|
|
||||||
/// Attach a local blob store. Enables the `Blob*` methods; nodes
|
/// Attach a local blob store. Enables the `Blob*` methods; nodes
|
||||||
/// without a store return [`ErrorCode::NotConfigured`] for those.
|
/// without a store return [`ErrorCode::NotConfigured`] for those.
|
||||||
pub fn with_blob_store(mut self, store: Arc<BlobStore>) -> Self {
|
pub fn with_blob_store(mut self, store: Arc<BlobStore>) -> Self {
|
||||||
@@ -368,6 +382,7 @@ impl RpcRouter {
|
|||||||
MAX_MESSAGE_BYTES
|
MAX_MESSAGE_BYTES
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
self.metrics.record_blob_get_bytes(bytes.len() as u64);
|
||||||
Ok(HandlerOutcome::Reply(bytes))
|
Ok(HandlerOutcome::Reply(bytes))
|
||||||
}
|
}
|
||||||
None => Ok(HandlerOutcome::Error(ErrorCode::NotFound)),
|
None => Ok(HandlerOutcome::Error(ErrorCode::NotFound)),
|
||||||
@@ -382,6 +397,7 @@ impl RpcRouter {
|
|||||||
// through to store.put_bytes(&[]) which yields the
|
// through to store.put_bytes(&[]) which yields the
|
||||||
// hash of the empty byte sequence.
|
// hash of the empty byte sequence.
|
||||||
let id = store.put_bytes(payload).await?;
|
let id = store.put_bytes(payload).await?;
|
||||||
|
self.metrics.record_blob_put_bytes(payload.len() as u64);
|
||||||
Ok(HandlerOutcome::Reply(id.as_bytes().to_vec()))
|
Ok(HandlerOutcome::Reply(id.as_bytes().to_vec()))
|
||||||
}
|
}
|
||||||
Method::BlobLoadManifest => {
|
Method::BlobLoadManifest => {
|
||||||
@@ -422,8 +438,10 @@ impl RpcRouter {
|
|||||||
None => return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest)),
|
None => return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest)),
|
||||||
};
|
};
|
||||||
if store.has_chunk(&hash).await? {
|
if store.has_chunk(&hash).await? {
|
||||||
|
self.metrics.record_has_chunk_hit();
|
||||||
Ok(HandlerOutcome::Reply(vec![STREAM_STATUS_OK]))
|
Ok(HandlerOutcome::Reply(vec![STREAM_STATUS_OK]))
|
||||||
} else {
|
} else {
|
||||||
|
self.metrics.record_has_chunk_miss();
|
||||||
Ok(HandlerOutcome::Error(ErrorCode::NotFound))
|
Ok(HandlerOutcome::Error(ErrorCode::NotFound))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -461,12 +479,17 @@ impl RpcRouter {
|
|||||||
// STREAM_STATUS_OK prefix so a legitimate first
|
// STREAM_STATUS_OK prefix so a legitimate first
|
||||||
// content byte of 0xf3 isn't confused with
|
// content byte of 0xf3 isn't confused with
|
||||||
// NotFound. Fixed 1-byte overhead.
|
// NotFound. Fixed 1-byte overhead.
|
||||||
|
self.metrics.record_get_chunk_hit();
|
||||||
|
self.metrics.record_blob_get_bytes(bytes.len() as u64);
|
||||||
let mut reply = Vec::with_capacity(1 + bytes.len());
|
let mut reply = Vec::with_capacity(1 + bytes.len());
|
||||||
reply.push(STREAM_STATUS_OK);
|
reply.push(STREAM_STATUS_OK);
|
||||||
reply.extend_from_slice(&bytes);
|
reply.extend_from_slice(&bytes);
|
||||||
Ok(HandlerOutcome::Reply(reply))
|
Ok(HandlerOutcome::Reply(reply))
|
||||||
}
|
}
|
||||||
None => Ok(HandlerOutcome::Error(ErrorCode::NotFound)),
|
None => {
|
||||||
|
self.metrics.record_get_chunk_miss();
|
||||||
|
Ok(HandlerOutcome::Error(ErrorCode::NotFound))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Method::PutManifest => {
|
Method::PutManifest => {
|
||||||
@@ -497,8 +520,14 @@ impl RpcRouter {
|
|||||||
None => return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest)),
|
None => return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest)),
|
||||||
};
|
};
|
||||||
match store.get(&key).await? {
|
match store.get(&key).await? {
|
||||||
Some(value) => Ok(HandlerOutcome::Reply(value.to_vec())),
|
Some(value) => {
|
||||||
None => Ok(HandlerOutcome::Error(ErrorCode::NotFound)),
|
self.metrics.record_get_ref_hit();
|
||||||
|
Ok(HandlerOutcome::Reply(value.to_vec()))
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
self.metrics.record_get_ref_miss();
|
||||||
|
Ok(HandlerOutcome::Error(ErrorCode::NotFound))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Method::PutRef => {
|
Method::PutRef => {
|
||||||
@@ -543,8 +572,14 @@ impl RpcRouter {
|
|||||||
_ => return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest)),
|
_ => return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest)),
|
||||||
};
|
};
|
||||||
match store.get(key).await? {
|
match store.get(key).await? {
|
||||||
Some(value) => Ok(HandlerOutcome::Reply(value.to_vec())),
|
Some(value) => {
|
||||||
None => Ok(HandlerOutcome::Error(ErrorCode::NotFound)),
|
self.metrics.record_get_tag_hit();
|
||||||
|
Ok(HandlerOutcome::Reply(value.to_vec()))
|
||||||
|
}
|
||||||
|
None => {
|
||||||
|
self.metrics.record_get_tag_miss();
|
||||||
|
Ok(HandlerOutcome::Error(ErrorCode::NotFound))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Method::DeleteTag => {
|
Method::DeleteTag => {
|
||||||
@@ -572,6 +607,12 @@ impl RpcRouter {
|
|||||||
.context("encoding TagEntry list as JSON")?;
|
.context("encoding TagEntry list as JSON")?;
|
||||||
Ok(HandlerOutcome::Reply(json))
|
Ok(HandlerOutcome::Reply(json))
|
||||||
}
|
}
|
||||||
|
Method::GetMetrics => {
|
||||||
|
let snapshot = self.metrics.snapshot();
|
||||||
|
let json = serde_json::to_vec(&snapshot)
|
||||||
|
.context("encoding MetricsReply as JSON")?;
|
||||||
|
Ok(HandlerOutcome::Reply(json))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -454,6 +454,19 @@ pub async fn call_put_ref(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Phase 5g: cache metrics client helper ────────────────────────────
|
||||||
|
|
||||||
|
/// Fetch the peer's current cache-metrics snapshot.
|
||||||
|
pub async fn call_get_metrics(conn: &Connection) -> Result<MetricsReply> {
|
||||||
|
let reply = rpc_call(conn, Method::GetMetrics, &[]).await?;
|
||||||
|
if reply.len() == 1 {
|
||||||
|
if let Some(err) = decode_error(reply[0]) {
|
||||||
|
bail!("peer replied with error: {}", err.describe());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
serde_json::from_slice(&reply).context("decoding MetricsReply JSON")
|
||||||
|
}
|
||||||
|
|
||||||
// ── Phase 5d: tag-store client helpers ───────────────────────────────
|
// ── Phase 5d: tag-store client helpers ───────────────────────────────
|
||||||
|
|
||||||
/// Publish (or overwrite) a named tag pointing at a 32-byte value.
|
/// Publish (or overwrite) a named tag pointing at a 32-byte value.
|
||||||
|
|||||||
@@ -290,6 +290,189 @@ async fn list_tags_returns_json_sorted() {
|
|||||||
assert_eq!(decoded[0].decode_value().unwrap(), [1u8; 32]);
|
assert_eq!(decoded[0].decode_value().unwrap(), [1u8; 32]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Phase 5g: cache metrics RPC ──────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn phase_5g_method_byte_encoding() {
|
||||||
|
assert_eq!(Method::GetMetrics.as_byte(), 0x13);
|
||||||
|
assert_eq!(Method::from_byte(0x13), Some(Method::GetMetrics));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn get_metrics_returns_empty_snapshot_before_any_activity() {
|
||||||
|
let (_tmp, router) = router_with_full_stack("solo", next_port()).await;
|
||||||
|
let reply = dispatch(&router, &[Method::GetMetrics.as_byte()]).await;
|
||||||
|
let snapshot: crate::cluster::metrics::MetricsReply =
|
||||||
|
serde_json::from_slice(&reply).unwrap();
|
||||||
|
assert!(snapshot.started_unix > 0);
|
||||||
|
assert_eq!(snapshot.get_ref_hits, 0);
|
||||||
|
assert_eq!(snapshot.get_ref_misses, 0);
|
||||||
|
assert_eq!(snapshot.blob_get_bytes, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn get_ref_records_hit_and_miss_counters() {
|
||||||
|
let (_tmp, router) = router_with_full_stack("solo", next_port()).await;
|
||||||
|
let store = router.ref_store().unwrap().clone();
|
||||||
|
let key = [0x11u8; 32];
|
||||||
|
store.put(&key, &[0x22u8; 32]).await.unwrap();
|
||||||
|
|
||||||
|
// Two hits.
|
||||||
|
for _ in 0..2 {
|
||||||
|
let mut req = vec![Method::GetRef.as_byte()];
|
||||||
|
req.extend_from_slice(&key);
|
||||||
|
let reply = dispatch(&router, &req).await;
|
||||||
|
assert_eq!(reply.len(), 32);
|
||||||
|
}
|
||||||
|
// One miss.
|
||||||
|
let mut req = vec![Method::GetRef.as_byte()];
|
||||||
|
req.extend_from_slice(&[0xffu8; 32]);
|
||||||
|
let reply = dispatch(&router, &req).await;
|
||||||
|
assert_eq!(reply, vec![ErrorCode::NotFound.as_byte()]);
|
||||||
|
|
||||||
|
let snapshot = router.metrics().snapshot();
|
||||||
|
assert_eq!(snapshot.get_ref_hits, 2);
|
||||||
|
assert_eq!(snapshot.get_ref_misses, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn get_tag_records_hit_and_miss_counters() {
|
||||||
|
let (_tmp, router) = router_with_full_stack("solo", next_port()).await;
|
||||||
|
let store = router.tag_store().unwrap().clone();
|
||||||
|
store.put("clawverse:main", &[0u8; 32]).await.unwrap();
|
||||||
|
|
||||||
|
let mut hit = vec![Method::GetTag.as_byte()];
|
||||||
|
hit.extend_from_slice(b"clawverse:main");
|
||||||
|
dispatch(&router, &hit).await;
|
||||||
|
|
||||||
|
let mut miss = vec![Method::GetTag.as_byte()];
|
||||||
|
miss.extend_from_slice(b"never-set");
|
||||||
|
dispatch(&router, &miss).await;
|
||||||
|
|
||||||
|
let snapshot = router.metrics().snapshot();
|
||||||
|
assert_eq!(snapshot.get_tag_hits, 1);
|
||||||
|
assert_eq!(snapshot.get_tag_misses, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn blob_get_and_blob_put_record_byte_counts() {
|
||||||
|
let (_tmp, router) = router_with_full_stack("solo", next_port()).await;
|
||||||
|
let payload = b"metrics witness";
|
||||||
|
|
||||||
|
// Put — should record blob_put_bytes.
|
||||||
|
let mut put_req = vec![Method::BlobPut.as_byte()];
|
||||||
|
put_req.extend_from_slice(payload);
|
||||||
|
let put_reply = dispatch(&router, &put_req).await;
|
||||||
|
assert_eq!(put_reply.len(), 32);
|
||||||
|
let mut id_bytes = [0u8; 32];
|
||||||
|
id_bytes.copy_from_slice(&put_reply);
|
||||||
|
|
||||||
|
// Get — should record blob_get_bytes.
|
||||||
|
let mut get_req = vec![Method::BlobGet.as_byte()];
|
||||||
|
get_req.extend_from_slice(&id_bytes);
|
||||||
|
let get_reply = dispatch(&router, &get_req).await;
|
||||||
|
assert_eq!(get_reply, payload);
|
||||||
|
|
||||||
|
let snapshot = router.metrics().snapshot();
|
||||||
|
assert_eq!(snapshot.blob_put_bytes, payload.len() as u64);
|
||||||
|
assert_eq!(snapshot.blob_get_bytes, payload.len() as u64);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn has_chunk_and_get_chunk_record_hit_miss_counters() {
|
||||||
|
let (_tmp, router) = router_with_full_stack("solo", next_port()).await;
|
||||||
|
let store = router.blob_store().unwrap().clone();
|
||||||
|
let bytes = b"chunk-in-store";
|
||||||
|
let hash = crate::cluster::blob::ChunkHash::from_bytes(blake3::hash(bytes).into());
|
||||||
|
store.put_chunk(&hash, bytes).await.unwrap();
|
||||||
|
|
||||||
|
// HasChunk hit + miss.
|
||||||
|
let mut hit = vec![Method::HasChunk.as_byte()];
|
||||||
|
hit.extend_from_slice(hash.as_bytes());
|
||||||
|
dispatch(&router, &hit).await;
|
||||||
|
|
||||||
|
let mut miss = vec![Method::HasChunk.as_byte()];
|
||||||
|
miss.extend_from_slice(&[0u8; 32]);
|
||||||
|
dispatch(&router, &miss).await;
|
||||||
|
|
||||||
|
// GetChunk hit + miss.
|
||||||
|
let mut get_hit = vec![Method::GetChunk.as_byte()];
|
||||||
|
get_hit.extend_from_slice(hash.as_bytes());
|
||||||
|
dispatch(&router, &get_hit).await;
|
||||||
|
|
||||||
|
let mut get_miss = vec![Method::GetChunk.as_byte()];
|
||||||
|
get_miss.extend_from_slice(&[0u8; 32]);
|
||||||
|
dispatch(&router, &get_miss).await;
|
||||||
|
|
||||||
|
let snapshot = router.metrics().snapshot();
|
||||||
|
assert_eq!(snapshot.has_chunk_hits, 1);
|
||||||
|
assert_eq!(snapshot.has_chunk_misses, 1);
|
||||||
|
assert_eq!(snapshot.get_chunk_hits, 1);
|
||||||
|
assert_eq!(snapshot.get_chunk_misses, 1);
|
||||||
|
// Get_chunk hit also records blob_get_bytes.
|
||||||
|
assert_eq!(snapshot.blob_get_bytes, bytes.len() as u64);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn end_to_end_get_metrics_over_real_quic() {
|
||||||
|
// Exercise every counter, then fetch the metrics reply through
|
||||||
|
// real QUIC and verify each field.
|
||||||
|
let (id_a, id_b) = NodeIdentity::generate_test_pair("a", "b").unwrap();
|
||||||
|
let (_tmp, router) = router_with_full_stack("a", next_port()).await;
|
||||||
|
|
||||||
|
// Seed some activity locally so the counters have real values.
|
||||||
|
let store = router.ref_store().unwrap().clone();
|
||||||
|
store.put(&[0x11u8; 32], &[0x22u8; 32]).await.unwrap();
|
||||||
|
let tag_store = router.tag_store().unwrap().clone();
|
||||||
|
tag_store.put("hit-me", &[0u8; 32]).await.unwrap();
|
||||||
|
let blob_store = router.blob_store().unwrap().clone();
|
||||||
|
let payload = vec![0x5au8; 4096];
|
||||||
|
let blob_id = blob_store.put_bytes(&payload).await.unwrap();
|
||||||
|
|
||||||
|
// Fire dispatches to move the counters.
|
||||||
|
let mut ref_hit = vec![Method::GetRef.as_byte()];
|
||||||
|
ref_hit.extend_from_slice(&[0x11u8; 32]);
|
||||||
|
dispatch(&router, &ref_hit).await;
|
||||||
|
|
||||||
|
let mut ref_miss = vec![Method::GetRef.as_byte()];
|
||||||
|
ref_miss.extend_from_slice(&[0x99u8; 32]);
|
||||||
|
dispatch(&router, &ref_miss).await;
|
||||||
|
|
||||||
|
let mut tag_hit = vec![Method::GetTag.as_byte()];
|
||||||
|
tag_hit.extend_from_slice(b"hit-me");
|
||||||
|
dispatch(&router, &tag_hit).await;
|
||||||
|
|
||||||
|
let mut blob_get = vec![Method::BlobGet.as_byte()];
|
||||||
|
blob_get.extend_from_slice(blob_id.as_bytes());
|
||||||
|
dispatch(&router, &blob_get).await;
|
||||||
|
|
||||||
|
// Now start the server + fetch metrics over the wire.
|
||||||
|
let server = QuicServer::bind(loopback(0), id_a).unwrap();
|
||||||
|
let server_addr = server.local_addr().unwrap();
|
||||||
|
let router_srv = router.clone();
|
||||||
|
let accept_task = tokio::spawn(async move {
|
||||||
|
if let Some(Ok(conn)) = server.accept().await {
|
||||||
|
let _ = serve_connection(conn, router_srv).await;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let client = QuicClient::new(loopback(0), id_b).unwrap();
|
||||||
|
let conn = client.connect(server_addr, "a").await.unwrap();
|
||||||
|
|
||||||
|
let m = call_get_metrics(&conn).await.unwrap();
|
||||||
|
assert_eq!(m.get_ref_hits, 1);
|
||||||
|
assert_eq!(m.get_ref_misses, 1);
|
||||||
|
assert_eq!(m.get_tag_hits, 1);
|
||||||
|
assert_eq!(m.get_tag_misses, 0);
|
||||||
|
assert_eq!(m.blob_get_bytes, payload.len() as u64);
|
||||||
|
assert_eq!(m.get_ref_hit_rate(), Some(0.5));
|
||||||
|
|
||||||
|
conn.close(quinn::VarInt::from_u32(0), b"done");
|
||||||
|
client.shutdown().await;
|
||||||
|
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||||
|
accept_task.abort();
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn end_to_end_prewarm_copies_tagged_blob_between_two_peers() {
|
async fn end_to_end_prewarm_copies_tagged_blob_between_two_peers() {
|
||||||
// Phase 5f: `claw-cargo prewarm --from A --to C --pin tag`.
|
// Phase 5f: `claw-cargo prewarm --from A --to C --pin tag`.
|
||||||
|
|||||||
Reference in New Issue
Block a user