Phase 2b: Blob RPC (Stat/Get/Put/LoadManifest) #7

Merged
osobh merged 1 commits from phase-2b-blob-rpc into main 2026-07-12 06:38:35 +00:00
5 changed files with 654 additions and 24 deletions
+6
View File
@@ -386,6 +386,7 @@ mod tests {
bind_rpc_lan: None, bind_rpc_lan: None,
bind_rpc_tailscale: None, bind_rpc_tailscale: None,
tls: None, tls: None,
blob_store_root: None,
}; };
let g = ClusterGossip::bootstrap(&cfg, "solo").await.unwrap(); let g = ClusterGossip::bootstrap(&cfg, "solo").await.unwrap();
let id = g.self_chitchat_id().await; let id = g.self_chitchat_id().await;
@@ -404,6 +405,7 @@ mod tests {
bind_rpc_lan: None, bind_rpc_lan: None,
bind_rpc_tailscale: None, bind_rpc_tailscale: None,
tls: None, tls: None,
blob_store_root: None,
}; };
let err = ClusterGossip::bootstrap(&cfg, "") let err = ClusterGossip::bootstrap(&cfg, "")
.await .await
@@ -422,6 +424,7 @@ mod tests {
bind_rpc_lan: None, bind_rpc_lan: None,
bind_rpc_tailscale: None, bind_rpc_tailscale: None,
tls: None, tls: None,
blob_store_root: None,
}; };
// ClusterConfig::validate rejects this first — that's what we want: // ClusterConfig::validate rejects this first — that's what we want:
// the daemon should refuse to bootstrap gossip on a malformed config. // the daemon should refuse to bootstrap gossip on a malformed config.
@@ -448,6 +451,7 @@ mod tests {
bind_rpc_lan: None, bind_rpc_lan: None,
bind_rpc_tailscale: None, bind_rpc_tailscale: None,
tls: None, tls: None,
blob_store_root: None,
}; };
// Node B: uses A as seed. // Node B: uses A as seed.
let cfg_b = ClusterConfig { let cfg_b = ClusterConfig {
@@ -463,6 +467,7 @@ mod tests {
bind_rpc_lan: None, bind_rpc_lan: None,
bind_rpc_tailscale: None, bind_rpc_tailscale: None,
tls: None, tls: None,
blob_store_root: None,
}; };
let gossip_a = ClusterGossip::bootstrap(&cfg_a, "a").await.unwrap(); let gossip_a = ClusterGossip::bootstrap(&cfg_a, "a").await.unwrap();
@@ -534,6 +539,7 @@ mod tests {
bind_rpc_lan: None, bind_rpc_lan: None,
bind_rpc_tailscale: None, bind_rpc_tailscale: None,
tls: None, tls: None,
blob_store_root: None,
}; };
let g = ClusterGossip::bootstrap(&cfg, "solo").await.unwrap(); let g = ClusterGossip::bootstrap(&cfg, "solo").await.unwrap();
// Solo cluster — peers() must never include self. // Solo cluster — peers() must never include self.
+506 -10
View File
@@ -25,6 +25,7 @@
//! containing this node's local view of the cluster (its own //! containing this node's local view of the cluster (its own
//! name+zone, plus every peer it currently knows about via gossip). //! name+zone, plus every peer it currently knows about via gossip).
use crate::cluster::blob::{BlobId, BlobManifest, BlobStat, BlobStore};
use crate::cluster::gossip::{ClusterGossip, PeerView}; use crate::cluster::gossip::{ClusterGossip, PeerView};
use anyhow::{bail, Context, Result}; use anyhow::{bail, Context, Result};
use quinn::{Connection, ConnectionError}; use quinn::{Connection, ConnectionError};
@@ -32,9 +33,11 @@ use serde::{Deserialize, Serialize};
use std::sync::Arc; use std::sync::Arc;
/// Cap on a single request or response, including the method tag. /// Cap on a single request or response, including the method tag.
/// Matches `MAX_MESSAGE_BYTES` in [`super::transport`] so both layers /// 16 MiB is generous enough to hold one 4 MiB blob chunk with plenty
/// bound memory the same way. /// of framing headroom; multi-chunk / whole-blob transfers still fit
pub const MAX_MESSAGE_BYTES: usize = 16 * 1024; /// well under that ceiling for anything up to a few MB. Streaming
/// (many-GB) put/get lands in Phase 2c.
pub const MAX_MESSAGE_BYTES: usize = 16 * 1024 * 1024;
/// RPC method tag byte. /// RPC method tag byte.
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -42,6 +45,18 @@ pub const MAX_MESSAGE_BYTES: usize = 16 * 1024;
pub enum Method { pub enum Method {
Ping = 0x01, Ping = 0x01,
PeerStatus = 0x02, PeerStatus = 0x02,
/// `payload`: 32-byte `BlobId`. Reply: JSON `BlobStat` or
/// single-byte [`ErrorCode::NotFound`].
BlobStat = 0x03,
/// `payload`: 32-byte `BlobId`. Reply: raw blob bytes or
/// single-byte [`ErrorCode::NotFound`].
BlobGet = 0x04,
/// `payload`: raw blob bytes. Reply: 32-byte `BlobId` of the stored
/// content.
BlobPut = 0x05,
/// `payload`: 32-byte `BlobId`. Reply: JSON `BlobManifest` or
/// single-byte [`ErrorCode::NotFound`].
BlobLoadManifest = 0x06,
} }
impl Method { impl Method {
@@ -51,6 +66,10 @@ impl Method {
match b { match b {
0x01 => Some(Method::Ping), 0x01 => Some(Method::Ping),
0x02 => Some(Method::PeerStatus), 0x02 => Some(Method::PeerStatus),
0x03 => Some(Method::BlobStat),
0x04 => Some(Method::BlobGet),
0x05 => Some(Method::BlobPut),
0x06 => Some(Method::BlobLoadManifest),
_ => None, _ => None,
} }
} }
@@ -70,6 +89,15 @@ pub enum ErrorCode {
EmptyRequest = 0xf0, EmptyRequest = 0xf0,
UnknownMethod = 0xf1, UnknownMethod = 0xf1,
HandlerFailure = 0xf2, HandlerFailure = 0xf2,
/// Blob (or manifest) not present in the local store.
NotFound = 0xf3,
/// Request payload was structurally wrong (e.g. wrong length for a
/// 32-byte hash).
InvalidRequest = 0xf4,
/// Server-side subsystem required for this method wasn't
/// configured (e.g. Blob RPCs called on a node with no local
/// blob store).
NotConfigured = 0xf5,
} }
impl ErrorCode { impl ErrorCode {
@@ -81,6 +109,9 @@ impl ErrorCode {
ErrorCode::EmptyRequest => "empty request", ErrorCode::EmptyRequest => "empty request",
ErrorCode::UnknownMethod => "unknown method", ErrorCode::UnknownMethod => "unknown method",
ErrorCode::HandlerFailure => "handler failure", ErrorCode::HandlerFailure => "handler failure",
ErrorCode::NotFound => "not found",
ErrorCode::InvalidRequest => "invalid request",
ErrorCode::NotConfigured => "server subsystem not configured",
} }
} }
} }
@@ -99,36 +130,64 @@ pub struct PeerStatusReply {
} }
/// The concrete RPC handler used by the daemon. Holds Arc references /// The concrete RPC handler used by the daemon. Holds Arc references
/// to the state a request might need to read (currently just the /// to the state a request might need to read: the gossip service
/// gossip service; Phase 2+ adds the blob store, metadata index, etc). /// (always), and optionally a local blob store (for Blob* methods).
/// ///
/// Not `Clone` on its own — wrap in `Arc<RpcRouter>` so a single /// Not `Clone` on its own — wrap in `Arc<RpcRouter>` so a single
/// instance backs the accept loop plus any explicit dispatch calls. /// instance backs the accept loop plus any explicit dispatch calls.
pub struct RpcRouter { pub struct RpcRouter {
gossip: Arc<ClusterGossip>, gossip: Arc<ClusterGossip>,
blob_store: Option<Arc<BlobStore>>,
local_name: String, local_name: String,
local_zone: String, local_zone: String,
} }
/// Result of dispatching a request: either a real reply (`Ok`) or a
/// well-known error code the wire layer surfaces as a single byte.
/// Extracted so `handle` stays free of `Vec<u8>` shell games.
enum HandlerOutcome {
Reply(Vec<u8>),
Error(ErrorCode),
}
impl RpcRouter { impl RpcRouter {
pub fn new(gossip: Arc<ClusterGossip>, local_name: String, local_zone: String) -> Self { pub fn new(gossip: Arc<ClusterGossip>, local_name: String, local_zone: String) -> Self {
Self { Self {
gossip, gossip,
blob_store: None,
local_name, local_name,
local_zone, local_zone,
} }
} }
/// Attach a local blob store. Enables the `Blob*` methods; nodes
/// without a store return [`ErrorCode::NotConfigured`] for those.
pub fn with_blob_store(mut self, store: Arc<BlobStore>) -> Self {
self.blob_store = Some(store);
self
}
pub fn blob_store(&self) -> Option<&Arc<BlobStore>> {
self.blob_store.as_ref()
}
/// Dispatch a single request. Called by [`serve_connection`] for /// Dispatch a single request. Called by [`serve_connection`] for
/// every accepted bidi stream. Test code may call it directly to /// every accepted bidi stream. Test code may call it directly to
/// bypass the transport. /// bypass the transport.
pub async fn handle(&self, method: Method, payload: &[u8]) -> Result<Vec<u8>> { pub async fn handle(&self, method: Method, payload: &[u8]) -> Result<Vec<u8>> {
match self.handle_outcome(method, payload).await? {
HandlerOutcome::Reply(bytes) => Ok(bytes),
HandlerOutcome::Error(code) => Ok(vec![code.as_byte()]),
}
}
async fn handle_outcome(&self, method: Method, payload: &[u8]) -> Result<HandlerOutcome> {
match method { match method {
Method::Ping => { Method::Ping => {
let mut reply = Vec::with_capacity(5 + payload.len()); let mut reply = Vec::with_capacity(5 + payload.len());
reply.extend_from_slice(b"pong:"); reply.extend_from_slice(b"pong:");
reply.extend_from_slice(payload); reply.extend_from_slice(payload);
Ok(reply) Ok(HandlerOutcome::Reply(reply))
} }
Method::PeerStatus => { Method::PeerStatus => {
let peers = self.gossip.peers().await; let peers = self.gossip.peers().await;
@@ -146,11 +205,93 @@ impl RpcRouter {
MAX_MESSAGE_BYTES MAX_MESSAGE_BYTES
); );
} }
Ok(json) Ok(HandlerOutcome::Reply(json))
}
Method::BlobStat => {
let store = match &self.blob_store {
Some(s) => s,
None => return Ok(HandlerOutcome::Error(ErrorCode::NotConfigured)),
};
let id = match decode_blob_id(payload) {
Some(id) => id,
None => return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest)),
};
match store.stat(&id).await? {
Some(stat) => {
let json = serde_json::to_vec(&stat)
.context("encoding BlobStat as JSON")?;
Ok(HandlerOutcome::Reply(json))
}
None => Ok(HandlerOutcome::Error(ErrorCode::NotFound)),
}
}
Method::BlobGet => {
let store = match &self.blob_store {
Some(s) => s,
None => return Ok(HandlerOutcome::Error(ErrorCode::NotConfigured)),
};
let id = match decode_blob_id(payload) {
Some(id) => id,
None => return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest)),
};
match store.get_bytes(&id).await? {
Some(bytes) => {
if bytes.len() > MAX_MESSAGE_BYTES {
bail!(
"blob {} at {} bytes exceeds RPC cap {}; use streaming variant (Phase 2c)",
id.to_hex(),
bytes.len(),
MAX_MESSAGE_BYTES
);
}
Ok(HandlerOutcome::Reply(bytes))
}
None => Ok(HandlerOutcome::Error(ErrorCode::NotFound)),
}
}
Method::BlobPut => {
let store = match &self.blob_store {
Some(s) => s,
None => return Ok(HandlerOutcome::Error(ErrorCode::NotConfigured)),
};
// Empty payload is a legitimate empty-blob put — falls
// through to store.put_bytes(&[]) which yields the
// hash of the empty byte sequence.
let id = store.put_bytes(payload).await?;
Ok(HandlerOutcome::Reply(id.as_bytes().to_vec()))
}
Method::BlobLoadManifest => {
let store = match &self.blob_store {
Some(s) => s,
None => return Ok(HandlerOutcome::Error(ErrorCode::NotConfigured)),
};
let id = match decode_blob_id(payload) {
Some(id) => id,
None => return Ok(HandlerOutcome::Error(ErrorCode::InvalidRequest)),
};
match store.load_manifest(&id).await? {
Some(manifest) => {
let json = serde_json::to_vec(&manifest)
.context("encoding BlobManifest as JSON")?;
Ok(HandlerOutcome::Reply(json))
}
None => Ok(HandlerOutcome::Error(ErrorCode::NotFound)),
} }
} }
} }
} }
}
/// Parse a payload as a 32-byte BlobId. Returns `None` for any other
/// length so the caller can surface [`ErrorCode::InvalidRequest`].
fn decode_blob_id(payload: &[u8]) -> Option<BlobId> {
if payload.len() != 32 {
return None;
}
let mut buf = [0u8; 32];
buf.copy_from_slice(payload);
Some(BlobId::from_bytes(buf))
}
/// Server-side: loop accepting bidi streams on `conn`, dispatch to /// Server-side: loop accepting bidi streams on `conn`, dispatch to
/// `router`, write the reply. Returns cleanly when the peer closes the /// `router`, write the reply. Returns cleanly when the peer closes the
@@ -262,15 +403,106 @@ pub async fn call_peer_status(conn: &Connection) -> Result<PeerStatusReply> {
/// Recognise a single-byte reply as one of our error codes. Returns /// Recognise a single-byte reply as one of our error codes. Returns
/// `None` for any other single-byte value (which is a valid reply, /// `None` for any other single-byte value (which is a valid reply,
/// just an unusually short one). /// just an unusually short one).
fn decode_error(b: u8) -> Option<ErrorCode> { pub fn decode_error(b: u8) -> Option<ErrorCode> {
match b { match b {
0xf0 => Some(ErrorCode::EmptyRequest), 0xf0 => Some(ErrorCode::EmptyRequest),
0xf1 => Some(ErrorCode::UnknownMethod), 0xf1 => Some(ErrorCode::UnknownMethod),
0xf2 => Some(ErrorCode::HandlerFailure), 0xf2 => Some(ErrorCode::HandlerFailure),
0xf3 => Some(ErrorCode::NotFound),
0xf4 => Some(ErrorCode::InvalidRequest),
0xf5 => Some(ErrorCode::NotConfigured),
_ => None, _ => None,
} }
} }
// ── Blob RPC client helpers ───────────────────────────────────────────
/// Ask the peer for a blob's size + chunk count. `Ok(None)` when the
/// peer replies [`ErrorCode::NotFound`]; other error codes surface as
/// `Err`.
pub async fn call_blob_stat(
conn: &Connection,
id: &BlobId,
) -> Result<Option<BlobStat>> {
let reply = rpc_call(conn, Method::BlobStat, id.as_bytes()).await?;
if reply.len() == 1 {
match decode_error(reply[0]) {
Some(ErrorCode::NotFound) => return Ok(None),
Some(code) => bail!("peer replied with error: {}", code.describe()),
None => {} // single-byte JSON like "1" is technically possible; fall through
}
}
let stat = serde_json::from_slice(&reply).context("decoding BlobStat JSON")?;
Ok(Some(stat))
}
/// Fetch a blob's raw bytes. `Ok(None)` when the peer replies
/// [`ErrorCode::NotFound`]. Callers that need many-GB transfers should
/// use the streaming variant (Phase 2c); this helper caps at
/// [`MAX_MESSAGE_BYTES`].
pub async fn call_blob_get(
conn: &Connection,
id: &BlobId,
) -> Result<Option<Vec<u8>>> {
let reply = rpc_call(conn, Method::BlobGet, id.as_bytes()).await?;
if reply.len() == 1 {
match decode_error(reply[0]) {
Some(ErrorCode::NotFound) => return Ok(None),
Some(code) => bail!("peer replied with error: {}", code.describe()),
None => {} // single-byte blob is legitimate content
}
}
Ok(Some(reply))
}
/// Upload a blob to the peer's store. Returns the BlobId the peer
/// assigned; must match the local hash of `bytes` (proves the peer
/// stored what we sent).
pub async fn call_blob_put(conn: &Connection, bytes: &[u8]) -> Result<BlobId> {
let reply = rpc_call(conn, Method::BlobPut, bytes).await?;
if reply.len() == 1 {
if let Some(code) = decode_error(reply[0]) {
bail!("peer replied with error: {}", code.describe());
}
}
if reply.len() != 32 {
bail!(
"expected 32-byte BlobId in reply, got {} bytes",
reply.len()
);
}
let mut buf = [0u8; 32];
buf.copy_from_slice(&reply);
let assigned = BlobId::from_bytes(buf);
let expected = BlobId::from_bytes(blake3::hash(bytes).into());
if assigned != expected {
bail!(
"peer returned BlobId {} but content hashes to {}; corruption or protocol drift",
assigned.to_hex(),
expected.to_hex()
);
}
Ok(assigned)
}
/// Fetch a blob's manifest (chunk-list + size). `Ok(None)` on NotFound.
pub async fn call_blob_load_manifest(
conn: &Connection,
id: &BlobId,
) -> Result<Option<BlobManifest>> {
let reply = rpc_call(conn, Method::BlobLoadManifest, id.as_bytes()).await?;
if reply.len() == 1 {
match decode_error(reply[0]) {
Some(ErrorCode::NotFound) => return Ok(None),
Some(code) => bail!("peer replied with error: {}", code.describe()),
None => {}
}
}
let manifest =
serde_json::from_slice(&reply).context("decoding BlobManifest JSON")?;
Ok(Some(manifest))
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@@ -304,12 +536,40 @@ mod tests {
fn method_round_trips_byte_encoding() { fn method_round_trips_byte_encoding() {
assert_eq!(Method::Ping.as_byte(), 0x01); assert_eq!(Method::Ping.as_byte(), 0x01);
assert_eq!(Method::PeerStatus.as_byte(), 0x02); assert_eq!(Method::PeerStatus.as_byte(), 0x02);
assert_eq!(Method::from_byte(0x01), Some(Method::Ping)); assert_eq!(Method::BlobStat.as_byte(), 0x03);
assert_eq!(Method::from_byte(0x02), Some(Method::PeerStatus)); assert_eq!(Method::BlobGet.as_byte(), 0x04);
assert_eq!(Method::BlobPut.as_byte(), 0x05);
assert_eq!(Method::BlobLoadManifest.as_byte(), 0x06);
for m in [
Method::Ping,
Method::PeerStatus,
Method::BlobStat,
Method::BlobGet,
Method::BlobPut,
Method::BlobLoadManifest,
] {
assert_eq!(Method::from_byte(m.as_byte()), Some(m));
}
assert_eq!(Method::from_byte(0x00), None); assert_eq!(Method::from_byte(0x00), None);
assert_eq!(Method::from_byte(0xff), None); assert_eq!(Method::from_byte(0xff), None);
} }
fn open_blob_store() -> (tempfile::TempDir, Arc<BlobStore>) {
let tmp = tempfile::TempDir::new().unwrap();
let store = Arc::new(BlobStore::open(tmp.path().to_path_buf()).unwrap());
(tmp, store)
}
async fn router_with_blobs(name: &str, port: u16) -> (tempfile::TempDir, Arc<RpcRouter>) {
let gossip = bootstrap_gossip(name, port).await;
let (tmp, store) = open_blob_store();
let router = Arc::new(
RpcRouter::new(gossip, name.into(), "fabric-10g".into())
.with_blob_store(store),
);
(tmp, router)
}
#[tokio::test] #[tokio::test]
async fn dispatch_returns_pong_for_ping() { async fn dispatch_returns_pong_for_ping() {
let gossip = bootstrap_gossip("solo", next_port()).await; let gossip = bootstrap_gossip("solo", next_port()).await;
@@ -506,5 +766,241 @@ mod tests {
assert_eq!(ErrorCode::EmptyRequest.describe(), "empty request"); assert_eq!(ErrorCode::EmptyRequest.describe(), "empty request");
assert_eq!(ErrorCode::UnknownMethod.describe(), "unknown method"); assert_eq!(ErrorCode::UnknownMethod.describe(), "unknown method");
assert_eq!(ErrorCode::HandlerFailure.describe(), "handler failure"); assert_eq!(ErrorCode::HandlerFailure.describe(), "handler failure");
assert_eq!(ErrorCode::NotFound.describe(), "not found");
assert_eq!(ErrorCode::InvalidRequest.describe(), "invalid request");
assert_eq!(
ErrorCode::NotConfigured.describe(),
"server subsystem not configured"
);
}
#[test]
fn decode_error_covers_all_known_codes() {
for code in [
ErrorCode::EmptyRequest,
ErrorCode::UnknownMethod,
ErrorCode::HandlerFailure,
ErrorCode::NotFound,
ErrorCode::InvalidRequest,
ErrorCode::NotConfigured,
] {
assert_eq!(decode_error(code.as_byte()), Some(code));
}
assert_eq!(decode_error(0x00), None);
assert_eq!(decode_error(0xff), None);
}
// ── Phase 2b: Blob RPC ─────────────────────────────────────────────
#[tokio::test]
async fn blob_rpcs_return_not_configured_without_store() {
// Router built via `new` alone (no `.with_blob_store`) must
// refuse Blob* methods with a well-known error code.
let gossip = bootstrap_gossip("solo", next_port()).await;
let router = RpcRouter::new(gossip, "solo".into(), "z".into());
for method in [
Method::BlobStat,
Method::BlobGet,
Method::BlobPut,
Method::BlobLoadManifest,
] {
let mut req = vec![method.as_byte()];
req.extend_from_slice(&[0u8; 32]);
let reply = dispatch(&router, &req).await;
assert_eq!(
reply,
vec![ErrorCode::NotConfigured.as_byte()],
"method {method:?} should be NotConfigured without a store"
);
}
}
#[tokio::test]
async fn blob_stat_returns_not_found_for_missing() {
let (_tmp, router) = router_with_blobs("solo", next_port()).await;
let missing = BlobId::from_bytes([0u8; 32]);
let mut req = vec![Method::BlobStat.as_byte()];
req.extend_from_slice(missing.as_bytes());
let reply = dispatch(&router, &req).await;
assert_eq!(reply, vec![ErrorCode::NotFound.as_byte()]);
}
#[tokio::test]
async fn blob_stat_returns_json_for_existing() {
let (_tmp, router) = router_with_blobs("solo", next_port()).await;
let id = router
.blob_store()
.unwrap()
.put_bytes(b"tiny content")
.await
.unwrap();
let mut req = vec![Method::BlobStat.as_byte()];
req.extend_from_slice(id.as_bytes());
let reply = dispatch(&router, &req).await;
let stat: BlobStat = serde_json::from_slice(&reply).unwrap();
assert_eq!(stat.total_size, b"tiny content".len() as u64);
assert_eq!(stat.chunk_count, 1);
}
#[tokio::test]
async fn blob_stat_returns_invalid_request_for_bad_length() {
let (_tmp, router) = router_with_blobs("solo", next_port()).await;
// Payload is only 5 bytes; a valid BlobId is 32.
let req = vec![Method::BlobStat.as_byte(), 1, 2, 3, 4, 5];
let reply = dispatch(&router, &req).await;
assert_eq!(reply, vec![ErrorCode::InvalidRequest.as_byte()]);
}
#[tokio::test]
async fn blob_get_returns_content_bytes() {
let (_tmp, router) = router_with_blobs("solo", next_port()).await;
let payload: &[u8] = b"contents to retrieve";
let id = router.blob_store().unwrap().put_bytes(payload).await.unwrap();
let mut req = vec![Method::BlobGet.as_byte()];
req.extend_from_slice(id.as_bytes());
let reply = dispatch(&router, &req).await;
assert_eq!(reply, payload);
}
#[tokio::test]
async fn blob_put_stores_bytes_and_returns_hash() {
let (_tmp, router) = router_with_blobs("solo", next_port()).await;
let payload = b"put via rpc";
let mut req = vec![Method::BlobPut.as_byte()];
req.extend_from_slice(payload);
let reply = dispatch(&router, &req).await;
assert_eq!(reply.len(), 32);
let mut id_bytes = [0u8; 32];
id_bytes.copy_from_slice(&reply);
let assigned = BlobId::from_bytes(id_bytes);
let expected = BlobId::from_bytes(blake3::hash(payload).into());
assert_eq!(assigned, expected);
// Round-trip: the bytes are now readable via the store.
let round = router
.blob_store()
.unwrap()
.get_bytes(&assigned)
.await
.unwrap();
assert_eq!(round.as_deref(), Some(payload.as_slice()));
}
#[tokio::test]
async fn blob_load_manifest_returns_json_for_existing() {
let (_tmp, router) = router_with_blobs("solo", next_port()).await;
let data = vec![0x77u8; 4 * 1024 * 1024 + 1]; // 2 chunks
let id = router.blob_store().unwrap().put_bytes(&data).await.unwrap();
let mut req = vec![Method::BlobLoadManifest.as_byte()];
req.extend_from_slice(id.as_bytes());
let reply = dispatch(&router, &req).await;
let manifest: BlobManifest = serde_json::from_slice(&reply).unwrap();
assert_eq!(manifest.blob_id, id);
assert_eq!(manifest.total_size, data.len() as u64);
assert_eq!(manifest.chunks.len(), 2);
}
#[tokio::test]
async fn blob_load_manifest_returns_not_found_for_missing() {
let (_tmp, router) = router_with_blobs("solo", next_port()).await;
let missing = BlobId::from_bytes([0u8; 32]);
let mut req = vec![Method::BlobLoadManifest.as_byte()];
req.extend_from_slice(missing.as_bytes());
let reply = dispatch(&router, &req).await;
assert_eq!(reply, vec![ErrorCode::NotFound.as_byte()]);
}
#[tokio::test]
async fn end_to_end_blob_put_stat_get_over_real_quic() {
// The full loop: B → A over real QUIC + mTLS.
// 1. B puts a blob on A (BlobPut).
// 2. B queries stat + fetches it back (BlobStat + BlobGet).
// 3. B asks for the manifest (BlobLoadManifest).
// Every step goes through the actual wire, no shortcuts.
let (id_a, id_b) = NodeIdentity::generate_test_pair("a", "b").unwrap();
let (_tmp, router) = router_with_blobs("a", next_port()).await;
let server = QuicServer::bind(loopback(0), id_a).unwrap();
let server_addr = server.local_addr().unwrap();
let router_srv = router.clone();
let accept_task = tokio::spawn(async move {
if let Some(Ok(conn)) = server.accept().await {
let _ = serve_connection(conn, router_srv).await;
}
});
let client = QuicClient::new(loopback(0), id_b).unwrap();
let conn = client.connect(server_addr, "a").await.unwrap();
let payload: &[u8] = b"cross-node payload";
let assigned = call_blob_put(&conn, payload).await.unwrap();
let expected = BlobId::from_bytes(blake3::hash(payload).into());
assert_eq!(assigned, expected);
let stat = call_blob_stat(&conn, &assigned).await.unwrap().unwrap();
assert_eq!(stat.total_size, payload.len() as u64);
assert_eq!(stat.chunk_count, 1);
let round = call_blob_get(&conn, &assigned).await.unwrap().unwrap();
assert_eq!(round, payload);
let manifest = call_blob_load_manifest(&conn, &assigned)
.await
.unwrap()
.unwrap();
assert_eq!(manifest.blob_id, assigned);
assert_eq!(manifest.chunks.len(), 1);
// NotFound path also works over the wire.
let ghost = BlobId::from_bytes([0u8; 32]);
assert!(call_blob_stat(&conn, &ghost).await.unwrap().is_none());
assert!(call_blob_get(&conn, &ghost).await.unwrap().is_none());
assert!(call_blob_load_manifest(&conn, &ghost)
.await
.unwrap()
.is_none());
conn.close(quinn::VarInt::from_u32(0), b"done");
client.shutdown().await;
tokio::time::sleep(Duration::from_millis(50)).await;
accept_task.abort();
}
#[tokio::test]
async fn end_to_end_multi_chunk_blob_over_real_quic() {
// 6 MB blob → 2 chunks. Round-trips whole via BlobPut/Get.
// Also confirms the RPC layer's MAX_MESSAGE_BYTES bump from
// 16 KiB to 16 MiB actually took effect end-to-end.
let (id_a, id_b) = NodeIdentity::generate_test_pair("a", "b").unwrap();
let (_tmp, router) = router_with_blobs("a", next_port()).await;
let server = QuicServer::bind(loopback(0), id_a).unwrap();
let server_addr = server.local_addr().unwrap();
let router_srv = router.clone();
let accept_task = tokio::spawn(async move {
if let Some(Ok(conn)) = server.accept().await {
let _ = serve_connection(conn, router_srv).await;
}
});
let client = QuicClient::new(loopback(0), id_b).unwrap();
let conn = client.connect(server_addr, "a").await.unwrap();
let payload: Vec<u8> = (0..6 * 1024 * 1024)
.map(|i| (i % 251) as u8)
.collect();
let assigned = call_blob_put(&conn, &payload).await.unwrap();
let manifest = call_blob_load_manifest(&conn, &assigned)
.await
.unwrap()
.unwrap();
assert_eq!(manifest.chunks.len(), 2, "6 MB should split into 2 chunks");
let round = call_blob_get(&conn, &assigned).await.unwrap().unwrap();
assert_eq!(round.len(), payload.len());
assert_eq!(round, payload);
conn.close(quinn::VarInt::from_u32(0), b"done");
client.shutdown().await;
tokio::time::sleep(Duration::from_millis(50)).await;
accept_task.abort();
} }
} }
+123 -6
View File
@@ -12,6 +12,7 @@
//! When `[cluster]` is absent from the config the daemon runs exactly //! When `[cluster]` is absent from the config the daemon runs exactly
//! as it did pre-v2: no gossip, no RPC, no metric ticker. //! as it did pre-v2: no gossip, no RPC, no metric ticker.
use crate::cluster::blob::BlobStore;
use crate::cluster::gossip::ClusterGossip; use crate::cluster::gossip::ClusterGossip;
use crate::cluster::rpc::{serve_connection, RpcRouter}; use crate::cluster::rpc::{serve_connection, RpcRouter};
use crate::cluster::transport::{NodeIdentity, QuicServer}; use crate::cluster::transport::{NodeIdentity, QuicServer};
@@ -39,6 +40,10 @@ pub struct ClusterServices {
/// Live gossip service. `Arc` so read-only consumers on other /// Live gossip service. `Arc` so read-only consumers on other
/// subsystems can hold references without blocking shutdown. /// subsystems can hold references without blocking shutdown.
pub gossip: Arc<ClusterGossip>, pub gossip: Arc<ClusterGossip>,
/// Local blob store, when `blob_store_root` was provided at start
/// time. `None` means this node runs gossip-only (Blob RPCs return
/// [`crate::cluster::rpc::ErrorCode::NotConfigured`]).
pub blob_store: Option<Arc<BlobStore>>,
/// QUIC accept-loop task. `None` when `[cluster.tls]` was absent /// QUIC accept-loop task. `None` when `[cluster.tls]` was absent
/// and RPC therefore didn't come up. /// and RPC therefore didn't come up.
accept_task: Option<JoinHandle<()>>, accept_task: Option<JoinHandle<()>>,
@@ -70,6 +75,7 @@ impl ClusterServices {
local_name: String, local_name: String,
hot_dir: PathBuf, hot_dir: PathBuf,
hot_max_bytes: u64, hot_max_bytes: u64,
blob_store_root: Option<PathBuf>,
) -> Result<Self> { ) -> Result<Self> {
let gossip = Arc::new( let gossip = Arc::new(
ClusterGossip::bootstrap(cluster, &local_name) ClusterGossip::bootstrap(cluster, &local_name)
@@ -80,6 +86,23 @@ impl ClusterServices {
// Publish the static config value once. Used-bytes updates every tick. // Publish the static config value once. Used-bytes updates every tick.
gossip.set_hot_max(hot_max_bytes).await; gossip.set_hot_max(hot_max_bytes).await;
// Open the local blob store if a root path was supplied. Kept
// outside the TLS branch: a node can serve blobs to callers
// without RPC (via in-process API) or over RPC (once TLS is
// configured too).
let blob_store: Option<Arc<BlobStore>> = match blob_store_root {
Some(root) => {
let store = BlobStore::open(root.clone())
.with_context(|| format!("opening blob store at {}", root.display()))?;
tracing::info!("blob store opened at {}", root.display());
Some(Arc::new(store))
}
None => {
tracing::info!("no blob store configured; Blob RPCs will return NotConfigured");
None
}
};
// RPC server + accept loop — only when TLS material is configured. // RPC server + accept loop — only when TLS material is configured.
let accept_task = match &cluster.tls { let accept_task = match &cluster.tls {
Some(tls) => { Some(tls) => {
@@ -92,11 +115,15 @@ impl ClusterServices {
.context("no RPC bind address (need bind_lan or bind_tailscale)")?; .context("no RPC bind address (need bind_lan or bind_tailscale)")?;
let server = QuicServer::bind(bind, identity) let server = QuicServer::bind(bind, identity)
.context("binding QUIC RPC server")?; .context("binding QUIC RPC server")?;
let router = Arc::new(RpcRouter::new( let mut router = RpcRouter::new(
gossip.clone(), gossip.clone(),
local_name.clone(), local_name.clone(),
cluster.zone.clone(), cluster.zone.clone(),
)); );
if let Some(store) = &blob_store {
router = router.with_blob_store(store.clone());
}
let router = Arc::new(router);
tracing::info!("cluster RPC server listening on {}", bind); tracing::info!("cluster RPC server listening on {}", bind);
Some(tokio::spawn(async move { Some(tokio::spawn(async move {
accept_forever(server, router).await; accept_forever(server, router).await;
@@ -124,11 +151,17 @@ impl ClusterServices {
Ok(Self { Ok(Self {
gossip, gossip,
blob_store,
accept_task, accept_task,
metric_task, metric_task,
}) })
} }
/// Whether a local blob store was configured at start time.
pub fn blob_store_enabled(&self) -> bool {
self.blob_store.is_some()
}
/// Whether the QUIC RPC server is running. `false` when `[cluster.tls]` /// Whether the QUIC RPC server is running. `false` when `[cluster.tls]`
/// was absent at start time. /// was absent at start time.
pub fn rpc_enabled(&self) -> bool { pub fn rpc_enabled(&self) -> bool {
@@ -223,9 +256,14 @@ mod tests {
/// Dedicated port range for services tests (44000+) so we don't /// Dedicated port range for services tests (44000+) so we don't
/// collide with gossip (41000+), transport (42000+), rpc (43000+). /// collide with gossip (41000+), transport (42000+), rpc (43000+).
///
/// Bump by 2 so each returned port `p` also implicitly reserves
/// `p + 1` — RPC binds to `bind_lan.port + 1` (see
/// `ClusterConfig::rpc_lan`), so returning consecutive ports would
/// have one test's RPC step on the next test's gossip.
static NEXT_PORT: AtomicU16 = AtomicU16::new(44001); static NEXT_PORT: AtomicU16 = AtomicU16::new(44001);
fn next_port() -> u16 { fn next_port() -> u16 {
NEXT_PORT.fetch_add(1, Ordering::Relaxed) NEXT_PORT.fetch_add(2, Ordering::Relaxed)
} }
fn loopback(port: u16) -> SocketAddr { fn loopback(port: u16) -> SocketAddr {
@@ -262,10 +300,12 @@ mod tests {
"test-node".into(), "test-node".into(),
tmp.path().to_path_buf(), tmp.path().to_path_buf(),
1_000_000, 1_000_000,
None,
) )
.await .await
.unwrap(); .unwrap();
assert!(!svc.rpc_enabled(), "no [cluster.tls] → RPC disabled"); assert!(!svc.rpc_enabled(), "no [cluster.tls] → RPC disabled");
assert!(!svc.blob_store_enabled(), "no blob root → blob store disabled");
// Gossip must still be functional. // Gossip must still be functional.
let self_id = svc.gossip.self_chitchat_id().await; let self_id = svc.gossip.self_chitchat_id().await;
assert_eq!(self_id.node_id.as_ref(), "test-node"); assert_eq!(self_id.node_id.as_ref(), "test-node");
@@ -304,7 +344,8 @@ mod tests {
std::fs::create_dir_all(&hot_dir).unwrap(); std::fs::create_dir_all(&hot_dir).unwrap();
std::fs::write(hot_dir.join("blob"), vec![0u8; 4096]).unwrap(); std::fs::write(hot_dir.join("blob"), vec![0u8; 4096]).unwrap();
let svc = ClusterServices::start(&cfg_a, "a".into(), hot_dir.clone(), 1_000_000) let svc =
ClusterServices::start(&cfg_a, "a".into(), hot_dir.clone(), 1_000_000, None)
.await .await
.unwrap(); .unwrap();
assert!(svc.rpc_enabled()); assert!(svc.rpc_enabled());
@@ -352,6 +393,7 @@ mod tests {
"publisher".into(), "publisher".into(),
tmp.path().to_path_buf(), tmp.path().to_path_buf(),
10 * 1024 * 1024, 10 * 1024 * 1024,
None,
) )
.await .await
.unwrap(); .unwrap();
@@ -391,11 +433,11 @@ mod tests {
}; };
let tmp = tempfile::TempDir::new().unwrap(); let tmp = tempfile::TempDir::new().unwrap();
let svc_a = let svc_a =
ClusterServices::start(&cfg_a, "a".into(), tmp.path().to_path_buf(), 1_000_000) ClusterServices::start(&cfg_a, "a".into(), tmp.path().to_path_buf(), 1_000_000, None)
.await .await
.unwrap(); .unwrap();
let svc_b = let svc_b =
ClusterServices::start(&cfg_b, "b".into(), tmp.path().to_path_buf(), 1_000_000) ClusterServices::start(&cfg_b, "b".into(), tmp.path().to_path_buf(), 1_000_000, None)
.await .await
.unwrap(); .unwrap();
@@ -415,4 +457,79 @@ mod tests {
svc_a.shutdown(); svc_a.shutdown();
svc_b.shutdown(); svc_b.shutdown();
} }
#[tokio::test]
async fn services_with_blob_store_serves_blob_rpc_end_to_end() {
// Prove that config → services → RPC path plumbs BlobStore
// through correctly. Node A runs with a blob store; B dials it
// and puts + gets a blob over real QUIC + mTLS.
use crate::cluster::rpc::{call_blob_get, call_blob_put};
use crate::cluster::transport::{FleetCa, NodeIdentity, QuicClient};
use crate::config::ClusterTlsConfig;
let tmp = tempfile::TempDir::new().unwrap();
let ca_dir = tmp.path().join("ca");
let a_tls_dir = tmp.path().join("a-tls");
let b_tls_dir = tmp.path().join("b-tls");
let blob_root = tmp.path().join("a-blobs");
let ca = FleetCa::generate("test CA").unwrap();
ca.save(&ca_dir).unwrap();
ca.sign_leaf_to_pem("a", &a_tls_dir).unwrap();
ca.sign_leaf_to_pem("b", &b_tls_dir).unwrap();
let port_a = next_port();
let cfg_a = ClusterConfig {
zone: "fabric-10g".into(),
bind_lan: Some(loopback(port_a)),
tls: Some(ClusterTlsConfig {
ca_cert: a_tls_dir.join("ca.crt"),
node_cert: a_tls_dir.join("node.crt"),
node_key: a_tls_dir.join("node.key"),
}),
blob_store_root: Some(blob_root.clone()),
..Default::default()
};
let hot_dir = tmp.path().join("a-hot");
std::fs::create_dir_all(&hot_dir).unwrap();
let svc = ClusterServices::start(
&cfg_a,
"a".into(),
hot_dir,
1_000_000,
Some(blob_root.clone()),
)
.await
.unwrap();
assert!(svc.rpc_enabled());
assert!(svc.blob_store_enabled());
let id_b = NodeIdentity::from_pem_dir(&b_tls_dir).unwrap();
let client = QuicClient::new(loopback(0), id_b).unwrap();
tokio::time::sleep(Duration::from_millis(50)).await;
let rpc_addr = cfg_a.rpc_lan().unwrap();
let conn = client.connect(rpc_addr, "a").await.unwrap();
let payload: &[u8] = b"cross-node blob over rpc";
let id = call_blob_put(&conn, payload).await.unwrap();
let round = call_blob_get(&conn, &id).await.unwrap().unwrap();
assert_eq!(round, payload);
// Independently confirm the bytes are physically on disk in A's
// store — proves the RPC didn't just echo back but actually
// wrote through to BlobStore.
let local_round = svc
.blob_store
.as_ref()
.unwrap()
.get_bytes(&id)
.await
.unwrap();
assert_eq!(local_round.as_deref(), Some(payload));
conn.close(quinn::VarInt::from_u32(0), b"done");
client.shutdown().await;
svc.shutdown();
}
} }
+8
View File
@@ -137,6 +137,12 @@ pub struct ClusterConfig {
/// used; absent means "gossip only, no RPC" for now. /// used; absent means "gossip only, no RPC" for now.
#[serde(default)] #[serde(default)]
pub tls: Option<ClusterTlsConfig>, pub tls: Option<ClusterTlsConfig>,
/// Optional local blob-store root (Phase 2). When set, the daemon
/// opens a content-addressed store at this path and serves it via
/// the Blob* RPC methods. Absent means the node participates in
/// gossip + peer-status but returns `NotConfigured` for Blob RPCs.
#[serde(default)]
pub blob_store_root: Option<PathBuf>,
} }
/// Compute the default RPC address for a gossip address: same IP, port + 1. /// Compute the default RPC address for a gossip address: same IP, port + 1.
@@ -395,6 +401,7 @@ tailscale_addr = "100.64.1.5:7701"
bind_rpc_lan: None, bind_rpc_lan: None,
bind_rpc_tailscale: None, bind_rpc_tailscale: None,
tls: None, tls: None,
blob_store_root: None,
}; };
let err = cluster.validate().unwrap_err().to_string(); let err = cluster.validate().unwrap_err().to_string();
assert!( assert!(
@@ -426,6 +433,7 @@ tailscale_addr = "100.64.1.5:7701"
bind_rpc_lan: None, bind_rpc_lan: None,
bind_rpc_tailscale: None, bind_rpc_tailscale: None,
tls: None, tls: None,
blob_store_root: None,
}; };
let err = cluster.validate().unwrap_err().to_string(); let err = cluster.validate().unwrap_err().to_string();
assert!( assert!(
+3
View File
@@ -35,17 +35,20 @@ pub async fn run(cfg: Config, mut manifest: Manifest) -> Result<()> {
Some(cluster_cfg) => { Some(cluster_cfg) => {
let hot_dir = cfg.hot.path.clone(); let hot_dir = cfg.hot.path.clone();
let hot_max_bytes = cfg.hot.max_gb.saturating_mul(1024 * 1024 * 1024); let hot_max_bytes = cfg.hot.max_gb.saturating_mul(1024 * 1024 * 1024);
let blob_root = cluster_cfg.blob_store_root.clone();
match ClusterServices::start( match ClusterServices::start(
cluster_cfg, cluster_cfg,
cfg.node.name.clone(), cfg.node.name.clone(),
hot_dir, hot_dir,
hot_max_bytes, hot_max_bytes,
blob_root,
) )
.await .await
{ {
Ok(svc) => { Ok(svc) => {
tracing::info!( tracing::info!(
rpc_enabled = svc.rpc_enabled(), rpc_enabled = svc.rpc_enabled(),
blob_store_enabled = svc.blob_store_enabled(),
zone = %cluster_cfg.zone, zone = %cluster_cfg.zone,
"cluster services online" "cluster services online"
); );