Phase 5k: parallel-fanout chunk transfer for prewarm
Pilot 2026-07-12 measured 109 MiB/s on the sequential prewarm path — ~11% of a 10G fabric. `quinn::Connection` is cheap-Clone (internal Arc), so we can run the has→get→put pipeline per chunk in concurrent tasks under a bounded semaphore. - `prewarm_missing_chunks_between_parallel(up, down, id, concurrency)` in `rpc/client.rs`. `concurrency <= 1` degrades to the sequential path (kept for diagnostic parity). - `claw-cargo prewarm --parallel N` (default 8). Ignored with `--buffered`. Memory ceiling: 4 MiB × in-flight = 32 MiB @ 8, 128 MiB @ 32. - Uses `tokio::task::JoinSet` + `Arc<Semaphore>`; permit held for the whole per-chunk pipeline so we never over-commit. - Retry pass on `put_manifest` mismatch stays sequential — small, correctness-critical. - Errors: JoinSet drains completely + returns first task error so a mid-fanout failure doesn't leave zombie tasks. +1 test: `end_to_end_parallel_prewarm_copies_chunks_and_matches_sequential` runs 5-chunk payload with concurrency=3, verifies byte-equal restore, then reruns with concurrency=8 → 0 uploads (has_chunk dedup), then concurrency=0 → 0 uploads (sequential fallback path). 254 tests pass (+1 from previous). Pre-existing macOS failure unchanged.
This commit is contained in:
@@ -1105,3 +1105,102 @@ async fn streaming_prewarm_skips_chunks_already_present_downstream() {
|
||||
acc_a.abort();
|
||||
acc_c.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn end_to_end_parallel_prewarm_copies_chunks_and_matches_sequential() {
|
||||
// Phase 5k: parallel prewarm variant produces the same downstream
|
||||
// state as the sequential path. Also proves the JoinSet fanout
|
||||
// doesn't lose or duplicate chunks.
|
||||
use crate::cluster::blob::CHUNK_SIZE;
|
||||
|
||||
let (id_a, id_b_for_a) = NodeIdentity::generate_test_pair("a", "b_a").unwrap();
|
||||
let (id_c, id_b_for_c) = NodeIdentity::generate_test_pair("c", "b_c").unwrap();
|
||||
|
||||
let (_tmp_a, router_a) = router_with_full_stack("a", next_port()).await;
|
||||
let (_tmp_c, router_c) = router_with_full_stack("c", next_port()).await;
|
||||
|
||||
// 5 chunks so parallelism (concurrency=3) actually queues work.
|
||||
let payload: Vec<u8> = (0..(4 * CHUNK_SIZE + CHUNK_SIZE / 3))
|
||||
.map(|i| ((i * 7) % 251) as u8)
|
||||
.collect();
|
||||
let blob_id = router_a
|
||||
.blob_store()
|
||||
.unwrap()
|
||||
.put_bytes(&payload)
|
||||
.await
|
||||
.unwrap();
|
||||
let manifest = router_a
|
||||
.blob_store()
|
||||
.unwrap()
|
||||
.load_manifest(&blob_id)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(manifest.chunks.len(), 5, "expected 5 chunks in payload");
|
||||
|
||||
let server_a = QuicServer::bind(loopback(0), id_a).unwrap();
|
||||
let addr_a = server_a.local_addr().unwrap();
|
||||
let ra = router_a.clone();
|
||||
let acc_a = tokio::spawn(async move {
|
||||
while let Some(Ok(conn)) = server_a.accept().await {
|
||||
let r = ra.clone();
|
||||
tokio::spawn(async move {
|
||||
let _ = serve_connection(conn, r).await;
|
||||
});
|
||||
}
|
||||
});
|
||||
let server_c = QuicServer::bind(loopback(0), id_c).unwrap();
|
||||
let addr_c = server_c.local_addr().unwrap();
|
||||
let rc = router_c.clone();
|
||||
let acc_c = tokio::spawn(async move {
|
||||
while let Some(Ok(conn)) = server_c.accept().await {
|
||||
let r = rc.clone();
|
||||
tokio::spawn(async move {
|
||||
let _ = serve_connection(conn, r).await;
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
let up_client = QuicClient::new(loopback(0), id_b_for_a).unwrap();
|
||||
let up_conn = up_client.connect(addr_a, "a").await.unwrap();
|
||||
let down_client = QuicClient::new(loopback(0), id_b_for_c).unwrap();
|
||||
let down_conn = down_client.connect(addr_c, "c").await.unwrap();
|
||||
|
||||
let (uploaded, total) =
|
||||
prewarm_missing_chunks_between_parallel(&up_conn, &down_conn, &blob_id, 3)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(total, 5);
|
||||
assert_eq!(uploaded, 5, "cold downstream must receive every chunk");
|
||||
|
||||
// Round-trip proves manifest committed AND chunks landed.
|
||||
let round = router_c
|
||||
.blob_store()
|
||||
.unwrap()
|
||||
.get_bytes(&blob_id)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(round.as_deref(), Some(payload.as_slice()));
|
||||
|
||||
// Idempotency: re-running with concurrency=8 uploads 0 (full dedup).
|
||||
let (uploaded2, _) =
|
||||
prewarm_missing_chunks_between_parallel(&up_conn, &down_conn, &blob_id, 8)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(uploaded2, 0, "re-run should be a no-op via has_chunk dedup");
|
||||
|
||||
// concurrency=0 falls through to sequential.
|
||||
let (uploaded3, _) =
|
||||
prewarm_missing_chunks_between_parallel(&up_conn, &down_conn, &blob_id, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(uploaded3, 0, "sequential fallback should also see full dedup");
|
||||
|
||||
up_conn.close(quinn::VarInt::from_u32(0), b"done");
|
||||
down_conn.close(quinn::VarInt::from_u32(0), b"done");
|
||||
up_client.shutdown().await;
|
||||
down_client.shutdown().await;
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
acc_a.abort();
|
||||
acc_c.abort();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user