Phase 5k: parallel-fanout chunk transfer for prewarm

Pilot 2026-07-12 measured 109 MiB/s on the sequential prewarm path —
~11% of a 10G fabric. `quinn::Connection` is cheap-Clone (internal
Arc), so we can run the has→get→put pipeline per chunk in concurrent
tasks under a bounded semaphore.

- `prewarm_missing_chunks_between_parallel(up, down, id, concurrency)`
  in `rpc/client.rs`. `concurrency <= 1` degrades to the sequential
  path (kept for diagnostic parity).
- `claw-cargo prewarm --parallel N` (default 8). Ignored with
  `--buffered`. Memory ceiling: 4 MiB × in-flight = 32 MiB @ 8,
  128 MiB @ 32.
- Uses `tokio::task::JoinSet` + `Arc<Semaphore>`; permit held for
  the whole per-chunk pipeline so we never over-commit.
- Retry pass on `put_manifest` mismatch stays sequential — small,
  correctness-critical.
- Errors: JoinSet drains completely + returns first task error so a
  mid-fanout failure doesn't leave zombie tasks.

+1 test: `end_to_end_parallel_prewarm_copies_chunks_and_matches_sequential`
runs 5-chunk payload with concurrency=3, verifies byte-equal restore,
then reruns with concurrency=8 → 0 uploads (has_chunk dedup), then
concurrency=0 → 0 uploads (sequential fallback path).

254 tests pass (+1 from previous). Pre-existing macOS failure unchanged.
This commit is contained in:
Omar Sobh
2026-07-12 06:13:28 -07:00
parent a8fac47470
commit 54e9da4d62
3 changed files with 236 additions and 13 deletions
@@ -1105,3 +1105,102 @@ async fn streaming_prewarm_skips_chunks_already_present_downstream() {
acc_a.abort();
acc_c.abort();
}
#[tokio::test]
async fn end_to_end_parallel_prewarm_copies_chunks_and_matches_sequential() {
// Phase 5k: parallel prewarm variant produces the same downstream
// state as the sequential path. Also proves the JoinSet fanout
// doesn't lose or duplicate chunks.
use crate::cluster::blob::CHUNK_SIZE;
let (id_a, id_b_for_a) = NodeIdentity::generate_test_pair("a", "b_a").unwrap();
let (id_c, id_b_for_c) = NodeIdentity::generate_test_pair("c", "b_c").unwrap();
let (_tmp_a, router_a) = router_with_full_stack("a", next_port()).await;
let (_tmp_c, router_c) = router_with_full_stack("c", next_port()).await;
// 5 chunks so parallelism (concurrency=3) actually queues work.
let payload: Vec<u8> = (0..(4 * CHUNK_SIZE + CHUNK_SIZE / 3))
.map(|i| ((i * 7) % 251) as u8)
.collect();
let blob_id = router_a
.blob_store()
.unwrap()
.put_bytes(&payload)
.await
.unwrap();
let manifest = router_a
.blob_store()
.unwrap()
.load_manifest(&blob_id)
.await
.unwrap()
.unwrap();
assert_eq!(manifest.chunks.len(), 5, "expected 5 chunks in payload");
let server_a = QuicServer::bind(loopback(0), id_a).unwrap();
let addr_a = server_a.local_addr().unwrap();
let ra = router_a.clone();
let acc_a = tokio::spawn(async move {
while let Some(Ok(conn)) = server_a.accept().await {
let r = ra.clone();
tokio::spawn(async move {
let _ = serve_connection(conn, r).await;
});
}
});
let server_c = QuicServer::bind(loopback(0), id_c).unwrap();
let addr_c = server_c.local_addr().unwrap();
let rc = router_c.clone();
let acc_c = tokio::spawn(async move {
while let Some(Ok(conn)) = server_c.accept().await {
let r = rc.clone();
tokio::spawn(async move {
let _ = serve_connection(conn, r).await;
});
}
});
let up_client = QuicClient::new(loopback(0), id_b_for_a).unwrap();
let up_conn = up_client.connect(addr_a, "a").await.unwrap();
let down_client = QuicClient::new(loopback(0), id_b_for_c).unwrap();
let down_conn = down_client.connect(addr_c, "c").await.unwrap();
let (uploaded, total) =
prewarm_missing_chunks_between_parallel(&up_conn, &down_conn, &blob_id, 3)
.await
.unwrap();
assert_eq!(total, 5);
assert_eq!(uploaded, 5, "cold downstream must receive every chunk");
// Round-trip proves manifest committed AND chunks landed.
let round = router_c
.blob_store()
.unwrap()
.get_bytes(&blob_id)
.await
.unwrap();
assert_eq!(round.as_deref(), Some(payload.as_slice()));
// Idempotency: re-running with concurrency=8 uploads 0 (full dedup).
let (uploaded2, _) =
prewarm_missing_chunks_between_parallel(&up_conn, &down_conn, &blob_id, 8)
.await
.unwrap();
assert_eq!(uploaded2, 0, "re-run should be a no-op via has_chunk dedup");
// concurrency=0 falls through to sequential.
let (uploaded3, _) =
prewarm_missing_chunks_between_parallel(&up_conn, &down_conn, &blob_id, 0)
.await
.unwrap();
assert_eq!(uploaded3, 0, "sequential fallback should also see full dedup");
up_conn.close(quinn::VarInt::from_u32(0), b"done");
down_conn.close(quinn::VarInt::from_u32(0), b"done");
up_client.shutdown().await;
down_client.shutdown().await;
tokio::time::sleep(Duration::from_millis(50)).await;
acc_a.abort();
acc_c.abort();
}