Phase 7b: chunk-level repair library
Build with clawstor cache / Cargo build (clawstor-cached) (pull_request) Failing after 3s
Build with clawstor cache / Cargo build (clawstor-cached) (pull_request) Failing after 3s
New primitive: BlobStore::repair_chunks(chunks, fetch) → RepairReport. Consumer flow: cluster-scrub returns a list of (blob, chunk) bad pairs. cluster-repair (next slice) will hand the chunk hashes here with a fetcher that walks peers via HasChunk/GetChunk. This PR is the library-only half — no peer wiring — so it's testable in isolation and reusable by callers who already have a chunk source. Fetcher contract: * Ok(Some(bytes)) → put locally, count repaired * Ok(None) → nobody has it, record as unrecoverable * Err(e) → per-chunk error, batch continues Guardrails: * Bytes are re-hashed by put_chunk before writing. A peer that returns wrong bytes for a hash cannot corrupt us further. * Duplicate chunk hashes in the input dedupe → fetcher called exactly once per unique chunk. Matters because scrub reports shared chunks once per owning manifest. * Errors on one chunk never abort the batch — the remaining chunks still get their shot. * Repair overwrites a corrupt file: unlink-then-put_chunk, since put_chunk itself is write-if-absent. NotFound on unlink is fine (missing-chunk case). +4 tests: - repair_writes_fetched_bytes_and_marks_repaired (happy: corrupt → repair → post-scrub clean) - repair_records_unrecoverable_when_fetcher_returns_none - repair_records_error_and_continues_batch (batch survives one chunk's error) - repair_dedups_duplicate_chunks_in_input (fetcher called exactly once for 3 identical hashes) 345 tests pass (+4). Pre-existing macOS hot::tests::test_project_target_size_bytes failure unchanged.
This commit is contained in:
@@ -168,6 +168,19 @@ pub struct GcReport {
|
||||
pub bytes_reclaimed: u64,
|
||||
}
|
||||
|
||||
/// Phase 7b (2026-07-14): report from [`BlobStore::repair_chunks`].
|
||||
///
|
||||
/// For each corrupt/missing chunk the caller supplied, records what
|
||||
/// happened: successfully fetched + written, fetcher returned None
|
||||
/// (no peer had it), or the fetch itself errored (network, protocol).
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct RepairReport {
|
||||
pub attempted: usize,
|
||||
pub repaired: usize,
|
||||
pub unrecoverable: Vec<ChunkHash>,
|
||||
pub errors: Vec<(ChunkHash, String)>,
|
||||
}
|
||||
|
||||
/// Phase 7a (2026-07-14): report from [`BlobStore::scrub_all`].
|
||||
///
|
||||
/// A scrub walks every manifest, recomputes BLAKE3 for each referenced
|
||||
@@ -982,6 +995,72 @@ enum ChunkVerdict {
|
||||
Corrupt,
|
||||
}
|
||||
|
||||
impl BlobStore {
|
||||
/// Phase 7b (2026-07-14): re-fetch a batch of chunks from a
|
||||
/// caller-supplied source and write them locally. Intended
|
||||
/// consumer: `cluster-repair`, which calls `scrub_all` first and
|
||||
/// hands the missing+corrupt chunks in.
|
||||
///
|
||||
/// `fetch(hash)` returns:
|
||||
/// * `Ok(Some(bytes))` — bytes for the chunk (caller may pull
|
||||
/// them from any peer that has it; a wrapper walking all peers
|
||||
/// fits here)
|
||||
/// * `Ok(None)` — nobody has it; recorded as unrecoverable
|
||||
/// * `Err(e)` — network/protocol failure for this chunk;
|
||||
/// recorded per-chunk, doesn't abort the batch
|
||||
///
|
||||
/// Bytes are re-hashed by `put_chunk` before writing, so a peer
|
||||
/// that returns wrong bytes for a hash can't corrupt us further.
|
||||
pub async fn repair_chunks<F, Fut>(
|
||||
&self,
|
||||
chunks: &[ChunkHash],
|
||||
fetch: F,
|
||||
) -> RepairReport
|
||||
where
|
||||
F: Fn(ChunkHash) -> Fut,
|
||||
Fut: std::future::Future<Output = Result<Option<Vec<u8>>>>,
|
||||
{
|
||||
let mut report = RepairReport {
|
||||
attempted: chunks.len(),
|
||||
..RepairReport::default()
|
||||
};
|
||||
// Dedup: same chunk may be listed twice by scrub (shared).
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
for chunk in chunks {
|
||||
if !seen.insert(*chunk) {
|
||||
continue;
|
||||
}
|
||||
match fetch(*chunk).await {
|
||||
Ok(Some(bytes)) => {
|
||||
// `put_chunk` uses write-if-absent, but repair is
|
||||
// exactly the case where a corrupt file may already
|
||||
// occupy the path. Unlink first (NotFound OK), then
|
||||
// re-put; put_chunk still re-hashes the bytes so a
|
||||
// wrong-answer peer can't corrupt us.
|
||||
let path = self.chunk_path(chunk);
|
||||
match tokio::fs::remove_file(&path).await {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(e) => {
|
||||
report.errors.push((*chunk, format!("remove: {e}")));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
match self.put_chunk(chunk, &bytes).await {
|
||||
Ok(()) => report.repaired += 1,
|
||||
Err(e) => {
|
||||
report.errors.push((*chunk, format!("put_chunk: {e}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => report.unrecoverable.push(*chunk),
|
||||
Err(e) => report.errors.push((*chunk, e.to_string())),
|
||||
}
|
||||
}
|
||||
report
|
||||
}
|
||||
}
|
||||
|
||||
/// Monotonic counter to disambiguate temp file names within a single
|
||||
/// process. Combined with the process id, this makes tmp filenames
|
||||
/// unique across a fleet without needing `Math.random`.
|
||||
@@ -1753,6 +1832,121 @@ mod tests {
|
||||
assert!(owners.contains(&fake_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repair_writes_fetched_bytes_and_marks_repaired() {
|
||||
// Phase 7b happy path: caller passed a corrupt chunk, fetcher
|
||||
// returned real bytes → put_chunk overwrites and repair
|
||||
// count = 1.
|
||||
let (_tmp, store) = open_store();
|
||||
let id = store.put_bytes(b"phase-7b repair").await.unwrap();
|
||||
// Snapshot the manifest to learn what chunks we have.
|
||||
let manifest = store.load_manifest(&id).await.unwrap().unwrap();
|
||||
assert_eq!(manifest.chunks.len(), 1);
|
||||
let chunk = manifest.chunks[0];
|
||||
|
||||
// Corrupt on-disk, then repair.
|
||||
std::fs::write(store.chunk_path(&chunk), b"tampered").unwrap();
|
||||
// Prove scrub sees it before we repair.
|
||||
let pre = store.scrub_all().await.unwrap();
|
||||
assert_eq!(pre.chunks_corrupt, 1);
|
||||
|
||||
let real_bytes = b"phase-7b repair".to_vec();
|
||||
let bytes_for_fetcher = real_bytes.clone();
|
||||
let report = store
|
||||
.repair_chunks(&[chunk], |_h| {
|
||||
let b = bytes_for_fetcher.clone();
|
||||
async move { Ok(Some(b)) }
|
||||
})
|
||||
.await;
|
||||
assert_eq!(report.attempted, 1);
|
||||
assert_eq!(report.repaired, 1);
|
||||
assert!(report.unrecoverable.is_empty());
|
||||
assert!(report.errors.is_empty());
|
||||
|
||||
// Post-condition: scrub is clean again.
|
||||
let post = store.scrub_all().await.unwrap();
|
||||
assert_eq!(post.chunks_ok, 1);
|
||||
assert_eq!(post.chunks_corrupt, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repair_records_unrecoverable_when_fetcher_returns_none() {
|
||||
// Fetcher says nobody has this chunk. Report must
|
||||
// capture it as unrecoverable; no error.
|
||||
let (_tmp, store) = open_store();
|
||||
let id = store.put_bytes(b"unrecoverable payload").await.unwrap();
|
||||
let manifest = store.load_manifest(&id).await.unwrap().unwrap();
|
||||
let chunk = manifest.chunks[0];
|
||||
std::fs::remove_file(store.chunk_path(&chunk)).unwrap();
|
||||
|
||||
let report = store
|
||||
.repair_chunks(&[chunk], |_h| async { Ok(None) })
|
||||
.await;
|
||||
assert_eq!(report.attempted, 1);
|
||||
assert_eq!(report.repaired, 0);
|
||||
assert_eq!(report.unrecoverable, vec![chunk]);
|
||||
assert!(report.errors.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repair_records_error_and_continues_batch() {
|
||||
// Two chunks, fetcher errors on one, succeeds on other.
|
||||
// Batch must NOT abort: second chunk still repairs.
|
||||
let (_tmp, store) = open_store();
|
||||
let id1 = store.put_bytes(b"batch-repair alpha").await.unwrap();
|
||||
let id2 = store.put_bytes(b"batch-repair beta").await.unwrap();
|
||||
let m1 = store.load_manifest(&id1).await.unwrap().unwrap();
|
||||
let m2 = store.load_manifest(&id2).await.unwrap().unwrap();
|
||||
let c1 = m1.chunks[0];
|
||||
let c2 = m2.chunks[0];
|
||||
std::fs::write(store.chunk_path(&c1), b"corrupt-1").unwrap();
|
||||
std::fs::write(store.chunk_path(&c2), b"corrupt-2").unwrap();
|
||||
|
||||
let bad = c1;
|
||||
let report = store
|
||||
.repair_chunks(&[c1, c2], |h| async move {
|
||||
if h == bad {
|
||||
Err(anyhow::anyhow!("simulated network failure"))
|
||||
} else {
|
||||
Ok(Some(b"batch-repair beta".to_vec()))
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert_eq!(report.attempted, 2);
|
||||
assert_eq!(report.repaired, 1, "beta must repair despite alpha error");
|
||||
assert_eq!(report.errors.len(), 1);
|
||||
assert_eq!(report.errors[0].0, c1);
|
||||
assert!(report.errors[0].1.contains("simulated network failure"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repair_dedups_duplicate_chunks_in_input() {
|
||||
// Scrub reports a shared chunk twice (once per owning blob).
|
||||
// Repair must fetch it exactly once — otherwise we waste
|
||||
// a peer round-trip per reference.
|
||||
let (_tmp, store) = open_store();
|
||||
let id = store.put_bytes(b"dedup input").await.unwrap();
|
||||
let manifest = store.load_manifest(&id).await.unwrap().unwrap();
|
||||
let chunk = manifest.chunks[0];
|
||||
std::fs::remove_file(store.chunk_path(&chunk)).unwrap();
|
||||
|
||||
let call_count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let counter = call_count.clone();
|
||||
let report = store
|
||||
.repair_chunks(&[chunk, chunk, chunk], move |_h| {
|
||||
counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
async move { Ok(Some(b"dedup input".to_vec())) }
|
||||
})
|
||||
.await;
|
||||
assert_eq!(report.attempted, 3);
|
||||
assert_eq!(report.repaired, 1, "one unique chunk actually repaired");
|
||||
assert_eq!(
|
||||
call_count.load(std::sync::atomic::Ordering::SeqCst),
|
||||
1,
|
||||
"fetcher must be called exactly once"
|
||||
);
|
||||
}
|
||||
|
||||
/// Recursive count of regular files under `root`. Test helper.
|
||||
fn count_files_under(root: &Path) -> usize {
|
||||
if !root.exists() {
|
||||
|
||||
Reference in New Issue
Block a user