Apply rustfmt to entire workspace

Runs cargo fmt --all; all 573 tests still passing, clippy still clean.
No logic changes — formatting only.

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
osobh
2026-04-04 20:31:33 -05:00
co-authored by Claude Sonnet 4.6
parent 3d524e2d63
commit 1c107fe58a
69 changed files with 2790 additions and 1325 deletions
@@ -46,7 +46,9 @@ fn bench_epoch_gc(c: &mut Criterion) {
b.iter_with_setup( b.iter_with_setup(
|| make_onion(n), || make_onion(n),
|(_tmp, _h5, mut onion)| { |(_tmp, _h5, mut onion)| {
let _ = onion.gc(black_box(GcPolicy::KeepLastN((n / 2) as u64))).unwrap(); let _ = onion
.gc(black_box(GcPolicy::KeepLastN((n / 2) as u64)))
.unwrap();
}, },
); );
}); });
@@ -60,9 +62,11 @@ fn bench_epoch_gc(c: &mut Criterion) {
b.iter_with_setup( b.iter_with_setup(
|| make_onion(n), || make_onion(n),
|(_tmp, _h5, mut onion)| { |(_tmp, _h5, mut onion)| {
let _ = onion.gc(black_box(GcPolicy::EpochFlip(Box::new( let _ = onion
GcPolicy::KeepLastN((n / 2) as u64), .gc(black_box(GcPolicy::EpochFlip(Box::new(
)))).unwrap(); GcPolicy::KeepLastN((n / 2) as u64),
))))
.unwrap();
}, },
); );
}); });
@@ -76,9 +80,11 @@ fn bench_epoch_gc(c: &mut Criterion) {
b.iter_with_setup( b.iter_with_setup(
|| { || {
let (tmp, h5, mut onion) = make_onion(n); let (tmp, h5, mut onion) = make_onion(n);
onion.gc(GcPolicy::EpochFlip(Box::new( onion
GcPolicy::KeepLastN((n / 2) as u64), .gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(
))).unwrap(); (n / 2) as u64,
))))
.unwrap();
(tmp, h5, onion) (tmp, h5, onion)
}, },
|(_tmp, _h5, mut onion)| { |(_tmp, _h5, mut onion)| {
+34 -15
View File
@@ -51,21 +51,31 @@ fn bench_merkle(c: &mut Criterion) {
for &n in &ns { for &n in &ns {
let full = RevisionMerkleTree::build(&make_entries(n)); let full = RevisionMerkleTree::build(&make_entries(n));
let minus1 = RevisionMerkleTree::build(&make_entries(n - 1)); let minus1 = RevisionMerkleTree::build(&make_entries(n - 1));
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, minus1), |b, (f, m)| { group.bench_with_input(
b.iter(|| f.diff_missing_revisions(black_box(m))); BenchmarkId::from_parameter(n),
}); &(full, minus1),
|b, (f, m)| {
b.iter(|| f.diff_missing_revisions(black_box(m)));
},
);
} }
group.finish(); group.finish();
// ── Diff walk: D=10 ────────────────────────────────────────────────────── // ── Diff walk: D=10 ──────────────────────────────────────────────────────
let mut group = c.benchmark_group("merkle_index/diff_walk_d10"); let mut group = c.benchmark_group("merkle_index/diff_walk_d10");
for &n in &ns { for &n in &ns {
if n < 20 { continue; } if n < 20 {
let full = RevisionMerkleTree::build(&make_entries(n)); continue;
let base = RevisionMerkleTree::build(&make_entries(n - 10)); }
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, base), |b, (f, base)| { let full = RevisionMerkleTree::build(&make_entries(n));
b.iter(|| f.diff_missing_revisions(black_box(base))); let base = RevisionMerkleTree::build(&make_entries(n - 10));
}); group.bench_with_input(
BenchmarkId::from_parameter(n),
&(full, base),
|b, (f, base)| {
b.iter(|| f.diff_missing_revisions(black_box(base)));
},
);
} }
group.finish(); group.finish();
@@ -74,9 +84,13 @@ fn bench_merkle(c: &mut Criterion) {
for &n in &[1_000usize, 10_000] { for &n in &[1_000usize, 10_000] {
let full = RevisionMerkleTree::build(&make_entries(n)); let full = RevisionMerkleTree::build(&make_entries(n));
let base = RevisionMerkleTree::build(&make_entries(n - 100)); let base = RevisionMerkleTree::build(&make_entries(n - 100));
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, base), |b, (f, base)| { group.bench_with_input(
b.iter(|| f.diff_missing_revisions(black_box(base))); BenchmarkId::from_parameter(n),
}); &(full, base),
|b, (f, base)| {
b.iter(|| f.diff_missing_revisions(black_box(base)));
},
);
} }
group.finish(); group.finish();
@@ -102,14 +116,19 @@ fn bench_merkle(c: &mut Criterion) {
// ── Serialised size report (not timed) ─────────────────────────────────── // ── Serialised size report (not timed) ───────────────────────────────────
println!("\n=== Serialised size vs flat manifest (N × 60 B) ==="); println!("\n=== Serialised size vs flat manifest (N × 60 B) ===");
println!("{:>8} {:>12} {:>12} {:>10}", "N", "Merkle (B)", "Flat (B)", "ratio"); println!(
"{:>8} {:>12} {:>12} {:>10}",
"N", "Merkle (B)", "Flat (B)", "ratio"
);
for &n in &[10usize, 100, 1_000, 10_000] { for &n in &[10usize, 100, 1_000, 10_000] {
let tree = RevisionMerkleTree::build(&make_entries(n)); let tree = RevisionMerkleTree::build(&make_entries(n));
let merkle_sz = tree.serialise().len(); let merkle_sz = tree.serialise().len();
let flat_sz = n * 60; let flat_sz = n * 60;
println!( println!(
"{:>8} {:>12} {:>12} {:>10.2}x", "{:>8} {:>12} {:>12} {:>10.2}x",
n, merkle_sz, flat_sz, n,
merkle_sz,
flat_sz,
flat_sz as f64 / merkle_sz as f64 flat_sz as f64 / merkle_sz as f64
); );
} }
+22 -19
View File
@@ -66,7 +66,10 @@ fn bench_write_one_revision(c: &mut Criterion) {
b.iter(|| { b.iter(|| {
let mut s = onion.begin_session(None).unwrap(); let mut s = onion.begin_session(None).unwrap();
for i in 0u64..4 { for i in 0u64..4 {
s.record_page(i * PAGE_SIZE as u64, black_box(&vec![0xCDu8; PAGE_SIZE as usize])); s.record_page(
i * PAGE_SIZE as u64,
black_box(&vec![0xCDu8; PAGE_SIZE as usize]),
);
} }
black_box(onion.commit_session(s, None).unwrap()); black_box(onion.commit_session(s, None).unwrap());
}); });
@@ -83,15 +86,15 @@ fn bench_reconstruct_revision(c: &mut Criterion) {
let (_f, _h5, onion, base) = tmp_onion_with_n(depth); let (_f, _h5, onion, base) = tmp_onion_with_n(depth);
let target_rev = (depth - 1) as u64; let target_rev = (depth - 1) as u64;
group.bench_with_input( group.bench_with_input(BenchmarkId::new("no_snapshot", depth), &depth, |b, _| {
BenchmarkId::new("no_snapshot", depth), b.iter(|| {
&depth, black_box(
|b, _| { onion
b.iter(|| { .reconstruct_revision(target_rev, black_box(&base))
black_box(onion.reconstruct_revision(target_rev, black_box(&base)).unwrap()); .unwrap(),
}); );
}, });
); });
} }
// With snapshot: create a real snapshot at midpoint via create_snapshot() // With snapshot: create a real snapshot at midpoint via create_snapshot()
@@ -107,15 +110,15 @@ fn bench_reconstruct_revision(c: &mut Criterion) {
} }
let target_rev = onion.revision_count() - 1; let target_rev = onion.revision_count() - 1;
group.bench_with_input( group.bench_with_input(BenchmarkId::new("with_snapshot", depth), &depth, |b, _| {
BenchmarkId::new("with_snapshot", depth), b.iter(|| {
&depth, black_box(
|b, _| { onion
b.iter(|| { .reconstruct_revision(target_rev, black_box(&base))
black_box(onion.reconstruct_revision(target_rev, black_box(&base)).unwrap()); .unwrap(),
}); );
}, });
); });
} }
group.finish(); group.finish();
+21 -17
View File
@@ -35,7 +35,9 @@ fn make_f32_random(n_bytes: usize) -> Vec<u8> {
let mut state = 0x_dead_beef_u64; let mut state = 0x_dead_beef_u64;
let mut out = Vec::with_capacity(n_bytes); let mut out = Vec::with_capacity(n_bytes);
while out.len() < n_bytes { while out.len() < n_bytes {
state = state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); state = state
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
out.extend_from_slice(&(state as u32).to_le_bytes()); out.extend_from_slice(&(state as u32).to_le_bytes());
} }
out.truncate(n_bytes); out.truncate(n_bytes);
@@ -72,16 +74,13 @@ fn bench_tdt_compress(c: &mut Criterion) {
let sizes = [4096usize, 65536]; let sizes = [4096usize, 65536];
let datasets: &[(&str, fn(usize) -> Vec<u8>, usize)] = &[ let datasets: &[(&str, fn(usize) -> Vec<u8>, usize)] = &[
("f32_smooth", make_f32_smooth, 4), ("f32_smooth", make_f32_smooth, 4),
("f32_random", make_f32_random, 4), ("f32_random", make_f32_random, 4),
("int32_random", make_int32_random, 4), ("int32_random", make_int32_random, 4),
("int8_seq", make_int8_seq, 1), ("int8_seq", make_int8_seq, 1),
]; ];
let codecs = [ let codecs = [("zstd", Codec::Zstd), ("zstd_tdt", Codec::ZstdTdt)];
("zstd", Codec::Zstd),
("zstd_tdt", Codec::ZstdTdt),
];
// ── Throughput benchmark ────────────────────────────────────────────────── // ── Throughput benchmark ──────────────────────────────────────────────────
let mut group = c.benchmark_group("tdt_compress/throughput"); let mut group = c.benchmark_group("tdt_compress/throughput");
@@ -90,10 +89,7 @@ fn bench_tdt_compress(c: &mut Criterion) {
let data = make(size); let data = make(size);
group.throughput(Throughput::Bytes(size as u64)); group.throughput(Throughput::Bytes(size as u64));
for &(codec_name, codec) in &codecs { for &(codec_name, codec) in &codecs {
let id = BenchmarkId::new( let id = BenchmarkId::new(format!("{codec_name}/{dtype}"), format!("{size}B"));
format!("{codec_name}/{dtype}"),
format!("{size}B"),
);
group.bench_with_input(id, &data, |b, d| { group.bench_with_input(id, &data, |b, d| {
b.iter(|| bench_compress(d, codec)); b.iter(|| bench_compress(d, codec));
}); });
@@ -126,16 +122,24 @@ fn bench_tdt_compress(c: &mut Criterion) {
// ── Compression ratio summary (printed, not timed) ─────────────────────── // ── Compression ratio summary (printed, not timed) ───────────────────────
// Run once outside Criterion to print ratio comparison. // Run once outside Criterion to print ratio comparison.
println!("\n─── TDT compression ratio summary ───"); println!("\n─── TDT compression ratio summary ───");
println!("{:<20} {:>8} {:>10} {:>10} {:>8}", println!(
"dataset/size", "orig", "zstd", "zstd_tdt", "savings"); "{:<20} {:>8} {:>10} {:>10} {:>8}",
"dataset/size", "orig", "zstd", "zstd_tdt", "savings"
);
for &size in &sizes { for &size in &sizes {
for &(dtype, make, _w) in datasets { for &(dtype, make, _w) in datasets {
let data = make(size); let data = make(size);
let zstd_size = compress_page(&data, Codec::Zstd).unwrap().len(); let zstd_size = compress_page(&data, Codec::Zstd).unwrap().len();
let tdt_size = compress_page(&data, Codec::ZstdTdt).unwrap().len(); let tdt_size = compress_page(&data, Codec::ZstdTdt).unwrap().len();
let savings_pct = 100.0 * (1.0 - tdt_size as f64 / zstd_size as f64); let savings_pct = 100.0 * (1.0 - tdt_size as f64 / zstd_size as f64);
println!("{:<20} {:>8} {:>10} {:>10} {:>7.1}%", println!(
format!("{dtype}/{size}B"), size, zstd_size, tdt_size, savings_pct); "{:<20} {:>8} {:>10} {:>10} {:>7.1}%",
format!("{dtype}/{size}B"),
size,
zstd_size,
tdt_size,
savings_pct
);
} }
} }
println!(); println!();
+6 -10
View File
@@ -37,10 +37,7 @@ use crate::writer::OnionFile;
/// let (bytes, onion) = open_revision("agent.h5", 7)?; /// let (bytes, onion) = open_revision("agent.h5", 7)?;
/// let file = clawhdf5::File::from_bytes(bytes)?; /// let file = clawhdf5::File::from_bytes(bytes)?;
/// ``` /// ```
pub fn open_revision( pub fn open_revision(h5_path: &Path, revision: u64) -> Result<(Vec<u8>, OnionFile), OnionError> {
h5_path: &Path,
revision: u64,
) -> Result<(Vec<u8>, OnionFile), OnionError> {
let h5_base = std::fs::read(h5_path)?; let h5_base = std::fs::read(h5_path)?;
let onion = OnionFile::open(h5_path)?; let onion = OnionFile::open(h5_path)?;
let reconstructed = onion.reconstruct_revision(revision, &h5_base)?; let reconstructed = onion.reconstruct_revision(revision, &h5_base)?;
@@ -48,10 +45,7 @@ pub fn open_revision(
} }
/// Open the HEAD of a named branch. /// Open the HEAD of a named branch.
pub fn open_branch( pub fn open_branch(h5_path: &Path, branch: &str) -> Result<(Vec<u8>, OnionFile), OnionError> {
h5_path: &Path,
branch: &str,
) -> Result<(Vec<u8>, OnionFile), OnionError> {
let h5_base = std::fs::read(h5_path)?; let h5_base = std::fs::read(h5_path)?;
let onion = OnionFile::open(h5_path)?; let onion = OnionFile::open(h5_path)?;
let reconstructed = onion.open_rev(OpenRevision::Branch(branch.to_owned()), &h5_base)?; let reconstructed = onion.open_rev(OpenRevision::Branch(branch.to_owned()), &h5_base)?;
@@ -66,8 +60,10 @@ pub fn open_branch_at(
) -> Result<(Vec<u8>, OnionFile), OnionError> { ) -> Result<(Vec<u8>, OnionFile), OnionError> {
let h5_base = std::fs::read(h5_path)?; let h5_base = std::fs::read(h5_path)?;
let onion = OnionFile::open(h5_path)?; let onion = OnionFile::open(h5_path)?;
let reconstructed = let reconstructed = onion.open_rev(
onion.open_rev(OpenRevision::BranchAt(branch.to_owned(), revision), &h5_base)?; OpenRevision::BranchAt(branch.to_owned(), revision),
&h5_base,
)?;
Ok((reconstructed, onion)) Ok((reconstructed, onion))
} }
+101 -59
View File
@@ -41,11 +41,7 @@ impl OnionFile {
/// Create a new named branch forked from the current HEAD of `source_branch`. /// Create a new named branch forked from the current HEAD of `source_branch`.
/// ///
/// Returns the new branch ID. /// Returns the new branch ID.
pub fn create_branch( pub fn create_branch(&mut self, name: &str, source_branch: &str) -> Result<u32, OnionError> {
&mut self,
name: &str,
source_branch: &str,
) -> Result<u32, OnionError> {
if self.branch_by_name(name).is_some() { if self.branch_by_name(name).is_some() {
return Err(OnionError::BranchExists(name.to_string())); return Err(OnionError::BranchExists(name.to_string()));
} }
@@ -133,11 +129,7 @@ impl OnionFile {
} }
} }
fn merge_latest_wins( fn merge_latest_wins(&mut self, source_id: u32, target_id: u32) -> Result<u64, OnionError> {
&mut self,
source_id: u32,
target_id: u32,
) -> Result<u64, OnionError> {
let fork_rev = self let fork_rev = self
.branches .branches
.iter() .iter()
@@ -231,21 +223,21 @@ impl OnionFile {
self.commit_session(session, Some(&annotation)) self.commit_session(session, Some(&annotation))
} }
fn merge_three_way( fn merge_three_way(&mut self, source_id: u32, target_id: u32) -> Result<u64, OnionError> {
&mut self,
source_id: u32,
target_id: u32,
) -> Result<u64, OnionError> {
let source_head = self let source_head = self
.index .index
.branch_head(source_id) .branch_head(source_id)
.ok_or_else(|| OnionError::Malformed(format!("source branch {source_id} has no revisions")))? .ok_or_else(|| {
OnionError::Malformed(format!("source branch {source_id} has no revisions"))
})?
.revision; .revision;
let target_head = self let target_head = self
.index .index
.branch_head(target_id) .branch_head(target_id)
.ok_or_else(|| OnionError::Malformed(format!("target branch {target_id} has no revisions")))? .ok_or_else(|| {
OnionError::Malformed(format!("target branch {target_id} has no revisions"))
})?
.revision; .revision;
// Find common ancestor of the two branch HEADs // Find common ancestor of the two branch HEADs
@@ -253,13 +245,24 @@ impl OnionFile {
.index .index
.common_ancestor(source_head, target_head) .common_ancestor(source_head, target_head)
.ok_or_else(|| { .ok_or_else(|| {
let src_name = self.branches.iter().find(|b| b.id == source_id) let src_name = self
.branches
.iter()
.find(|b| b.id == source_id)
.and_then(|b| self.annotations.get(b.name_off)) .and_then(|b| self.annotations.get(b.name_off))
.unwrap_or("?").to_owned(); .unwrap_or("?")
let tgt_name = self.branches.iter().find(|b| b.id == target_id) .to_owned();
let tgt_name = self
.branches
.iter()
.find(|b| b.id == target_id)
.and_then(|b| self.annotations.get(b.name_off)) .and_then(|b| self.annotations.get(b.name_off))
.unwrap_or("?").to_owned(); .unwrap_or("?")
OnionError::NoCommonAncestor { a: src_name, b: tgt_name } .to_owned();
OnionError::NoCommonAncestor {
a: src_name,
b: tgt_name,
}
})?; })?;
// Collect source delta since ancestor (latest write per offset) // Collect source delta since ancestor (latest write per offset)
@@ -321,7 +324,8 @@ impl OnionFile {
if merged.is_empty() { if merged.is_empty() {
return Err(OnionError::Malformed( return Err(OnionError::Malformed(
"three-way merge: no source changes to apply (target already has all changes)".to_string(), "three-way merge: no source changes to apply (target already has all changes)"
.to_string(),
)); ));
} }
@@ -329,9 +333,8 @@ impl OnionFile {
for (h5_off, bytes) in &merged { for (h5_off, bytes) in &merged {
session.record_page(*h5_off, bytes); session.record_page(*h5_off, bytes);
} }
let annotation = format!( let annotation =
"3-way merge {source_id}{target_id} [ancestor rev {ancestor_rev}]" format!("3-way merge {source_id}{target_id} [ancestor rev {ancestor_rev}]");
);
self.commit_session(session, Some(&annotation)) self.commit_session(session, Some(&annotation))
} }
@@ -371,11 +374,7 @@ impl OnionFile {
.iter() .iter()
.map(|b| BranchInfo { .map(|b| BranchInfo {
id: b.id, id: b.id,
name: self name: self.annotations.get(b.name_off).unwrap_or("?").to_owned(),
.annotations
.get(b.name_off)
.unwrap_or("?")
.to_owned(),
head_rev: b.head_rev, head_rev: b.head_rev,
fork_rev: b.fork_rev, fork_rev: b.fork_rev,
}) })
@@ -383,7 +382,10 @@ impl OnionFile {
} }
/// Return revision entries for a named branch in chronological order. /// Return revision entries for a named branch in chronological order.
pub fn branch_history(&self, name: &str) -> Result<Vec<&crate::format::RevisionEntry>, OnionError> { pub fn branch_history(
&self,
name: &str,
) -> Result<Vec<&crate::format::RevisionEntry>, OnionError> {
let branch = self let branch = self
.branch_by_name(name) .branch_by_name(name)
.ok_or_else(|| OnionError::BranchNotFound(name.to_string()))?; .ok_or_else(|| OnionError::BranchNotFound(name.to_string()))?;
@@ -414,7 +416,9 @@ impl OnionFile {
.branch_by_name(name) .branch_by_name(name)
.ok_or_else(|| OnionError::BranchNotFound(name.to_string()))?; .ok_or_else(|| OnionError::BranchNotFound(name.to_string()))?;
if branch.id == crate::format::BRANCH_MAIN { if branch.id == crate::format::BRANCH_MAIN {
return Err(OnionError::Malformed("cannot delete the main branch".into())); return Err(OnionError::Malformed(
"cannot delete the main branch".into(),
));
} }
let id = branch.id; let id = branch.id;
self.branches.retain(|b| b.id != id); self.branches.retain(|b| b.id != id);
@@ -605,14 +609,18 @@ mod tests {
fn merge_latest_wins_produces_new_revision() { fn merge_latest_wins_produces_new_revision() {
let (_h5, mut onion, _feat_id) = make_fork_scenario(); let (_h5, mut onion, _feat_id) = make_fork_scenario();
let pre_count = onion.revision_count(); let pre_count = onion.revision_count();
onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap(); onion
.merge_into("feat", "main", MergeStrategy::LatestWins)
.unwrap();
assert_eq!(onion.revision_count(), pre_count + 1); assert_eq!(onion.revision_count(), pre_count + 1);
} }
#[test] #[test]
fn merge_latest_wins_page_content_correct() { fn merge_latest_wins_page_content_correct() {
let (h5, mut onion, _feat_id) = make_fork_scenario(); let (h5, mut onion, _feat_id) = make_fork_scenario();
onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap(); onion
.merge_into("feat", "main", MergeStrategy::LatestWins)
.unwrap();
onion.flush().unwrap(); onion.flush().unwrap();
// Reload from disk and reconstruct main HEAD // Reload from disk and reconstruct main HEAD
@@ -621,8 +629,10 @@ mod tests {
let base = std::fs::read(&h5).unwrap(); let base = std::fs::read(&h5).unwrap();
let state = onion2.reconstruct_revision(main_head, &base).unwrap(); let state = onion2.reconstruct_revision(main_head, &base).unwrap();
// Page 0 should now be BB (from feat) // Page 0 should now be BB (from feat)
assert!(state[0..4096].iter().all(|&b| b == 0xBB), assert!(
"page 0 should be BB after latest-wins merge"); state[0..4096].iter().all(|&b| b == 0xBB),
"page 0 should be BB after latest-wins merge"
);
} }
#[test] #[test]
@@ -646,15 +656,19 @@ mod tests {
s2.record_page(0, &vec![0x22u8; 4096]); s2.record_page(0, &vec![0x22u8; 4096]);
onion.commit_session(s2, None).unwrap(); onion.commit_session(s2, None).unwrap();
onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap(); onion
.merge_into("feat", "main", MergeStrategy::LatestWins)
.unwrap();
onion.flush().unwrap(); onion.flush().unwrap();
let onion2 = OnionFile::open(&h5).unwrap(); let onion2 = OnionFile::open(&h5).unwrap();
let head = onion2.revision_count() - 1; let head = onion2.revision_count() - 1;
let base = std::fs::read(&h5).unwrap(); let base = std::fs::read(&h5).unwrap();
let state = onion2.reconstruct_revision(head, &base).unwrap(); let state = onion2.reconstruct_revision(head, &base).unwrap();
assert!(state[0..4096].iter().all(|&b| b == 0x22), assert!(
"latest write (0x22) must win in LatestWins merge"); state[0..4096].iter().all(|&b| b == 0x22),
"latest write (0x22) must win in LatestWins merge"
);
} }
#[test] #[test]
@@ -666,7 +680,9 @@ mod tests {
onion.commit_session(s, None).unwrap(); onion.commit_session(s, None).unwrap();
// fork but make no commits on feat // fork but make no commits on feat
onion.create_branch("feat", "main").unwrap(); onion.create_branch("feat", "main").unwrap();
let err = onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap_err(); let err = onion
.merge_into("feat", "main", MergeStrategy::LatestWins)
.unwrap_err();
assert!(matches!(err, OnionError::Malformed(_))); assert!(matches!(err, OnionError::Malformed(_)));
} }
@@ -682,20 +698,30 @@ mod tests {
source.to_vec() // just return source source.to_vec() // just return source
}; };
onion onion
.merge_into("feat", "main", MergeStrategy::DatasetLevel(Box::new(resolver))) .merge_into(
"feat",
"main",
MergeStrategy::DatasetLevel(Box::new(resolver)),
)
.unwrap(); .unwrap();
assert!(called.load(std::sync::atomic::Ordering::SeqCst), "resolver must be called"); assert!(
called.load(std::sync::atomic::Ordering::SeqCst),
"resolver must be called"
);
} }
#[test] #[test]
fn merge_dataset_level_resolver_controls_output() { fn merge_dataset_level_resolver_controls_output() {
let (h5, mut onion, _feat_id) = make_fork_scenario(); let (h5, mut onion, _feat_id) = make_fork_scenario();
// Resolver always returns 0xCC regardless of inputs // Resolver always returns 0xCC regardless of inputs
let resolver = |_path: &str, _target: &[u8], _source: &[u8]| -> Vec<u8> { let resolver =
vec![0xCCu8; 4096] |_path: &str, _target: &[u8], _source: &[u8]| -> Vec<u8> { vec![0xCCu8; 4096] };
};
onion onion
.merge_into("feat", "main", MergeStrategy::DatasetLevel(Box::new(resolver))) .merge_into(
"feat",
"main",
MergeStrategy::DatasetLevel(Box::new(resolver)),
)
.unwrap(); .unwrap();
onion.flush().unwrap(); onion.flush().unwrap();
@@ -703,8 +729,10 @@ mod tests {
let head = onion2.revision_count() - 1; let head = onion2.revision_count() - 1;
let base = std::fs::read(&h5).unwrap(); let base = std::fs::read(&h5).unwrap();
let state = onion2.reconstruct_revision(head, &base).unwrap(); let state = onion2.reconstruct_revision(head, &base).unwrap();
assert!(state[0..4096].iter().all(|&b| b == 0xCC), assert!(
"resolver output 0xCC should be in merged state"); state[0..4096].iter().all(|&b| b == 0xCC),
"resolver output 0xCC should be in merged state"
);
} }
// ── Merge: ThreeWay ────────────────────────────────────────────────────── // ── Merge: ThreeWay ──────────────────────────────────────────────────────
@@ -736,7 +764,9 @@ mod tests {
sf.record_page(0, &vec![0xBBu8; 4096]); sf.record_page(0, &vec![0xBBu8; 4096]);
onion.commit_session(sf, None).unwrap(); onion.commit_session(sf, None).unwrap();
onion.merge_into("feat", "main", MergeStrategy::ThreeWay).unwrap(); onion
.merge_into("feat", "main", MergeStrategy::ThreeWay)
.unwrap();
onion.flush().unwrap(); onion.flush().unwrap();
let onion2 = OnionFile::open(&h5).unwrap(); let onion2 = OnionFile::open(&h5).unwrap();
@@ -744,10 +774,14 @@ mod tests {
let base = std::fs::read(&h5).unwrap(); let base = std::fs::read(&h5).unwrap();
let state = onion2.reconstruct_revision(head, &base).unwrap(); let state = onion2.reconstruct_revision(head, &base).unwrap();
assert!(state[0..4096].iter().all(|&b| b == 0xBB), assert!(
"page 0 should be BB (from feat)"); state[0..4096].iter().all(|&b| b == 0xBB),
assert!(state[4096..8192].iter().all(|&b| b == 0xCC), "page 0 should be BB (from feat)"
"page 4096 should stay CC (from main — not overwritten by 3-way)"); );
assert!(
state[4096..8192].iter().all(|&b| b == 0xCC),
"page 4096 should stay CC (from main — not overwritten by 3-way)"
);
} }
#[test] #[test]
@@ -772,22 +806,28 @@ mod tests {
sf.record_page(0, &vec![0xCCu8; 4096]); sf.record_page(0, &vec![0xCCu8; 4096]);
onion.commit_session(sf, None).unwrap(); onion.commit_session(sf, None).unwrap();
onion.merge_into("feat", "main", MergeStrategy::ThreeWay).unwrap(); onion
.merge_into("feat", "main", MergeStrategy::ThreeWay)
.unwrap();
onion.flush().unwrap(); onion.flush().unwrap();
let onion2 = OnionFile::open(&h5).unwrap(); let onion2 = OnionFile::open(&h5).unwrap();
let head = onion2.revision_count() - 1; let head = onion2.revision_count() - 1;
let base = std::fs::read(&h5).unwrap(); let base = std::fs::read(&h5).unwrap();
let state = onion2.reconstruct_revision(head, &base).unwrap(); let state = onion2.reconstruct_revision(head, &base).unwrap();
assert!(state[0..4096].iter().all(|&b| b == 0xCC), assert!(
"on conflict, source (CC) should win"); state[0..4096].iter().all(|&b| b == 0xCC),
"on conflict, source (CC) should win"
);
} }
#[test] #[test]
fn merge_nonexistent_source_errors() { fn merge_nonexistent_source_errors() {
let h5 = tmp_h5(); let h5 = tmp_h5();
let mut onion = OnionFile::create(&h5, 4096).unwrap(); let mut onion = OnionFile::create(&h5, 4096).unwrap();
let err = onion.merge_into("no-such", "main", MergeStrategy::LatestWins).unwrap_err(); let err = onion
.merge_into("no-such", "main", MergeStrategy::LatestWins)
.unwrap_err();
assert!(matches!(err, OnionError::BranchNotFound(_))); assert!(matches!(err, OnionError::BranchNotFound(_)));
} }
@@ -799,7 +839,9 @@ mod tests {
s.record_page(0, &vec![0u8; 4096]); s.record_page(0, &vec![0u8; 4096]);
onion.commit_session(s, None).unwrap(); onion.commit_session(s, None).unwrap();
onion.create_branch("feat", "main").unwrap(); onion.create_branch("feat", "main").unwrap();
let err = onion.merge_into("feat", "no-target", MergeStrategy::LatestWins).unwrap_err(); let err = onion
.merge_into("feat", "no-target", MergeStrategy::LatestWins)
.unwrap_err();
assert!(matches!(err, OnionError::BranchNotFound(_))); assert!(matches!(err, OnionError::BranchNotFound(_)));
} }
} }
+6 -10
View File
@@ -13,26 +13,22 @@ use crate::tdt;
pub fn compress_page(data: &[u8], codec: Codec) -> Result<Vec<u8>, OnionError> { pub fn compress_page(data: &[u8], codec: Codec) -> Result<Vec<u8>, OnionError> {
match codec { match codec {
Codec::None => Ok(data.to_vec()), Codec::None => Ok(data.to_vec()),
Codec::Zstd => zstd::bulk::compress(data, 3) Codec::Zstd => {
.map_err(|e| OnionError::Compress(e.to_string())), zstd::bulk::compress(data, 3).map_err(|e| OnionError::Compress(e.to_string()))
}
Codec::Lz4 => Ok(lz4_flex::compress_prepend_size(data)), Codec::Lz4 => Ok(lz4_flex::compress_prepend_size(data)),
Codec::Brotli => Err(OnionError::Compress( Codec::Brotli => Err(OnionError::Compress(
"brotli not yet implemented".to_string(), "brotli not yet implemented".to_string(),
)), )),
Codec::ZstdTdt => { Codec::ZstdTdt => {
let interleaved = tdt::encode(data, 4); let interleaved = tdt::encode(data, 4);
zstd::bulk::compress(&interleaved, 3) zstd::bulk::compress(&interleaved, 3).map_err(|e| OnionError::Compress(e.to_string()))
.map_err(|e| OnionError::Compress(e.to_string()))
} }
} }
} }
/// Decompress `data` using the given codec back to `orig_size` bytes. /// Decompress `data` using the given codec back to `orig_size` bytes.
pub fn decompress_page( pub fn decompress_page(data: &[u8], codec: Codec, orig_size: u32) -> Result<Vec<u8>, OnionError> {
data: &[u8],
codec: Codec,
orig_size: u32,
) -> Result<Vec<u8>, OnionError> {
match codec { match codec {
Codec::None => { Codec::None => {
if data.len() != orig_size as usize { if data.len() != orig_size as usize {
@@ -161,7 +157,7 @@ mod tests {
} }
let orig = data.len() as u32; let orig = data.len() as u32;
let zstd_size = compress_page(&data, Codec::Zstd).unwrap().len(); let zstd_size = compress_page(&data, Codec::Zstd).unwrap().len();
let tdt_size = compress_page(&data, Codec::ZstdTdt).unwrap().len(); let tdt_size = compress_page(&data, Codec::ZstdTdt).unwrap().len();
assert!( assert!(
tdt_size < zstd_size, tdt_size < zstd_size,
"ZstdTdt ({tdt_size} B) should beat plain Zstd ({zstd_size} B) on smooth f32 data ({orig} B)" "ZstdTdt ({tdt_size} B) should beat plain Zstd ({zstd_size} B) on smooth f32 data ({orig} B)"
+12 -3
View File
@@ -94,7 +94,10 @@ mod tests {
.unwrap(); .unwrap();
let f = vf.current().unwrap(); let f = vf.current().unwrap();
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![1.0, 2.0, 3.0]); assert_eq!(
f.dataset("data").unwrap().read_f64().unwrap(),
vec![1.0, 2.0, 3.0]
);
assert_eq!(vf.onion().revision_count(), 0); assert_eq!(vf.onion().revision_count(), 0);
} }
@@ -106,7 +109,10 @@ mod tests {
let _ = make_builder(&[1.0]).with_onion(&h5_path, 4096).unwrap(); let _ = make_builder(&[1.0]).with_onion(&h5_path, 4096).unwrap();
let onion_path = PathBuf::from(format!("{}.onion", h5_path.display())); let onion_path = PathBuf::from(format!("{}.onion", h5_path.display()));
assert!(onion_path.exists(), "sidecar should be flushed by with_onion"); assert!(
onion_path.exists(),
"sidecar should be flushed by with_onion"
);
} }
#[test] #[test]
@@ -140,6 +146,9 @@ mod tests {
vf.page_size().is_power_of_two(), vf.page_size().is_power_of_two(),
"auto page size must be a power of two" "auto page size must be a power of two"
); );
assert!(vf.page_size() >= 4096, "auto page size must be at least 4 KiB"); assert!(
vf.page_size() >= 4096,
"auto page size must be at least 4 KiB"
);
} }
} }
+53 -47
View File
@@ -49,9 +49,9 @@ pub const HEADER_SIZE: usize = 128;
pub mod feature_flags { pub mod feature_flags {
pub const COMPRESSION: u64 = 1 << 0; pub const COMPRESSION: u64 = 1 << 0;
pub const BRANCHING: u64 = 1 << 1; pub const BRANCHING: u64 = 1 << 1;
pub const PROVENANCE: u64 = 1 << 2; pub const PROVENANCE: u64 = 1 << 2;
pub const SNAPSHOTS: u64 = 1 << 3; pub const SNAPSHOTS: u64 = 1 << 3;
} }
pub const DEFAULT_FEATURE_FLAGS: u64 = pub const DEFAULT_FEATURE_FLAGS: u64 =
@@ -64,13 +64,13 @@ pub const DEFAULT_FEATURE_FLAGS: u64 =
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)] #[repr(u8)]
pub enum Codec { pub enum Codec {
None = 0, None = 0,
Zstd = 1, Zstd = 1,
Lz4 = 2, Lz4 = 2,
Brotli = 3, Brotli = 3,
/// zstd applied after TDT byte-interleaving transform (arXiv:2506.18062). /// zstd applied after TDT byte-interleaving transform (arXiv:2506.18062).
/// Improves compression ratio ~16% for `f32`/`f16` numeric pages. /// Improves compression ratio ~16% for `f32`/`f16` numeric pages.
ZstdTdt = 4, ZstdTdt = 4,
} }
impl Codec { impl Codec {
@@ -108,19 +108,19 @@ impl Codec {
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)] #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
#[repr(C)] #[repr(C)]
pub struct OnionHeader { pub struct OnionHeader {
pub magic: [u8; 9], pub magic: [u8; 9],
pub format_version: u8, pub format_version: u8,
pub _pad_align: [u8; 6], pub _pad_align: [u8; 6],
pub feature_flags: u64, pub feature_flags: u64,
pub page_size: u32, pub page_size: u32,
pub _pad_ps: [u8; 4], pub _pad_ps: [u8; 4],
pub revision_count: u64, pub revision_count: u64,
pub branch_count: u32, pub branch_count: u32,
pub _pad_bc: [u8; 4], pub _pad_bc: [u8; 4],
pub index_offset: u64, pub index_offset: u64,
pub branch_offset: u64, pub branch_offset: u64,
pub created_at: f64, pub created_at: f64,
pub reserved: [u8; 56], pub reserved: [u8; 56],
} }
const _: () = assert!(size_of::<OnionHeader>() == HEADER_SIZE); const _: () = assert!(size_of::<OnionHeader>() == HEADER_SIZE);
@@ -180,19 +180,19 @@ impl OnionHeader {
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)] #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
#[repr(C)] #[repr(C)]
pub struct RevisionEntry { pub struct RevisionEntry {
pub revision: u64, pub revision: u64,
pub branch_id: u32, pub branch_id: u32,
pub _pad_bi: [u8; 4], pub _pad_bi: [u8; 4],
pub parent_rev: u64, pub parent_rev: u64,
pub page_count: u32, pub page_count: u32,
pub _pad_pc: [u8; 4], pub _pad_pc: [u8; 4],
pub page_table_off: u64, pub page_table_off: u64,
pub timestamp: f64, pub timestamp: f64,
pub blake3: [u8; 32], pub blake3: [u8; 32],
pub session_uuid: [u8; 16], pub session_uuid: [u8; 16],
pub annotation_off: u64, pub annotation_off: u64,
pub flags: u8, pub flags: u8,
pub _pad_flags: [u8; 7], pub _pad_flags: [u8; 7],
} }
pub const REV_FLAG_SNAPSHOT: u8 = 1 << 0; pub const REV_FLAG_SNAPSHOT: u8 = 1 << 0;
@@ -233,11 +233,11 @@ impl RevisionEntry {
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)] #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
#[repr(C)] #[repr(C)]
pub struct BranchEntry { pub struct BranchEntry {
pub id: u32, pub id: u32,
pub _pad_id: [u8; 4], pub _pad_id: [u8; 4],
pub name_off: u64, pub name_off: u64,
pub head_rev: u64, pub head_rev: u64,
pub fork_rev: u64, pub fork_rev: u64,
pub created_at: f64, pub created_at: f64,
} }
@@ -255,25 +255,25 @@ pub struct BranchEntry {
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)] #[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
#[repr(C)] #[repr(C)]
pub struct PageTableEntry { pub struct PageTableEntry {
pub h5_offset: u64, pub h5_offset: u64,
pub data_offset: u64, pub data_offset: u64,
pub orig_size: u32, pub orig_size: u32,
pub data_size: u32, pub data_size: u32,
pub codec: u8, pub codec: u8,
pub _pad: [u8; 7], pub _pad: [u8; 7],
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
// Compile-time size/alignment assertions // Compile-time size/alignment assertions
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
const _: () = assert!(size_of::<OnionHeader>() == 128); const _: () = assert!(size_of::<OnionHeader>() == 128);
const _: () = assert!(size_of::<RevisionEntry>() == 112); const _: () = assert!(size_of::<RevisionEntry>() == 112);
const _: () = assert!(size_of::<BranchEntry>() == 40); const _: () = assert!(size_of::<BranchEntry>() == 40);
const _: () = assert!(size_of::<PageTableEntry>() == 32); const _: () = assert!(size_of::<PageTableEntry>() == 32);
const _: () = assert!(size_of::<RevisionEntry>() % 8 == 0); const _: () = assert!(size_of::<RevisionEntry>() % 8 == 0);
const _: () = assert!(size_of::<BranchEntry>() % 8 == 0); const _: () = assert!(size_of::<BranchEntry>() % 8 == 0);
const _: () = assert!(size_of::<PageTableEntry>() % 8 == 0); const _: () = assert!(size_of::<PageTableEntry>() % 8 == 0);
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@@ -321,7 +321,10 @@ mod tests {
fn header_unknown_version_rejected() { fn header_unknown_version_rejected() {
let mut hdr = OnionHeader::new(4096, DEFAULT_FEATURE_FLAGS, 0.0); let mut hdr = OnionHeader::new(4096, DEFAULT_FEATURE_FLAGS, 0.0);
hdr.format_version = 42; hdr.format_version = 42;
assert!(matches!(hdr.validate(), Err(OnionError::UnknownVersion(42)))); assert!(matches!(
hdr.validate(),
Err(OnionError::UnknownVersion(42))
));
} }
#[test] #[test]
@@ -488,7 +491,10 @@ mod tests {
#[test] #[test]
fn codec_unknown_returns_error() { fn codec_unknown_returns_error() {
assert!(matches!(Codec::from_u8(99), Err(OnionError::UnknownCodec(99)))); assert!(matches!(
Codec::from_u8(99),
Err(OnionError::UnknownCodec(99))
));
} }
#[test] #[test]
+58 -25
View File
@@ -69,7 +69,10 @@ impl OnionFile {
entry.set_epoch(EPOCH_DEAD); entry.set_epoch(EPOCH_DEAD);
} }
} }
Ok(GcStats { revisions_removed: count, bytes_reclaimed: 0 }) Ok(GcStats {
revisions_removed: count,
bytes_reclaimed: 0,
})
} }
_ => { _ => {
// ── Immediate path: compact now ─────────────────────────── // ── Immediate path: compact now ───────────────────────────
@@ -100,11 +103,7 @@ impl OnionFile {
GcPolicy::KeepSince(cutoff) => all_revs GcPolicy::KeepSince(cutoff) => all_revs
.iter() .iter()
.copied() .copied()
.filter(|&rev| { .filter(|&rev| self.index.get(rev).is_some_and(|e| e.timestamp >= *cutoff))
self.index
.get(rev)
.is_some_and(|e| e.timestamp >= *cutoff)
})
.collect(), .collect(),
GcPolicy::KeepRevisions(explicit) => { GcPolicy::KeepRevisions(explicit) => {
let mut set: HashSet<u64> = explicit.iter().copied().collect(); let mut set: HashSet<u64> = explicit.iter().copied().collect();
@@ -118,7 +117,11 @@ impl OnionFile {
GcPolicy::EpochFlip(inner) => return self.compute_to_remove(inner), GcPolicy::EpochFlip(inner) => return self.compute_to_remove(inner),
}; };
all_revs.iter().copied().filter(|rev| !keep.contains(rev)).collect() all_revs
.iter()
.copied()
.filter(|rev| !keep.contains(rev))
.collect()
} }
/// Consolidate + compact the given revision set immediately. /// Consolidate + compact the given revision set immediately.
@@ -204,8 +207,7 @@ impl OnionFile {
} }
// Compact page_data blob. // Compact page_data blob.
let surviving_revs: Vec<u64> = let surviving_revs: Vec<u64> = self.index.entries().iter().map(|e| e.revision).collect();
self.index.entries().iter().map(|e| e.revision).collect();
let mut new_page_data: Vec<u8> = Vec::new(); let mut new_page_data: Vec<u8> = Vec::new();
for &rev in &surviving_revs { for &rev in &surviving_revs {
if let Some(table) = self.page_tables.get_mut(rev as usize) { if let Some(table) = self.page_tables.get_mut(rev as usize) {
@@ -222,7 +224,10 @@ impl OnionFile {
} }
self.page_data = new_page_data; self.page_data = new_page_data;
Ok(GcStats { revisions_removed, bytes_reclaimed }) Ok(GcStats {
revisions_removed,
bytes_reclaimed,
})
} }
/// Find all entries marked [`EPOCH_DEAD`] and compact them. /// Find all entries marked [`EPOCH_DEAD`] and compact them.
@@ -270,9 +275,7 @@ mod tests {
} else { } else {
None None
}; };
onion onion.commit_session(s, annotation.as_deref()).unwrap();
.commit_session(s, annotation.as_deref())
.unwrap();
} }
onion onion
} }
@@ -369,7 +372,10 @@ mod tests {
assert_eq!(stats.revisions_removed, 4); assert_eq!(stats.revisions_removed, 4);
let after = onion.page_data.len(); let after = onion.page_data.len();
// page_data must be smaller (or at most equal for all-same pages that compress to 0) // page_data must be smaller (or at most equal for all-same pages that compress to 0)
assert!(after <= before, "page_data must shrink after GC: {before} -> {after}"); assert!(
after <= before,
"page_data must shrink after GC: {before} -> {after}"
);
} }
/// After GC + flush, the on-disk file is smaller than before. /// After GC + flush, the on-disk file is smaller than before.
@@ -421,7 +427,10 @@ mod tests {
// The three surviving revisions must still reconstruct without error. // The three surviving revisions must still reconstruct without error.
for rev in 3..6u64 { for rev in 3..6u64 {
let bytes = onion.reconstruct_revision(rev, &h5_base).unwrap(); let bytes = onion.reconstruct_revision(rev, &h5_base).unwrap();
assert!(!bytes.is_empty(), "rev {rev} should produce non-empty bytes"); assert!(
!bytes.is_empty(),
"rev {rev} should produce non-empty bytes"
);
} }
} }
@@ -456,7 +465,11 @@ mod tests {
let before_gc = onion.reconstruct_revision(2, &h5_base).unwrap(); let before_gc = onion.reconstruct_revision(2, &h5_base).unwrap();
assert_eq!(&before_gc[0..4096], &vec![0xAA_u8; 4096], "pre-GC page0"); assert_eq!(&before_gc[0..4096], &vec![0xAA_u8; 4096], "pre-GC page0");
assert_eq!(&before_gc[4096..8192], &vec![0xBB_u8; 4096], "pre-GC page1"); assert_eq!(&before_gc[4096..8192], &vec![0xBB_u8; 4096], "pre-GC page1");
assert_eq!(&before_gc[8192..12288], &vec![0xCC_u8; 4096], "pre-GC page2"); assert_eq!(
&before_gc[8192..12288],
&vec![0xCC_u8; 4096],
"pre-GC page2"
);
// GC: keep only rev 2 — revs 0 and 1 are ancestors that will be pruned. // GC: keep only rev 2 — revs 0 and 1 are ancestors that will be pruned.
onion.gc(GcPolicy::KeepLastN(1)).unwrap(); onion.gc(GcPolicy::KeepLastN(1)).unwrap();
@@ -490,15 +503,20 @@ mod tests {
let mut onion = make_onion_with_n_revisions(10); let mut onion = make_onion_with_n_revisions(10);
let page_data_len_before = onion.page_data.len(); let page_data_len_before = onion.page_data.len();
let stats = onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(3)))).unwrap(); let stats = onion
.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(3))))
.unwrap();
// Reports the count that will be removed. // Reports the count that will be removed.
assert_eq!(stats.revisions_removed, 7); assert_eq!(stats.revisions_removed, 7);
// bytes_reclaimed is 0 until flush compacts. // bytes_reclaimed is 0 until flush compacts.
assert_eq!(stats.bytes_reclaimed, 0); assert_eq!(stats.bytes_reclaimed, 0);
// page_data must NOT have changed yet. // page_data must NOT have changed yet.
assert_eq!(onion.page_data.len(), page_data_len_before, assert_eq!(
"epoch flip must not compact page_data immediately"); onion.page_data.len(),
page_data_len_before,
"epoch flip must not compact page_data immediately"
);
// Index still has all 10 entries (removal deferred). // Index still has all 10 entries (removal deferred).
assert_eq!(onion.revision_count(), 10); assert_eq!(onion.revision_count(), 10);
// Entries 0..6 should be marked EPOCH_DEAD. // Entries 0..6 should be marked EPOCH_DEAD.
@@ -525,7 +543,9 @@ mod tests {
} }
// Epoch flip: defer compaction. // Epoch flip: defer compaction.
onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(2)))).unwrap(); onion
.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(2))))
.unwrap();
assert_eq!(onion.revision_count(), 5, "not compacted yet"); assert_eq!(onion.revision_count(), 5, "not compacted yet");
// After flush, deferred compaction runs. // After flush, deferred compaction runs.
@@ -533,7 +553,11 @@ mod tests {
// Reload and verify. // Reload and verify.
let reloaded = OnionFile::open(&h5).unwrap(); let reloaded = OnionFile::open(&h5).unwrap();
assert_eq!(reloaded.revision_count(), 2, "2 revisions survive after flush"); assert_eq!(
reloaded.revision_count(),
2,
"2 revisions survive after flush"
);
// Surviving revisions are still reconstructable. // Surviving revisions are still reconstructable.
for rev in 3u64..5 { for rev in 3u64..5 {
@@ -559,16 +583,25 @@ mod tests {
#[test] #[test]
fn gc_epoch_flip_nested_policy_keep_tagged() { fn gc_epoch_flip_nested_policy_keep_tagged() {
let mut onion = make_onion_with_n_revisions(9); // tagged at 0,3,6 let mut onion = make_onion_with_n_revisions(9); // tagged at 0,3,6
let stats = onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepTagged))).unwrap(); let stats = onion
.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepTagged)))
.unwrap();
assert_eq!(stats.revisions_removed, 6); // keeps 0,3,6 assert_eq!(stats.revisions_removed, 6); // keeps 0,3,6
assert_eq!(onion.revision_count(), 9, "deferred — index intact"); assert_eq!(onion.revision_count(), 9, "deferred — index intact");
// 0,3,6 are live; rest are dead. // 0,3,6 are live; rest are dead.
for rev in [0u64, 3, 6] { for rev in [0u64, 3, 6] {
assert_eq!(onion.index.get(rev).unwrap().epoch(), 0, "tagged rev {rev} must be live"); assert_eq!(
onion.index.get(rev).unwrap().epoch(),
0,
"tagged rev {rev} must be live"
);
} }
for rev in [1u64, 2, 4, 5, 7, 8] { for rev in [1u64, 2, 4, 5, 7, 8] {
assert_eq!(onion.index.get(rev).unwrap().epoch(), EPOCH_DEAD, assert_eq!(
"untagged rev {rev} must be EPOCH_DEAD"); onion.index.get(rev).unwrap().epoch(),
EPOCH_DEAD,
"untagged rev {rev} must be EPOCH_DEAD"
);
} }
} }
+14 -10
View File
@@ -1,6 +1,6 @@
//! RevisionIndex: O(1) lookup + branch-filtered iteration. //! RevisionIndex: O(1) lookup + branch-filtered iteration.
use crate::format::{RevisionEntry, NO_PARENT}; use crate::format::{NO_PARENT, RevisionEntry};
/// In-memory index of all revision entries. /// In-memory index of all revision entries.
/// ///
@@ -53,7 +53,9 @@ impl RevisionIndex {
/// monotonically greater than the current maximum. /// monotonically greater than the current maximum.
pub fn append(&mut self, entry: RevisionEntry) { pub fn append(&mut self, entry: RevisionEntry) {
debug_assert!( debug_assert!(
self.entries.last().is_none_or(|e| e.revision < entry.revision), self.entries
.last()
.is_none_or(|e| e.revision < entry.revision),
"revision numbers must be monotonically increasing" "revision numbers must be monotonically increasing"
); );
self.entries.push(entry); self.entries.push(entry);
@@ -61,15 +63,14 @@ impl RevisionIndex {
/// Iterate over all revisions on a specific branch (by `branch_id`). /// Iterate over all revisions on a specific branch (by `branch_id`).
pub fn branch_revisions(&self, branch_id: u32) -> impl Iterator<Item = &RevisionEntry> { pub fn branch_revisions(&self, branch_id: u32) -> impl Iterator<Item = &RevisionEntry> {
self.entries.iter().filter(move |e| e.branch_id == branch_id) self.entries
.iter()
.filter(move |e| e.branch_id == branch_id)
} }
/// Return the HEAD revision entry for a branch (highest revision number). /// Return the HEAD revision entry for a branch (highest revision number).
pub fn branch_head(&self, branch_id: u32) -> Option<&RevisionEntry> { pub fn branch_head(&self, branch_id: u32) -> Option<&RevisionEntry> {
self.entries self.entries.iter().rev().find(|e| e.branch_id == branch_id)
.iter()
.rev()
.find(|e| e.branch_id == branch_id)
} }
/// Walk the DAG from `start_rev` to the root, following `parent_rev`. /// Walk the DAG from `start_rev` to the root, following `parent_rev`.
@@ -191,8 +192,7 @@ mod tests {
.collect(); .collect();
assert_eq!(main_revs, vec![0, 1, 4]); assert_eq!(main_revs, vec![0, 1, 4]);
let branch1_revs: Vec<u64> = let branch1_revs: Vec<u64> = idx.branch_revisions(1).map(|e| e.revision).collect();
idx.branch_revisions(1).map(|e| e.revision).collect();
assert_eq!(branch1_revs, vec![2, 3]); assert_eq!(branch1_revs, vec![2, 3]);
} }
@@ -252,7 +252,11 @@ mod tests {
fn remove_revisions() { fn remove_revisions() {
let mut idx = RevisionIndex::new(); let mut idx = RevisionIndex::new();
for i in 0u64..5 { for i in 0u64..5 {
idx.append(make_entry(i, BRANCH_MAIN, if i == 0 { NO_PARENT } else { i - 1 })); idx.append(make_entry(
i,
BRANCH_MAIN,
if i == 0 { NO_PARENT } else { i - 1 },
));
} }
let to_remove = [1u64, 2].into_iter().collect(); let to_remove = [1u64, 2].into_iter().collect();
idx.remove_revisions(&to_remove); idx.remove_revisions(&to_remove);
+10 -11
View File
@@ -36,19 +36,18 @@ pub mod tdt;
pub mod versioned_file; pub mod versioned_file;
pub mod writer; pub mod writer;
pub use error::OnionError;
pub use format::{
BranchEntry, Codec, OnionHeader, PageTableEntry, RevisionEntry,
BRANCH_MAIN, DEFAULT_FEATURE_FLAGS, EPOCH_DEAD, FORMAT_VERSION, HEADER_SIZE, MAGIC, NO_PARENT,
REV_FLAG_SNAPSHOT, feature_flags,
};
pub use api::{ pub use api::{
open_revision, open_branch, open_branch_at, create_branch, delete_branch, list_branches, list_revisions, open_branch, open_branch_at,
list_revisions, rollback, open_revision, rename_branch, rollback,
list_branches, create_branch, delete_branch, rename_branch,
}; };
pub use branch::{BranchInfo, DatasetResolver, MergeStrategy}; pub use branch::{BranchInfo, DatasetResolver, MergeStrategy};
pub use merkle::{MerkleError, MerkleNode, RevisionMerkleTree, WalkStep}; pub use error::OnionError;
pub use ext::FileBuilderExt; pub use ext::FileBuilderExt;
pub use versioned_file::{VersionedFile, open_at_revision, open_at_branch}; pub use format::{
BRANCH_MAIN, BranchEntry, Codec, DEFAULT_FEATURE_FLAGS, EPOCH_DEAD, FORMAT_VERSION,
HEADER_SIZE, MAGIC, NO_PARENT, OnionHeader, PageTableEntry, REV_FLAG_SNAPSHOT, RevisionEntry,
feature_flags,
};
pub use merkle::{MerkleError, MerkleNode, RevisionMerkleTree, WalkStep};
pub use versioned_file::{VersionedFile, open_at_branch, open_at_revision};
pub use writer::eof_to_page_size; pub use writer::eof_to_page_size;
+16 -4
View File
@@ -97,7 +97,11 @@ impl RevisionMerkleTree {
/// The slice **must** be sorted by `revision` in ascending order. /// The slice **must** be sorted by `revision` in ascending order.
pub fn build(entries: &[(u64, [u8; 32])]) -> Self { pub fn build(entries: &[(u64, [u8; 32])]) -> Self {
let leaf_count = entries.len(); let leaf_count = entries.len();
let capacity = if leaf_count == 0 { 1 } else { leaf_count.next_power_of_two() }; let capacity = if leaf_count == 0 {
1
} else {
leaf_count.next_power_of_two()
};
// 1-indexed BFS array; index 0 unused. // 1-indexed BFS array; index 0 unused.
let mut hashes = vec![[0u8; 32]; 2 * capacity + 1]; let mut hashes = vec![[0u8; 32]; 2 * capacity + 1];
@@ -113,7 +117,12 @@ impl RevisionMerkleTree {
// Build internal nodes bottom-up // Build internal nodes bottom-up
build_internal(&mut hashes, capacity); build_internal(&mut hashes, capacity);
RevisionMerkleTree { hashes, leaf_revisions, leaf_count, capacity } RevisionMerkleTree {
hashes,
leaf_revisions,
leaf_count,
capacity,
}
} }
// ── Queries ─────────────────────────────────────────────────────────────── // ── Queries ───────────────────────────────────────────────────────────────
@@ -311,7 +320,7 @@ impl RevisionMerkleTree {
/// `hashes` must have length `2 * capacity + 1`. /// `hashes` must have length `2 * capacity + 1`.
fn build_internal(hashes: &mut [[u8; 32]], capacity: usize) { fn build_internal(hashes: &mut [[u8; 32]], capacity: usize) {
for i in (1..capacity).rev() { for i in (1..capacity).rev() {
let left = hashes[2 * i]; let left = hashes[2 * i];
let right = hashes[2 * i + 1]; let right = hashes[2 * i + 1];
hashes[i] = node_hash(&left, &right); hashes[i] = node_hash(&left, &right);
} }
@@ -484,7 +493,10 @@ mod tests {
let header_only = &full[..13]; // 5 + 8 bytes let header_only = &full[..13]; // 5 + 8 bytes
assert_eq!( assert_eq!(
RevisionMerkleTree::deserialise(header_only), RevisionMerkleTree::deserialise(header_only),
Err(MerkleError::TruncatedPayload { expected: 5 + 8 + 5 * 40, got: 13 }) Err(MerkleError::TruncatedPayload {
expected: 5 + 8 + 5 * 40,
got: 13
})
); );
} }
+4 -1
View File
@@ -96,7 +96,10 @@ mod tests {
// Same pages, different insertion order // Same pages, different insertion order
let h1 = hash_pages(&[(0, a.as_ref()), (4096, b.as_ref())]); let h1 = hash_pages(&[(0, a.as_ref()), (4096, b.as_ref())]);
let h2 = hash_pages(&[(4096, b.as_ref()), (0, a.as_ref())]); let h2 = hash_pages(&[(4096, b.as_ref()), (0, a.as_ref())]);
assert_eq!(h1, h2, "hash must be order-independent (sorted by h5_offset)"); assert_eq!(
h1, h2,
"hash must be order-independent (sorted by h5_offset)"
);
} }
#[test] #[test]
+21 -25
View File
@@ -3,11 +3,11 @@
//! Opening a historical revision reads only the `.onion` sidecar — //! Opening a historical revision reads only the `.onion` sidecar —
//! the primary `.h5` file is always at the latest committed state. //! the primary `.h5` file is always at the latest committed state.
use std::collections::BTreeMap;
use crate::compress::decompress_page; use crate::compress::decompress_page;
use crate::error::OnionError; use crate::error::OnionError;
use crate::format::Codec; use crate::format::Codec;
use crate::writer::OnionFile; use crate::writer::OnionFile;
use std::collections::BTreeMap;
/// `(h5_offset, compressed_data, codec_byte, orig_size)` — raw page as stored. /// `(h5_offset, compressed_data, codec_byte, orig_size)` — raw page as stored.
pub type RawPage = (u64, Vec<u8>, u8, u32); pub type RawPage = (u64, Vec<u8>, u8, u32);
@@ -41,7 +41,11 @@ impl OnionFile {
// Collect all ancestor revisions in order from oldest → newest // Collect all ancestor revisions in order from oldest → newest
let ancestors: Vec<u64> = { let ancestors: Vec<u64> = {
let mut chain = self.index.ancestors(rev).map(|e| e.revision).collect::<Vec<_>>(); let mut chain = self
.index
.ancestors(rev)
.map(|e| e.revision)
.collect::<Vec<_>>();
chain.reverse(); // oldest first chain.reverse(); // oldest first
chain chain
}; };
@@ -53,13 +57,11 @@ impl OnionFile {
// Optimisation: find the newest snapshot in the ancestor chain and // Optimisation: find the newest snapshot in the ancestor chain and
// start from there instead of from `h5_base`. This bounds // start from there instead of from `h5_base`. This bounds
// reconstruction depth to O(N_since_snapshot · P). // reconstruction depth to O(N_since_snapshot · P).
let snapshot_start_idx = ancestors let snapshot_start_idx = ancestors.iter().rposition(|&r| {
.iter() self.index
.rposition(|&r| { .get(r)
self.index .is_some_and(|e| e.flags & REV_FLAG_SNAPSHOT != 0)
.get(r) });
.is_some_and(|e| e.flags & REV_FLAG_SNAPSHOT != 0)
});
let (start_idx, mut file_bytes) = match snapshot_start_idx { let (start_idx, mut file_bytes) = match snapshot_start_idx {
Some(idx) => { Some(idx) => {
@@ -76,7 +78,9 @@ impl OnionFile {
let table = self let table = self
.page_tables .page_tables
.get(*ancestor_rev as usize) .get(*ancestor_rev as usize)
.ok_or_else(|| OnionError::Malformed(format!("missing page table for rev {ancestor_rev}")))?; .ok_or_else(|| {
OnionError::Malformed(format!("missing page table for rev {ancestor_rev}"))
})?;
for pt_entry in table { for pt_entry in table {
let codec = Codec::from_u8(pt_entry.codec)?; let codec = Codec::from_u8(pt_entry.codec)?;
@@ -111,11 +115,7 @@ impl OnionFile {
} }
/// Open a specific revision using an [`OpenRevision`] selector. /// Open a specific revision using an [`OpenRevision`] selector.
pub fn open_rev( pub fn open_rev(&self, selector: OpenRevision, h5_base: &[u8]) -> Result<Vec<u8>, OnionError> {
&self,
selector: OpenRevision,
h5_base: &[u8],
) -> Result<Vec<u8>, OnionError> {
let rev = self.resolve_selector(selector)?; let rev = self.resolve_selector(selector)?;
self.reconstruct_revision(rev, h5_base) self.reconstruct_revision(rev, h5_base)
} }
@@ -128,12 +128,11 @@ impl OnionFile {
self.branch_head_rev(crate::format::BRANCH_MAIN) self.branch_head_rev(crate::format::BRANCH_MAIN)
.ok_or(OnionError::RevisionNotFound(0)) .ok_or(OnionError::RevisionNotFound(0))
} }
OpenRevision::At(rev) => { OpenRevision::At(rev) => self
self.index .index
.get(rev) .get(rev)
.map(|_| rev) .map(|_| rev)
.ok_or(OnionError::RevisionNotFound(rev)) .ok_or(OnionError::RevisionNotFound(rev)),
}
OpenRevision::Branch(name) => { OpenRevision::Branch(name) => {
let branch = self let branch = self
.branch_by_name(&name) .branch_by_name(&name)
@@ -201,10 +200,7 @@ impl OnionFile {
/// compressed bytes directly and let the receiver decompress. /// compressed bytes directly and let the receiver decompress.
/// ///
/// Returns `(h5_offset, compressed_data, codec_byte, orig_size)`. /// Returns `(h5_offset, compressed_data, codec_byte, orig_size)`.
pub fn revision_pages_raw( pub fn revision_pages_raw(&self, rev: u64) -> Result<Vec<RawPage>, OnionError> {
&self,
rev: u64,
) -> Result<Vec<RawPage>, OnionError> {
let table = self let table = self
.page_tables .page_tables
.get(rev as usize) .get(rev as usize)
+6 -2
View File
@@ -117,7 +117,10 @@ mod tests {
// byte_pos=1: elements 1,5,9,13 // byte_pos=1: elements 1,5,9,13
// byte_pos=2: elements 2,6,10,14 // byte_pos=2: elements 2,6,10,14
// byte_pos=3: elements 3,7,11,15 // byte_pos=3: elements 3,7,11,15
assert_eq!(enc, vec![0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15]); assert_eq!(
enc,
vec![0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15]
);
let dec = decode(&enc, 4, 16); let dec = decode(&enc, 4, 16);
assert_eq!(dec, data); assert_eq!(dec, data);
} }
@@ -180,7 +183,8 @@ mod tests {
assert!( assert!(
tdt_compressed.len() <= raw_compressed.len(), tdt_compressed.len() <= raw_compressed.len(),
"TDT+zstd ({} bytes) should be no worse than raw zstd ({} bytes) on structured float data", "TDT+zstd ({} bytes) should be no worse than raw zstd ({} bytes) on structured float data",
tdt_compressed.len(), raw_compressed.len() tdt_compressed.len(),
raw_compressed.len()
); );
} }
+167 -53
View File
@@ -57,21 +57,21 @@ use crate::writer::OnionFile;
/// [`export_revision`](VersionedFile::export_revision) to materialise a /// [`export_revision`](VersionedFile::export_revision) to materialise a
/// specific revision to disk. /// specific revision to disk.
pub struct VersionedFile { pub struct VersionedFile {
onion: OnionFile, onion: OnionFile,
h5_path: PathBuf, h5_path: PathBuf,
/// Original `.h5` bytes — the reconstruction base. **Never mutated.** /// Original `.h5` bytes — the reconstruction base. **Never mutated.**
h5_base: Vec<u8>, h5_base: Vec<u8>,
/// In-memory state of the most recently committed revision on the **default** /// In-memory state of the most recently committed revision on the **default**
/// branch. Kept in sync with `branch_state[BRANCH_MAIN]`. /// branch. Kept in sync with `branch_state[BRANCH_MAIN]`.
/// Used by [`snapshot`](VersionedFile::snapshot) and /// Used by [`snapshot`](VersionedFile::snapshot) and
/// [`auto_snapshot`](VersionedFile::auto_snapshot). /// [`auto_snapshot`](VersionedFile::auto_snapshot).
current_h5: Vec<u8>, current_h5: Vec<u8>,
/// Per-branch diff baseline: `branch_id → last-committed bytes on that branch`. /// Per-branch diff baseline: `branch_id → last-committed bytes on that branch`.
/// ///
/// Ensures that interleaved commits on different branches always diff against /// Ensures that interleaved commits on different branches always diff against
/// the correct prior state for each branch, not just the globally last commit. /// the correct prior state for each branch, not just the globally last commit.
branch_state: HashMap<u32, Vec<u8>>, branch_state: HashMap<u32, Vec<u8>>,
page_size: u32, page_size: u32,
} }
impl VersionedFile { impl VersionedFile {
@@ -105,7 +105,14 @@ impl VersionedFile {
.cloned() .cloned()
.unwrap_or_else(|| h5_base.clone()); .unwrap_or_else(|| h5_base.clone());
Ok(Self { onion, h5_path, h5_base, current_h5, branch_state, page_size }) Ok(Self {
onion,
h5_path,
h5_base,
current_h5,
branch_state,
page_size,
})
} }
/// Create a new versioned HDF5 file from a [`clawhdf5::FileBuilder`]. /// Create a new versioned HDF5 file from a [`clawhdf5::FileBuilder`].
@@ -144,7 +151,14 @@ impl VersionedFile {
let h5_base = std::fs::read(&h5_path)?; let h5_base = std::fs::read(&h5_path)?;
let current_h5 = h5_base.clone(); let current_h5 = h5_base.clone();
let onion = OnionFile::create(&h5_path, page_size)?; let onion = OnionFile::create(&h5_path, page_size)?;
Ok(Self { onion, h5_path, h5_base, current_h5, branch_state: HashMap::new(), page_size }) Ok(Self {
onion,
h5_path,
h5_base,
current_h5,
branch_state: HashMap::new(),
page_size,
})
} }
/// Create a new versioned HDF5 file, automatically choosing the page size. /// Create a new versioned HDF5 file, automatically choosing the page size.
@@ -163,7 +177,14 @@ impl VersionedFile {
let current_h5 = h5_base.clone(); let current_h5 = h5_base.clone();
let onion = OnionFile::create_auto(&h5_path)?; let onion = OnionFile::create_auto(&h5_path)?;
let page_size = onion.page_size(); let page_size = onion.page_size();
Ok(Self { onion, h5_path, h5_base, current_h5, branch_state: HashMap::new(), page_size }) Ok(Self {
onion,
h5_path,
h5_base,
current_h5,
branch_state: HashMap::new(),
page_size,
})
} }
// ── Read API ────────────────────────────────────────────────────────────── // ── Read API ──────────────────────────────────────────────────────────────
@@ -352,7 +373,9 @@ impl VersionedFile {
/// vf.auto_snapshot(500)?; // snapshot every 500 revisions /// vf.auto_snapshot(500)?; // snapshot every 500 revisions
/// ``` /// ```
pub fn auto_snapshot(&mut self, interval: u64) -> Result<Option<u64>, OnionError> { pub fn auto_snapshot(&mut self, interval: u64) -> Result<Option<u64>, OnionError> {
let result = self.onion.auto_snapshot_if_needed(&self.current_h5, interval)?; let result = self
.onion
.auto_snapshot_if_needed(&self.current_h5, interval)?;
if result.is_some() { if result.is_some() {
self.onion.flush()?; self.onion.flush()?;
} }
@@ -383,7 +406,8 @@ impl VersionedFile {
// Refresh the target branch's cached state so future commits on it // Refresh the target branch's cached state so future commits on it
// use the merged content as their diff baseline. // use the merged content as their diff baseline.
let (target_id, target_head) = { let (target_id, target_head) = {
let entry = self.onion let entry = self
.onion
.branch_by_name(target) .branch_by_name(target)
.ok_or_else(|| OnionError::BranchNotFound(target.to_string()))?; .ok_or_else(|| OnionError::BranchNotFound(target.to_string()))?;
(entry.id, entry.head_rev) (entry.id, entry.head_rev)
@@ -435,8 +459,7 @@ impl VersionedFile {
// ── Internal ───────────────────────────────────────────────────────────── // ── Internal ─────────────────────────────────────────────────────────────
fn hdf5_from_bytes(&self, bytes: Vec<u8>) -> Result<clawhdf5::File, OnionError> { fn hdf5_from_bytes(&self, bytes: Vec<u8>) -> Result<clawhdf5::File, OnionError> {
clawhdf5::File::from_bytes(bytes) clawhdf5::File::from_bytes(bytes).map_err(|e| OnionError::Hdf5(e.to_string()))
.map_err(|e| OnionError::Hdf5(e.to_string()))
} }
} }
@@ -510,11 +533,17 @@ mod tests {
// Read back rev 0 — should still be [1.0, 2.0] // Read back rev 0 — should still be [1.0, 2.0]
let f0 = vf.revision(0).unwrap(); let f0 = vf.revision(0).unwrap();
assert_eq!(f0.dataset("data").unwrap().read_f64().unwrap(), vec![1.0, 2.0]); assert_eq!(
f0.dataset("data").unwrap().read_f64().unwrap(),
vec![1.0, 2.0]
);
// Read back rev 1 — should be [3.0, 4.0] // Read back rev 1 — should be [3.0, 4.0]
let f1 = vf.revision(1).unwrap(); let f1 = vf.revision(1).unwrap();
assert_eq!(f1.dataset("data").unwrap().read_f64().unwrap(), vec![3.0, 4.0]); assert_eq!(
f1.dataset("data").unwrap().read_f64().unwrap(),
vec![3.0, 4.0]
);
} }
#[test] #[test]
@@ -580,9 +609,15 @@ mod tests {
let vf2 = VersionedFile::open(&h5_path).unwrap(); let vf2 = VersionedFile::open(&h5_path).unwrap();
assert_eq!(vf2.onion().revision_count(), 2); assert_eq!(vf2.onion().revision_count(), 2);
let f0 = vf2.revision(0).unwrap(); let f0 = vf2.revision(0).unwrap();
assert_eq!(f0.dataset("data").unwrap().read_f64().unwrap(), vec![10.0, 20.0]); assert_eq!(
f0.dataset("data").unwrap().read_f64().unwrap(),
vec![10.0, 20.0]
);
let f1 = vf2.revision(1).unwrap(); let f1 = vf2.revision(1).unwrap();
assert_eq!(f1.dataset("data").unwrap().read_f64().unwrap(), vec![30.0, 40.0]); assert_eq!(
f1.dataset("data").unwrap().read_f64().unwrap(),
vec![30.0, 40.0]
);
} }
// ── merge ──────────────────────────────────────────────────────────────── // ── merge ────────────────────────────────────────────────────────────────
@@ -598,7 +633,8 @@ mod tests {
// Fork feat // Fork feat
let feat_id = vf.onion_mut().create_branch("feat", "main").unwrap(); let feat_id = vf.onion_mut().create_branch("feat", "main").unwrap();
// Rev 1: feat writes [3.0, 4.0] // Rev 1: feat writes [3.0, 4.0]
vf.commit_on(Some("feat"), make_h5(&[3.0, 4.0]), Some("feat-v1")).unwrap(); vf.commit_on(Some("feat"), make_h5(&[3.0, 4.0]), Some("feat-v1"))
.unwrap();
// Merge feat → main // Merge feat → main
let merge_rev = vf.merge("feat", "main", MergeStrategy::LatestWins).unwrap(); let merge_rev = vf.merge("feat", "main", MergeStrategy::LatestWins).unwrap();
@@ -610,7 +646,10 @@ mod tests {
// Content at merge revision reflects feat's state (latest wins) // Content at merge revision reflects feat's state (latest wins)
let f = vf.revision(merge_rev).unwrap(); let f = vf.revision(merge_rev).unwrap();
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![3.0, 4.0]); assert_eq!(
f.dataset("data").unwrap().read_f64().unwrap(),
vec![3.0, 4.0]
);
let _ = feat_id; let _ = feat_id;
} }
@@ -622,7 +661,8 @@ mod tests {
vf.commit(make_h5(&[1.0]), Some("main-v1")).unwrap(); vf.commit(make_h5(&[1.0]), Some("main-v1")).unwrap();
vf.onion_mut().create_branch("feat", "main").unwrap(); vf.onion_mut().create_branch("feat", "main").unwrap();
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1")).unwrap(); vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1"))
.unwrap();
vf.merge("feat", "main", MergeStrategy::LatestWins).unwrap(); vf.merge("feat", "main", MergeStrategy::LatestWins).unwrap();
@@ -640,7 +680,9 @@ mod tests {
let mut vf = VersionedFile::create(&h5_path, 4096).unwrap(); let mut vf = VersionedFile::create(&h5_path, 4096).unwrap();
vf.commit(make_h5(&[1.0]), None).unwrap(); vf.commit(make_h5(&[1.0]), None).unwrap();
let err = vf.merge("ghost", "main", MergeStrategy::LatestWins).unwrap_err(); let err = vf
.merge("ghost", "main", MergeStrategy::LatestWins)
.unwrap_err();
assert!(matches!(err, OnionError::BranchNotFound(_))); assert!(matches!(err, OnionError::BranchNotFound(_)));
} }
@@ -653,11 +695,12 @@ mod tests {
vf.commit(make_h5(&[1.0]), Some("main-v1")).unwrap(); vf.commit(make_h5(&[1.0]), Some("main-v1")).unwrap();
let feat_id = vf.onion_mut().create_branch("feat", "main").unwrap(); let feat_id = vf.onion_mut().create_branch("feat", "main").unwrap();
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1")).unwrap(); vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1"))
.unwrap();
let revs = vf.onion().list_revisions(); let revs = vf.onion().list_revisions();
assert_eq!(revs.len(), 2); assert_eq!(revs.len(), 2);
assert_eq!(revs[0].branch_id, 0, "rev 0 is on main"); assert_eq!(revs[0].branch_id, 0, "rev 0 is on main");
assert_eq!(revs[1].branch_id, feat_id, "rev 1 is on feat"); assert_eq!(revs[1].branch_id, feat_id, "rev 1 is on feat");
} }
@@ -679,7 +722,9 @@ mod tests {
let (_tmp, h5_path) = tmp_h5(&[1.0]); let (_tmp, h5_path) = tmp_h5(&[1.0]);
let mut vf = VersionedFile::create(&h5_path, 4096).unwrap(); let mut vf = VersionedFile::create(&h5_path, 4096).unwrap();
let err = vf.commit_on(Some("ghost"), make_h5(&[2.0]), None).unwrap_err(); let err = vf
.commit_on(Some("ghost"), make_h5(&[2.0]), None)
.unwrap_err();
assert!(matches!(err, OnionError::BranchNotFound(_))); assert!(matches!(err, OnionError::BranchNotFound(_)));
} }
@@ -698,7 +743,8 @@ mod tests {
vf.onion_mut().create_branch("feat", "main").unwrap(); vf.onion_mut().create_branch("feat", "main").unwrap();
// Commit v2 on feat (rev 1: [2.0]) // Commit v2 on feat (rev 1: [2.0])
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1")).unwrap(); vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1"))
.unwrap();
// Commit v2 on main (rev 2: [3.0]) — diff must be against main's last // Commit v2 on main (rev 2: [3.0]) — diff must be against main's last
// commit ([1.0]), not feat's last commit ([2.0]). // commit ([1.0]), not feat's last commit ([2.0]).
@@ -706,16 +752,25 @@ mod tests {
// Reconstruct each revision and verify round-trip fidelity // Reconstruct each revision and verify round-trip fidelity
let f0 = vf.revision(0).unwrap(); let f0 = vf.revision(0).unwrap();
assert_eq!(f0.dataset("data").unwrap().read_f64().unwrap(), vec![1.0], assert_eq!(
"rev 0 (main-v1)"); f0.dataset("data").unwrap().read_f64().unwrap(),
vec![1.0],
"rev 0 (main-v1)"
);
let f1 = vf.revision(1).unwrap(); let f1 = vf.revision(1).unwrap();
assert_eq!(f1.dataset("data").unwrap().read_f64().unwrap(), vec![2.0], assert_eq!(
"rev 1 (feat-v1)"); f1.dataset("data").unwrap().read_f64().unwrap(),
vec![2.0],
"rev 1 (feat-v1)"
);
let f2 = vf.revision(2).unwrap(); let f2 = vf.revision(2).unwrap();
assert_eq!(f2.dataset("data").unwrap().read_f64().unwrap(), vec![3.0], assert_eq!(
"rev 2 (main-v2) — would be wrong if diffed against feat state"); f2.dataset("data").unwrap().read_f64().unwrap(),
vec![3.0],
"rev 2 (main-v2) — would be wrong if diffed against feat state"
);
} }
/// After a reload, `open()` must restore per-branch state from the sidecar /// After a reload, `open()` must restore per-branch state from the sidecar
@@ -730,13 +785,21 @@ mod tests {
let mut s = vf.onion_mut().begin_session(Some(feat_id)).unwrap(); let mut s = vf.onion_mut().begin_session(Some(feat_id)).unwrap();
let feat_bytes = make_h5(&[9.0]); let feat_bytes = make_h5(&[9.0]);
let ps = 4096usize; let ps = 4096usize;
let base = vf.onion().reconstruct_revision(0, let base = vf
&std::fs::read(&h5_path).unwrap()).unwrap(); .onion()
.reconstruct_revision(0, &std::fs::read(&h5_path).unwrap())
.unwrap();
for i in 0..(feat_bytes.len().max(base.len())).div_ceil(ps) { for i in 0..(feat_bytes.len().max(base.len())).div_ceil(ps) {
let start = i * ps; let start = i * ps;
if start >= feat_bytes.len() { break; } if start >= feat_bytes.len() {
let ns = &feat_bytes[start..feat_bytes.len().min(start+ps)]; break;
let os: &[u8] = if start < base.len() { &base[start..base.len().min(start+ps)] } else { &[] }; }
let ns = &feat_bytes[start..feat_bytes.len().min(start + ps)];
let os: &[u8] = if start < base.len() {
&base[start..base.len().min(start + ps)]
} else {
&[]
};
if ns != os { if ns != os {
let mut pad = vec![0u8; ps]; let mut pad = vec![0u8; ps];
pad[..ns.len()].copy_from_slice(ns); pad[..ns.len()].copy_from_slice(ns);
@@ -752,8 +815,11 @@ mod tests {
vf2.commit(make_h5(&[2.0]), Some("main-v2")).unwrap(); vf2.commit(make_h5(&[2.0]), Some("main-v2")).unwrap();
let f_main = vf2.revision(2).unwrap(); let f_main = vf2.revision(2).unwrap();
assert_eq!(f_main.dataset("data").unwrap().read_f64().unwrap(), vec![2.0], assert_eq!(
"main-v2 content must be [2.0] after reload"); f_main.dataset("data").unwrap().read_f64().unwrap(),
vec![2.0],
"main-v2 content must be [2.0] after reload"
);
} }
// ── create_auto ────────────────────────────────────────────────────────── // ── create_auto ──────────────────────────────────────────────────────────
@@ -767,7 +833,10 @@ mod tests {
"auto page size must be a power of two, got {}", "auto page size must be a power of two, got {}",
vf.page_size() vf.page_size()
); );
assert!(vf.page_size() >= 4096, "auto page size must be at least 4 KiB"); assert!(
vf.page_size() >= 4096,
"auto page size must be at least 4 KiB"
);
} }
#[test] #[test]
@@ -778,7 +847,10 @@ mod tests {
vf.commit(make_h5(&[7.0, 8.0]), Some("auto-v1")).unwrap(); vf.commit(make_h5(&[7.0, 8.0]), Some("auto-v1")).unwrap();
let f = vf.revision(0).unwrap(); let f = vf.revision(0).unwrap();
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![7.0, 8.0]); assert_eq!(
f.dataset("data").unwrap().read_f64().unwrap(),
vec![7.0, 8.0]
);
} }
// ── snapshot / auto_snapshot ───────────────────────────────────────────── // ── snapshot / auto_snapshot ─────────────────────────────────────────────
@@ -797,7 +869,10 @@ mod tests {
// The snapshot revision must be flagged as a snapshot. // The snapshot revision must be flagged as a snapshot.
let revs = vf.onion().list_revisions(); let revs = vf.onion().list_revisions();
let snap = revs.iter().find(|r| r.revision == snap_rev).unwrap(); let snap = revs.iter().find(|r| r.revision == snap_rev).unwrap();
assert!(snap.is_snapshot, "snapshot revision must have is_snapshot == true"); assert!(
snap.is_snapshot,
"snapshot revision must have is_snapshot == true"
);
} }
#[test] #[test]
@@ -843,7 +918,10 @@ mod tests {
vf.commit(make_h5(&[i as f64]), None).unwrap(); vf.commit(make_h5(&[i as f64]), None).unwrap();
} }
let result = vf.auto_snapshot(5).unwrap(); let result = vf.auto_snapshot(5).unwrap();
assert!(result.is_some(), "auto_snapshot should fire when count % interval == 0"); assert!(
result.is_some(),
"auto_snapshot should fire when count % interval == 0"
);
} }
#[test] #[test]
@@ -855,7 +933,10 @@ mod tests {
vf.commit(make_h5(&[2.0]), None).unwrap(); vf.commit(make_h5(&[2.0]), None).unwrap();
// 2 revisions, interval 5 — should not trigger // 2 revisions, interval 5 — should not trigger
let result = vf.auto_snapshot(5).unwrap(); let result = vf.auto_snapshot(5).unwrap();
assert!(result.is_none(), "auto_snapshot should be a no-op when count % interval != 0"); assert!(
result.is_none(),
"auto_snapshot should be a no-op when count % interval != 0"
);
} }
#[test] #[test]
@@ -908,12 +989,21 @@ mod tests {
// Record a changed page for the feat branch rev // Record a changed page for the feat branch rev
let feat_bytes = make_h5(&[3.0]); let feat_bytes = make_h5(&[3.0]);
let ps = 4096usize; let ps = 4096usize;
let current = vf.onion().reconstruct_revision(1, &std::fs::read(&h5_path).unwrap()).unwrap(); let current = vf
.onion()
.reconstruct_revision(1, &std::fs::read(&h5_path).unwrap())
.unwrap();
for i in 0..(feat_bytes.len().max(current.len())).div_ceil(ps) { for i in 0..(feat_bytes.len().max(current.len())).div_ceil(ps) {
let start = i * ps; let start = i * ps;
if start >= feat_bytes.len() { break; } if start >= feat_bytes.len() {
break;
}
let new_sl = &feat_bytes[start..feat_bytes.len().min(start + ps)]; let new_sl = &feat_bytes[start..feat_bytes.len().min(start + ps)];
let old_sl: &[u8] = if start < current.len() { &current[start..current.len().min(start + ps)] } else { &[] }; let old_sl: &[u8] = if start < current.len() {
&current[start..current.len().min(start + ps)]
} else {
&[]
};
if new_sl != old_sl { if new_sl != old_sl {
let mut pad = vec![0u8; ps]; let mut pad = vec![0u8; ps];
pad[..new_sl.len()].copy_from_slice(new_sl); pad[..new_sl.len()].copy_from_slice(new_sl);
@@ -1009,7 +1099,10 @@ mod tests {
p.set_extension(&new_ext); p.set_extension(&new_ext);
p p
}; };
assert!(onion_path.exists(), ".onion sidecar should exist after from_builder: checked {onion_path:?}"); assert!(
onion_path.exists(),
".onion sidecar should exist after from_builder: checked {onion_path:?}"
);
} }
#[test] #[test]
@@ -1026,7 +1119,10 @@ mod tests {
assert_eq!(rev, 0); assert_eq!(rev, 0);
let f = vf.revision(0).unwrap(); let f = vf.revision(0).unwrap();
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![3.0, 4.0]); assert_eq!(
f.dataset("data").unwrap().read_f64().unwrap(),
vec![3.0, 4.0]
);
} }
// ── export_revision ─────────────────────────────────────────────────────── // ── export_revision ───────────────────────────────────────────────────────
@@ -1046,7 +1142,10 @@ mod tests {
// Re-open the exported file as a raw clawhdf5::File // Re-open the exported file as a raw clawhdf5::File
let bytes = std::fs::read(&export_path).unwrap(); let bytes = std::fs::read(&export_path).unwrap();
let f = clawhdf5::File::from_bytes(bytes).unwrap(); let f = clawhdf5::File::from_bytes(bytes).unwrap();
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![1.0, 2.0]); assert_eq!(
f.dataset("data").unwrap().read_f64().unwrap(),
vec![1.0, 2.0]
);
} }
#[test] #[test]
@@ -1075,7 +1174,10 @@ mod tests {
let bytes = std::fs::read(&export_path).unwrap(); let bytes = std::fs::read(&export_path).unwrap();
let f = clawhdf5::File::from_bytes(bytes).unwrap(); let f = clawhdf5::File::from_bytes(bytes).unwrap();
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![8.0, 9.0]); assert_eq!(
f.dataset("data").unwrap().read_f64().unwrap(),
vec![8.0, 9.0]
);
} }
// ── multiple datasets ───────────────────────────────────────────────────── // ── multiple datasets ─────────────────────────────────────────────────────
@@ -1098,11 +1200,23 @@ mod tests {
vf.commit(bytes_v2, Some("update x only")).unwrap(); vf.commit(bytes_v2, Some("update x only")).unwrap();
let f0 = vf.revision(0).unwrap(); let f0 = vf.revision(0).unwrap();
assert_eq!(f0.dataset("x").unwrap().read_f64().unwrap(), vec![1.0, 2.0, 3.0]); assert_eq!(
assert_eq!(f0.dataset("y").unwrap().read_f64().unwrap(), vec![4.0, 5.0, 6.0]); f0.dataset("x").unwrap().read_f64().unwrap(),
vec![1.0, 2.0, 3.0]
);
assert_eq!(
f0.dataset("y").unwrap().read_f64().unwrap(),
vec![4.0, 5.0, 6.0]
);
let f1 = vf.revision(1).unwrap(); let f1 = vf.revision(1).unwrap();
assert_eq!(f1.dataset("x").unwrap().read_f64().unwrap(), vec![10.0, 20.0, 30.0]); assert_eq!(
assert_eq!(f1.dataset("y").unwrap().read_f64().unwrap(), vec![4.0, 5.0, 6.0]); f1.dataset("x").unwrap().read_f64().unwrap(),
vec![10.0, 20.0, 30.0]
);
assert_eq!(
f1.dataset("y").unwrap().read_f64().unwrap(),
vec![4.0, 5.0, 6.0]
);
} }
} }
+24 -17
View File
@@ -18,8 +18,8 @@ use crate::annotation::AnnotationHeap;
use crate::compress::compress_page; use crate::compress::compress_page;
use crate::error::OnionError; use crate::error::OnionError;
use crate::format::{ use crate::format::{
BranchEntry, Codec, OnionHeader, PageTableEntry, RevisionEntry, BRANCH_MAIN, BranchEntry, Codec, DEFAULT_FEATURE_FLAGS, HEADER_SIZE, NO_PARENT, OnionHeader,
BRANCH_MAIN, DEFAULT_FEATURE_FLAGS, HEADER_SIZE, NO_PARENT, PageTableEntry, RevisionEntry,
}; };
use crate::index::RevisionIndex; use crate::index::RevisionIndex;
use crate::provenance::{SessionId, hash_pages}; use crate::provenance::{SessionId, hash_pages};
@@ -204,9 +204,7 @@ impl OnionFile {
annotation: Option<&str>, annotation: Option<&str>,
) -> Result<u64, OnionError> { ) -> Result<u64, OnionError> {
let revision = self.index.len() as u64; let revision = self.index.len() as u64;
let parent_rev = self let parent_rev = self.branch_head_rev(session.branch_id).unwrap_or(NO_PARENT);
.branch_head_rev(session.branch_id)
.unwrap_or(NO_PARENT);
// Compress pages and build BLAKE3 input // Compress pages and build BLAKE3 input
let mut pages: Vec<(u64, Vec<u8>, u32, Codec)> = Vec::new(); let mut pages: Vec<(u64, Vec<u8>, u32, Codec)> = Vec::new();
@@ -226,9 +224,7 @@ impl OnionFile {
); );
// Annotation // Annotation
let annotation_off = annotation let annotation_off = annotation.map(|a| self.annotations.push(a)).unwrap_or(0);
.map(|a| self.annotations.push(a))
.unwrap_or(0);
// Append page table entries // Append page table entries
let mut table: Vec<PageTableEntry> = Vec::new(); let mut table: Vec<PageTableEntry> = Vec::new();
@@ -451,9 +447,9 @@ impl OnionFile {
} }
pub fn branch_by_name(&self, name: &str) -> Option<&BranchEntry> { pub fn branch_by_name(&self, name: &str) -> Option<&BranchEntry> {
self.branches.iter().find(|b| { self.branches
self.annotations.get(b.name_off) == Some(name) .iter()
}) .find(|b| self.annotations.get(b.name_off) == Some(name))
} }
/// Serialise the entire `.onion` file to bytes. /// Serialise the entire `.onion` file to bytes.
@@ -480,8 +476,7 @@ impl OnionFile {
let branch_offset = index_offset + (n_revisions * rev_entry_size) as u64; let branch_offset = index_offset + (n_revisions * rev_entry_size) as u64;
// Calculate PageTable section start // Calculate PageTable section start
let page_table_section_start = let page_table_section_start = branch_offset + (n_branches * branch_entry_size) as u64;
branch_offset + (n_branches * branch_entry_size) as u64;
// Calculate per-revision page_table_off values and total page table size // Calculate per-revision page_table_off values and total page table size
let mut page_table_offsets: Vec<u64> = Vec::with_capacity(n_revisions); let mut page_table_offsets: Vec<u64> = Vec::with_capacity(n_revisions);
@@ -599,7 +594,11 @@ impl OnionFile {
// 0-4). We extend the Vec with empty entries so that // 0-4). We extend the Vec with empty entries so that
// page_tables[rev_number] is always valid. // page_tables[rev_number] is always valid.
let max_rev = entries.iter().map(|e| e.revision).max().unwrap_or(0); let max_rev = entries.iter().map(|e| e.revision).max().unwrap_or(0);
let table_len = if entries.is_empty() { 0 } else { max_rev as usize + 1 }; let table_len = if entries.is_empty() {
0
} else {
max_rev as usize + 1
};
let mut page_tables: Vec<Vec<PageTableEntry>> = vec![Vec::new(); table_len]; let mut page_tables: Vec<Vec<PageTableEntry>> = vec![Vec::new(); table_len];
for entry in &entries { for entry in &entries {
let pt_start = entry.page_table_off as usize; let pt_start = entry.page_table_off as usize;
@@ -792,7 +791,10 @@ mod tests {
fn create_accepts_valid_page_sizes() { fn create_accepts_valid_page_sizes() {
for &size in &[512u32, 1024, 4096, 65536] { for &size in &[512u32, 1024, 4096, 65536] {
let path = Path::new("/tmp/dummy.h5"); let path = Path::new("/tmp/dummy.h5");
assert!(OnionFile::create(path, size).is_ok(), "size {size} rejected"); assert!(
OnionFile::create(path, size).is_ok(),
"size {size} rejected"
);
} }
} }
@@ -973,7 +975,9 @@ mod tests {
.list_revisions() .list_revisions()
.into_iter() .into_iter()
.filter(|r| { .filter(|r| {
onion.index.get(r.revision) onion
.index
.get(r.revision)
.map_or(false, |e| e.flags & crate::format::REV_FLAG_SNAPSHOT != 0) .map_or(false, |e| e.flags & crate::format::REV_FLAG_SNAPSHOT != 0)
}) })
.collect(); .collect();
@@ -1041,7 +1045,10 @@ mod tests {
fn eof_to_page_size_always_power_of_two() { fn eof_to_page_size_always_power_of_two() {
for eof in [1, 100, 1_000, 10_000, 100_000, 1_000_000, 100_000_000u64] { for eof in [1, 100, 1_000, 10_000, 100_000, 1_000_000, 100_000_000u64] {
let ps = eof_to_page_size(eof); let ps = eof_to_page_size(eof);
assert!(ps.is_power_of_two(), "page_size {ps} is not a power of two for eof={eof}"); assert!(
ps.is_power_of_two(),
"page_size {ps} is not a power of two for eof={eof}"
);
} }
} }
+159 -76
View File
@@ -20,23 +20,27 @@ use tempfile::NamedTempFile;
// Layout constants (must match CLAWONION_SPEC.md §2 + §3) // Layout constants (must match CLAWONION_SPEC.md §2 + §3)
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
const HEADER_SIZE: usize = 128; const HEADER_SIZE: usize = 128;
const REV_ENTRY_SIZE: usize = 112; const REV_ENTRY_SIZE: usize = 112;
const BRANCH_ENTRY_SIZE: usize = 40; const BRANCH_ENTRY_SIZE: usize = 40;
const PAGE_ENTRY_SIZE: usize = 32; const PAGE_ENTRY_SIZE: usize = 32;
const PAGE_SIZE: u32 = 4096; const PAGE_SIZE: u32 = 4096;
const INDEX_OFFSET: usize = HEADER_SIZE; // 128 const INDEX_OFFSET: usize = HEADER_SIZE; // 128
const BRANCH_OFFSET: usize = INDEX_OFFSET + REV_ENTRY_SIZE; // 240 const BRANCH_OFFSET: usize = INDEX_OFFSET + REV_ENTRY_SIZE; // 240
const PT_OFFSET: usize = BRANCH_OFFSET + BRANCH_ENTRY_SIZE; // 280 const PT_OFFSET: usize = BRANCH_OFFSET + BRANCH_ENTRY_SIZE; // 280
const PD_OFFSET: usize = PT_OFFSET + PAGE_ENTRY_SIZE; // 312 const PD_OFFSET: usize = PT_OFFSET + PAGE_ENTRY_SIZE; // 312
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
// Helpers // Helpers
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
fn le64(v: u64) -> [u8; 8] { v.to_le_bytes() } fn le64(v: u64) -> [u8; 8] {
fn le32(v: u32) -> [u8; 4] { v.to_le_bytes() } v.to_le_bytes()
}
fn le32(v: u32) -> [u8; 4] {
v.to_le_bytes()
}
/// Build the expected BLAKE3 for one 4096-byte page of 0xAB at h5_offset = 0. /// Build the expected BLAKE3 for one 4096-byte page of 0xAB at h5_offset = 0.
/// ///
@@ -82,7 +86,8 @@ fn golden_minimal_onion_byte_layout() {
let ann_heap: &[u8] = &[0x00, 0x04, 0x00, 0x00, 0x00, b'm', b'a', b'i', b'n']; let ann_heap: &[u8] = &[0x00, 0x04, 0x00, 0x00, 0x00, b'm', b'a', b'i', b'n'];
let expected_total = PD_OFFSET + PAGE_SIZE as usize + ann_heap.len(); // 4417 let expected_total = PD_OFFSET + PAGE_SIZE as usize + ann_heap.len(); // 4417
assert_eq!( assert_eq!(
bytes.len(), expected_total, bytes.len(),
expected_total,
"total file size mismatch (got {}, expected {expected_total})", "total file size mismatch (got {}, expected {expected_total})",
bytes.len() bytes.len()
); );
@@ -127,10 +132,17 @@ fn golden_minimal_onion_byte_layout() {
// [64..72] created_at — f64, time-dependent; skip exact value check, // [64..72] created_at — f64, time-dependent; skip exact value check,
// but assert it's non-zero (a real Unix timestamp). // but assert it's non-zero (a real Unix timestamp).
let created_at = f64::from_le_bytes(bytes[64..72].try_into().unwrap()); let created_at = f64::from_le_bytes(bytes[64..72].try_into().unwrap());
assert!(created_at > 0.0, "created_at should be a positive Unix timestamp"); assert!(
created_at > 0.0,
"created_at should be a positive Unix timestamp"
);
// [72..128] reserved — 56 bytes, must all be zero // [72..128] reserved — 56 bytes, must all be zero
assert_eq!(&bytes[72..128], &[0u8; 56], "reserved header bytes must be zero"); assert_eq!(
&bytes[72..128],
&[0u8; 56],
"reserved header bytes must be zero"
);
// ══════════════════════════════════════════════════════════════════════════ // ══════════════════════════════════════════════════════════════════════════
// §3.2 RevisionEntry — 112 bytes @ offset 128 // §3.2 RevisionEntry — 112 bytes @ offset 128
@@ -139,45 +151,69 @@ fn golden_minimal_onion_byte_layout() {
let re = INDEX_OFFSET; // 128 let re = INDEX_OFFSET; // 128
// [re+0..re+8] revision = 0 // [re+0..re+8] revision = 0
assert_eq!(&bytes[re..re+8], &le64(0), "revision number"); assert_eq!(&bytes[re..re + 8], &le64(0), "revision number");
// [re+8..re+12] branch_id = 0 (main) // [re+8..re+12] branch_id = 0 (main)
assert_eq!(&bytes[re+8..re+12], &le32(0), "branch_id"); assert_eq!(&bytes[re + 8..re + 12], &le32(0), "branch_id");
// [re+12..re+16] _pad_bi — must be zero // [re+12..re+16] _pad_bi — must be zero
assert_eq!(&bytes[re+12..re+16], &[0u8; 4], "_pad_bi must be zero"); assert_eq!(&bytes[re + 12..re + 16], &[0u8; 4], "_pad_bi must be zero");
// [re+16..re+24] parent_rev = u64::MAX (root has no parent) // [re+16..re+24] parent_rev = u64::MAX (root has no parent)
assert_eq!(&bytes[re+16..re+24], &le64(u64::MAX), "parent_rev (NO_PARENT sentinel)"); assert_eq!(
&bytes[re + 16..re + 24],
&le64(u64::MAX),
"parent_rev (NO_PARENT sentinel)"
);
// [re+24..re+28] page_count = 1 // [re+24..re+28] page_count = 1
assert_eq!(&bytes[re+24..re+28], &le32(1), "page_count"); assert_eq!(&bytes[re + 24..re + 28], &le32(1), "page_count");
// [re+28..re+32] _pad_pc — must be zero // [re+28..re+32] _pad_pc — must be zero
assert_eq!(&bytes[re+28..re+32], &[0u8; 4], "_pad_pc must be zero"); assert_eq!(&bytes[re + 28..re + 32], &[0u8; 4], "_pad_pc must be zero");
// [re+32..re+40] page_table_off = 280 (PT_OFFSET) // [re+32..re+40] page_table_off = 280 (PT_OFFSET)
assert_eq!(&bytes[re+32..re+40], &le64(PT_OFFSET as u64), "page_table_off"); assert_eq!(
&bytes[re + 32..re + 40],
&le64(PT_OFFSET as u64),
"page_table_off"
);
// [re+40..re+48] timestamp — f64, time-dependent; assert positive // [re+40..re+48] timestamp — f64, time-dependent; assert positive
let ts = f64::from_le_bytes(bytes[re+40..re+48].try_into().unwrap()); let ts = f64::from_le_bytes(bytes[re + 40..re + 48].try_into().unwrap());
assert!(ts > 0.0, "revision timestamp should be positive"); assert!(ts > 0.0, "revision timestamp should be positive");
// [re+48..re+80] blake3 — verify against spec §5 algorithm // [re+48..re+80] blake3 — verify against spec §5 algorithm
let computed_blake3 = expected_blake3(&page_bytes); let computed_blake3 = expected_blake3(&page_bytes);
assert_eq!(&bytes[re+48..re+80], &computed_blake3, "BLAKE3 hash mismatch"); assert_eq!(
&bytes[re + 48..re + 80],
&computed_blake3,
"BLAKE3 hash mismatch"
);
// [re+80..re+96] session_uuid — 16 bytes UUIDv7; must be non-zero // [re+80..re+96] session_uuid — 16 bytes UUIDv7; must be non-zero
assert_ne!(&bytes[re+80..re+96], &[0u8; 16], "session_uuid must not be all zeros"); assert_ne!(
&bytes[re + 80..re + 96],
&[0u8; 16],
"session_uuid must not be all zeros"
);
// [re+96..re+104] annotation_off = 0 (no annotation) // [re+96..re+104] annotation_off = 0 (no annotation)
assert_eq!(&bytes[re+96..re+104], &le64(0), "annotation_off must be 0 (no annotation)"); assert_eq!(
&bytes[re + 96..re + 104],
&le64(0),
"annotation_off must be 0 (no annotation)"
);
// [re+104] flags = 0 (not a snapshot) // [re+104] flags = 0 (not a snapshot)
assert_eq!(bytes[re+104], 0x00, "revision flags"); assert_eq!(bytes[re + 104], 0x00, "revision flags");
// [re+105..re+112] _pad_flags — must be zero // [re+105..re+112] _pad_flags — must be zero
assert_eq!(&bytes[re+105..re+112], &[0u8; 7], "_pad_flags must be zero"); assert_eq!(
&bytes[re + 105..re + 112],
&[0u8; 7],
"_pad_flags must be zero"
);
// ══════════════════════════════════════════════════════════════════════════ // ══════════════════════════════════════════════════════════════════════════
// §3.3 BranchEntry — 40 bytes @ offset 240 // §3.3 BranchEntry — 40 bytes @ offset 240
@@ -186,22 +222,34 @@ fn golden_minimal_onion_byte_layout() {
let be = BRANCH_OFFSET; // 240 let be = BRANCH_OFFSET; // 240
// [be+0..be+4] id = 0 (main) // [be+0..be+4] id = 0 (main)
assert_eq!(&bytes[be..be+4], &le32(0), "branch id"); assert_eq!(&bytes[be..be + 4], &le32(0), "branch id");
// [be+4..be+8] _pad_id — must be zero // [be+4..be+8] _pad_id — must be zero
assert_eq!(&bytes[be+4..be+8], &[0u8; 4], "_pad_id must be zero"); assert_eq!(&bytes[be + 4..be + 8], &[0u8; 4], "_pad_id must be zero");
// [be+8..be+16] name_off = 1 (heap-relative; first string after reserved null) // [be+8..be+16] name_off = 1 (heap-relative; first string after reserved null)
assert_eq!(&bytes[be+8..be+16], &le64(1), "name_off for main branch"); assert_eq!(
&bytes[be + 8..be + 16],
&le64(1),
"name_off for main branch"
);
// [be+16..be+24] head_rev = 0 (updated after commit) // [be+16..be+24] head_rev = 0 (updated after commit)
assert_eq!(&bytes[be+16..be+24], &le64(0), "head_rev after first commit"); assert_eq!(
&bytes[be + 16..be + 24],
&le64(0),
"head_rev after first commit"
);
// [be+24..be+32] fork_rev = u64::MAX (main is not forked) // [be+24..be+32] fork_rev = u64::MAX (main is not forked)
assert_eq!(&bytes[be+24..be+32], &le64(u64::MAX), "fork_rev (NO_PARENT for main)"); assert_eq!(
&bytes[be + 24..be + 32],
&le64(u64::MAX),
"fork_rev (NO_PARENT for main)"
);
// [be+32..be+40] created_at — f64, time-dependent; assert positive // [be+32..be+40] created_at — f64, time-dependent; assert positive
let branch_ts = f64::from_le_bytes(bytes[be+32..be+40].try_into().unwrap()); let branch_ts = f64::from_le_bytes(bytes[be + 32..be + 40].try_into().unwrap());
assert!(branch_ts > 0.0, "branch created_at should be positive"); assert!(branch_ts > 0.0, "branch created_at should be positive");
// ══════════════════════════════════════════════════════════════════════════ // ══════════════════════════════════════════════════════════════════════════
@@ -211,36 +259,51 @@ fn golden_minimal_onion_byte_layout() {
let pt = PT_OFFSET; // 280 let pt = PT_OFFSET; // 280
// [pt+0..pt+8] h5_offset = 0 // [pt+0..pt+8] h5_offset = 0
assert_eq!(&bytes[pt..pt+8], &le64(0), "h5_offset"); assert_eq!(&bytes[pt..pt + 8], &le64(0), "h5_offset");
// [pt+8..pt+16] data_offset = 312 (PD_OFFSET, absolute file position) // [pt+8..pt+16] data_offset = 312 (PD_OFFSET, absolute file position)
assert_eq!(&bytes[pt+8..pt+16], &le64(PD_OFFSET as u64), "data_offset (absolute)"); assert_eq!(
&bytes[pt + 8..pt + 16],
&le64(PD_OFFSET as u64),
"data_offset (absolute)"
);
// [pt+16..pt+20] orig_size = 4096 // [pt+16..pt+20] orig_size = 4096
assert_eq!(&bytes[pt+16..pt+20], &le32(4096), "orig_size"); assert_eq!(&bytes[pt + 16..pt + 20], &le32(4096), "orig_size");
// [pt+20..pt+24] data_size = 4096 (codec=None → no compression) // [pt+20..pt+24] data_size = 4096 (codec=None → no compression)
assert_eq!(&bytes[pt+20..pt+24], &le32(4096), "data_size (codec=None, uncompressed)"); assert_eq!(
&bytes[pt + 20..pt + 24],
&le32(4096),
"data_size (codec=None, uncompressed)"
);
// [pt+24] codec = 0 (None) // [pt+24] codec = 0 (None)
assert_eq!(bytes[pt+24], 0x00, "codec = None (0)"); assert_eq!(bytes[pt + 24], 0x00, "codec = None (0)");
// [pt+25..pt+32] _pad — must be zero // [pt+25..pt+32] _pad — must be zero
assert_eq!(&bytes[pt+25..pt+32], &[0u8; 7], "_pad must be zero"); assert_eq!(&bytes[pt + 25..pt + 32], &[0u8; 7], "_pad must be zero");
// ══════════════════════════════════════════════════════════════════════════ // ══════════════════════════════════════════════════════════════════════════
// PageData — 4096 bytes @ offset 312 // PageData — 4096 bytes @ offset 312
// ══════════════════════════════════════════════════════════════════════════ // ══════════════════════════════════════════════════════════════════════════
assert_eq!(&bytes[PD_OFFSET..PD_OFFSET + 4096], page_bytes.as_slice(), assert_eq!(
"page data must equal the uncompressed original bytes"); &bytes[PD_OFFSET..PD_OFFSET + 4096],
page_bytes.as_slice(),
"page data must equal the uncompressed original bytes"
);
// ══════════════════════════════════════════════════════════════════════════ // ══════════════════════════════════════════════════════════════════════════
// §3.5 AnnotationHeap — tail of file // §3.5 AnnotationHeap — tail of file
// ══════════════════════════════════════════════════════════════════════════ // ══════════════════════════════════════════════════════════════════════════
// Last 9 bytes: [0x00 reserved] [u32 LE length=4] ["main"] // Last 9 bytes: [0x00 reserved] [u32 LE length=4] ["main"]
assert_eq!(&bytes[bytes.len()-9..], ann_heap, "annotation heap content"); assert_eq!(
&bytes[bytes.len() - 9..],
ann_heap,
"annotation heap content"
);
// ══════════════════════════════════════════════════════════════════════════ // ══════════════════════════════════════════════════════════════════════════
// §7 Reconstruction — verify round-trip // §7 Reconstruction — verify round-trip
@@ -270,10 +333,10 @@ fn golden_minimal_onion_byte_layout() {
#[test] #[test]
fn byte_exact_format_freeze() { fn byte_exact_format_freeze() {
let re = INDEX_OFFSET; // 128 — RevisionEntry start let re = INDEX_OFFSET; // 128 — RevisionEntry start
let be = BRANCH_OFFSET; // 240 — BranchEntry start let be = BRANCH_OFFSET; // 240 — BranchEntry start
let pt = PT_OFFSET; // 280 — PageTableEntry start let pt = PT_OFFSET; // 280 — PageTableEntry start
let pd = PD_OFFSET; // 312 — PageData start let pd = PD_OFFSET; // 312 — PageData start
// ── Build the actual serialised file ──────────────────────────────────── // ── Build the actual serialised file ────────────────────────────────────
@@ -298,71 +361,75 @@ fn byte_exact_format_freeze() {
let mut expected = vec![0u8; total]; let mut expected = vec![0u8; total];
// §3.1 OnionHeader (128 bytes) // §3.1 OnionHeader (128 bytes)
expected[0..9].copy_from_slice(b"CLAWONION"); // magic expected[0..9].copy_from_slice(b"CLAWONION"); // magic
expected[9] = 0x01; // format_version expected[9] = 0x01; // format_version
// [10..16] = 0 (_pad_align) // [10..16] = 0 (_pad_align)
expected[16..24].copy_from_slice(&le64(0x07)); // feature_flags expected[16..24].copy_from_slice(&le64(0x07)); // feature_flags
expected[24..28].copy_from_slice(&le32(4096)); // page_size expected[24..28].copy_from_slice(&le32(4096)); // page_size
// [28..32] = 0 (_pad_ps) // [28..32] = 0 (_pad_ps)
expected[32..40].copy_from_slice(&le64(1)); // revision_count = 1 expected[32..40].copy_from_slice(&le64(1)); // revision_count = 1
expected[40..44].copy_from_slice(&le32(1)); // branch_count = 1 expected[40..44].copy_from_slice(&le32(1)); // branch_count = 1
// [44..48] = 0 (_pad_bc) // [44..48] = 0 (_pad_bc)
expected[48..56].copy_from_slice(&le64(128)); // index_offset expected[48..56].copy_from_slice(&le64(128)); // index_offset
expected[56..64].copy_from_slice(&le64(240)); // branch_offset expected[56..64].copy_from_slice(&le64(240)); // branch_offset
// [64..72] = 0 (created_at — zeroed, dynamic) // [64..72] = 0 (created_at — zeroed, dynamic)
// [72..128] = 0 (reserved) // [72..128] = 0 (reserved)
// §3.2 RevisionEntry (112 bytes @ 128) // §3.2 RevisionEntry (112 bytes @ 128)
expected[re..re+8] .copy_from_slice(&le64(0)); // revision = 0 expected[re..re + 8].copy_from_slice(&le64(0)); // revision = 0
expected[re+8..re+12].copy_from_slice(&le32(0)); // branch_id = 0 expected[re + 8..re + 12].copy_from_slice(&le32(0)); // branch_id = 0
// [re+12..re+16] = 0 (_pad_bi) // [re+12..re+16] = 0 (_pad_bi)
expected[re+16..re+24].copy_from_slice(&le64(u64::MAX)); // parent_rev = NO_PARENT expected[re + 16..re + 24].copy_from_slice(&le64(u64::MAX)); // parent_rev = NO_PARENT
expected[re+24..re+28].copy_from_slice(&le32(1)); // page_count = 1 expected[re + 24..re + 28].copy_from_slice(&le32(1)); // page_count = 1
// [re+28..re+32] = 0 (_pad_pc) // [re+28..re+32] = 0 (_pad_pc)
expected[re+32..re+40].copy_from_slice(&le64(pt as u64)); // page_table_off expected[re + 32..re + 40].copy_from_slice(&le64(pt as u64)); // page_table_off
// [re+40..re+48] = 0 (timestamp — zeroed, dynamic) // [re+40..re+48] = 0 (timestamp — zeroed, dynamic)
expected[re+48..re+80].copy_from_slice(&expected_blake3(&page_bytes)); // blake3 expected[re + 48..re + 80].copy_from_slice(&expected_blake3(&page_bytes)); // blake3
// [re+80..re+96] = 0 (session_uuid — zeroed, dynamic) // [re+80..re+96] = 0 (session_uuid — zeroed, dynamic)
// [re+96..re+104] = 0 (annotation_off = 0, no annotation) // [re+96..re+104] = 0 (annotation_off = 0, no annotation)
// [re+104] = 0 (flags = 0, not a snapshot) // [re+104] = 0 (flags = 0, not a snapshot)
// [re+105..re+112] = 0 (_pad_flags) // [re+105..re+112] = 0 (_pad_flags)
// §3.3 BranchEntry (40 bytes @ 240) // §3.3 BranchEntry (40 bytes @ 240)
expected[be..be+4] .copy_from_slice(&le32(0)); // id = 0 (main) expected[be..be + 4].copy_from_slice(&le32(0)); // id = 0 (main)
// [be+4..be+8] = 0 (_pad_id) // [be+4..be+8] = 0 (_pad_id)
expected[be+8..be+16].copy_from_slice(&le64(1)); // name_off = 1 (heap offset of "main") expected[be + 8..be + 16].copy_from_slice(&le64(1)); // name_off = 1 (heap offset of "main")
expected[be+16..be+24].copy_from_slice(&le64(0)); // head_rev = 0 expected[be + 16..be + 24].copy_from_slice(&le64(0)); // head_rev = 0
expected[be+24..be+32].copy_from_slice(&le64(u64::MAX)); // fork_rev = NO_PARENT expected[be + 24..be + 32].copy_from_slice(&le64(u64::MAX)); // fork_rev = NO_PARENT
// [be+32..be+40] = 0 (created_at — zeroed, dynamic) // [be+32..be+40] = 0 (created_at — zeroed, dynamic)
// §3.4 PageTableEntry (32 bytes @ 280) // §3.4 PageTableEntry (32 bytes @ 280)
expected[pt..pt+8] .copy_from_slice(&le64(0)); // h5_offset = 0 expected[pt..pt + 8].copy_from_slice(&le64(0)); // h5_offset = 0
expected[pt+8..pt+16].copy_from_slice(&le64(pd as u64)); // data_offset expected[pt + 8..pt + 16].copy_from_slice(&le64(pd as u64)); // data_offset
expected[pt+16..pt+20].copy_from_slice(&le32(4096)); // orig_size expected[pt + 16..pt + 20].copy_from_slice(&le32(4096)); // orig_size
expected[pt+20..pt+24].copy_from_slice(&le32(4096)); // data_size (no compression) expected[pt + 20..pt + 24].copy_from_slice(&le32(4096)); // data_size (no compression)
// [pt+24] = 0 (codec = None) // [pt+24] = 0 (codec = None)
// [pt+25..pt+32] = 0 (_pad) // [pt+25..pt+32] = 0 (_pad)
// PageData (4096 bytes @ 312) // PageData (4096 bytes @ 312)
expected[pd..pd+4096].fill(0xAB); expected[pd..pd + 4096].fill(0xAB);
// §3.5 AnnotationHeap (9 bytes at end) // §3.5 AnnotationHeap (9 bytes at end)
expected[pd+4096..].copy_from_slice(ann_heap); expected[pd + 4096..].copy_from_slice(ann_heap);
// ── Zero dynamic fields in both buffers ────────────────────────────────── // ── Zero dynamic fields in both buffers ──────────────────────────────────
// OnionHeader.created_at // OnionHeader.created_at
actual[64..72].fill(0); actual[64..72].fill(0);
// RevisionEntry.timestamp // RevisionEntry.timestamp
actual[re+40..re+48].fill(0); actual[re + 40..re + 48].fill(0);
// RevisionEntry.session_uuid // RevisionEntry.session_uuid
actual[re+80..re+96].fill(0); actual[re + 80..re + 96].fill(0);
// BranchEntry.created_at // BranchEntry.created_at
actual[be+32..be+40].fill(0); actual[be + 32..be + 40].fill(0);
// ── Compare byte-for-byte ──────────────────────────────────────────────── // ── Compare byte-for-byte ────────────────────────────────────────────────
assert_eq!(actual.len(), expected.len(), assert_eq!(
actual.len(),
expected.len(),
"total file size mismatch: actual={} expected={}", "total file size mismatch: actual={} expected={}",
actual.len(), expected.len()); actual.len(),
expected.len()
);
// Find the first differing byte for a useful failure message. // Find the first differing byte for a useful failure message.
if actual != expected { if actual != expected {
@@ -387,13 +454,29 @@ fn byte_exact_format_freeze() {
#[test] #[test]
fn struct_sizes_match_spec() { fn struct_sizes_match_spec() {
use clawhdf5_onion::format::{BranchEntry, OnionHeader, PageTableEntry, RevisionEntry};
use std::mem::size_of; use std::mem::size_of;
use clawhdf5_onion::format::{OnionHeader, RevisionEntry, BranchEntry, PageTableEntry};
assert_eq!(size_of::<OnionHeader>(), 128, "OnionHeader must be 128 bytes (§3.1)"); assert_eq!(
assert_eq!(size_of::<RevisionEntry>(), 112, "RevisionEntry must be 112 bytes (§3.2)"); size_of::<OnionHeader>(),
assert_eq!(size_of::<BranchEntry>(), 40, "BranchEntry must be 40 bytes (§3.3)"); 128,
assert_eq!(size_of::<PageTableEntry>(), 32, "PageTableEntry must be 32 bytes (§3.4)"); "OnionHeader must be 128 bytes (§3.1)"
);
assert_eq!(
size_of::<RevisionEntry>(),
112,
"RevisionEntry must be 112 bytes (§3.2)"
);
assert_eq!(
size_of::<BranchEntry>(),
40,
"BranchEntry must be 40 bytes (§3.3)"
);
assert_eq!(
size_of::<PageTableEntry>(),
32,
"PageTableEntry must be 32 bytes (§3.4)"
);
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@@ -258,8 +258,13 @@ fn assert_gc_invariants(onion: &OnionFile) {
// 5. blake3_hex has correct length (64 chars) // 5. blake3_hex has correct length (64 chars)
for rev in &revisions { for rev in &revisions {
assert_eq!(rev.blake3_hex.len(), 64, assert_eq!(
"revision {} has blake3_hex of wrong length {}", rev.revision, rev.blake3_hex.len()); rev.blake3_hex.len(),
64,
"revision {} has blake3_hex of wrong length {}",
rev.revision,
rev.blake3_hex.len()
);
} }
// 6. Pages are accessible for every revision // 6. Pages are accessible for every revision
@@ -48,7 +48,11 @@ struct Round {
fn arb_round() -> impl Strategy<Value = Round> { fn arb_round() -> impl Strategy<Value = Round> {
(any::<bool>(), 0usize..8usize, 1u8..=255u8).prop_map(|(create_branch, branch_idx, fill)| { (any::<bool>(), 0usize..8usize, 1u8..=255u8).prop_map(|(create_branch, branch_idx, fill)| {
Round { create_branch, branch_idx, fill } Round {
create_branch,
branch_idx,
fill,
}
}) })
} }
@@ -72,10 +76,7 @@ fn make_vf() -> (TempDir, std::path::PathBuf, VersionedFile) {
/// Apply `rounds` to `vf`, returning a Vec of `(revision_number, expected_page_bytes)`. /// Apply `rounds` to `vf`, returning a Vec of `(revision_number, expected_page_bytes)`.
/// ///
/// `branches` starts as `["main"]` and grows as new branches are created. /// `branches` starts as `["main"]` and grows as new branches are created.
fn apply_rounds( fn apply_rounds(vf: &mut VersionedFile, rounds: &[Round]) -> Vec<(u64, Vec<u8>)> {
vf: &mut VersionedFile,
rounds: &[Round],
) -> Vec<(u64, Vec<u8>)> {
let mut branches: Vec<String> = vec!["main".to_string()]; let mut branches: Vec<String> = vec!["main".to_string()];
let mut expectations: Vec<(u64, Vec<u8>)> = Vec::new(); let mut expectations: Vec<(u64, Vec<u8>)> = Vec::new();
let mut branch_counter: usize = 0; let mut branch_counter: usize = 0;
@@ -124,7 +125,8 @@ fn verify_expectations(
prop_assert_eq!( prop_assert_eq!(
&actual[..page_end], &actual[..page_end],
expected.as_slice(), expected.as_slice(),
"revision {}: page 0 mismatch", rev "revision {}: page 0 mismatch",
rev
); );
} }
Ok(()) Ok(())
@@ -78,17 +78,25 @@ async fn serve_one(server: TcpServer, server_onion_path: PathBuf) {
let server_onion = OnionFile::open(&server_onion_path).unwrap(); let server_onion = OnionFile::open(&server_onion_path).unwrap();
let server_rev_count = server_onion.revision_count(); let server_rev_count = server_onion.revision_count();
let manifest = ClawSyncManifest::from_onion("bench-server", &server_onion, H5_MAGIC); let manifest = ClawSyncManifest::from_onion("bench-server", &server_onion, H5_MAGIC);
conn.send(&SyncMessage::ManifestResponse { manifest }).await.unwrap(); conn.send(&SyncMessage::ManifestResponse { manifest })
.await
.unwrap();
if server_rev_count > client_rev_count { if server_rev_count > client_rev_count {
// Server is ahead — push its extra revisions to client // Server is ahead — push its extra revisions to client
let client_head = if client_rev_count == 0 { NO_PARENT } else { client_rev_count - 1 }; let client_head = if client_rev_count == 0 {
NO_PARENT
} else {
client_rev_count - 1
};
let packets = diff_revisions(&server_onion, client_head).unwrap(); let packets = diff_revisions(&server_onion, client_head).unwrap();
let total = packets.len() as u64; let total = packets.len() as u64;
let mut bytes_sent = 0u64; let mut bytes_sent = 0u64;
for packet in packets { for packet in packets {
bytes_sent += packet.page_data_size() as u64; bytes_sent += packet.page_data_size() as u64;
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap(); conn.send(&SyncMessage::LayerPacket { packet })
.await
.unwrap();
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::Ack { .. } => {} SyncMessage::Ack { .. } => {}
other => panic!("server: expected Ack, got {other:?}"), other => panic!("server: expected Ack, got {other:?}"),
@@ -96,8 +104,10 @@ async fn serve_one(server: TcpServer, server_onion_path: PathBuf) {
} }
conn.send(&SyncMessage::SyncComplete { conn.send(&SyncMessage::SyncComplete {
revisions_transferred: total, revisions_transferred: total,
bytes_transferred: bytes_sent, bytes_transferred: bytes_sent,
}).await.unwrap(); })
.await
.unwrap();
} else { } else {
// Server is behind or equal — receive packets from client // Server is behind or equal — receive packets from client
let mut server_onion_rw = OnionFile::open(&server_onion_path).unwrap(); let mut server_onion_rw = OnionFile::open(&server_onion_path).unwrap();
@@ -106,7 +116,9 @@ async fn serve_one(server: TcpServer, server_onion_path: PathBuf) {
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::LayerPacket { packet } => { SyncMessage::LayerPacket { packet } => {
let rev = packet.revision; let rev = packet.revision;
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap(); conn.send(&SyncMessage::Ack { revision: rev })
.await
.unwrap();
packets.push(packet); packets.push(packet);
} }
SyncMessage::SyncComplete { .. } => { SyncMessage::SyncComplete { .. } => {
@@ -157,7 +169,10 @@ fn bench_e2e(c: &mut Criterion) {
rt.block_on(async { rt.block_on(async {
let server_task = tokio::spawn(serve_one(server, dst_h5)); let server_task = tokio::spawn(serve_one(server, dst_h5));
let backend = TcpSyncBackend::new(addr, "bench-agent"); let backend = TcpSyncBackend::new(addr, "bench-agent");
backend.push(Path::new(&src_h5), &SyncSelector::All).await.unwrap(); backend
.push(Path::new(&src_h5), &SyncSelector::All)
.await
.unwrap();
server_task.await.unwrap(); server_task.await.unwrap();
}); });
drop(src_dir); drop(src_dir);
@@ -275,7 +290,10 @@ fn bench_e2e(c: &mut Criterion) {
let addr = server.local_addr; let addr = server.local_addr;
let server_task = tokio::spawn(serve_one(server, dst_h5)); let server_task = tokio::spawn(serve_one(server, dst_h5));
let backend = TcpSyncBackend::new(addr, "bench-agent"); let backend = TcpSyncBackend::new(addr, "bench-agent");
let stats = backend.push(Path::new(&src_h5), &SyncSelector::All).await.unwrap(); let stats = backend
.push(Path::new(&src_h5), &SyncSelector::All)
.await
.unwrap();
server_task.await.unwrap(); server_task.await.unwrap();
stats.revisions_transferred stats.revisions_transferred
}); });
@@ -78,13 +78,13 @@ async fn serve_with_latency(server: TcpServer, rtt_ms: f64) {
conn.send(&SyncMessage::ManifestResponse { conn.send(&SyncMessage::ManifestResponse {
manifest: ClawSyncManifest { manifest: ClawSyncManifest {
agent_id: "bench-server".to_string(), agent_id: "bench-server".to_string(),
file_blake3: [0u8; 32], file_blake3: [0u8; 32],
revision_count: 0, revision_count: 0,
head_revision: 0, head_revision: 0,
head_blake3: [0u8; 32], head_blake3: [0u8; 32],
revisions: vec![], revisions: vec![],
last_write: 0.0, last_write: 0.0,
}, },
}) })
.await .await
@@ -98,7 +98,9 @@ async fn serve_with_latency(server: TcpServer, rtt_ms: f64) {
let rev = packet.revision; let rev = packet.revision;
// Inject simulated WAN RTT before sending Ack. // Inject simulated WAN RTT before sending Ack.
tokio::time::sleep(delay).await; tokio::time::sleep(delay).await;
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap(); conn.send(&SyncMessage::Ack { revision: rev })
.await
.unwrap();
received += 1; received += 1;
} }
SyncMessage::SyncComplete { .. } => break, SyncMessage::SyncComplete { .. } => break,
@@ -120,8 +122,8 @@ async fn pipelined_push(addr: SocketAddr, onion: &OnionFile, window: usize) {
let rev_count = onion.revision_count(); let rev_count = onion.revision_count();
conn.send(&SyncMessage::ManifestRequest { conn.send(&SyncMessage::ManifestRequest {
agent_id: "bench-client".to_string(), agent_id: "bench-client".to_string(),
head_revision: rev_count.saturating_sub(1), head_revision: rev_count.saturating_sub(1),
revision_count: rev_count, revision_count: rev_count,
}) })
.await .await
@@ -170,7 +172,7 @@ async fn pipelined_push(addr: SocketAddr, onion: &OnionFile, window: usize) {
write_half write_half
.send(&SyncMessage::SyncComplete { .send(&SyncMessage::SyncComplete {
revisions_transferred: sent, revisions_transferred: sent,
bytes_transferred: bytes_sent, bytes_transferred: bytes_sent,
}) })
.await .await
.unwrap(); .unwrap();
@@ -197,12 +199,12 @@ fn bench_wan_pipeline(c: &mut Criterion) {
// At 1 ms RTT: W=1 ≈ 100 ms/iter, W=16 ≈ 7 ms/iter // At 1 ms RTT: W=1 ≈ 100 ms/iter, W=16 ≈ 7 ms/iter
// At 5 ms RTT: W=1 ≈ 500 ms/iter, W=16 ≈ 32 ms/iter // At 5 ms RTT: W=1 ≈ 500 ms/iter, W=16 ≈ 32 ms/iter
let params: &[(f64, usize, &str)] = &[ let params: &[(f64, usize, &str)] = &[
(1.0, 1, "rtt_1ms_W1"), (1.0, 1, "rtt_1ms_W1"),
(1.0, 4, "rtt_1ms_W4"), (1.0, 4, "rtt_1ms_W4"),
(1.0, 16, "rtt_1ms_W16"), (1.0, 16, "rtt_1ms_W16"),
(5.0, 1, "rtt_5ms_W1"), (5.0, 1, "rtt_5ms_W1"),
(5.0, 4, "rtt_5ms_W4"), (5.0, 4, "rtt_5ms_W4"),
(5.0, 16, "rtt_5ms_W16"), (5.0, 16, "rtt_5ms_W16"),
]; ];
for &(rtt_ms, window, label) in params { for &(rtt_ms, window, label) in params {
@@ -223,8 +225,7 @@ fn bench_wan_pipeline(c: &mut Criterion) {
// TIMED: full push with simulated WAN latency // TIMED: full push with simulated WAN latency
let _ = (dir, path); // keep TempDir alive let _ = (dir, path); // keep TempDir alive
rt.block_on(async { rt.block_on(async {
let server_task = let server_task = tokio::spawn(serve_with_latency(server, rtt_ms));
tokio::spawn(serve_with_latency(server, rtt_ms));
pipelined_push(addr, &onion, window).await; pipelined_push(addr, &onion, window).await;
server_task.await.unwrap(); server_task.await.unwrap();
}); });
+74 -28
View File
@@ -42,7 +42,11 @@ pub struct PeerCapabilities {
impl Default for PeerCapabilities { impl Default for PeerCapabilities {
fn default() -> Self { fn default() -> Self {
Self { branching: true, dataset_diff: false, protocol_version: 1 } Self {
branching: true,
dataset_diff: false,
protocol_version: 1,
}
} }
} }
@@ -54,10 +58,17 @@ impl Default for PeerCapabilities {
/// transport (e.g., QUIC, S3, local copy). /// transport (e.g., QUIC, S3, local copy).
pub trait SyncBackend: Send + Sync { pub trait SyncBackend: Send + Sync {
/// Push any revisions the remote lacks from `local_path` to `remote`. /// Push any revisions the remote lacks from `local_path` to `remote`.
fn push(&self, local_path: &Path, selector: &SyncSelector) -> impl std::future::Future<Output = Result<SyncStats, AgentSyncError>> + Send; fn push(
&self,
local_path: &Path,
selector: &SyncSelector,
) -> impl std::future::Future<Output = Result<SyncStats, AgentSyncError>> + Send;
/// Pull any revisions the local file lacks from the remote into `local_path`. /// Pull any revisions the local file lacks from the remote into `local_path`.
fn pull(&self, local_path: &Path) -> impl std::future::Future<Output = Result<SyncStats, AgentSyncError>> + Send; fn pull(
&self,
local_path: &Path,
) -> impl std::future::Future<Output = Result<SyncStats, AgentSyncError>> + Send;
/// Build a manifest for the given path. /// Build a manifest for the given path.
fn local_manifest(&self, local_path: &Path) -> Result<ClawSyncManifest, AgentSyncError>; fn local_manifest(&self, local_path: &Path) -> Result<ClawSyncManifest, AgentSyncError>;
@@ -75,14 +86,20 @@ pub struct TcpSyncBackend {
impl TcpSyncBackend { impl TcpSyncBackend {
pub fn new(remote_addr: SocketAddr, agent_id: impl Into<String>) -> Self { pub fn new(remote_addr: SocketAddr, agent_id: impl Into<String>) -> Self {
Self { remote_addr, agent_id: agent_id.into() } Self {
remote_addr,
agent_id: agent_id.into(),
}
} }
} }
impl SyncBackend for TcpSyncBackend { impl SyncBackend for TcpSyncBackend {
async fn push(&self, local_path: &Path, selector: &SyncSelector) -> Result<SyncStats, AgentSyncError> { async fn push(
let local_onion = OnionFile::open(local_path) &self,
.map_err(AgentSyncError::Onion)?; local_path: &Path,
selector: &SyncSelector,
) -> Result<SyncStats, AgentSyncError> {
let local_onion = OnionFile::open(local_path).map_err(AgentSyncError::Onion)?;
let h5_base = std::fs::read(local_path).map_err(AgentSyncError::Io)?; let h5_base = std::fs::read(local_path).map_err(AgentSyncError::Io)?;
let local_manifest = ClawSyncManifest::from_onion(&self.agent_id, &local_onion, &h5_base); let local_manifest = ClawSyncManifest::from_onion(&self.agent_id, &local_onion, &h5_base);
@@ -98,21 +115,26 @@ impl SyncBackend for TcpSyncBackend {
.collect(); .collect();
let sketch = IbltSketch::from_keys(&local_rev_numbers, IBLT_SYNC_SEED); let sketch = IbltSketch::from_keys(&local_rev_numbers, IBLT_SYNC_SEED);
let iblt_manifest = IbltManifest { let iblt_manifest = IbltManifest {
agent_id: self.agent_id.clone(), agent_id: self.agent_id.clone(),
file_blake3: local_manifest.file_blake3, file_blake3: local_manifest.file_blake3,
revision_count: local_manifest.revision_count, revision_count: local_manifest.revision_count,
head_revision: local_manifest.head_revision, head_revision: local_manifest.head_revision,
head_blake3: local_manifest.head_blake3, head_blake3: local_manifest.head_blake3,
last_write: local_manifest.last_write, last_write: local_manifest.last_write,
sketch_cells: sketch.cell_count() as u32, sketch_cells: sketch.cell_count() as u32,
sketch: sketch.to_bytes(), sketch: sketch.to_bytes(),
}; };
conn.send(&SyncMessage::IbltRequest { sketch: iblt_manifest }) conn.send(&SyncMessage::IbltRequest {
.await sketch: iblt_manifest,
.map_err(AgentSyncError::Transport)?; })
.await
.map_err(AgentSyncError::Transport)?;
let missing_from_remote = match conn.recv().await.map_err(AgentSyncError::Transport)? { let missing_from_remote = match conn.recv().await.map_err(AgentSyncError::Transport)? {
SyncMessage::IbltResponse { missing_from_remote, .. } => missing_from_remote, SyncMessage::IbltResponse {
missing_from_remote,
..
} => missing_from_remote,
SyncMessage::Error { message } => return Err(AgentSyncError::Remote(message)), SyncMessage::Error { message } => return Err(AgentSyncError::Remote(message)),
other => return Err(AgentSyncError::Protocol(format!("unexpected: {other:?}"))), other => return Err(AgentSyncError::Protocol(format!("unexpected: {other:?}"))),
}; };
@@ -127,9 +149,13 @@ impl SyncBackend for TcpSyncBackend {
for packet in packets { for packet in packets {
let sz = packet.page_data_size() as u64; let sz = packet.page_data_size() as u64;
conn.send(&SyncMessage::LayerPacket { packet }).await.map_err(AgentSyncError::Transport)?; conn.send(&SyncMessage::LayerPacket { packet })
.await
.map_err(AgentSyncError::Transport)?;
match conn.recv().await.map_err(AgentSyncError::Transport)? { match conn.recv().await.map_err(AgentSyncError::Transport)? {
SyncMessage::Ack { .. } => { bytes_sent += sz; } SyncMessage::Ack { .. } => {
bytes_sent += sz;
}
SyncMessage::Error { message } => return Err(AgentSyncError::Remote(message)), SyncMessage::Error { message } => return Err(AgentSyncError::Remote(message)),
other => return Err(AgentSyncError::Protocol(format!("unexpected: {other:?}"))), other => return Err(AgentSyncError::Protocol(format!("unexpected: {other:?}"))),
} }
@@ -138,10 +164,16 @@ impl SyncBackend for TcpSyncBackend {
conn.send(&SyncMessage::SyncComplete { conn.send(&SyncMessage::SyncComplete {
revisions_transferred: total, revisions_transferred: total,
bytes_transferred: bytes_sent, bytes_transferred: bytes_sent,
}).await.map_err(AgentSyncError::Transport)?; })
.await
.map_err(AgentSyncError::Transport)?;
conn.shutdown().await.map_err(AgentSyncError::Transport)?; conn.shutdown().await.map_err(AgentSyncError::Transport)?;
Ok(SyncStats { revisions_transferred: total, bytes_transferred: bytes_sent, revisions_skipped: 0 }) Ok(SyncStats {
revisions_transferred: total,
bytes_transferred: bytes_sent,
revisions_skipped: 0,
})
} }
async fn pull(&self, local_path: &Path) -> Result<SyncStats, AgentSyncError> { async fn pull(&self, local_path: &Path) -> Result<SyncStats, AgentSyncError> {
@@ -157,7 +189,9 @@ impl SyncBackend for TcpSyncBackend {
agent_id: self.agent_id.clone(), agent_id: self.agent_id.clone(),
head_revision: local_rev_count.saturating_sub(1), head_revision: local_rev_count.saturating_sub(1),
revision_count: local_rev_count, revision_count: local_rev_count,
}).await.map_err(AgentSyncError::Transport)?; })
.await
.map_err(AgentSyncError::Transport)?;
// Get the server's revision count to detect "already in sync". // Get the server's revision count to detect "already in sync".
let server_rev_count = match conn.recv().await.map_err(AgentSyncError::Transport)? { let server_rev_count = match conn.recv().await.map_err(AgentSyncError::Transport)? {
@@ -171,12 +205,15 @@ impl SyncBackend for TcpSyncBackend {
conn.send(&SyncMessage::SyncComplete { conn.send(&SyncMessage::SyncComplete {
revisions_transferred: 0, revisions_transferred: 0,
bytes_transferred: 0, bytes_transferred: 0,
}).await.map_err(AgentSyncError::Transport)?; })
.await
.map_err(AgentSyncError::Transport)?;
return Ok(SyncStats::default()); return Ok(SyncStats::default());
} }
let mut local_onion = if local_path.exists() { let mut local_onion = if local_path.exists() {
OnionFile::open(local_path).or_else(|_| OnionFile::create_auto(local_path)) OnionFile::open(local_path)
.or_else(|_| OnionFile::create_auto(local_path))
.map_err(AgentSyncError::Onion)? .map_err(AgentSyncError::Onion)?
} else { } else {
std::fs::write(local_path, b"\x89HDF\r\n\x1a\n").map_err(AgentSyncError::Io)?; std::fs::write(local_path, b"\x89HDF\r\n\x1a\n").map_err(AgentSyncError::Io)?;
@@ -189,10 +226,15 @@ impl SyncBackend for TcpSyncBackend {
match conn.recv().await.map_err(AgentSyncError::Transport)? { match conn.recv().await.map_err(AgentSyncError::Transport)? {
SyncMessage::LayerPacket { packet } => { SyncMessage::LayerPacket { packet } => {
let rev = packet.revision; let rev = packet.revision;
conn.send(&SyncMessage::Ack { revision: rev }).await.map_err(AgentSyncError::Transport)?; conn.send(&SyncMessage::Ack { revision: rev })
.await
.map_err(AgentSyncError::Transport)?;
packets.push(packet); packets.push(packet);
} }
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => { SyncMessage::SyncComplete {
revisions_transferred,
bytes_transferred,
} => {
let stats = merge_packets(&mut local_onion, packets, true) let stats = merge_packets(&mut local_onion, packets, true)
.map_err(AgentSyncError::SyncOnion)?; .map_err(AgentSyncError::SyncOnion)?;
return Ok(SyncStats { return Ok(SyncStats {
@@ -210,7 +252,11 @@ impl SyncBackend for TcpSyncBackend {
fn local_manifest(&self, local_path: &Path) -> Result<ClawSyncManifest, AgentSyncError> { fn local_manifest(&self, local_path: &Path) -> Result<ClawSyncManifest, AgentSyncError> {
let onion = OnionFile::open(local_path).map_err(AgentSyncError::Onion)?; let onion = OnionFile::open(local_path).map_err(AgentSyncError::Onion)?;
let h5_base = std::fs::read(local_path).map_err(AgentSyncError::Io)?; let h5_base = std::fs::read(local_path).map_err(AgentSyncError::Io)?;
Ok(ClawSyncManifest::from_onion(&self.agent_id, &onion, &h5_base)) Ok(ClawSyncManifest::from_onion(
&self.agent_id,
&onion,
&h5_base,
))
} }
} }
+37 -9
View File
@@ -77,8 +77,12 @@ impl PeerCapabilities {
let mut out = [0u8; 8]; let mut out = [0u8; 8];
out[..4].copy_from_slice(&self.protocol_version.to_le_bytes()); out[..4].copy_from_slice(&self.protocol_version.to_le_bytes());
let mut flags: u32 = 0; let mut flags: u32 = 0;
if self.branching { flags |= 1 << 0; } if self.branching {
if self.dataset_diff { flags |= 1 << 1; } flags |= 1 << 0;
}
if self.dataset_diff {
flags |= 1 << 1;
}
out[4..8].copy_from_slice(&flags.to_le_bytes()); out[4..8].copy_from_slice(&flags.to_le_bytes());
out out
} }
@@ -105,7 +109,11 @@ mod tests {
use crate::backend::PeerCapabilities; use crate::backend::PeerCapabilities;
fn caps(branching: bool, dataset_diff: bool, version: u32) -> PeerCapabilities { fn caps(branching: bool, dataset_diff: bool, version: u32) -> PeerCapabilities {
PeerCapabilities { branching, dataset_diff, protocol_version: version } PeerCapabilities {
branching,
dataset_diff,
protocol_version: version,
}
} }
#[test] #[test]
@@ -123,7 +131,7 @@ mod tests {
let a = caps(true, true, 2); let a = caps(true, true, 2);
let b = caps(false, true, 1); let b = caps(false, true, 1);
let s = negotiate(&a, &b); let s = negotiate(&a, &b);
assert!(!s.branching); // b doesn't support it assert!(!s.branching); // b doesn't support it
assert!(s.dataset_diff); // both support assert!(s.dataset_diff); // both support
assert_eq!(s.protocol_version, 1); // min assert_eq!(s.protocol_version, 1); // min
} }
@@ -140,27 +148,43 @@ mod tests {
#[test] #[test]
fn assert_compatible_ok() { fn assert_compatible_ok() {
let s = SessionCapabilities { branching: true, dataset_diff: false, protocol_version: 1 }; let s = SessionCapabilities {
branching: true,
dataset_diff: false,
protocol_version: 1,
};
assert!(assert_compatible(&s, true, false, 1).is_ok()); assert!(assert_compatible(&s, true, false, 1).is_ok());
} }
#[test] #[test]
fn assert_compatible_missing_branching() { fn assert_compatible_missing_branching() {
let s = SessionCapabilities { branching: false, dataset_diff: false, protocol_version: 1 }; let s = SessionCapabilities {
branching: false,
dataset_diff: false,
protocol_version: 1,
};
let err = assert_compatible(&s, true, false, 1).unwrap_err(); let err = assert_compatible(&s, true, false, 1).unwrap_err();
assert!(err.contains("branching")); assert!(err.contains("branching"));
} }
#[test] #[test]
fn assert_compatible_version_too_low() { fn assert_compatible_version_too_low() {
let s = SessionCapabilities { branching: true, dataset_diff: false, protocol_version: 1 }; let s = SessionCapabilities {
branching: true,
dataset_diff: false,
protocol_version: 1,
};
let err = assert_compatible(&s, false, false, 2).unwrap_err(); let err = assert_compatible(&s, false, false, 2).unwrap_err();
assert!(err.contains("protocol_version")); assert!(err.contains("protocol_version"));
} }
#[test] #[test]
fn capabilities_wire_roundtrip() { fn capabilities_wire_roundtrip() {
let orig = PeerCapabilities { branching: true, dataset_diff: true, protocol_version: 3 }; let orig = PeerCapabilities {
branching: true,
dataset_diff: true,
protocol_version: 3,
};
let bytes = orig.to_bytes(); let bytes = orig.to_bytes();
let decoded = PeerCapabilities::from_bytes(&bytes); let decoded = PeerCapabilities::from_bytes(&bytes);
assert_eq!(decoded.branching, orig.branching); assert_eq!(decoded.branching, orig.branching);
@@ -170,7 +194,11 @@ mod tests {
#[test] #[test]
fn capabilities_wire_no_features() { fn capabilities_wire_no_features() {
let orig = PeerCapabilities { branching: false, dataset_diff: false, protocol_version: 0 }; let orig = PeerCapabilities {
branching: false,
dataset_diff: false,
protocol_version: 0,
};
let bytes = orig.to_bytes(); let bytes = orig.to_bytes();
let decoded = PeerCapabilities::from_bytes(&bytes); let decoded = PeerCapabilities::from_bytes(&bytes);
assert!(!decoded.branching); assert!(!decoded.branching);
+42 -15
View File
@@ -47,16 +47,28 @@ impl OnionMemory {
/// Create a new `OnionMemory` at `h5_path`. /// Create a new `OnionMemory` at `h5_path`.
pub fn create(config: MemoryConfig) -> Result<Self, AgentSyncError> { pub fn create(config: MemoryConfig) -> Result<Self, AgentSyncError> {
let h5_path = config.path.clone(); let h5_path = config.path.clone();
let inner = HDF5Memory::create(config).map_err(|e| AgentSyncError::AgentMemory(e.to_string()))?; let inner =
HDF5Memory::create(config).map_err(|e| AgentSyncError::AgentMemory(e.to_string()))?;
let onion = OnionFile::create(&h5_path, 4096).map_err(AgentSyncError::Onion)?; let onion = OnionFile::create(&h5_path, 4096).map_err(AgentSyncError::Onion)?;
Ok(Self { inner, onion, h5_path, auto_checkpoint: true }) Ok(Self {
inner,
onion,
h5_path,
auto_checkpoint: true,
})
} }
/// Open an existing `OnionMemory` from `h5_path`. /// Open an existing `OnionMemory` from `h5_path`.
pub fn open(h5_path: &Path) -> Result<Self, AgentSyncError> { pub fn open(h5_path: &Path) -> Result<Self, AgentSyncError> {
let inner = HDF5Memory::open(h5_path).map_err(|e| AgentSyncError::AgentMemory(e.to_string()))?; let inner =
HDF5Memory::open(h5_path).map_err(|e| AgentSyncError::AgentMemory(e.to_string()))?;
let onion = OnionFile::open(h5_path).map_err(AgentSyncError::Onion)?; let onion = OnionFile::open(h5_path).map_err(AgentSyncError::Onion)?;
Ok(Self { inner, onion, h5_path: h5_path.to_owned(), auto_checkpoint: true }) Ok(Self {
inner,
onion,
h5_path: h5_path.to_owned(),
auto_checkpoint: true,
})
} }
/// Commit a new onion revision of the current HDF5 state. /// Commit a new onion revision of the current HDF5 state.
@@ -71,14 +83,20 @@ impl OnionMemory {
let page_size = self.onion.page_size() as usize; let page_size = self.onion.page_size() as usize;
// Record all pages from the HDF5 file // Record all pages from the HDF5 file
let mut session = self.onion.begin_session(None).map_err(AgentSyncError::Onion)?; let mut session = self
.onion
.begin_session(None)
.map_err(AgentSyncError::Onion)?;
for (chunk_idx, chunk) in h5_bytes.chunks(page_size).enumerate() { for (chunk_idx, chunk) in h5_bytes.chunks(page_size).enumerate() {
let offset = (chunk_idx * page_size) as u64; let offset = (chunk_idx * page_size) as u64;
let mut page = vec![0u8; page_size]; let mut page = vec![0u8; page_size];
page[..chunk.len()].copy_from_slice(chunk); page[..chunk.len()].copy_from_slice(chunk);
session.record_page(offset, &page); session.record_page(offset, &page);
} }
let rev = self.onion.commit_session(session, annotation).map_err(AgentSyncError::Onion)?; let rev = self
.onion
.commit_session(session, annotation)
.map_err(AgentSyncError::Onion)?;
self.onion.flush().map_err(AgentSyncError::Onion)?; self.onion.flush().map_err(AgentSyncError::Onion)?;
Ok(rev) Ok(rev)
} }
@@ -89,12 +107,11 @@ impl OnionMemory {
/// current file, opens an `HDF5Memory` from it, then removes the temp /// current file, opens an `HDF5Memory` from it, then removes the temp
/// file. Returns the restored memory (detached from the onion sidecar). /// file. Returns the restored memory (detached from the onion sidecar).
pub fn restore_revision(&self, revision: u64) -> Result<HDF5Memory, AgentSyncError> { pub fn restore_revision(&self, revision: u64) -> Result<HDF5Memory, AgentSyncError> {
let (h5_bytes, _) = open_revision(&self.h5_path, revision) let (h5_bytes, _) =
.map_err(AgentSyncError::Onion)?; open_revision(&self.h5_path, revision).map_err(AgentSyncError::Onion)?;
let tmp = self.h5_path.with_extension(format!("h5.rev{revision}.tmp")); let tmp = self.h5_path.with_extension(format!("h5.rev{revision}.tmp"));
std::fs::write(&tmp, &h5_bytes).map_err(AgentSyncError::Io)?; std::fs::write(&tmp, &h5_bytes).map_err(AgentSyncError::Io)?;
let result = HDF5Memory::open(&tmp) let result = HDF5Memory::open(&tmp).map_err(|e| AgentSyncError::AgentMemory(e.to_string()));
.map_err(|e| AgentSyncError::AgentMemory(e.to_string()));
let _ = std::fs::remove_file(&tmp); let _ = std::fs::remove_file(&tmp);
result result
} }
@@ -353,7 +370,11 @@ mod tests {
let head = mem.checkpoint(Some("three")).unwrap(); let head = mem.checkpoint(Some("three")).unwrap();
let restored = mem.restore_revision(head).unwrap(); let restored = mem.restore_revision(head).unwrap();
assert_eq!(restored.count(), mem.count(), "restoring HEAD gives same count"); assert_eq!(
restored.count(),
mem.count(),
"restoring HEAD gives same count"
);
} }
/// Multiple checkpoints can each be individually restored. /// Multiple checkpoints can each be individually restored.
@@ -373,8 +394,10 @@ mod tests {
for i in 0..5u64 { for i in 0..5u64 {
let r = mem.restore_revision(i).unwrap(); let r = mem.restore_revision(i).unwrap();
assert_eq!( assert_eq!(
r.count() as u64, i + 1, r.count() as u64,
"rev {i} should have {} entries", i + 1 i + 1,
"rev {i} should have {} entries",
i + 1
); );
} }
} }
@@ -387,7 +410,10 @@ mod tests {
mem.checkpoint(None).unwrap(); // only rev 0 exists mem.checkpoint(None).unwrap(); // only rev 0 exists
let result = mem.restore_revision(99); let result = mem.restore_revision(99);
assert!(result.is_err(), "restoring a non-existent revision must error"); assert!(
result.is_err(),
"restoring a non-existent revision must error"
);
} }
/// `save_batch` with auto_checkpoint creates exactly one snapshot per call. /// `save_batch` with auto_checkpoint creates exactly one snapshot per call.
@@ -417,7 +443,8 @@ mod tests {
mem.checkpoint(Some("original")).unwrap(); mem.checkpoint(Some("original")).unwrap();
// Rev 1: replace with entirely different text // Rev 1: replace with entirely different text
mem.save(sample_entry("completely-different-content")).unwrap(); mem.save(sample_entry("completely-different-content"))
.unwrap();
mem.checkpoint(Some("updated")).unwrap(); mem.checkpoint(Some("updated")).unwrap();
assert_eq!(mem.count(), 2); assert_eq!(mem.count(), 2);
+23 -5
View File
@@ -96,7 +96,11 @@ pub fn tcp_scheduler(
let addr = remote let addr = remote
.parse() .parse()
.map_err(|_| AgentSyncError::Protocol(format!("invalid address: {remote}")))?; .map_err(|_| AgentSyncError::Protocol(format!("invalid address: {remote}")))?;
Ok(SyncScheduler::new(TcpSyncBackend::new(addr, agent_id), h5_path, selector)) Ok(SyncScheduler::new(
TcpSyncBackend::new(addr, agent_id),
h5_path,
selector,
))
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@@ -134,9 +138,19 @@ mod tests {
} }
impl SyncBackend for StubBackend { impl SyncBackend for StubBackend {
async fn push(&self, path: &Path, _sel: &SyncSelector) -> Result<SyncStats, AgentSyncError> { async fn push(
self.push_calls.lock().unwrap().push(path.to_string_lossy().to_string()); &self,
Ok(SyncStats { revisions_transferred: 1, ..Default::default() }) path: &Path,
_sel: &SyncSelector,
) -> Result<SyncStats, AgentSyncError> {
self.push_calls
.lock()
.unwrap()
.push(path.to_string_lossy().to_string());
Ok(SyncStats {
revisions_transferred: 1,
..Default::default()
})
} }
async fn pull(&self, _path: &Path) -> Result<SyncStats, AgentSyncError> { async fn pull(&self, _path: &Path) -> Result<SyncStats, AgentSyncError> {
Ok(SyncStats::default()) Ok(SyncStats::default())
@@ -189,7 +203,11 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn push_force_always_pushes() { async fn push_force_always_pushes() {
let backend = StubBackend::new(2); let backend = StubBackend::new(2);
let sched = SyncScheduler::new(backend.clone(), PathBuf::from("/tmp/x.h5"), SyncSelector::All); let sched = SyncScheduler::new(
backend.clone(),
PathBuf::from("/tmp/x.h5"),
SyncSelector::All,
);
sched.mark_pushed(1); // up-to-date sched.mark_pushed(1); // up-to-date
// force push anyway // force push anyway
@@ -48,7 +48,9 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
// ── IBLT pre-flight (push direction) ───────────────────────────── // ── IBLT pre-flight (push direction) ─────────────────────────────
SyncMessage::IbltRequest { sketch: client_iblt } => { SyncMessage::IbltRequest {
sketch: client_iblt,
} => {
let server_revs: Vec<u64> = dst.list_revisions().iter().map(|s| s.revision).collect(); let server_revs: Vec<u64> = dst.list_revisions().iter().map(|s| s.revision).collect();
// Decode diff: what client has that server lacks // Decode diff: what client has that server lacks
@@ -59,26 +61,30 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
let seed = client_sketch.seed(); let seed = client_sketch.seed();
let srv_sketch = IbltSketch::from_keys(&server_revs, seed); let srv_sketch = IbltSketch::from_keys(&server_revs, seed);
let server_iblt = IbltManifest { let server_iblt = IbltManifest {
agent_id: "test-server".to_string(), agent_id: "test-server".to_string(),
file_blake3: [0u8; 32], file_blake3: [0u8; 32],
revision_count: server_revs.len() as u64, revision_count: server_revs.len() as u64,
head_revision: server_revs.last().copied().unwrap_or(0), head_revision: server_revs.last().copied().unwrap_or(0),
head_blake3: [0u8; 32], head_blake3: [0u8; 32],
last_write: 0.0, last_write: 0.0,
sketch_cells: srv_sketch.cell_count() as u32, sketch_cells: srv_sketch.cell_count() as u32,
sketch: srv_sketch.to_bytes(), sketch: srv_sketch.to_bytes(),
}; };
conn.send(&SyncMessage::IbltResponse { conn.send(&SyncMessage::IbltResponse {
sketch: server_iblt, sketch: server_iblt,
missing_from_remote: diff.only_in_b.clone(), // client has, server lacks missing_from_remote: diff.only_in_b.clone(), // client has, server lacks
}).await.unwrap(); })
.await
.unwrap();
let mut packets = Vec::new(); let mut packets = Vec::new();
loop { loop {
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::LayerPacket { packet } => { SyncMessage::LayerPacket { packet } => {
let rev = packet.revision; let rev = packet.revision;
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap(); conn.send(&SyncMessage::Ack { revision: rev })
.await
.unwrap();
packets.push(packet); packets.push(packet);
} }
SyncMessage::SyncComplete { .. } => break, SyncMessage::SyncComplete { .. } => break,
@@ -91,19 +97,30 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
} }
// ── Flat manifest (pull direction) ──────────────────────────────── // ── Flat manifest (pull direction) ────────────────────────────────
SyncMessage::ManifestRequest { revision_count: remote_rev_count, .. } => { SyncMessage::ManifestRequest {
revision_count: remote_rev_count,
..
} => {
let local_rev_count = dst.revision_count(); let local_rev_count = dst.revision_count();
let manifest = ClawSyncManifest::from_onion("server", &dst, H5_MAGIC); let manifest = ClawSyncManifest::from_onion("server", &dst, H5_MAGIC);
conn.send(&SyncMessage::ManifestResponse { manifest }).await.unwrap(); conn.send(&SyncMessage::ManifestResponse { manifest })
.await
.unwrap();
if local_rev_count > remote_rev_count { if local_rev_count > remote_rev_count {
let remote_head = if remote_rev_count == 0 { NO_PARENT } else { remote_rev_count - 1 }; let remote_head = if remote_rev_count == 0 {
NO_PARENT
} else {
remote_rev_count - 1
};
let packets = diff_revisions(&dst, remote_head).unwrap(); let packets = diff_revisions(&dst, remote_head).unwrap();
let total = packets.len() as u64; let total = packets.len() as u64;
let mut bytes_sent = 0u64; let mut bytes_sent = 0u64;
for packet in packets { for packet in packets {
bytes_sent += packet.page_data_size() as u64; bytes_sent += packet.page_data_size() as u64;
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap(); conn.send(&SyncMessage::LayerPacket { packet })
.await
.unwrap();
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::Ack { .. } => {} SyncMessage::Ack { .. } => {}
other => panic!("expected Ack, got {other:?}"), other => panic!("expected Ack, got {other:?}"),
@@ -112,14 +129,18 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
conn.send(&SyncMessage::SyncComplete { conn.send(&SyncMessage::SyncComplete {
revisions_transferred: total, revisions_transferred: total,
bytes_transferred: bytes_sent, bytes_transferred: bytes_sent,
}).await.unwrap(); })
.await
.unwrap();
} else { } else {
let mut packets = Vec::new(); let mut packets = Vec::new();
loop { loop {
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::LayerPacket { packet } => { SyncMessage::LayerPacket { packet } => {
let rev = packet.revision; let rev = packet.revision;
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap(); conn.send(&SyncMessage::Ack { revision: rev })
.await
.unwrap();
packets.push(packet); packets.push(packet);
} }
SyncMessage::SyncComplete { .. } => break, SyncMessage::SyncComplete { .. } => break,
@@ -241,7 +262,10 @@ async fn backend_pull_already_in_sync() {
let backend = TcpSyncBackend::new(addr, "agent"); let backend = TcpSyncBackend::new(addr, "agent");
let stats = backend.pull(&cli_h5).await.unwrap(); let stats = backend.pull(&cli_h5).await.unwrap();
assert_eq!(stats.revisions_transferred, 0, "nothing to pull when in sync"); assert_eq!(
stats.revisions_transferred, 0,
"nothing to pull when in sync"
);
let _ = (cli_onion, server_task.await.unwrap()); let _ = (cli_onion, server_task.await.unwrap());
} }
@@ -34,7 +34,9 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
// ── IBLT pre-flight (push direction) ───────────────────────────── // ── IBLT pre-flight (push direction) ─────────────────────────────
SyncMessage::IbltRequest { sketch: client_iblt } => { SyncMessage::IbltRequest {
sketch: client_iblt,
} => {
let server_revs: Vec<u64> = dst.list_revisions().iter().map(|s| s.revision).collect(); let server_revs: Vec<u64> = dst.list_revisions().iter().map(|s| s.revision).collect();
let diff = client_iblt.diff_against(&server_revs).unwrap(); let diff = client_iblt.diff_against(&server_revs).unwrap();
@@ -43,26 +45,30 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
let seed = client_sketch.seed(); let seed = client_sketch.seed();
let srv_sketch = IbltSketch::from_keys(&server_revs, seed); let srv_sketch = IbltSketch::from_keys(&server_revs, seed);
let server_iblt = IbltManifest { let server_iblt = IbltManifest {
agent_id: "test-server".to_string(), agent_id: "test-server".to_string(),
file_blake3: [0u8; 32], file_blake3: [0u8; 32],
revision_count: server_revs.len() as u64, revision_count: server_revs.len() as u64,
head_revision: server_revs.last().copied().unwrap_or(0), head_revision: server_revs.last().copied().unwrap_or(0),
head_blake3: [0u8; 32], head_blake3: [0u8; 32],
last_write: 0.0, last_write: 0.0,
sketch_cells: srv_sketch.cell_count() as u32, sketch_cells: srv_sketch.cell_count() as u32,
sketch: srv_sketch.to_bytes(), sketch: srv_sketch.to_bytes(),
}; };
conn.send(&SyncMessage::IbltResponse { conn.send(&SyncMessage::IbltResponse {
sketch: server_iblt, sketch: server_iblt,
missing_from_remote: diff.only_in_b.clone(), missing_from_remote: diff.only_in_b.clone(),
}).await.unwrap(); })
.await
.unwrap();
let mut packets = Vec::new(); let mut packets = Vec::new();
loop { loop {
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::LayerPacket { packet } => { SyncMessage::LayerPacket { packet } => {
let rev = packet.revision; let rev = packet.revision;
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap(); conn.send(&SyncMessage::Ack { revision: rev })
.await
.unwrap();
packets.push(packet); packets.push(packet);
} }
SyncMessage::SyncComplete { .. } => break, SyncMessage::SyncComplete { .. } => break,
@@ -75,19 +81,30 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
} }
// ── Flat manifest (pull direction) ──────────────────────────────── // ── Flat manifest (pull direction) ────────────────────────────────
SyncMessage::ManifestRequest { revision_count: remote_rev_count, .. } => { SyncMessage::ManifestRequest {
revision_count: remote_rev_count,
..
} => {
let local_rev_count = dst.revision_count(); let local_rev_count = dst.revision_count();
let manifest = ClawSyncManifest::from_onion("server", &dst, H5_MAGIC); let manifest = ClawSyncManifest::from_onion("server", &dst, H5_MAGIC);
conn.send(&SyncMessage::ManifestResponse { manifest }).await.unwrap(); conn.send(&SyncMessage::ManifestResponse { manifest })
.await
.unwrap();
if local_rev_count > remote_rev_count { if local_rev_count > remote_rev_count {
let remote_head = if remote_rev_count == 0 { NO_PARENT } else { remote_rev_count - 1 }; let remote_head = if remote_rev_count == 0 {
NO_PARENT
} else {
remote_rev_count - 1
};
let packets = diff_revisions(&dst, remote_head).unwrap(); let packets = diff_revisions(&dst, remote_head).unwrap();
let total = packets.len() as u64; let total = packets.len() as u64;
let mut bytes_sent = 0u64; let mut bytes_sent = 0u64;
for packet in packets { for packet in packets {
bytes_sent += packet.page_data_size() as u64; bytes_sent += packet.page_data_size() as u64;
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap(); conn.send(&SyncMessage::LayerPacket { packet })
.await
.unwrap();
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::Ack { .. } => {} SyncMessage::Ack { .. } => {}
other => panic!("expected Ack, got {other:?}"), other => panic!("expected Ack, got {other:?}"),
@@ -96,14 +113,18 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
conn.send(&SyncMessage::SyncComplete { conn.send(&SyncMessage::SyncComplete {
revisions_transferred: total, revisions_transferred: total,
bytes_transferred: bytes_sent, bytes_transferred: bytes_sent,
}).await.unwrap(); })
.await
.unwrap();
} else { } else {
let mut packets = Vec::new(); let mut packets = Vec::new();
loop { loop {
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::LayerPacket { packet } => { SyncMessage::LayerPacket { packet } => {
let rev = packet.revision; let rev = packet.revision;
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap(); conn.send(&SyncMessage::Ack { revision: rev })
.await
.unwrap();
packets.push(packet); packets.push(packet);
} }
SyncMessage::SyncComplete { .. } => break, SyncMessage::SyncComplete { .. } => break,
@@ -124,8 +145,13 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
/// Create a temp HDF5 file with `n` committed onion revisions. /// Create a temp HDF5 file with `n` committed onion revisions.
fn make_onion(n: u8, dir: &TempDir) -> (PathBuf, OnionFile) { fn make_onion(n: u8, dir: &TempDir) -> (PathBuf, OnionFile) {
let h5 = dir.path().join(format!("data_{n}_{}.h5", std::time::SystemTime::now() let h5 = dir.path().join(format!(
.duration_since(std::time::UNIX_EPOCH).unwrap().subsec_nanos())); "data_{n}_{}.h5",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.subsec_nanos()
));
std::fs::write(&h5, H5_MAGIC).unwrap(); std::fs::write(&h5, H5_MAGIC).unwrap();
let mut onion = OnionFile::create(&h5, 4096).unwrap(); let mut onion = OnionFile::create(&h5, 4096).unwrap();
for i in 0..n { for i in 0..n {
@@ -288,7 +314,8 @@ async fn tcp_scheduler_constructor_and_push() {
src_h5.clone(), src_h5.clone(),
"test-agent", "test-agent",
SyncSelector::All, SyncSelector::All,
).unwrap(); )
.unwrap();
let stats = sched.push_force().await.unwrap(); let stats = sched.push_force().await.unwrap();
assert_eq!(stats.revisions_transferred, 3); assert_eq!(stats.revisions_transferred, 3);
+259 -148
View File
@@ -26,21 +26,21 @@ use clawhdf5_onion::api::{
create_branch, delete_branch, list_branches, list_revisions, rename_branch, rollback, create_branch, delete_branch, list_branches, list_revisions, rename_branch, rollback,
}; };
use clawhdf5_onion::branch::MergeStrategy; use clawhdf5_onion::branch::MergeStrategy;
use clawhdf5_onion::gc::GcPolicy;
use clawhdf5_onion::format::NO_PARENT; use clawhdf5_onion::format::NO_PARENT;
use clawhdf5_onion::gc::GcPolicy;
use clawhdf5_onion::writer::OnionFile; use clawhdf5_onion::writer::OnionFile;
use clawsync_onion::differ::{diff_revisions, packets_for_revisions}; use clawsync_onion::differ::{diff_revisions, packets_for_revisions};
use clawsync_onion::iblt::{IbltSketch, IBLT_SYNC_SEED}; use clawsync_onion::iblt::{IBLT_SYNC_SEED, IbltSketch};
use clawsync_onion::manifest::{ClawSyncManifest, IbltManifest}; use clawsync_onion::manifest::{ClawSyncManifest, IbltManifest};
use clawsync_onion::merger::merge_packets; use clawsync_onion::merger::merge_packets;
use std::sync::Arc; use std::sync::Arc;
use clawsync_fs::{FsSyncClient, FsSyncServer};
use clawsync_hdf5::{DatasetManifest, ReceivedDataset, apply_received_payloads};
use clawsync_transport::peer::{PipeWriteHalf, SyncPeer}; use clawsync_transport::peer::{PipeWriteHalf, SyncPeer};
use clawsync_transport::protocol::SyncMessage; use clawsync_transport::protocol::SyncMessage;
use clawsync_transport::quic::{QuicConfig, QuicServer, quic_connect}; use clawsync_transport::quic::{QuicConfig, QuicServer, quic_connect};
use clawsync_transport::tcp::{TcpConnection, TcpServer}; use clawsync_transport::tcp::{TcpConnection, TcpServer};
use clawsync_fs::{FsSyncClient, FsSyncServer};
use clawsync_hdf5::{DatasetManifest, ReceivedDataset, apply_received_payloads};
/// Maximum number of `LayerPacket`s in flight before the sender blocks waiting /// Maximum number of `LayerPacket`s in flight before the sender blocks waiting
/// for acks. A window of 16 reduces WAN round-trip overhead by up to 16× /// for acks. A window of 16 reduces WAN round-trip overhead by up to 16×
@@ -326,7 +326,9 @@ enum BranchAction {
/// Parse `host:port/path` into `(SocketAddr, PathBuf)`. /// Parse `host:port/path` into `(SocketAddr, PathBuf)`.
fn parse_remote(remote: &str) -> Result<(SocketAddr, PathBuf)> { fn parse_remote(remote: &str) -> Result<(SocketAddr, PathBuf)> {
let slash = remote.find('/').context("remote must be 'host:port/path'")?; let slash = remote
.find('/')
.context("remote must be 'host:port/path'")?;
let addr_str = &remote[..slash]; let addr_str = &remote[..slash];
let path_str = &remote[slash + 1..]; let path_str = &remote[slash + 1..];
let addr: SocketAddr = addr_str let addr: SocketAddr = addr_str
@@ -356,7 +358,9 @@ async fn cmd_list_revisions(h5_path: PathBuf, verbose: bool) -> Result<()> {
print!("{:>6} {:>4} {:<30} {}", r.revision, r.branch_id, ann, ts); print!("{:>6} {:>4} {:<30} {}", r.revision, r.branch_id, ann, ts);
if verbose { if verbose {
print!(" blake3={}", &r.blake3_hex[..12]); print!(" blake3={}", &r.blake3_hex[..12]);
if r.is_snapshot { print!(" [snapshot]"); } if r.is_snapshot {
print!(" [snapshot]");
}
} }
println!(); println!();
} }
@@ -365,7 +369,11 @@ async fn cmd_list_revisions(h5_path: PathBuf, verbose: bool) -> Result<()> {
} }
async fn cmd_rollback(h5_path: PathBuf, revision: u64) -> Result<()> { async fn cmd_rollback(h5_path: PathBuf, revision: u64) -> Result<()> {
println!("Rolling back {} to revision {}...", h5_path.display(), revision); println!(
"Rolling back {} to revision {}...",
h5_path.display(),
revision
);
rollback(&h5_path, revision) rollback(&h5_path, revision)
.with_context(|| format!("rollback failed for {}", h5_path.display()))?; .with_context(|| format!("rollback failed for {}", h5_path.display()))?;
println!("Done."); println!("Done.");
@@ -389,13 +397,13 @@ async fn cmd_push(
); );
// Load local onion — O(sidecar size), does NOT read the large h5 base file. // Load local onion — O(sidecar size), does NOT read the large h5 base file.
let local_onion = OnionFile::open(&local) let local_onion =
.with_context(|| format!("cannot open {}", local.display()))?; OnionFile::open(&local).with_context(|| format!("cannot open {}", local.display()))?;
// Connect to remote — TCP or QUIC depending on flag. // Connect to remote — TCP or QUIC depending on flag.
let mut peer: SyncPeer = if quic { let mut peer: SyncPeer = if quic {
let cfg = QuicConfig::self_signed() let cfg =
.with_context(|| "failed to build QUIC self-signed config")?; QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
let conn = quic_connect(addr, "localhost", cfg) let conn = quic_connect(addr, "localhost", cfg)
.await .await
.with_context(|| format!("QUIC connect to {addr} failed"))?; .with_context(|| format!("QUIC connect to {addr} failed"))?;
@@ -418,20 +426,26 @@ async fn cmd_push(
.collect(); .collect();
let sketch = IbltSketch::from_keys(&local_rev_numbers, IBLT_SYNC_SEED); let sketch = IbltSketch::from_keys(&local_rev_numbers, IBLT_SYNC_SEED);
let iblt_manifest = IbltManifest { let iblt_manifest = IbltManifest {
agent_id: "push-client".to_string(), agent_id: "push-client".to_string(),
file_blake3: [0u8; 32], // filled lazily; not needed for IBLT pre-flight file_blake3: [0u8; 32], // filled lazily; not needed for IBLT pre-flight
revision_count: local_rev_numbers.len() as u64, revision_count: local_rev_numbers.len() as u64,
head_revision: local_rev_numbers.last().copied().unwrap_or(0), head_revision: local_rev_numbers.last().copied().unwrap_or(0),
head_blake3: [0u8; 32], head_blake3: [0u8; 32],
last_write: 0.0, last_write: 0.0,
sketch_cells: sketch.cell_count() as u32, sketch_cells: sketch.cell_count() as u32,
sketch: sketch.to_bytes(), sketch: sketch.to_bytes(),
}; };
peer.send(&SyncMessage::IbltRequest { sketch: iblt_manifest }).await?; peer.send(&SyncMessage::IbltRequest {
sketch: iblt_manifest,
})
.await?;
// Receive the server's IBLT response: which revisions we should push. // Receive the server's IBLT response: which revisions we should push.
let missing_from_remote = match peer.recv().await? { let missing_from_remote = match peer.recv().await? {
SyncMessage::IbltResponse { sketch: server_iblt, missing_from_remote } => { SyncMessage::IbltResponse {
sketch: server_iblt,
missing_from_remote,
} => {
println!( println!(
"Remote has {} revision(s) (IBLT). Pushing {} revision(s).", "Remote has {} revision(s) (IBLT). Pushing {} revision(s).",
server_iblt.revision_count, server_iblt.revision_count,
@@ -476,20 +490,23 @@ async fn cmd_push(
let sem_writer = semaphore.clone(); let sem_writer = semaphore.clone();
// Return write_half from the task so we can send SyncComplete after the loop. // Return write_half from the task so we can send SyncComplete after the loop.
let writer_task: tokio::task::JoinHandle<Result<PipeWriteHalf>> = let writer_task: tokio::task::JoinHandle<Result<PipeWriteHalf>> = tokio::spawn(async move {
tokio::spawn(async move { for packet in packets {
for packet in packets { let rev = packet.revision;
let rev = packet.revision; let pkt_bytes = packet.page_data_size() as u64;
let pkt_bytes = packet.page_data_size() as u64; sem_writer
sem_writer.acquire().await.expect("semaphore closed").forget(); .acquire()
meta_tx.send((rev, pkt_bytes)).expect("meta channel closed"); .await
write_half .expect("semaphore closed")
.send(&SyncMessage::LayerPacket { packet }) .forget();
.await meta_tx.send((rev, pkt_bytes)).expect("meta channel closed");
.map_err(|e| anyhow::anyhow!("send error: {e}"))?; write_half
} .send(&SyncMessage::LayerPacket { packet })
Ok(write_half) .await
}); .map_err(|e| anyhow::anyhow!("send error: {e}"))?;
}
Ok(write_half)
});
let mut sent = 0u64; let mut sent = 0u64;
let mut bytes_sent = 0u64; let mut bytes_sent = 0u64;
@@ -516,7 +533,7 @@ async fn cmd_push(
write_half write_half
.send(&SyncMessage::SyncComplete { .send(&SyncMessage::SyncComplete {
revisions_transferred: sent, revisions_transferred: sent,
bytes_transferred: bytes_sent, bytes_transferred: bytes_sent,
}) })
.await?; .await?;
write_half.shutdown().await?; write_half.shutdown().await?;
@@ -548,8 +565,7 @@ async fn cmd_pull(
.unwrap_or(0) .unwrap_or(0)
} else { } else {
// Create empty base file // Create empty base file
std::fs::write(&local, b"\x89HDF\r\n\x1a\n") std::fs::write(&local, b"\x89HDF\r\n\x1a\n").context("cannot create local h5")?;
.context("cannot create local h5")?;
0 0
}; };
@@ -557,8 +573,8 @@ async fn cmd_pull(
// Connect — TCP or QUIC. // Connect — TCP or QUIC.
let mut peer: SyncPeer = if quic { let mut peer: SyncPeer = if quic {
let cfg = QuicConfig::self_signed() let cfg =
.with_context(|| "failed to build QUIC self-signed config")?; QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
SyncPeer::Quic(Arc::new( SyncPeer::Quic(Arc::new(
quic_connect(addr, "localhost", cfg) quic_connect(addr, "localhost", cfg)
.await .await
@@ -623,7 +639,10 @@ async fn cmd_pull(
peer.send(&SyncMessage::Ack { revision: rev }).await?; peer.send(&SyncMessage::Ack { revision: rev }).await?;
packets.push(packet); packets.push(packet);
} }
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => { SyncMessage::SyncComplete {
revisions_transferred,
bytes_transferred,
} => {
println!( println!(
"Received {} revision(s), {} bytes.", "Received {} revision(s), {} bytes.",
revisions_transferred, bytes_transferred revisions_transferred, bytes_transferred
@@ -637,8 +656,7 @@ async fn cmd_pull(
// Optionally filter to a specific branch before applying. // Optionally filter to a specific branch before applying.
let packets_to_apply = if let Some(ref branch_name) = branch { let packets_to_apply = if let Some(ref branch_name) = branch {
let selector = let selector = clawsync_onion::selector::SyncSelector::Branch(branch_name.clone());
clawsync_onion::selector::SyncSelector::Branch(branch_name.clone());
clawsync_onion::selector::filter_packets(packets, &selector, &local_onion) clawsync_onion::selector::filter_packets(packets, &selector, &local_onion)
.context("branch filter failed")? .context("branch filter failed")?
} else { } else {
@@ -646,8 +664,7 @@ async fn cmd_pull(
}; };
// Apply packets // Apply packets
let stats = merge_packets(&mut local_onion, packets_to_apply, true) let stats = merge_packets(&mut local_onion, packets_to_apply, true).context("merge failed")?;
.context("merge failed")?;
if let Some(b) = &branch { if let Some(b) = &branch {
println!( println!(
"Pull complete (branch {b:?}): {} revision(s) merged, {} skipped.", "Pull complete (branch {b:?}): {} revision(s) merged, {} skipped.",
@@ -668,8 +685,8 @@ async fn cmd_pull(
/// Serve a local HDF5 file, handling one client connection at a time. /// Serve a local HDF5 file, handling one client connection at a time.
async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()> { async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()> {
if quic { if quic {
let cfg = QuicConfig::self_signed() let cfg =
.with_context(|| "failed to build QUIC self-signed config")?; QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
let server = QuicServer::bind(bind, cfg) let server = QuicServer::bind(bind, cfg)
.await .await
.with_context(|| format!("cannot bind QUIC to {bind}"))?; .with_context(|| format!("cannot bind QUIC to {bind}"))?;
@@ -685,7 +702,11 @@ async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()>
tokio::spawn(async move { tokio::spawn(async move {
if let Err(e) = handle_client(&mut peer, &h5).await { if let Err(e) = handle_client(&mut peer, &h5).await {
eprintln!("QUIC client error: {e}"); eprintln!("QUIC client error: {e}");
let _ = peer.send(&SyncMessage::Error { message: e.to_string() }).await; let _ = peer
.send(&SyncMessage::Error {
message: e.to_string(),
})
.await;
} }
}); });
} }
@@ -693,7 +714,11 @@ async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()>
let server = TcpServer::bind(bind) let server = TcpServer::bind(bind)
.await .await
.with_context(|| format!("cannot bind to {bind}"))?; .with_context(|| format!("cannot bind to {bind}"))?;
println!("Serving {} on {} (Ctrl-C to stop)", h5_path.display(), server.local_addr); println!(
"Serving {} on {} (Ctrl-C to stop)",
h5_path.display(),
server.local_addr
);
loop { loop {
let (conn, peer_addr) = server.accept().await?; let (conn, peer_addr) = server.accept().await?;
println!("Connection from {peer_addr}"); println!("Connection from {peer_addr}");
@@ -702,7 +727,11 @@ async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()>
tokio::spawn(async move { tokio::spawn(async move {
if let Err(e) = handle_client(&mut peer, &h5).await { if let Err(e) = handle_client(&mut peer, &h5).await {
eprintln!("Client {peer_addr} error: {e}"); eprintln!("Client {peer_addr} error: {e}");
let _ = peer.send(&SyncMessage::Error { message: e.to_string() }).await; let _ = peer
.send(&SyncMessage::Error {
message: e.to_string(),
})
.await;
} }
}); });
} }
@@ -722,9 +751,11 @@ async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()>
async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> { async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
match conn.recv().await? { match conn.recv().await? {
// ── IBLT pre-flight (push direction only) ───────────────────────── // ── IBLT pre-flight (push direction only) ─────────────────────────
SyncMessage::IbltRequest { sketch: client_iblt } => { SyncMessage::IbltRequest {
let mut local_onion = OnionFile::open(h5_path) sketch: client_iblt,
.or_else(|_| OnionFile::create_auto(h5_path))?; } => {
let mut local_onion =
OnionFile::open(h5_path).or_else(|_| OnionFile::create_auto(h5_path))?;
let local_revisions: Vec<u64> = local_onion let local_revisions: Vec<u64> = local_onion
.list_revisions() .list_revisions()
@@ -749,25 +780,26 @@ async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
// not validated during IBLT pre-flight, so we use zeros to avoid // not validated during IBLT pre-flight, so we use zeros to avoid
// an O(file_size) read of the h5 base. // an O(file_size) read of the h5 base.
let server_iblt = IbltManifest { let server_iblt = IbltManifest {
agent_id: "server".to_string(), agent_id: "server".to_string(),
file_blake3: [0u8; 32], file_blake3: [0u8; 32],
revision_count: local_revisions.len() as u64, revision_count: local_revisions.len() as u64,
head_revision: local_revisions.last().copied().unwrap_or(0), head_revision: local_revisions.last().copied().unwrap_or(0),
head_blake3: [0u8; 32], head_blake3: [0u8; 32],
last_write: 0.0, last_write: 0.0,
sketch_cells: server_sketch.cell_count() as u32, sketch_cells: server_sketch.cell_count() as u32,
sketch: server_sketch.to_bytes(), sketch: server_sketch.to_bytes(),
}; };
conn.send(&SyncMessage::IbltResponse { conn.send(&SyncMessage::IbltResponse {
sketch: server_iblt, sketch: server_iblt,
missing_from_remote: diff.only_in_b.clone(), // client has, server lacks missing_from_remote: diff.only_in_b.clone(), // client has, server lacks
}).await?; })
.await?;
println!( println!(
" IBLT: client missing {}, server missing {} revision(s).", " IBLT: client missing {}, server missing {} revision(s).",
diff.only_in_a.len(), // server has, client lacks diff.only_in_a.len(), // server has, client lacks
diff.only_in_b.len(), // client has, server lacks diff.only_in_b.len(), // client has, server lacks
); );
// Receive exactly the packets the client is pushing. // Receive exactly the packets the client is pushing.
@@ -779,7 +811,10 @@ async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
conn.send(&SyncMessage::Ack { revision: rev }).await?; conn.send(&SyncMessage::Ack { revision: rev }).await?;
packets.push(packet); packets.push(packet);
} }
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => { SyncMessage::SyncComplete {
revisions_transferred,
bytes_transferred,
} => {
println!( println!(
" Client sent {revisions_transferred} revision(s), \ " Client sent {revisions_transferred} revision(s), \
{bytes_transferred} bytes (IBLT)." {bytes_transferred} bytes (IBLT)."
@@ -800,14 +835,18 @@ async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
} }
// ── Flat manifest (pull direction or legacy push) ────────────────── // ── Flat manifest (pull direction or legacy push) ──────────────────
SyncMessage::ManifestRequest { revision_count: remote_rev_count, .. } => { SyncMessage::ManifestRequest {
let mut local_onion = OnionFile::open(h5_path) revision_count: remote_rev_count,
.or_else(|_| OnionFile::create_auto(h5_path))?; ..
} => {
let mut local_onion =
OnionFile::open(h5_path).or_else(|_| OnionFile::create_auto(h5_path))?;
let h5_base = std::fs::read(h5_path)?; let h5_base = std::fs::read(h5_path)?;
let manifest = ClawSyncManifest::from_onion("server", &local_onion, &h5_base); let manifest = ClawSyncManifest::from_onion("server", &local_onion, &h5_base);
let local_rev_count = manifest.revision_count; let local_rev_count = manifest.revision_count;
conn.send(&SyncMessage::ManifestResponse { manifest }).await?; conn.send(&SyncMessage::ManifestResponse { manifest })
.await?;
if local_rev_count > remote_rev_count { if local_rev_count > remote_rev_count {
// ── Server pushes (client is behind) ────────────────────── // ── Server pushes (client is behind) ──────────────────────
@@ -816,8 +855,8 @@ async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
} else { } else {
remote_rev_count - 1 remote_rev_count - 1
}; };
let packets = diff_revisions(&local_onion, remote_head) let packets =
.context("diff_revisions failed")?; diff_revisions(&local_onion, remote_head).context("diff_revisions failed")?;
let total = packets.len() as u64; let total = packets.len() as u64;
let mut bytes_sent = 0u64; let mut bytes_sent = 0u64;
@@ -846,7 +885,10 @@ async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
conn.send(&SyncMessage::Ack { revision: rev }).await?; conn.send(&SyncMessage::Ack { revision: rev }).await?;
packets.push(packet); packets.push(packet);
} }
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => { SyncMessage::SyncComplete {
revisions_transferred,
bytes_transferred,
} => {
println!( println!(
" Client sent {revisions_transferred} revision(s), \ " Client sent {revisions_transferred} revision(s), \
{bytes_transferred} bytes." {bytes_transferred} bytes."
@@ -880,11 +922,7 @@ async fn cmd_export_revision(h5_path: PathBuf, revision: u64, dest: PathBuf) ->
.with_context(|| format!("cannot open {}", h5_path.display()))?; .with_context(|| format!("cannot open {}", h5_path.display()))?;
vf.export_revision(revision, &dest) vf.export_revision(revision, &dest)
.with_context(|| format!("export of revision {revision} failed"))?; .with_context(|| format!("export of revision {revision} failed"))?;
println!( println!("Exported revision {}{}", revision, dest.display());
"Exported revision {}{}",
revision,
dest.display()
);
Ok(()) Ok(())
} }
@@ -921,8 +959,8 @@ async fn cmd_gc(
); );
}; };
let mut onion = OnionFile::open(&h5_path) let mut onion =
.with_context(|| format!("cannot open {}", h5_path.display()))?; OnionFile::open(&h5_path).with_context(|| format!("cannot open {}", h5_path.display()))?;
let stats = onion.gc(policy).context("GC failed")?; let stats = onion.gc(policy).context("GC failed")?;
onion.flush().context("flush after GC failed")?; onion.flush().context("flush after GC failed")?;
@@ -935,10 +973,10 @@ async fn cmd_gc(
} }
async fn cmd_snapshot(h5_path: PathBuf, message: Option<String>) -> Result<()> { async fn cmd_snapshot(h5_path: PathBuf, message: Option<String>) -> Result<()> {
let mut onion = OnionFile::open(&h5_path) let mut onion =
.with_context(|| format!("cannot open {}", h5_path.display()))?; OnionFile::open(&h5_path).with_context(|| format!("cannot open {}", h5_path.display()))?;
let h5_base = std::fs::read(&h5_path) let h5_base =
.with_context(|| format!("cannot read {}", h5_path.display()))?; std::fs::read(&h5_path).with_context(|| format!("cannot read {}", h5_path.display()))?;
let annotation = message.as_deref().or(Some("manual snapshot")); let annotation = message.as_deref().or(Some("manual snapshot"));
let snap_rev = onion let snap_rev = onion
@@ -960,7 +998,7 @@ async fn cmd_branch_merge(
let merge_strategy = match strategy.as_str() { let merge_strategy = match strategy.as_str() {
"latest-wins" => MergeStrategy::LatestWins, "latest-wins" => MergeStrategy::LatestWins,
"three-way" => MergeStrategy::ThreeWay, "three-way" => MergeStrategy::ThreeWay,
other => anyhow::bail!("unknown strategy {other:?}; use latest-wins or three-way"), other => anyhow::bail!("unknown strategy {other:?}; use latest-wins or three-way"),
}; };
@@ -980,7 +1018,11 @@ async fn cmd_branch_merge(
async fn cmd_branch(action: BranchAction) -> Result<()> { async fn cmd_branch(action: BranchAction) -> Result<()> {
match action { match action {
BranchAction::Create { h5_path, name, from } => { BranchAction::Create {
h5_path,
name,
from,
} => {
let id = create_branch(&h5_path, &name, &from) let id = create_branch(&h5_path, &name, &from)
.with_context(|| format!("failed to create branch {name:?}"))?; .with_context(|| format!("failed to create branch {name:?}"))?;
println!("Branch {name:?} created (id={id}), forked from {from:?}."); println!("Branch {name:?} created (id={id}), forked from {from:?}.");
@@ -1023,7 +1065,12 @@ async fn cmd_branch(action: BranchAction) -> Result<()> {
println!("Branch {from:?} renamed to {to:?}."); println!("Branch {from:?} renamed to {to:?}.");
} }
BranchAction::Merge { h5_path, source, into, strategy } => { BranchAction::Merge {
h5_path,
source,
into,
strategy,
} => {
cmd_branch_merge(h5_path, source, into, strategy).await?; cmd_branch_merge(h5_path, source, into, strategy).await?;
} }
} }
@@ -1037,10 +1084,7 @@ async fn cmd_branch(action: BranchAction) -> Result<()> {
fn build_glob_set(patterns: &[String]) -> Result<globset::GlobSet> { fn build_glob_set(patterns: &[String]) -> Result<globset::GlobSet> {
let mut builder = globset::GlobSetBuilder::new(); let mut builder = globset::GlobSetBuilder::new();
for p in patterns { for p in patterns {
builder.add( builder.add(globset::Glob::new(p).with_context(|| format!("invalid glob pattern: {p}"))?);
globset::Glob::new(p)
.with_context(|| format!("invalid glob pattern: {p}"))?,
);
} }
builder.build().context("failed to build glob set") builder.build().context("failed to build glob set")
} }
@@ -1063,8 +1107,8 @@ async fn cmd_sync(
); );
let peer: SyncPeer = if quic { let peer: SyncPeer = if quic {
let cfg = QuicConfig::self_signed() let cfg =
.with_context(|| "failed to build QUIC self-signed config")?; QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
SyncPeer::Quic(Arc::new( SyncPeer::Quic(Arc::new(
quic_connect(addr, "localhost", cfg) quic_connect(addr, "localhost", cfg)
.await .await
@@ -1099,8 +1143,8 @@ async fn cmd_serve_fs(
let excludes = build_glob_set(&exclude)?; let excludes = build_glob_set(&exclude)?;
if quic { if quic {
let cfg = QuicConfig::self_signed() let cfg =
.with_context(|| "failed to build QUIC self-signed config")?; QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
let server = clawsync_transport::quic::QuicServer::bind(bind, cfg) let server = clawsync_transport::quic::QuicServer::bind(bind, cfg)
.await .await
.with_context(|| format!("cannot bind QUIC to {bind}"))?; .with_context(|| format!("cannot bind QUIC to {bind}"))?;
@@ -1148,12 +1192,7 @@ async fn cmd_serve_fs(
// HDF5 dataset-granular sync commands // HDF5 dataset-granular sync commands
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
async fn cmd_hdf5_sync( async fn cmd_hdf5_sync(local: PathBuf, remote: String, delete: bool, quic: bool) -> Result<()> {
local: PathBuf,
remote: String,
delete: bool,
quic: bool,
) -> Result<()> {
let (addr, remote_path) = parse_remote(&remote)?; let (addr, remote_path) = parse_remote(&remote)?;
let remote_path_str = remote_path.to_string_lossy().into_owned(); let remote_path_str = remote_path.to_string_lossy().into_owned();
@@ -1185,18 +1224,18 @@ async fn cmd_hdf5_sync(
.datasets .datasets
.values() .values()
.map(|e| clawsync_transport::protocol::Hdf5DatasetEntry { .map(|e| clawsync_transport::protocol::Hdf5DatasetEntry {
path: e.path.clone(), path: e.path.clone(),
blake3: e.blake3, blake3: e.blake3,
byte_len: e.byte_len, byte_len: e.byte_len,
shape: e.shape.clone(), shape: e.shape.clone(),
dtype: e.dtype.clone(), dtype: e.dtype.clone(),
}) })
.collect(); .collect();
// Connect. // Connect.
let mut peer: SyncPeer = if quic { let mut peer: SyncPeer = if quic {
let cfg = QuicConfig::self_signed() let cfg =
.with_context(|| "failed to build QUIC self-signed config")?; QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
SyncPeer::Quic(Arc::new( SyncPeer::Quic(Arc::new(
quic_connect(addr, "localhost", cfg) quic_connect(addr, "localhost", cfg)
.await .await
@@ -1221,7 +1260,10 @@ async fn cmd_hdf5_sync(
// Receive which datasets the server needs. // Receive which datasets the server needs.
let (datasets_needed, _to_delete) = match peer.recv().await? { let (datasets_needed, _to_delete) = match peer.recv().await? {
SyncMessage::Hdf5ManifestResponse { datasets_needed, to_delete } => { SyncMessage::Hdf5ManifestResponse {
datasets_needed,
to_delete,
} => {
println!( println!(
"Server needs {} dataset(s), {} to delete.", "Server needs {} dataset(s), {} to delete.",
datasets_needed.len(), datasets_needed.len(),
@@ -1237,7 +1279,10 @@ async fn cmd_hdf5_sync(
// Nothing to send; wait for completion. // Nothing to send; wait for completion.
match peer.recv().await? { match peer.recv().await? {
SyncMessage::Hdf5SyncComplete { SyncMessage::Hdf5SyncComplete {
datasets_added, datasets_modified, datasets_removed, bytes_transferred datasets_added,
datasets_modified,
datasets_removed,
bytes_transferred,
} => { } => {
println!( println!(
"HDF5-sync complete: +{datasets_added} ~{datasets_modified} \ "HDF5-sync complete: +{datasets_added} ~{datasets_modified} \
@@ -1298,7 +1343,10 @@ async fn cmd_hdf5_sync(
// Wait for final completion message. // Wait for final completion message.
match peer.recv().await? { match peer.recv().await? {
SyncMessage::Hdf5SyncComplete { SyncMessage::Hdf5SyncComplete {
datasets_added, datasets_modified, datasets_removed, bytes_transferred datasets_added,
datasets_modified,
datasets_removed,
bytes_transferred,
} => { } => {
println!( println!(
"HDF5-sync complete: +{datasets_added} ~{datasets_modified} \ "HDF5-sync complete: +{datasets_added} ~{datasets_modified} \
@@ -1321,9 +1369,12 @@ async fn handle_hdf5_client(
) -> Result<()> { ) -> Result<()> {
// Receive the client's manifest. // Receive the client's manifest.
let (remote_path, _client_file_blake3, client_entries, want_delete) = match conn.recv().await? { let (remote_path, _client_file_blake3, client_entries, want_delete) = match conn.recv().await? {
SyncMessage::Hdf5ManifestRequest { remote_path, file_blake3, entries, want_delete } => { SyncMessage::Hdf5ManifestRequest {
(remote_path, file_blake3, entries, want_delete) remote_path,
} file_blake3,
entries,
want_delete,
} => (remote_path, file_blake3, entries, want_delete),
other => anyhow::bail!("expected Hdf5ManifestRequest, got {other:?}"), other => anyhow::bail!("expected Hdf5ManifestRequest, got {other:?}"),
}; };
@@ -1359,8 +1410,13 @@ async fn handle_hdf5_client(
let mut to_delete = Vec::new(); let mut to_delete = Vec::new();
// Build a quick lookup of client entries by path. // Build a quick lookup of client entries by path.
let client_map: std::collections::HashMap<&str, &clawsync_transport::protocol::Hdf5DatasetEntry> = let client_map: std::collections::HashMap<
client_entries.iter().map(|e| (e.path.as_str(), e)).collect(); &str,
&clawsync_transport::protocol::Hdf5DatasetEntry,
> = client_entries
.iter()
.map(|e| (e.path.as_str(), e))
.collect();
// Added or Modified: in client manifest but not on server, or hash differs. // Added or Modified: in client manifest but not on server, or hash differs.
for entry in &client_entries { for entry in &client_entries {
@@ -1394,9 +1450,19 @@ async fn handle_hdf5_client(
for _ in 0..total_needed { for _ in 0..total_needed {
match conn.recv().await? { match conn.recv().await? {
SyncMessage::Hdf5DataPayload { path, data, dtype, shape } => { SyncMessage::Hdf5DataPayload {
path,
data,
dtype,
shape,
} => {
bytes_received += data.len() as u64; bytes_received += data.len() as u64;
payloads.push(ReceivedDataset { path: path.clone(), data, dtype, shape }); payloads.push(ReceivedDataset {
path: path.clone(),
data,
dtype,
shape,
});
conn.send(&SyncMessage::Hdf5DataAck { path }).await?; conn.send(&SyncMessage::Hdf5DataAck { path }).await?;
} }
other => anyhow::bail!("expected Hdf5DataPayload, got {other:?}"), other => anyhow::bail!("expected Hdf5DataPayload, got {other:?}"),
@@ -1419,9 +1485,9 @@ async fn handle_hdf5_client(
tokio::fs::rename(&tmp_path, &target_path).await?; tokio::fs::rename(&tmp_path, &target_path).await?;
conn.send(&SyncMessage::Hdf5SyncComplete { conn.send(&SyncMessage::Hdf5SyncComplete {
datasets_added: stats.datasets_added, datasets_added: stats.datasets_added,
datasets_modified: stats.datasets_modified, datasets_modified: stats.datasets_modified,
datasets_removed: stats.datasets_removed, datasets_removed: stats.datasets_removed,
bytes_transferred: bytes_received, bytes_transferred: bytes_received,
}) })
.await?; .await?;
@@ -1440,8 +1506,8 @@ async fn cmd_serve_hdf5(
quic: bool, quic: bool,
) -> Result<()> { ) -> Result<()> {
if quic { if quic {
let cfg = QuicConfig::self_signed() let cfg =
.with_context(|| "failed to build QUIC self-signed config")?; QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
let server = QuicServer::bind(bind, cfg) let server = QuicServer::bind(bind, cfg)
.await .await
.with_context(|| format!("cannot bind QUIC to {bind}"))?; .with_context(|| format!("cannot bind QUIC to {bind}"))?;
@@ -1453,7 +1519,11 @@ async fn cmd_serve_hdf5(
tokio::spawn(async move { tokio::spawn(async move {
if let Err(e) = handle_hdf5_client(&mut peer, &root, allow_delete).await { if let Err(e) = handle_hdf5_client(&mut peer, &root, allow_delete).await {
eprintln!("HDF5 QUIC client error: {e}"); eprintln!("HDF5 QUIC client error: {e}");
let _ = peer.send(&SyncMessage::Error { message: e.to_string() }).await; let _ = peer
.send(&SyncMessage::Error {
message: e.to_string(),
})
.await;
} }
}); });
} }
@@ -1469,7 +1539,11 @@ async fn cmd_serve_hdf5(
tokio::spawn(async move { tokio::spawn(async move {
if let Err(e) = handle_hdf5_client(&mut peer, &root, allow_delete).await { if let Err(e) = handle_hdf5_client(&mut peer, &root, allow_delete).await {
eprintln!("HDF5 client {peer_addr} error: {e}"); eprintln!("HDF5 client {peer_addr} error: {e}");
let _ = peer.send(&SyncMessage::Error { message: e.to_string() }).await; let _ = peer
.send(&SyncMessage::Error {
message: e.to_string(),
})
.await;
} }
}); });
} }
@@ -1498,36 +1572,73 @@ fn main() -> Result<()> {
Commands::ListRevisions { h5_path, verbose } => { Commands::ListRevisions { h5_path, verbose } => {
cmd_list_revisions(h5_path, verbose).await cmd_list_revisions(h5_path, verbose).await
} }
Commands::Rollback { h5_path, revision } => { Commands::Rollback { h5_path, revision } => cmd_rollback(h5_path, revision).await,
cmd_rollback(h5_path, revision).await Commands::Push {
} local,
Commands::Push { local, remote, branch, quic } => { remote,
cmd_push(local, remote, branch, quic).await branch,
} quic,
Commands::Pull { remote, local, branch, quic } => { } => cmd_push(local, remote, branch, quic).await,
cmd_pull(remote, local, branch, quic).await Commands::Pull {
} remote,
Commands::Serve { h5_path, bind, quic } => cmd_serve(h5_path, bind, quic).await, local,
branch,
quic,
} => cmd_pull(remote, local, branch, quic).await,
Commands::Serve {
h5_path,
bind,
quic,
} => cmd_serve(h5_path, bind, quic).await,
Commands::Branch { action } => cmd_branch(action).await, Commands::Branch { action } => cmd_branch(action).await,
Commands::ExportRevision { h5_path, revision, dest } => { Commands::ExportRevision {
cmd_export_revision(h5_path, revision, dest).await h5_path,
} revision,
Commands::Gc { h5_path, keep_last, keep_tagged, keep_since_days, keep_revisions } => { dest,
cmd_gc(h5_path, keep_last, keep_tagged, keep_since_days, keep_revisions).await } => cmd_export_revision(h5_path, revision, dest).await,
Commands::Gc {
h5_path,
keep_last,
keep_tagged,
keep_since_days,
keep_revisions,
} => {
cmd_gc(
h5_path,
keep_last,
keep_tagged,
keep_since_days,
keep_revisions,
)
.await
} }
Commands::Snapshot { h5_path, message } => cmd_snapshot(h5_path, message).await, Commands::Snapshot { h5_path, message } => cmd_snapshot(h5_path, message).await,
Commands::Sync { local, remote, delete, exclude, quic } => { Commands::Sync {
cmd_sync(local, remote, delete, exclude, quic).await local,
} remote,
Commands::ServeFs { dir, bind, allow_delete, exclude, quic } => { delete,
cmd_serve_fs(dir, bind, allow_delete, exclude, quic).await exclude,
} quic,
Commands::Hdf5Sync { local, remote, delete, quic } => { } => cmd_sync(local, remote, delete, exclude, quic).await,
cmd_hdf5_sync(local, remote, delete, quic).await Commands::ServeFs {
} dir,
Commands::ServeHdf5 { dir, bind, allow_delete, quic } => { bind,
cmd_serve_hdf5(dir, bind, allow_delete, quic).await allow_delete,
} exclude,
quic,
} => cmd_serve_fs(dir, bind, allow_delete, exclude, quic).await,
Commands::Hdf5Sync {
local,
remote,
delete,
quic,
} => cmd_hdf5_sync(local, remote, delete, quic).await,
Commands::ServeHdf5 {
dir,
bind,
allow_delete,
quic,
} => cmd_serve_hdf5(dir, bind, allow_delete, quic).await,
} }
}) })
} }
+177 -56
View File
@@ -9,8 +9,8 @@ use std::path::PathBuf;
use std::time::Duration; use std::time::Duration;
use clawhdf5_onion::api::{create_branch, delete_branch, list_branches, rename_branch}; use clawhdf5_onion::api::{create_branch, delete_branch, list_branches, rename_branch};
use clawhdf5_onion::gc::GcPolicy;
use clawhdf5_onion::format::NO_PARENT; use clawhdf5_onion::format::NO_PARENT;
use clawhdf5_onion::gc::GcPolicy;
use clawhdf5_onion::writer::OnionFile; use clawhdf5_onion::writer::OnionFile;
use clawsync_onion::differ::diff_revisions; use clawsync_onion::differ::diff_revisions;
use clawsync_onion::manifest::ClawSyncManifest; use clawsync_onion::manifest::ClawSyncManifest;
@@ -62,7 +62,9 @@ async fn run_server_one_connection(server: TcpServer, mut dst: OnionFile) -> Oni
let local_rev_count = dst.revision_count(); let local_rev_count = dst.revision_count();
let manifest = ClawSyncManifest::from_onion("server", &dst, H5_MAGIC); let manifest = ClawSyncManifest::from_onion("server", &dst, H5_MAGIC);
conn.send(&SyncMessage::ManifestResponse { manifest }).await.unwrap(); conn.send(&SyncMessage::ManifestResponse { manifest })
.await
.unwrap();
if local_rev_count > remote_rev_count { if local_rev_count > remote_rev_count {
// Push to client // Push to client
@@ -76,7 +78,9 @@ async fn run_server_one_connection(server: TcpServer, mut dst: OnionFile) -> Oni
let mut bytes_sent = 0u64; let mut bytes_sent = 0u64;
for packet in packets { for packet in packets {
bytes_sent += packet.page_data_size() as u64; bytes_sent += packet.page_data_size() as u64;
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap(); conn.send(&SyncMessage::LayerPacket { packet })
.await
.unwrap();
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::Ack { .. } => {} SyncMessage::Ack { .. } => {}
other => panic!("expected Ack, got {other:?}"), other => panic!("expected Ack, got {other:?}"),
@@ -95,7 +99,9 @@ async fn run_server_one_connection(server: TcpServer, mut dst: OnionFile) -> Oni
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::LayerPacket { packet } => { SyncMessage::LayerPacket { packet } => {
let rev = packet.revision; let rev = packet.revision;
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap(); conn.send(&SyncMessage::Ack { revision: rev })
.await
.unwrap();
packets.push(packet); packets.push(packet);
} }
SyncMessage::SyncComplete { .. } => break, SyncMessage::SyncComplete { .. } => break,
@@ -139,7 +145,11 @@ async fn push_all_to_empty_server() {
let remote_head = match conn.recv().await.unwrap() { let remote_head = match conn.recv().await.unwrap() {
SyncMessage::ManifestResponse { manifest } => { SyncMessage::ManifestResponse { manifest } => {
if manifest.revision_count == 0 { clawhdf5_onion::format::NO_PARENT } else { manifest.head_revision } if manifest.revision_count == 0 {
clawhdf5_onion::format::NO_PARENT
} else {
manifest.head_revision
}
} }
other => panic!("{other:?}"), other => panic!("{other:?}"),
}; };
@@ -148,14 +158,20 @@ async fn push_all_to_empty_server() {
let total = packets.len() as u64; let total = packets.len() as u64;
for packet in packets { for packet in packets {
let rev = packet.revision; let rev = packet.revision;
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap(); conn.send(&SyncMessage::LayerPacket { packet })
.await
.unwrap();
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::Ack { revision } => assert_eq!(revision, rev), SyncMessage::Ack { revision } => assert_eq!(revision, rev),
other => panic!("{other:?}"), other => panic!("{other:?}"),
} }
} }
conn.send(&SyncMessage::SyncComplete { revisions_transferred: total, bytes_transferred: 0 }) conn.send(&SyncMessage::SyncComplete {
.await.unwrap(); revisions_transferred: total,
bytes_transferred: 0,
})
.await
.unwrap();
let dst_final = server_task.await.unwrap(); let dst_final = server_task.await.unwrap();
assert_eq!(dst_final.revision_count(), 5); assert_eq!(dst_final.revision_count(), 5);
@@ -180,7 +196,9 @@ async fn push_already_in_sync_is_noop() {
agent_id: "c".into(), agent_id: "c".into(),
head_revision: mf.head_revision, head_revision: mf.head_revision,
revision_count: mf.revision_count, revision_count: mf.revision_count,
}).await.unwrap(); })
.await
.unwrap();
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::ManifestResponse { .. } => {} SyncMessage::ManifestResponse { .. } => {}
@@ -188,8 +206,12 @@ async fn push_already_in_sync_is_noop() {
} }
// diff = 0; send SyncComplete immediately // diff = 0; send SyncComplete immediately
conn.send(&SyncMessage::SyncComplete { revisions_transferred: 0, bytes_transferred: 0 }) conn.send(&SyncMessage::SyncComplete {
.await.unwrap(); revisions_transferred: 0,
bytes_transferred: 0,
})
.await
.unwrap();
let dst_final = server_task.await.unwrap(); let dst_final = server_task.await.unwrap();
assert_eq!(dst_final.revision_count(), 3, "no duplicate revisions"); assert_eq!(dst_final.revision_count(), 3, "no duplicate revisions");
@@ -215,7 +237,9 @@ async fn pull_all_from_server() {
agent_id: "pull-client".into(), agent_id: "pull-client".into(),
head_revision: 0, head_revision: 0,
revision_count: 0, revision_count: 0,
}).await.unwrap(); })
.await
.unwrap();
let server_count = match conn.recv().await.unwrap() { let server_count = match conn.recv().await.unwrap() {
SyncMessage::ManifestResponse { manifest } => manifest.revision_count, SyncMessage::ManifestResponse { manifest } => manifest.revision_count,
@@ -230,7 +254,9 @@ async fn pull_all_from_server() {
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::LayerPacket { packet } => { SyncMessage::LayerPacket { packet } => {
let rev = packet.revision; let rev = packet.revision;
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap(); conn.send(&SyncMessage::Ack { revision: rev })
.await
.unwrap();
packets.push(packet); packets.push(packet);
} }
SyncMessage::SyncComplete { .. } => break, SyncMessage::SyncComplete { .. } => break,
@@ -260,7 +286,9 @@ async fn pull_already_in_sync() {
agent_id: "pull-client".into(), agent_id: "pull-client".into(),
head_revision: local_count.saturating_sub(1), head_revision: local_count.saturating_sub(1),
revision_count: local_count, revision_count: local_count,
}).await.unwrap(); })
.await
.unwrap();
let server_count = match conn.recv().await.unwrap() { let server_count = match conn.recv().await.unwrap() {
SyncMessage::ManifestResponse { manifest } => manifest.revision_count, SyncMessage::ManifestResponse { manifest } => manifest.revision_count,
@@ -269,8 +297,12 @@ async fn pull_already_in_sync() {
assert!(server_count <= local_count, "already in sync"); assert!(server_count <= local_count, "already in sync");
// Client detects it's up to date — sends SyncComplete immediately. // Client detects it's up to date — sends SyncComplete immediately.
conn.send(&SyncMessage::SyncComplete { revisions_transferred: 0, bytes_transferred: 0 }) conn.send(&SyncMessage::SyncComplete {
.await.unwrap(); revisions_transferred: 0,
bytes_transferred: 0,
})
.await
.unwrap();
server_task.await.unwrap(); server_task.await.unwrap();
let _ = cli_h5; let _ = cli_h5;
@@ -294,7 +326,9 @@ async fn pull_partial_delta() {
agent_id: "c".into(), agent_id: "c".into(),
head_revision: K as u64 - 1, head_revision: K as u64 - 1,
revision_count: K as u64, revision_count: K as u64,
}).await.unwrap(); })
.await
.unwrap();
let server_count = match conn.recv().await.unwrap() { let server_count = match conn.recv().await.unwrap() {
SyncMessage::ManifestResponse { manifest } => manifest.revision_count, SyncMessage::ManifestResponse { manifest } => manifest.revision_count,
@@ -308,7 +342,9 @@ async fn pull_partial_delta() {
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::LayerPacket { packet } => { SyncMessage::LayerPacket { packet } => {
let rev = packet.revision; let rev = packet.revision;
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap(); conn.send(&SyncMessage::Ack { revision: rev })
.await
.unwrap();
packets.push(packet); packets.push(packet);
received += 1; received += 1;
} }
@@ -376,7 +412,9 @@ async fn push_one_revision_within_timeout() {
agent_id: "c".into(), agent_id: "c".into(),
head_revision: mf.head_revision, head_revision: mf.head_revision,
revision_count: mf.revision_count, revision_count: mf.revision_count,
}).await.unwrap(); })
.await
.unwrap();
let remote_head = match conn.recv().await.unwrap() { let remote_head = match conn.recv().await.unwrap() {
SyncMessage::ManifestResponse { manifest } => { SyncMessage::ManifestResponse { manifest } => {
@@ -393,18 +431,27 @@ async fn push_one_revision_within_timeout() {
let total = packets.len() as u64; let total = packets.len() as u64;
for packet in packets { for packet in packets {
let rev = packet.revision; let rev = packet.revision;
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap(); conn.send(&SyncMessage::LayerPacket { packet })
.await
.unwrap();
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::Ack { revision } => assert_eq!(revision, rev), SyncMessage::Ack { revision } => assert_eq!(revision, rev),
other => panic!("{other:?}"), other => panic!("{other:?}"),
} }
} }
conn.send(&SyncMessage::SyncComplete { revisions_transferred: total, bytes_transferred: 0 }) conn.send(&SyncMessage::SyncComplete {
.await.unwrap(); revisions_transferred: total,
bytes_transferred: 0,
})
.await
.unwrap();
}) })
.await; .await;
assert!(client_result.is_ok(), "push timed out — possible protocol deadlock"); assert!(
client_result.is_ok(),
"push timed out — possible protocol deadlock"
);
let dst_final = server_handle.await.unwrap(); let dst_final = server_handle.await.unwrap();
assert_eq!(dst_final.revision_count(), 1); assert_eq!(dst_final.revision_count(), 1);
} }
@@ -495,11 +542,17 @@ async fn push_with_branch_filter_sends_only_branch_revisions() {
agent_id: "c".into(), agent_id: "c".into(),
head_revision: mf.head_revision, head_revision: mf.head_revision,
revision_count: mf.revision_count, revision_count: mf.revision_count,
}).await.unwrap(); })
.await
.unwrap();
let remote_head = match conn.recv().await.unwrap() { let remote_head = match conn.recv().await.unwrap() {
SyncMessage::ManifestResponse { manifest } => { SyncMessage::ManifestResponse { manifest } => {
if manifest.revision_count == 0 { clawhdf5_onion::format::NO_PARENT } else { manifest.head_revision } if manifest.revision_count == 0 {
clawhdf5_onion::format::NO_PARENT
} else {
manifest.head_revision
}
} }
other => panic!("{other:?}"), other => panic!("{other:?}"),
}; };
@@ -509,22 +562,36 @@ async fn push_with_branch_filter_sends_only_branch_revisions() {
let packets = filter_packets(all_packets, &selector, &src).unwrap(); let packets = filter_packets(all_packets, &selector, &src).unwrap();
// Only the 2 feature revisions should be sent (not the 3 main ones). // Only the 2 feature revisions should be sent (not the 3 main ones).
assert_eq!(packets.len(), 2, "only feature branch packets should be sent"); assert_eq!(
packets.len(),
2,
"only feature branch packets should be sent"
);
let total = packets.len() as u64; let total = packets.len() as u64;
for packet in packets { for packet in packets {
let rev = packet.revision; let rev = packet.revision;
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap(); conn.send(&SyncMessage::LayerPacket { packet })
.await
.unwrap();
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::Ack { revision } => assert_eq!(revision, rev), SyncMessage::Ack { revision } => assert_eq!(revision, rev),
other => panic!("{other:?}"), other => panic!("{other:?}"),
} }
} }
conn.send(&SyncMessage::SyncComplete { revisions_transferred: total, bytes_transferred: 0 }) conn.send(&SyncMessage::SyncComplete {
.await.unwrap(); revisions_transferred: total,
bytes_transferred: 0,
})
.await
.unwrap();
let dst_final = server_task.await.unwrap(); let dst_final = server_task.await.unwrap();
assert_eq!(dst_final.revision_count(), 2, "destination should have only the 2 feature revisions"); assert_eq!(
dst_final.revision_count(),
2,
"destination should have only the 2 feature revisions"
);
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@@ -567,7 +634,9 @@ async fn push_to_server(src: &OnionFile, src_h5: &PathBuf, addr: SocketAddr) ->
let sent = packets.len() as u64; let sent = packets.len() as u64;
for packet in packets { for packet in packets {
let rev = packet.revision; let rev = packet.revision;
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap(); conn.send(&SyncMessage::LayerPacket { packet })
.await
.unwrap();
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::Ack { revision } => assert_eq!(revision, rev), SyncMessage::Ack { revision } => assert_eq!(revision, rev),
other => panic!("client: expected Ack, got {other:?}"), other => panic!("client: expected Ack, got {other:?}"),
@@ -597,7 +666,9 @@ async fn run_disk_server(server: TcpServer, h5_path: PathBuf, n: usize) {
let local_rev_count = onion.revision_count(); let local_rev_count = onion.revision_count();
let manifest = ClawSyncManifest::from_onion("server", &onion, H5_MAGIC); let manifest = ClawSyncManifest::from_onion("server", &onion, H5_MAGIC);
conn.send(&SyncMessage::ManifestResponse { manifest }).await.unwrap(); conn.send(&SyncMessage::ManifestResponse { manifest })
.await
.unwrap();
if local_rev_count > remote_rev_count { if local_rev_count > remote_rev_count {
// Server pushes (pull direction) // Server pushes (pull direction)
@@ -611,7 +682,9 @@ async fn run_disk_server(server: TcpServer, h5_path: PathBuf, n: usize) {
let mut bytes_sent = 0u64; let mut bytes_sent = 0u64;
for packet in packets { for packet in packets {
bytes_sent += packet.page_data_size() as u64; bytes_sent += packet.page_data_size() as u64;
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap(); conn.send(&SyncMessage::LayerPacket { packet })
.await
.unwrap();
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::Ack { .. } => {} SyncMessage::Ack { .. } => {}
other => panic!("server: expected Ack, got {other:?}"), other => panic!("server: expected Ack, got {other:?}"),
@@ -630,7 +703,9 @@ async fn run_disk_server(server: TcpServer, h5_path: PathBuf, n: usize) {
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::LayerPacket { packet } => { SyncMessage::LayerPacket { packet } => {
let rev = packet.revision; let rev = packet.revision;
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap(); conn.send(&SyncMessage::Ack { revision: rev })
.await
.unwrap();
packets.push(packet); packets.push(packet);
} }
SyncMessage::SyncComplete { .. } => break, SyncMessage::SyncComplete { .. } => break,
@@ -687,7 +762,11 @@ async fn serve_accumulates_across_two_sequential_pushes() {
// The on-disk server file must now have all 5 revisions. // The on-disk server file must now have all 5 revisions.
let final_onion = OnionFile::open(&srv_h5).unwrap(); let final_onion = OnionFile::open(&srv_h5).unwrap();
assert_eq!(final_onion.revision_count(), 5, "server should have accumulated all 5 revisions"); assert_eq!(
final_onion.revision_count(),
5,
"server should have accumulated all 5 revisions"
);
let _ = srv_dir; let _ = srv_dir;
} }
@@ -713,7 +792,10 @@ async fn serve_noop_when_second_client_already_in_sync() {
// Connection 2: same client — already in sync, should send 0 packets. // Connection 2: same client — already in sync, should send 0 packets.
let sent2 = push_to_server(&src, &src_h5, addr).await; let sent2 = push_to_server(&src, &src_h5, addr).await;
assert_eq!(sent2, 0, "second identical push should transfer 0 revisions"); assert_eq!(
sent2, 0,
"second identical push should transfer 0 revisions"
);
server_task.await.unwrap(); server_task.await.unwrap();
@@ -738,8 +820,11 @@ fn export_revision_writes_correct_historical_state() {
let bytes = std::fs::read(&dest).unwrap(); let bytes = std::fs::read(&dest).unwrap();
// The reconstructed bytes at offset 0 should be the rev-1 page (all 0x01) // The reconstructed bytes at offset 0 should be the rev-1 page (all 0x01)
assert_eq!(bytes[0..4096], vec![0x01u8; 4096][..], assert_eq!(
"exported rev 1 should contain the page written in rev 1"); bytes[0..4096],
vec![0x01u8; 4096][..],
"exported rev 1 should contain the page written in rev 1"
);
} }
#[test] #[test]
@@ -761,7 +846,10 @@ fn export_revision_does_not_modify_source_sidecar() {
vf.export_revision(0, &dest).unwrap(); vf.export_revision(0, &dest).unwrap();
let after_len = std::fs::metadata(&onion_path).unwrap().len(); let after_len = std::fs::metadata(&onion_path).unwrap().len();
assert_eq!(before_len, after_len, "sidecar must be unchanged after export"); assert_eq!(
before_len, after_len,
"sidecar must be unchanged after export"
);
} }
#[test] #[test]
@@ -771,7 +859,10 @@ fn export_revision_nonexistent_errors() {
let dest = dest_dir.path().join("out.h5"); let dest = dest_dir.path().join("out.h5");
let vf = clawhdf5_onion::versioned_file::VersionedFile::open(&h5).unwrap(); let vf = clawhdf5_onion::versioned_file::VersionedFile::open(&h5).unwrap();
let err = vf.export_revision(999, &dest).unwrap_err(); let err = vf.export_revision(999, &dest).unwrap_err();
assert!(matches!(err, clawhdf5_onion::OnionError::RevisionNotFound(999))); assert!(matches!(
err,
clawhdf5_onion::OnionError::RevisionNotFound(999)
));
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@@ -802,10 +893,18 @@ fn branch_merge_latest_wins_creates_merge_revision() {
.unwrap(); .unwrap();
let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap(); let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap();
assert!(revs.iter().any(|r| r.revision == merge_rev), assert!(
"merge revision must appear in list_revisions"); revs.iter().any(|r| r.revision == merge_rev),
assert_eq!(revs.iter().find(|r| r.revision == merge_rev).unwrap().branch_id, 0, "merge revision must appear in list_revisions"
"merge revision is on main (branch_id 0)"); );
assert_eq!(
revs.iter()
.find(|r| r.revision == merge_rev)
.unwrap()
.branch_id,
0,
"merge revision is on main (branch_id 0)"
);
let _ = h5_base; let _ = h5_base;
} }
@@ -865,8 +964,10 @@ fn gc_keep_tagged_retains_annotated_only() {
assert_eq!(stats.revisions_removed, 2); assert_eq!(stats.revisions_removed, 2);
let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap(); let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap();
assert!(revs.iter().all(|r| r.annotation.is_some()), assert!(
"all remaining revisions should be annotated"); revs.iter().all(|r| r.annotation.is_some()),
"all remaining revisions should be annotated"
);
} }
#[test] #[test]
@@ -887,8 +988,10 @@ fn gc_keep_since_days_zero_removes_all_old_revisions() {
let stats = onion.gc(GcPolicy::KeepSince(future_cutoff)).unwrap(); let stats = onion.gc(GcPolicy::KeepSince(future_cutoff)).unwrap();
onion.flush().unwrap(); onion.flush().unwrap();
assert_eq!(stats.revisions_removed, 4, assert_eq!(
"all 4 revisions should be pruned when cutoff is in the future"); stats.revisions_removed, 4,
"all 4 revisions should be pruned when cutoff is in the future"
);
} }
#[test] #[test]
@@ -898,8 +1001,10 @@ fn gc_keep_since_days_past_keeps_all() {
let stats = onion.gc(GcPolicy::KeepSince(0.0)).unwrap(); let stats = onion.gc(GcPolicy::KeepSince(0.0)).unwrap();
// All revisions are after Unix epoch 0 → 0 removed // All revisions are after Unix epoch 0 → 0 removed
assert_eq!(stats.revisions_removed, 0, assert_eq!(
"no revisions should be removed when cutoff is epoch 0"); stats.revisions_removed, 0,
"no revisions should be removed when cutoff is epoch 0"
);
assert_eq!(onion.revision_count(), 4); assert_eq!(onion.revision_count(), 4);
} }
@@ -926,19 +1031,29 @@ fn gc_keep_revisions_retains_only_listed_and_ancestors() {
onion.flush().unwrap(); onion.flush().unwrap();
// Revisions 3 and 4 are pruned; 0, 1, 2 are kept (2 plus its ancestors). // Revisions 3 and 4 are pruned; 0, 1, 2 are kept (2 plus its ancestors).
assert_eq!(stats.revisions_removed, 2, "revisions 3 and 4 should be removed"); assert_eq!(
stats.revisions_removed, 2,
"revisions 3 and 4 should be removed"
);
let reloaded = OnionFile::open(&h5).unwrap(); let reloaded = OnionFile::open(&h5).unwrap();
assert_eq!(reloaded.revision_count(), 3, "revisions 0, 1, 2 remain"); assert_eq!(reloaded.revision_count(), 3, "revisions 0, 1, 2 remain");
let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap(); let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap();
let annotations: Vec<_> = revs.iter() let annotations: Vec<_> = revs
.iter()
.map(|r| r.annotation.as_deref().unwrap_or("")) .map(|r| r.annotation.as_deref().unwrap_or(""))
.collect(); .collect();
assert!(annotations.contains(&"rev 2"), "rev 2 must survive"); assert!(annotations.contains(&"rev 2"), "rev 2 must survive");
// Ancestors are retained too // Ancestors are retained too
assert!(annotations.contains(&"rev 0"), "ancestor rev 0 must survive"); assert!(
assert!(annotations.contains(&"rev 1"), "ancestor rev 1 must survive"); annotations.contains(&"rev 0"),
"ancestor rev 0 must survive"
);
assert!(
annotations.contains(&"rev 1"),
"ancestor rev 1 must survive"
);
} }
#[test] #[test]
@@ -948,7 +1063,10 @@ fn gc_keep_revisions_empty_list_removes_all() {
let stats = onion.gc(GcPolicy::KeepRevisions(vec![])).unwrap(); let stats = onion.gc(GcPolicy::KeepRevisions(vec![])).unwrap();
onion.flush().unwrap(); onion.flush().unwrap();
assert_eq!(stats.revisions_removed, 4, "all revisions pruned when keep list is empty"); assert_eq!(
stats.revisions_removed, 4,
"all revisions pruned when keep list is empty"
);
let reloaded = OnionFile::open(&h5).unwrap(); let reloaded = OnionFile::open(&h5).unwrap();
assert_eq!(reloaded.revision_count(), 0); assert_eq!(reloaded.revision_count(), 0);
} }
@@ -967,7 +1085,10 @@ fn snapshot_creates_flagged_revision() {
let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap(); let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap();
let snap = revs.iter().find(|r| r.revision == snap_rev).unwrap(); let snap = revs.iter().find(|r| r.revision == snap_rev).unwrap();
assert!(snap.is_snapshot, "snapshot revision must have is_snapshot=true"); assert!(
snap.is_snapshot,
"snapshot revision must have is_snapshot=true"
);
assert_eq!(snap.annotation.as_deref(), Some("snap")); assert_eq!(snap.annotation.as_deref(), Some("snap"));
} }
+54 -19
View File
@@ -15,8 +15,8 @@ use std::io::{BufRead, BufReader};
use std::net::SocketAddr; use std::net::SocketAddr;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio}; use std::process::{Child, Command, Stdio};
use std::time::Duration;
use std::thread; use std::thread;
use std::time::Duration;
use tempfile::TempDir; use tempfile::TempDir;
@@ -51,13 +51,13 @@ impl FsServer {
// Wait for: "ClawSync FS server listening on 127.0.0.1:PORT" // Wait for: "ClawSync FS server listening on 127.0.0.1:PORT"
let mut line = String::new(); let mut line = String::new();
reader.read_line(&mut line).expect("failed to read serve-fs startup line"); reader
.read_line(&mut line)
.expect("failed to read serve-fs startup line");
let addr = parse_fs_serve_addr(&line); let addr = parse_fs_serve_addr(&line);
// Drain remaining stdout in a background thread so the process doesn't block. // Drain remaining stdout in a background thread so the process doesn't block.
thread::spawn(move || { thread::spawn(move || for _ in reader.lines() {});
for _ in reader.lines() {}
});
Self { child, addr } Self { child, addr }
} }
@@ -85,11 +85,11 @@ fn parse_fs_serve_addr(line: &str) -> String {
#[derive(Debug)] #[derive(Debug)]
struct SyncResult { struct SyncResult {
files_added: u32, files_added: u32,
files_modified: u32, files_modified: u32,
files_removed: u32, files_removed: u32,
bytes_transferred: u64, bytes_transferred: u64,
success: bool, success: bool,
} }
fn run_sync(src: &Path, server_addr: &str, delete: bool, excludes: &[&str]) -> SyncResult { fn run_sync(src: &Path, server_addr: &str, delete: bool, excludes: &[&str]) -> SyncResult {
@@ -136,7 +136,13 @@ fn parse_sync_output(out: &str, success: bool) -> SyncResult {
} }
} }
SyncResult { files_added, files_modified, files_removed, bytes_transferred, success } SyncResult {
files_added,
files_modified,
files_removed,
bytes_transferred,
success,
}
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@@ -161,7 +167,9 @@ fn assert_files_equal(a: &Path, b: &Path) {
fn assert_dir_equal(src: &Path, dst: &Path) { fn assert_dir_equal(src: &Path, dst: &Path) {
for entry in walkdir::WalkDir::new(src).sort_by_file_name() { for entry in walkdir::WalkDir::new(src).sort_by_file_name() {
let entry: walkdir::DirEntry = entry.unwrap(); let entry: walkdir::DirEntry = entry.unwrap();
if !entry.file_type().is_file() { continue; } if !entry.file_type().is_file() {
continue;
}
let rel = entry.path().strip_prefix(src).unwrap(); let rel = entry.path().strip_prefix(src).unwrap();
let dst_path = dst.join(rel); let dst_path = dst.join(rel);
assert!(dst_path.exists(), "missing in dst: {}", rel.display()); assert!(dst_path.exists(), "missing in dst: {}", rel.display());
@@ -242,7 +250,11 @@ fn fs_sync_cdc_insertion_minimizes_transfer() {
// Base: 512 KB of pseudo-random data // Base: 512 KB of pseudo-random data
let base: Vec<u8> = (0..512 * 1024) let base: Vec<u8> = (0..512 * 1024)
.map(|i| (i as u64).wrapping_mul(6364136223846793005).wrapping_shr(56) as u8) .map(|i| {
(i as u64)
.wrapping_mul(6364136223846793005)
.wrapping_shr(56) as u8
})
.collect(); .collect();
// Server has the base. // Server has the base.
@@ -260,7 +272,10 @@ fn fs_sync_cdc_insertion_minimizes_transfer() {
assert_eq!(result.files_modified, 1); assert_eq!(result.files_modified, 1);
// Verify the content is correct after sync. // Verify the content is correct after sync.
assert_files_equal(&src.path().join("weights.bin"), &dst.path().join("weights.bin")); assert_files_equal(
&src.path().join("weights.bin"),
&dst.path().join("weights.bin"),
);
// CDC should transfer much less than the full file (< 40% = 205 KB). // CDC should transfer much less than the full file (< 40% = 205 KB).
// The insertion only invalidates ~2-3 chunk boundaries. // The insertion only invalidates ~2-3 chunk boundaries.
@@ -286,7 +301,10 @@ fn fs_sync_delete_removes_server_only_files() {
assert!(result.success); assert!(result.success);
assert_eq!(result.files_removed, 1); assert_eq!(result.files_removed, 1);
assert!(!dst.path().join("extra.bin").exists(), "extra.bin should have been deleted"); assert!(
!dst.path().join("extra.bin").exists(),
"extra.bin should have been deleted"
);
assert!(dst.path().join("keep.bin").exists()); assert!(dst.path().join("keep.bin").exists());
} }
@@ -304,7 +322,10 @@ fn fs_sync_no_delete_preserves_server_only_files() {
assert!(result.success); assert!(result.success);
// extra.bin must still be there // extra.bin must still be there
assert!(dst.path().join("extra.bin").exists(), "extra.bin should be preserved (no --delete)"); assert!(
dst.path().join("extra.bin").exists(),
"extra.bin should be preserved (no --delete)"
);
} }
#[test] #[test]
@@ -319,8 +340,14 @@ fn fs_sync_exclude_pattern_skips_matching_files() {
let result = run_sync(src.path(), &server.addr, false, &["*.tmp"]); let result = run_sync(src.path(), &server.addr, false, &["*.tmp"]);
assert!(result.success); assert!(result.success);
assert!(dst.path().join("keep.bin").exists(), "keep.bin should be transferred"); assert!(
assert!(!dst.path().join("skip.tmp").exists(), "skip.tmp should be excluded"); dst.path().join("keep.bin").exists(),
"keep.bin should be transferred"
);
assert!(
!dst.path().join("skip.tmp").exists(),
"skip.tmp should be excluded"
);
} }
#[test] #[test]
@@ -356,7 +383,11 @@ fn fs_sync_large_file_roundtrip() {
let dst = TempDir::new().unwrap(); let dst = TempDir::new().unwrap();
let data: Vec<u8> = (0..8 * 1024 * 1024) let data: Vec<u8> = (0..8 * 1024 * 1024)
.map(|i| (i as u64).wrapping_mul(6364136223846793005).wrapping_shr(40) as u8) .map(|i| {
(i as u64)
.wrapping_mul(6364136223846793005)
.wrapping_shr(40) as u8
})
.collect(); .collect();
fs::write(src.path().join("large.bin"), &data).unwrap(); fs::write(src.path().join("large.bin"), &data).unwrap();
@@ -376,7 +407,11 @@ fn fs_sync_append_to_file_sends_minimal_chunks() {
let dst = TempDir::new().unwrap(); let dst = TempDir::new().unwrap();
let base: Vec<u8> = (0..256 * 1024) let base: Vec<u8> = (0..256 * 1024)
.map(|i| (i as u64).wrapping_mul(6364136223846793005).wrapping_shr(48) as u8) .map(|i| {
(i as u64)
.wrapping_mul(6364136223846793005)
.wrapping_shr(48) as u8
})
.collect(); .collect();
fs::write(dst.path().join("model.bin"), &base).unwrap(); fs::write(dst.path().join("model.bin"), &base).unwrap();
+28 -16
View File
@@ -99,10 +99,13 @@ fn run_hdf5_sync(local: &Path, addr: SocketAddr, rel_path: &str, delete: bool) {
fn hdf5_sync_cold_copy() { fn hdf5_sync_cold_copy() {
let src = TempDir::new().unwrap(); let src = TempDir::new().unwrap();
let local = src.path().join("model.h5"); let local = src.path().join("model.h5");
write_h5(&local, &[ write_h5(
("weights", vec![1.0, 2.0, 3.0, 4.0]), &local,
("biases", vec![0.1, 0.2]), &[
]); ("weights", vec![1.0, 2.0, 3.0, 4.0]),
("biases", vec![0.1, 0.2]),
],
);
let (mut server, addr, serve_dir) = spawn_server(false); let (mut server, addr, serve_dir) = spawn_server(false);
run_hdf5_sync(&local, addr, "model.h5", false); run_hdf5_sync(&local, addr, "model.h5", false);
@@ -150,19 +153,19 @@ fn hdf5_sync_warm_noop() {
fn hdf5_sync_incremental_modified_dataset() { fn hdf5_sync_incremental_modified_dataset() {
let src = TempDir::new().unwrap(); let src = TempDir::new().unwrap();
let local = src.path().join("model.h5"); let local = src.path().join("model.h5");
write_h5(&local, &[ write_h5(
("weights", vec![1.0, 2.0]), &local,
("biases", vec![0.5]), &[("weights", vec![1.0, 2.0]), ("biases", vec![0.5])],
]); );
let (mut server, addr, serve_dir) = spawn_server(false); let (mut server, addr, serve_dir) = spawn_server(false);
run_hdf5_sync(&local, addr, "model.h5", false); run_hdf5_sync(&local, addr, "model.h5", false);
// Modify weights; biases unchanged. // Modify weights; biases unchanged.
write_h5(&local, &[ write_h5(
("weights", vec![9.0, 9.0]), &local,
("biases", vec![0.5]), &[("weights", vec![9.0, 9.0]), ("biases", vec![0.5])],
]); );
run_hdf5_sync(&local, addr, "model.h5", false); run_hdf5_sync(&local, addr, "model.h5", false);
server.kill().unwrap(); server.kill().unwrap();
server.wait().unwrap(); server.wait().unwrap();
@@ -201,7 +204,10 @@ fn hdf5_sync_added_dataset() {
let synced = serve_dir.path().join("model.h5"); let synced = serve_dir.path().join("model.h5");
let m_server = DatasetManifest::from_path(&synced).unwrap(); let m_server = DatasetManifest::from_path(&synced).unwrap();
assert!(m_server.get("/alpha").is_some(), "alpha should exist"); assert!(m_server.get("/alpha").is_some(), "alpha should exist");
assert!(m_server.get("/beta").is_some(), "beta should have been added"); assert!(
m_server.get("/beta").is_some(),
"beta should have been added"
);
} }
/// Delete with `--delete`: dataset absent from client → removed from server. /// Delete with `--delete`: dataset absent from client → removed from server.
@@ -223,7 +229,10 @@ fn hdf5_sync_delete_removes_server_only_dataset() {
let synced = serve_dir.path().join("model.h5"); let synced = serve_dir.path().join("model.h5");
let m_server = DatasetManifest::from_path(&synced).unwrap(); let m_server = DatasetManifest::from_path(&synced).unwrap();
assert!(m_server.get("/keep").is_some(), "keep should remain"); assert!(m_server.get("/keep").is_some(), "keep should remain");
assert!(m_server.get("/drop").is_none(), "drop should have been removed"); assert!(
m_server.get("/drop").is_none(),
"drop should have been removed"
);
} }
/// No-delete: dataset absent from client is preserved on server when `--delete` not set. /// No-delete: dataset absent from client is preserved on server when `--delete` not set.
@@ -244,8 +253,11 @@ fn hdf5_sync_no_delete_preserves_server_dataset() {
let synced = serve_dir.path().join("model.h5"); let synced = serve_dir.path().join("model.h5");
let m_server = DatasetManifest::from_path(&synced).unwrap(); let m_server = DatasetManifest::from_path(&synced).unwrap();
assert!(m_server.get("/keep").is_some(), "keep should remain"); assert!(m_server.get("/keep").is_some(), "keep should remain");
assert!(m_server.get("/extra").is_some(), "extra should be preserved (no --delete)"); assert!(
m_server.get("/extra").is_some(),
"extra should be preserved (no --delete)"
);
} }
/// Nested path: server creates subdirectory as needed. /// Nested path: server creates subdirectory as needed.
+90 -31
View File
@@ -64,7 +64,10 @@ fn make_large_h5(dir: &TempDir, name: &str, n_pages: usize, fill: u8) -> PathBuf
let path = dir.path().join(name); let path = dir.path().join(name);
let mut data = Vec::with_capacity(n_pages * PAGE_SIZE); let mut data = Vec::with_capacity(n_pages * PAGE_SIZE);
data.extend_from_slice(H5_MAGIC); data.extend_from_slice(H5_MAGIC);
data.extend(std::iter::repeat_n(fill, n_pages * PAGE_SIZE - H5_MAGIC.len())); data.extend(std::iter::repeat_n(
fill,
n_pages * PAGE_SIZE - H5_MAGIC.len(),
));
std::fs::write(&path, &data).unwrap(); std::fs::write(&path, &data).unwrap();
path path
} }
@@ -93,7 +96,10 @@ fn make_versioned_large_h5(
// Base file: n_pages worth of data // Base file: n_pages worth of data
let mut base = Vec::with_capacity(n_pages * PAGE_SIZE); let mut base = Vec::with_capacity(n_pages * PAGE_SIZE);
base.extend_from_slice(H5_MAGIC); base.extend_from_slice(H5_MAGIC);
base.extend(std::iter::repeat_n(0u8, n_pages * PAGE_SIZE - H5_MAGIC.len())); base.extend(std::iter::repeat_n(
0u8,
n_pages * PAGE_SIZE - H5_MAGIC.len(),
));
std::fs::write(&path, &base).unwrap(); std::fs::write(&path, &base).unwrap();
let mut onion = OnionFile::create(&path, PAGE_SIZE as u32).unwrap(); let mut onion = OnionFile::create(&path, PAGE_SIZE as u32).unwrap();
@@ -104,7 +110,9 @@ fn make_versioned_large_h5(
let page_idx = (rev * pages_per_rev + p) % n_pages; let page_idx = (rev * pages_per_rev + p) % n_pages;
s.record_page((page_idx * PAGE_SIZE) as u64, &vec![fill; PAGE_SIZE]); s.record_page((page_idx * PAGE_SIZE) as u64, &vec![fill; PAGE_SIZE]);
} }
onion.commit_session(s, Some(&format!("rev {rev}"))).unwrap(); onion
.commit_session(s, Some(&format!("rev {rev}")))
.unwrap();
} }
onion.flush().unwrap(); onion.flush().unwrap();
path path
@@ -165,7 +173,10 @@ fn run_rsync(src: &Path, dst: &Path) -> RsyncStats {
assert!(output.status.success(), "rsync failed: {}", output.status); assert!(output.status.success(), "rsync failed: {}", output.status);
let stdout = String::from_utf8_lossy(&output.stdout); let stdout = String::from_utf8_lossy(&output.stdout);
let mut stats = RsyncStats { elapsed, ..Default::default() }; let mut stats = RsyncStats {
elapsed,
..Default::default()
};
for line in stdout.lines() { for line in stdout.lines() {
let line = line.trim(); let line = line.trim();
@@ -227,8 +238,8 @@ fn start_clawsync_server(h5_path: &Path) -> (std::process::Child, String) {
let mut line = String::new(); let mut line = String::new();
reader.read_line(&mut line).expect("server startup line"); reader.read_line(&mut line).expect("server startup line");
let addr = parse_serve_addr(&line) let addr =
.unwrap_or_else(|| panic!("could not parse addr from: {line:?}")); parse_serve_addr(&line).unwrap_or_else(|| panic!("could not parse addr from: {line:?}"));
// Drain remaining stdout so server doesn't get SIGPIPE. // Drain remaining stdout so server doesn't get SIGPIPE.
std::thread::spawn(move || { std::thread::spawn(move || {
@@ -255,11 +266,17 @@ fn run_clawsync_push(src_h5: &Path, dst_h5: &Path) -> ClawStats {
server.kill().ok(); server.kill().ok();
server.wait().ok(); server.wait().ok();
assert!(output.status.success(), "clawsync push failed: {}", assert!(
String::from_utf8_lossy(&output.stderr)); output.status.success(),
"clawsync push failed: {}",
String::from_utf8_lossy(&output.stderr)
);
let stdout = String::from_utf8_lossy(&output.stdout); let stdout = String::from_utf8_lossy(&output.stdout);
let mut stats = ClawStats { elapsed, ..Default::default() }; let mut stats = ClawStats {
elapsed,
..Default::default()
};
// "Push complete: N revision(s), M bytes" // "Push complete: N revision(s), M bytes"
if let Some(line) = stdout.lines().find(|l| l.starts_with("Push complete:")) { if let Some(line) = stdout.lines().find(|l| l.starts_with("Push complete:")) {
@@ -341,7 +358,7 @@ fn clawsync_vs_rsync_comparison() {
let src = make_large_h5(&src_dir, "src.h5", n_pages, 0xAB); let src = make_large_h5(&src_dir, "src.h5", n_pages, 0xAB);
let dst_rsync = dst_dir.path().join("dst_rsync.h5"); let dst_rsync = dst_dir.path().join("dst_rsync.h5");
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 0, 0); let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 0, 0);
// src as clawsync source: 1 revision with all pages // src as clawsync source: 1 revision with all pages
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, 1, n_pages); let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, 1, n_pages);
@@ -387,12 +404,18 @@ fn clawsync_vs_rsync_comparison() {
// clawsync: server has 1 revision, source adds 1 more (1 page) // clawsync: server has 1 revision, source adds 1 more (1 page)
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, 1, n_pages); let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, 1, n_pages);
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 1, n_pages); let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 1, n_pages);
add_revision(&src_claw, 1, 0xFF); // 1 new page in new revision add_revision(&src_claw, 1, 0xFF); // 1 new page in new revision
let rs = run_rsync(&src, &dst_rsync); let rs = run_rsync(&src, &dst_rsync);
let cs = run_clawsync_push(&src_claw, &dst_claw); let cs = run_clawsync_push(&src_claw, &dst_claw);
print_row("incremental, 1 page changed (0.4%)", &rs, &cs, n_pages, total); print_row(
"incremental, 1 page changed (0.4%)",
&rs,
&cs,
n_pages,
total,
);
} }
// ── 4. Incremental: 25 pages changed (10%) ───────────────────────────── // ── 4. Incremental: 25 pages changed (10%) ─────────────────────────────
@@ -409,7 +432,7 @@ fn clawsync_vs_rsync_comparison() {
overwrite_pages(&src, 0, changed, 0xFF); overwrite_pages(&src, 0, changed, 0xFF);
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, 1, n_pages); let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, 1, n_pages);
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 1, n_pages); let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 1, n_pages);
add_revision(&src_claw, changed, 0xFF); add_revision(&src_claw, changed, 0xFF);
let rs = run_rsync(&src, &dst_rsync); let rs = run_rsync(&src, &dst_rsync);
@@ -430,7 +453,7 @@ fn clawsync_vs_rsync_comparison() {
overwrite_pages(&src, 512, 1, 0xFF); overwrite_pages(&src, 512, 1, 0xFF);
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, 1, n_pages); let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, 1, n_pages);
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 1, n_pages); let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 1, n_pages);
add_revision(&src_claw, 1, 0xFF); add_revision(&src_claw, 1, 0xFF);
let rs = run_rsync(&src, &dst_rsync); let rs = run_rsync(&src, &dst_rsync);
@@ -453,12 +476,15 @@ fn clawsync_vs_rsync_comparison() {
// clawsync: transfers all 10 revisions with full history // clawsync: transfers all 10 revisions with full history
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, n_revs, 1); let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, n_revs, 1);
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 0, 0); let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 0, 0);
let cs = run_clawsync_push(&src_claw, &dst_claw); let cs = run_clawsync_push(&src_claw, &dst_claw);
print_row( print_row(
&format!("history: {n_revs} revisions (1 page/rev)"), &format!("history: {n_revs} revisions (1 page/rev)"),
&rs, &cs, n_pages, total, &rs,
&cs,
n_pages,
total,
); );
} }
@@ -475,12 +501,15 @@ fn clawsync_vs_rsync_comparison() {
let rs = run_rsync(&src_final, &dst_rsync); let rs = run_rsync(&src_final, &dst_rsync);
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, n_revs, 1); let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, n_revs, 1);
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 0, 0); let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 0, 0);
let cs = run_clawsync_push(&src_claw, &dst_claw); let cs = run_clawsync_push(&src_claw, &dst_claw);
print_row( print_row(
&format!("history: {n_revs} revisions (1 page/rev)"), &format!("history: {n_revs} revisions (1 page/rev)"),
&rs, &cs, n_pages, total, &rs,
&cs,
n_pages,
total,
); );
} }
@@ -528,7 +557,10 @@ fn make_large_file_small_sidecar(
let path = dir.path().join(name); let path = dir.path().join(name);
let mut data = Vec::with_capacity(n_pages * PAGE_SIZE); let mut data = Vec::with_capacity(n_pages * PAGE_SIZE);
data.extend_from_slice(H5_MAGIC); data.extend_from_slice(H5_MAGIC);
data.extend(std::iter::repeat_n(fill, n_pages * PAGE_SIZE - H5_MAGIC.len())); data.extend(std::iter::repeat_n(
fill,
n_pages * PAGE_SIZE - H5_MAGIC.len(),
));
std::fs::write(&path, &data).unwrap(); std::fs::write(&path, &data).unwrap();
let mut onion = OnionFile::create(&path, PAGE_SIZE as u32).unwrap(); let mut onion = OnionFile::create(&path, PAGE_SIZE as u32).unwrap();
@@ -536,7 +568,9 @@ fn make_large_file_small_sidecar(
let mut s = onion.begin_session(None).unwrap(); let mut s = onion.begin_session(None).unwrap();
// Only record page 0 — keeps the sidecar tiny regardless of file size. // Only record page 0 — keeps the sidecar tiny regardless of file size.
s.record_page(0, &vec![(rev as u8).wrapping_add(1); PAGE_SIZE]); s.record_page(0, &vec![(rev as u8).wrapping_add(1); PAGE_SIZE]);
onion.commit_session(s, Some(&format!("rev {rev}"))).unwrap(); onion
.commit_session(s, Some(&format!("rev {rev}")))
.unwrap();
} }
onion.flush().unwrap(); onion.flush().unwrap();
path path
@@ -564,13 +598,16 @@ fn clawsync_preflight_scaling() {
"| {:<20} | {:>10} | {:>13} | {:>13} | {:>9} |", "| {:<20} | {:>10} | {:>13} | {:>13} | {:>9} |",
"File size", "Revisions", "rsync no-op", "claw no-op", "Speedup" "File size", "Revisions", "rsync no-op", "claw no-op", "Speedup"
); );
println!("|{:-<22}|{:-<12}|{:-<15}|{:-<15}|{:-<11}|", "", "", "", "", ""); println!(
"|{:-<22}|{:-<12}|{:-<15}|{:-<15}|{:-<11}|",
"", "", "", "", ""
);
// (n_pages, n_revisions): keep n_revisions small (1) so IBLT sketch is constant. // (n_pages, n_revisions): keep n_revisions small (1) so IBLT sketch is constant.
// Only n_pages grows — that's what drives rsync cost. // Only n_pages grows — that's what drives rsync cost.
let cases: &[(usize, usize, &str)] = &[ let cases: &[(usize, usize, &str)] = &[
(64, 1, " 256 KiB"), (64, 1, " 256 KiB"),
(256, 1, " 1 MiB"), (256, 1, " 1 MiB"),
(1024, 1, " 4 MiB"), (1024, 1, " 4 MiB"),
(4096, 1, " 16 MiB"), (4096, 1, " 16 MiB"),
]; ];
@@ -584,7 +621,10 @@ fn clawsync_preflight_scaling() {
{ {
let mut data = Vec::with_capacity(n_pages * PAGE_SIZE); let mut data = Vec::with_capacity(n_pages * PAGE_SIZE);
data.extend_from_slice(H5_MAGIC); data.extend_from_slice(H5_MAGIC);
data.extend(std::iter::repeat_n(0xABu8, n_pages * PAGE_SIZE - H5_MAGIC.len())); data.extend(std::iter::repeat_n(
0xABu8,
n_pages * PAGE_SIZE - H5_MAGIC.len(),
));
std::fs::write(&src_h5, &data).unwrap(); std::fs::write(&src_h5, &data).unwrap();
std::fs::write(&dst_h5, &data).unwrap(); std::fs::write(&dst_h5, &data).unwrap();
} }
@@ -671,7 +711,11 @@ fn clawsync_unique_features() {
.lines() .lines()
.filter(|l| l.contains("experiment rev")) .filter(|l| l.contains("experiment rev"))
.collect(); .collect();
assert_eq!(rev_lines.len(), 5, "expected 5 revision lines, got: {revs_output}"); assert_eq!(
rev_lines.len(),
5,
"expected 5 revision lines, got: {revs_output}"
);
println!("rsync equivalent: **none** (rsync tracks no history)"); println!("rsync equivalent: **none** (rsync tracks no history)");
println!(); println!();
@@ -697,7 +741,10 @@ fn clawsync_unique_features() {
println!("Exported revision 2 → {} bytes", export_bytes); println!("Exported revision 2 → {} bytes", export_bytes);
// Exported file starts with HDF5 magic. // Exported file starts with HDF5 magic.
let exported = std::fs::read(&export_path).unwrap(); let exported = std::fs::read(&export_path).unwrap();
assert!(exported.starts_with(H5_MAGIC), "exported file missing HDF5 magic"); assert!(
exported.starts_with(H5_MAGIC),
"exported file missing HDF5 magic"
);
// Page 0 in revision 2 was filled with 2*10+1=21. // Page 0 in revision 2 was filled with 2*10+1=21.
let page0_byte = exported[PAGE_SIZE]; // skip magic, first full page let page0_byte = exported[PAGE_SIZE]; // skip magic, first full page
assert_eq!(page0_byte, 21u8, "exported revision 2 page0 byte mismatch"); assert_eq!(page0_byte, 21u8, "exported revision 2 page0 byte mismatch");
@@ -714,12 +761,18 @@ fn clawsync_unique_features() {
.args(["rollback", h5.to_str().unwrap(), "4"]) .args(["rollback", h5.to_str().unwrap(), "4"])
.output() .output()
.expect("rollback to head failed"); .expect("rollback to head failed");
assert!(out.status.success(), "rollback to head failed: {}", assert!(
String::from_utf8_lossy(&out.stderr)); out.status.success(),
"rollback to head failed: {}",
String::from_utf8_lossy(&out.stderr)
);
let head_bytes = std::fs::read(&h5).unwrap(); let head_bytes = std::fs::read(&h5).unwrap();
let head_byte = head_bytes[PAGE_SIZE]; let head_byte = head_bytes[PAGE_SIZE];
// Rev 4: page filled with 4*10+1=41 // Rev 4: page filled with 4*10+1=41
assert_eq!(head_byte, 41u8, "HEAD page byte should be 41 at rev 4, got {head_byte}"); assert_eq!(
head_byte, 41u8,
"HEAD page byte should be 41 at rev 4, got {head_byte}"
);
println!("HEAD (rev 4) data-page byte = {}", head_byte); println!("HEAD (rev 4) data-page byte = {}", head_byte);
let out = Command::new(BIN) let out = Command::new(BIN)
@@ -735,8 +788,14 @@ fn clawsync_unique_features() {
let rolled_bytes = std::fs::read(&h5).unwrap(); let rolled_bytes = std::fs::read(&h5).unwrap();
let rolled_byte = rolled_bytes[PAGE_SIZE]; let rolled_byte = rolled_bytes[PAGE_SIZE];
// After rollback to rev 1: page filled with 1*10+1=11 // After rollback to rev 1: page filled with 1*10+1=11
assert_eq!(rolled_byte, 11u8, "after rollback to rev 1, data-page byte should be 11, got {rolled_byte}"); assert_eq!(
println!("After rollback to rev 1: data-page byte = {} ✓ (expected 11)", rolled_byte); rolled_byte, 11u8,
"after rollback to rev 1, data-page byte should be 11, got {rolled_byte}"
);
println!(
"After rollback to rev 1: data-page byte = {} ✓ (expected 11)",
rolled_byte
);
println!("rsync equivalent: **none** (rsync cannot revert to a past state)"); println!("rsync equivalent: **none** (rsync cannot revert to a past state)");
println!(); println!();
+95 -27
View File
@@ -93,12 +93,7 @@ fn parse_serve_addr(line: &str) -> Option<String> {
/// pipe error when it tries to print subsequent log lines. /// pipe error when it tries to print subsequent log lines.
fn start_server(h5_path: &PathBuf) -> (std::process::Child, String) { fn start_server(h5_path: &PathBuf) -> (std::process::Child, String) {
let mut server = Command::new(BIN) let mut server = Command::new(BIN)
.args([ .args(["serve", h5_path.to_str().unwrap(), "--bind", "127.0.0.1:0"])
"serve",
h5_path.to_str().unwrap(),
"--bind",
"127.0.0.1:0",
])
.stdout(Stdio::piped()) .stdout(Stdio::piped())
.stderr(Stdio::null()) .stderr(Stdio::null())
.spawn() .spawn()
@@ -213,7 +208,8 @@ fn subprocess_pull_fetches_all_revisions_from_server() {
let reloaded = OnionFile::open(&client_h5).expect("cannot open client onion after pull"); let reloaded = OnionFile::open(&client_h5).expect("cannot open client onion after pull");
assert_eq!( assert_eq!(
reloaded.revision_count(), 4, reloaded.revision_count(),
4,
"client should have all 4 pulled revisions" "client should have all 4 pulled revisions"
); );
@@ -231,7 +227,11 @@ fn subprocess_list_revisions_output_is_correct() {
.output() .output()
.expect("failed to run clawsync list-revisions"); .expect("failed to run clawsync list-revisions");
assert!(output.status.success(), "list-revisions exited with {}", output.status); assert!(
output.status.success(),
"list-revisions exited with {}",
output.status
);
let stdout = String::from_utf8_lossy(&output.stdout); let stdout = String::from_utf8_lossy(&output.stdout);
assert!( assert!(
@@ -255,7 +255,11 @@ fn subprocess_list_revisions_timestamp_is_iso_date() {
.output() .output()
.expect("failed to run clawsync list-revisions"); .expect("failed to run clawsync list-revisions");
assert!(output.status.success(), "list-revisions exited with {}", output.status); assert!(
output.status.success(),
"list-revisions exited with {}",
output.status
);
let stdout = String::from_utf8_lossy(&output.stdout); let stdout = String::from_utf8_lossy(&output.stdout);
assert!( assert!(
@@ -311,7 +315,11 @@ fn subprocess_rollback_restores_h5_content_and_preserves_history() {
// Sidecar history must be fully preserved — all 4 revisions still readable. // Sidecar history must be fully preserved — all 4 revisions still readable.
let onion = OnionFile::open(&h5).expect("cannot open onion after rollback"); let onion = OnionFile::open(&h5).expect("cannot open onion after rollback");
assert_eq!(onion.revision_count(), 4, "rollback must not delete sidecar history"); assert_eq!(
onion.revision_count(),
4,
"rollback must not delete sidecar history"
);
// The h5 file must reflect the page content at revision 1. // The h5 file must reflect the page content at revision 1.
// make_versioned_h5 writes page 0 = vec![i; 4096] for each revision i, // make_versioned_h5 writes page 0 = vec![i; 4096] for each revision i,
@@ -340,7 +348,11 @@ fn subprocess_gc_keep_last_prunes_old_revisions() {
assert!(status.success(), "gc exited with {status}"); assert!(status.success(), "gc exited with {status}");
let onion = OnionFile::open(&h5).expect("cannot open onion after gc"); let onion = OnionFile::open(&h5).expect("cannot open onion after gc");
assert_eq!(onion.revision_count(), 2, "gc --keep-last 2 should leave exactly 2 revisions"); assert_eq!(
onion.revision_count(),
2,
"gc --keep-last 2 should leave exactly 2 revisions"
);
} }
/// `clawsync export-revision`: exports a specific revision to a new file /// `clawsync export-revision`: exports a specific revision to a new file
@@ -369,7 +381,11 @@ fn subprocess_export_revision_creates_standalone_file() {
// Source sidecar must be unchanged — still 3 revisions. // Source sidecar must be unchanged — still 3 revisions.
let onion = OnionFile::open(&h5).expect("cannot open source onion after export"); let onion = OnionFile::open(&h5).expect("cannot open source onion after export");
assert_eq!(onion.revision_count(), 3, "source sidecar must not be modified by export"); assert_eq!(
onion.revision_count(),
3,
"source sidecar must not be modified by export"
);
} }
/// `clawsync snapshot`: creates a snapshot checkpoint revision, incrementing /// `clawsync snapshot`: creates a snapshot checkpoint revision, incrementing
@@ -379,13 +395,22 @@ fn subprocess_snapshot_increments_revision_count() {
let (_dir, h5, _) = make_versioned_h5(3); let (_dir, h5, _) = make_versioned_h5(3);
let status = Command::new(BIN) let status = Command::new(BIN)
.args(["snapshot", h5.to_str().unwrap(), "--message", "test-snapshot"]) .args([
"snapshot",
h5.to_str().unwrap(),
"--message",
"test-snapshot",
])
.status() .status()
.expect("failed to run clawsync snapshot"); .expect("failed to run clawsync snapshot");
assert!(status.success(), "snapshot exited with {status}"); assert!(status.success(), "snapshot exited with {status}");
let onion = OnionFile::open(&h5).expect("cannot open onion after snapshot"); let onion = OnionFile::open(&h5).expect("cannot open onion after snapshot");
assert_eq!(onion.revision_count(), 4, "snapshot should add exactly one revision"); assert_eq!(
onion.revision_count(),
4,
"snapshot should add exactly one revision"
);
} }
/// `clawsync branch create / list / rename / delete`: full lifecycle in order. /// `clawsync branch create / list / rename / delete`: full lifecycle in order.
@@ -406,7 +431,11 @@ fn subprocess_branch_lifecycle() {
.args(["branch", "list", h5_str]) .args(["branch", "list", h5_str])
.output() .output()
.expect("failed to run branch list"); .expect("failed to run branch list");
assert!(output.status.success(), "branch list exited with {}", output.status); assert!(
output.status.success(),
"branch list exited with {}",
output.status
);
let stdout = String::from_utf8_lossy(&output.stdout); let stdout = String::from_utf8_lossy(&output.stdout);
assert!( assert!(
stdout.contains("experiment"), stdout.contains("experiment"),
@@ -427,8 +456,14 @@ fn subprocess_branch_lifecycle() {
.expect("failed to run branch list after rename"); .expect("failed to run branch list after rename");
assert!(output.status.success()); assert!(output.status.success());
let stdout = String::from_utf8_lossy(&output.stdout); let stdout = String::from_utf8_lossy(&output.stdout);
assert!(stdout.contains("feature-x"), "expected 'feature-x' after rename:\n{stdout}"); assert!(
assert!(!stdout.contains("experiment"), "old name 'experiment' must be gone:\n{stdout}"); stdout.contains("feature-x"),
"expected 'feature-x' after rename:\n{stdout}"
);
assert!(
!stdout.contains("experiment"),
"old name 'experiment' must be gone:\n{stdout}"
);
// Delete "feature-x". // Delete "feature-x".
let status = Command::new(BIN) let status = Command::new(BIN)
@@ -444,7 +479,10 @@ fn subprocess_branch_lifecycle() {
.expect("failed to run branch list after delete"); .expect("failed to run branch list after delete");
assert!(output.status.success()); assert!(output.status.success());
let stdout = String::from_utf8_lossy(&output.stdout); let stdout = String::from_utf8_lossy(&output.stdout);
assert!(!stdout.contains("feature-x"), "deleted branch must not appear in list:\n{stdout}"); assert!(
!stdout.contains("feature-x"),
"deleted branch must not appear in list:\n{stdout}"
);
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@@ -483,13 +521,20 @@ fn subprocess_push_delta_transfers_exact_count() {
.output() .output()
.expect("failed to run clawsync push"); .expect("failed to run clawsync push");
assert!(output.status.success(), "push exited with {}", output.status); assert!(
output.status.success(),
"push exited with {}",
output.status
);
let stdout = String::from_utf8_lossy(&output.stdout); let stdout = String::from_utf8_lossy(&output.stdout);
let transferred = parse_push_revision_count(&stdout) let transferred = parse_push_revision_count(&stdout)
.unwrap_or_else(|| panic!("could not parse push count from:\n{stdout}")); .unwrap_or_else(|| panic!("could not parse push count from:\n{stdout}"));
assert_eq!(transferred, 4, "expected exactly 4 delta revisions, got {transferred}"); assert_eq!(
transferred, 4,
"expected exactly 4 delta revisions, got {transferred}"
);
// Server must eventually reach 7 revisions total. // Server must eventually reach 7 revisions total.
let count = await_revisions(&server_h5, 7); let count = await_revisions(&server_h5, 7);
@@ -515,7 +560,9 @@ fn subprocess_pull_delta_transfers_exact_count() {
for i in 0u8..2 { for i in 0u8..2 {
let mut s = onion.begin_session(None).unwrap(); let mut s = onion.begin_session(None).unwrap();
s.record_page(0, &vec![i; 4096]); s.record_page(0, &vec![i; 4096]);
onion.commit_session(s, Some(&format!("local {i}"))).unwrap(); onion
.commit_session(s, Some(&format!("local {i}")))
.unwrap();
} }
onion.flush().unwrap(); onion.flush().unwrap();
} }
@@ -527,20 +574,31 @@ fn subprocess_pull_delta_transfers_exact_count() {
.output() .output()
.expect("failed to run clawsync pull"); .expect("failed to run clawsync pull");
assert!(output.status.success(), "pull exited with {}", output.status); assert!(
output.status.success(),
"pull exited with {}",
output.status
);
let stdout = String::from_utf8_lossy(&output.stdout); let stdout = String::from_utf8_lossy(&output.stdout);
let received = parse_pull_revision_count(&stdout) let received = parse_pull_revision_count(&stdout)
.unwrap_or_else(|| panic!("could not parse pull count from:\n{stdout}")); .unwrap_or_else(|| panic!("could not parse pull count from:\n{stdout}"));
assert_eq!(received, 4, "expected exactly 4 delta revisions, got {received}"); assert_eq!(
received, 4,
"expected exactly 4 delta revisions, got {received}"
);
server.kill().ok(); server.kill().ok();
server.wait().ok(); server.wait().ok();
// Client must end up with 6 revisions (2 original + 4 pulled). // Client must end up with 6 revisions (2 original + 4 pulled).
let onion = OnionFile::open(&client_h5).expect("cannot open client onion after pull"); let onion = OnionFile::open(&client_h5).expect("cannot open client onion after pull");
assert_eq!(onion.revision_count(), 6, "client should have 6 revisions after delta pull"); assert_eq!(
onion.revision_count(),
6,
"client should have 6 revisions after delta pull"
);
let _ = (server_dir, client_dir); let _ = (server_dir, client_dir);
} }
@@ -571,18 +629,28 @@ fn subprocess_push_large_delta_correct_count() {
.output() .output()
.expect("failed to run clawsync push"); .expect("failed to run clawsync push");
assert!(output.status.success(), "push exited with {}", output.status); assert!(
output.status.success(),
"push exited with {}",
output.status
);
let stdout = String::from_utf8_lossy(&output.stdout); let stdout = String::from_utf8_lossy(&output.stdout);
let transferred = parse_push_revision_count(&stdout) let transferred = parse_push_revision_count(&stdout)
.unwrap_or_else(|| panic!("could not parse push count from:\n{stdout}")); .unwrap_or_else(|| panic!("could not parse push count from:\n{stdout}"));
assert_eq!(transferred, 40, "expected exactly 40 delta revisions, got {transferred}"); assert_eq!(
transferred, 40,
"expected exactly 40 delta revisions, got {transferred}"
);
let count = await_revisions(&server_h5, 50); let count = await_revisions(&server_h5, 50);
server.kill().ok(); server.kill().ok();
server.wait().ok(); server.wait().ok();
assert_eq!(count, 50, "server should have 50 revisions after large delta push"); assert_eq!(
count, 50,
"server should have 50 revisions after large delta push"
);
} }
+3 -3
View File
@@ -39,7 +39,7 @@ fn float_data(n: usize) -> Vec<u8> {
let mant0 = (i.wrapping_mul(2654435769) >> 24) as u8; let mant0 = (i.wrapping_mul(2654435769) >> 24) as u8;
let mant1 = (i.wrapping_mul(0x811c9dc5) >> 16) as u8; let mant1 = (i.wrapping_mul(0x811c9dc5) >> 16) as u8;
let mant2 = (i.wrapping_mul(0x01000193)) as u8; let mant2 = (i.wrapping_mul(0x01000193)) as u8;
v.push(mant2); // byte 0: low mantissa (often < 64 → hot) v.push(mant2); // byte 0: low mantissa (often < 64 → hot)
v.push(mant1); v.push(mant1);
v.push(mant0); v.push(mant0);
v.push(exp_byte); // byte 3: exponent (often 0x3F-0x45 ≥ 64 → cold) v.push(exp_byte); // byte 3: exponent (often 0x3F-0x45 ≥ 64 → cold)
@@ -60,8 +60,8 @@ fn cold_data(n: usize) -> Vec<u8> {
fn bench_cdc(c: &mut Criterion) { fn bench_cdc(c: &mut Criterion) {
let datasets: &[(&str, fn(usize) -> Vec<u8>)] = &[ let datasets: &[(&str, fn(usize) -> Vec<u8>)] = &[
("random", random_data), ("random", random_data),
("float", float_data), ("float", float_data),
("cold_only", cold_data), ("cold_only", cold_data),
]; ];
let sizes = [1 << 20, 4 << 20, 16 << 20]; // 1 MB, 4 MB, 16 MB let sizes = [1 << 20, 4 << 20, 16 << 20]; // 1 MB, 4 MB, 16 MB
+13 -9
View File
@@ -35,12 +35,7 @@ pub fn chunk_data(data: &[u8]) -> Vec<Chunk> {
} }
/// Split `data` into content-defined chunks with explicit size parameters. /// Split `data` into content-defined chunks with explicit size parameters.
pub fn chunk_data_sized( pub fn chunk_data_sized(data: &[u8], min_size: u32, avg_size: u32, max_size: u32) -> Vec<Chunk> {
data: &[u8],
min_size: u32,
avg_size: u32,
max_size: u32,
) -> Vec<Chunk> {
if data.is_empty() { if data.is_empty() {
return Vec::new(); return Vec::new();
} }
@@ -152,8 +147,14 @@ mod tests {
for (i, c) in chunks.iter().enumerate() { for (i, c) in chunks.iter().enumerate() {
let is_last = i == chunks.len() - 1; let is_last = i == chunks.len() - 1;
if !is_last { if !is_last {
assert!(c.length >= MIN_CHUNK_SIZE as usize, "chunk below min at {i}"); assert!(
assert!(c.length <= MAX_CHUNK_SIZE as usize, "chunk above max at {i}"); c.length >= MIN_CHUNK_SIZE as usize,
"chunk below min at {i}"
);
assert!(
c.length <= MAX_CHUNK_SIZE as usize,
"chunk above max at {i}"
);
} }
} }
} }
@@ -189,7 +190,10 @@ mod tests {
let chunks = chunk_data(&data); let chunks = chunk_data(&data);
let source_set = chunk_hash_set(&chunks); let source_set = chunk_hash_set(&chunks);
let diff = changed_chunks(&source_set, &chunks); let diff = changed_chunks(&source_set, &chunks);
assert!(diff.is_empty(), "identical data should have no changed chunks"); assert!(
diff.is_empty(),
"identical data should have no changed chunks"
);
} }
#[test] #[test]
+3 -6
View File
@@ -63,10 +63,7 @@ pub fn blake3_hash_large(data: &[u8]) -> [u8; 32] {
/// order and page offsets must be incorporated (use `hash_pages` instead). /// order and page offsets must be incorporated (use `hash_pages` instead).
pub fn blake3_hash_batch(chunks: &[&[u8]]) -> Vec<[u8; 32]> { pub fn blake3_hash_batch(chunks: &[&[u8]]) -> Vec<[u8; 32]> {
use rayon::prelude::*; use rayon::prelude::*;
chunks chunks.par_iter().map(|c| blake3_hash_large(c)).collect()
.par_iter()
.map(|c| blake3_hash_large(c))
.collect()
} }
/// Verify that `data` matches an expected BLAKE3 digest. /// Verify that `data` matches an expected BLAKE3 digest.
@@ -83,7 +80,7 @@ pub fn blake3_hex(data: &[u8]) -> String {
let hash = blake3_hash(data); let hash = blake3_hash(data);
let mut out = vec![0u8; 64]; let mut out = vec![0u8; 64];
for (i, &byte) in hash.iter().enumerate() { for (i, &byte) in hash.iter().enumerate() {
out[i * 2] = HEX[(byte >> 4) as usize]; out[i * 2] = HEX[(byte >> 4) as usize];
out[i * 2 + 1] = HEX[(byte & 0xf) as usize]; out[i * 2 + 1] = HEX[(byte & 0xf) as usize];
} }
// Every byte is a valid ASCII hex character — from_utf8 cannot fail. // Every byte is a valid ASCII hex character — from_utf8 cannot fail.
@@ -95,7 +92,7 @@ pub fn hash_to_hex(hash: &[u8; 32]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef"; const HEX: &[u8; 16] = b"0123456789abcdef";
let mut out = vec![0u8; 64]; let mut out = vec![0u8; 64];
for (i, &byte) in hash.iter().enumerate() { for (i, &byte) in hash.iter().enumerate() {
out[i * 2] = HEX[(byte >> 4) as usize]; out[i * 2] = HEX[(byte >> 4) as usize];
out[i * 2 + 1] = HEX[(byte & 0xf) as usize]; out[i * 2 + 1] = HEX[(byte & 0xf) as usize];
} }
String::from_utf8(out).expect("hex output is always valid UTF-8") String::from_utf8(out).expect("hex output is always valid UTF-8")
+2 -6
View File
@@ -34,8 +34,7 @@ pub fn lz4_compress(data: &[u8]) -> Result<Vec<u8>, CoreError> {
/// Decompress lz4-compressed `data` (expects prepended original size). /// Decompress lz4-compressed `data` (expects prepended original size).
pub fn lz4_decompress(data: &[u8]) -> Result<Vec<u8>, CoreError> { pub fn lz4_decompress(data: &[u8]) -> Result<Vec<u8>, CoreError> {
lz4_flex::decompress_size_prepended(data) lz4_flex::decompress_size_prepended(data).map_err(|e| CoreError::Decompress(e.to_string()))
.map_err(|e| CoreError::Decompress(e.to_string()))
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@@ -64,10 +63,7 @@ mod tests {
fn zstd_compress_reduces_size() { fn zstd_compress_reduces_size() {
let data = vec![0u8; 65536]; // highly compressible let data = vec![0u8; 65536]; // highly compressible
let compressed = zstd_compress(&data, ZSTD_DEFAULT_LEVEL).unwrap(); let compressed = zstd_compress(&data, ZSTD_DEFAULT_LEVEL).unwrap();
assert!( assert!(compressed.len() < data.len(), "zstd should compress zeroes");
compressed.len() < data.len(),
"zstd should compress zeroes"
);
} }
#[test] #[test]
+22 -8
View File
@@ -58,7 +58,11 @@ pub enum DeltaOp {
/// This produces a simple but correct delta — it does not do rolling-hash /// This produces a simple but correct delta — it does not do rolling-hash
/// sliding-window matching (that is CDC's job). For block-aligned page data /// sliding-window matching (that is CDC's job). For block-aligned page data
/// (the primary use case) this is optimal. /// (the primary use case) this is optimal.
pub fn compute_delta(source_hashes: &[BlockHash], target: &[u8], block_size: usize) -> Vec<DeltaOp> { pub fn compute_delta(
source_hashes: &[BlockHash],
target: &[u8],
block_size: usize,
) -> Vec<DeltaOp> {
// Build a hash → offset map for O(1) lookups // Build a hash → offset map for O(1) lookups
let lookup: HashMap<u64, usize> = source_hashes let lookup: HashMap<u64, usize> = source_hashes
.iter() .iter()
@@ -74,21 +78,28 @@ pub fn compute_delta(source_hashes: &[BlockHash], target: &[u8], block_size: usi
if let Some(&src_offset) = lookup.get(&hash) { if let Some(&src_offset) = lookup.get(&hash) {
// Coalesce adjacent copies // Coalesce adjacent copies
if let Some(DeltaOp::Copy { offset, length }) = ops.last_mut() { if let Some(DeltaOp::Copy { offset, length }) = ops.last_mut() {
if *offset + *length == src_offset && target_offset == { if *offset + *length == src_offset
let prev_target_block = i.saturating_sub(1) * block_size; && target_offset == {
prev_target_block + block_size let prev_target_block = i.saturating_sub(1) * block_size;
} { prev_target_block + block_size
}
{
*length += chunk.len(); *length += chunk.len();
continue; continue;
} }
} }
ops.push(DeltaOp::Copy { offset: src_offset, length: chunk.len() }); ops.push(DeltaOp::Copy {
offset: src_offset,
length: chunk.len(),
});
} else { } else {
// Coalesce adjacent inserts // Coalesce adjacent inserts
if let Some(DeltaOp::Insert { data }) = ops.last_mut() { if let Some(DeltaOp::Insert { data }) = ops.last_mut() {
data.extend_from_slice(chunk); data.extend_from_slice(chunk);
} else { } else {
ops.push(DeltaOp::Insert { data: chunk.to_vec() }); ops.push(DeltaOp::Insert {
data: chunk.to_vec(),
});
} }
} }
} }
@@ -149,7 +160,10 @@ mod tests {
"identical data should produce only Copy ops" "identical data should produce only Copy ops"
); );
let transfer = delta_transfer_size(&ops); let transfer = delta_transfer_size(&ops);
assert_eq!(transfer, 0, "no bytes should be transferred for identical data"); assert_eq!(
transfer, 0,
"no bytes should be transferred for identical data"
);
} }
#[test] #[test]
+2 -2
View File
@@ -10,11 +10,11 @@
//! - [`compress`]: zstd and lz4 streaming wrappers //! - [`compress`]: zstd and lz4 streaming wrappers
//! - [`error`]: [`CoreError`] type //! - [`error`]: [`CoreError`] type
#[forbid(unsafe_code)]
pub mod checksum;
#[forbid(unsafe_code)] #[forbid(unsafe_code)]
pub mod cdc; pub mod cdc;
#[forbid(unsafe_code)] #[forbid(unsafe_code)]
pub mod checksum;
#[forbid(unsafe_code)]
pub mod compress; pub mod compress;
#[forbid(unsafe_code)] #[forbid(unsafe_code)]
pub mod delta; pub mod delta;
+61 -13
View File
@@ -49,7 +49,11 @@ const fn make_gear() -> [u64; 256] {
state ^= state >> 7; state ^= state >> 7;
state ^= state << 17; state ^= state << 17;
// ensure non-zero so hot bytes always change the hash // ensure non-zero so hot bytes always change the hash
t[i] = if state == 0 { 0x0BAD_CAFE_DEAD_BEEFu64 } else { state }; t[i] = if state == 0 {
0x0BAD_CAFE_DEAD_BEEFu64
} else {
state
};
i += 1; i += 1;
} }
t t
@@ -82,7 +86,12 @@ pub fn chunk_data_simd(data: &[u8]) -> Vec<Chunk> {
} }
/// Split `data` with explicit size parameters. /// Split `data` with explicit size parameters.
pub fn chunk_data_sized_simd(data: &[u8], min_size: u32, avg_size: u32, max_size: u32) -> Vec<Chunk> { pub fn chunk_data_sized_simd(
data: &[u8],
min_size: u32,
avg_size: u32,
max_size: u32,
) -> Vec<Chunk> {
let min = min_size as usize; let min = min_size as usize;
let avg = avg_size as usize; let avg = avg_size as usize;
let max = max_size as usize; let max = max_size as usize;
@@ -222,7 +231,17 @@ mod neon {
// Hot byte present, at/past min_size, or approaching max: // Hot byte present, at/past min_size, or approaching max:
// scalar fallback handles boundaries correctly. // scalar fallback handles boundaries correctly.
let window = &data[pos..pos + 16]; let window = &data[pos..pos + 16];
scalar_window(data, window, pos, &mut chunk_start, &mut h, min, max, mask, &mut chunks); scalar_window(
data,
window,
pos,
&mut chunk_start,
&mut h,
min,
max,
mask,
&mut chunks,
);
pos += 16; pos += 16;
} }
} }
@@ -230,7 +249,17 @@ mod neon {
// Tail bytes (< 16 remaining) // Tail bytes (< 16 remaining)
if pos < data.len() { if pos < data.len() {
let window = &data[pos..]; let window = &data[pos..];
scalar_window(data, window, pos, &mut chunk_start, &mut h, min, max, mask, &mut chunks); scalar_window(
data,
window,
pos,
&mut chunk_start,
&mut h,
min,
max,
mask,
&mut chunks,
);
} }
if chunk_start < data.len() { if chunk_start < data.len() {
@@ -292,14 +321,34 @@ mod sse2 {
pos += 16; pos += 16;
} else { } else {
let window = &data[pos..pos + 16]; let window = &data[pos..pos + 16];
scalar_window(data, window, pos, &mut chunk_start, &mut h, min, max, mask, &mut chunks); scalar_window(
data,
window,
pos,
&mut chunk_start,
&mut h,
min,
max,
mask,
&mut chunks,
);
pos += 16; pos += 16;
} }
} }
if pos < data.len() { if pos < data.len() {
let window = &data[pos..]; let window = &data[pos..];
scalar_window(data, window, pos, &mut chunk_start, &mut h, min, max, mask, &mut chunks); scalar_window(
data,
window,
pos,
&mut chunk_start,
&mut h,
min,
max,
mask,
&mut chunks,
);
} }
if chunk_start < data.len() { if chunk_start < data.len() {
@@ -410,12 +459,7 @@ mod tests {
#[test] #[test]
fn chunk_sizes_respect_min_max_except_last() { fn chunk_sizes_respect_min_max_except_last() {
let data = lcg_data(500_000, 7); let data = lcg_data(500_000, 7);
let chunks = chunk_data_sized_simd( let chunks = chunk_data_sized_simd(&data, MIN_CHUNK_SIZE, AVG_CHUNK_SIZE, MAX_CHUNK_SIZE);
&data,
MIN_CHUNK_SIZE,
AVG_CHUNK_SIZE,
MAX_CHUNK_SIZE,
);
for (i, c) in chunks.iter().enumerate() { for (i, c) in chunks.iter().enumerate() {
let is_last = i == chunks.len() - 1; let is_last = i == chunks.len() - 1;
if !is_last { if !is_last {
@@ -511,7 +555,11 @@ mod tests {
let data = lcg_data(200_000, 77); let data = lcg_data(200_000, 77);
for c in chunk_data_simd(&data) { for c in chunk_data_simd(&data) {
let expected = crate::checksum::xxh3_block(&data[c.offset..c.offset + c.length]); let expected = crate::checksum::xxh3_block(&data[c.offset..c.offset + c.length]);
assert_eq!(c.hash, expected, "chunk hash mismatch at offset {}", c.offset); assert_eq!(
c.hash, expected,
"chunk hash mismatch at offset {}",
c.offset
);
} }
} }
} }
+30 -19
View File
@@ -9,9 +9,9 @@
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
use clawsync_core::cdc::{chunk_data, Chunk}; use clawsync_core::cdc::{Chunk, chunk_data};
use clawsync_core::checksum::blake3_hash_large; use clawsync_core::checksum::blake3_hash_large;
use clawsync_core::compress::{zstd_compress, zstd_decompress, ZSTD_DEFAULT_LEVEL}; use clawsync_core::compress::{ZSTD_DEFAULT_LEVEL, zstd_compress, zstd_decompress};
use clawsync_transport::protocol::{FsChunk, FsChunkHash}; use clawsync_transport::protocol::{FsChunk, FsChunkHash};
use crate::error::FsSyncError; use crate::error::FsSyncError;
@@ -40,7 +40,10 @@ pub fn chunk_file_for_request(data: &[u8]) -> (Vec<Chunk>, Vec<FsChunkHash>) {
let chunks = chunk_data(data); let chunks = chunk_data(data);
let hashes = chunks let hashes = chunks
.iter() .iter()
.map(|c| FsChunkHash { hash: c.hash, length: c.length as u32 }) .map(|c| FsChunkHash {
hash: c.hash,
length: c.length as u32,
})
.collect(); .collect();
(chunks, hashes) (chunks, hashes)
} }
@@ -72,11 +75,7 @@ pub fn compute_needed_indices(
/// ///
/// Each chunk is zstd-compressed; raw bytes are used instead when the /// Each chunk is zstd-compressed; raw bytes are used instead when the
/// compressed output is not smaller. /// compressed output is not smaller.
pub fn build_chunk_data( pub fn build_chunk_data(data: &[u8], all_chunks: &[Chunk], needed_indices: &[u32]) -> Vec<FsChunk> {
data: &[u8],
all_chunks: &[Chunk],
needed_indices: &[u32],
) -> Vec<FsChunk> {
needed_indices needed_indices
.iter() .iter()
.map(|&idx| { .map(|&idx| {
@@ -90,7 +89,11 @@ pub fn build_chunk_data(
_ => (raw.to_vec(), false), _ => (raw.to_vec(), false),
}; };
FsChunk { hash: c.hash, data: payload, compressed } FsChunk {
hash: c.hash,
data: payload,
compressed,
}
}) })
.collect() .collect()
} }
@@ -127,9 +130,14 @@ pub fn reconstruct_file(
for desc in client_chunk_order { for desc in client_chunk_order {
if let Some(&src_offset) = server_chunk_map.get(&desc.hash) { if let Some(&src_offset) = server_chunk_map.get(&desc.hash) {
plan.push(ReconstructOp::LocalCopy { src_offset, length: desc.length }); plan.push(ReconstructOp::LocalCopy {
src_offset,
length: desc.length,
});
} else { } else {
plan.push(ReconstructOp::Literal { chunk_index: literal_idx }); plan.push(ReconstructOp::Literal {
chunk_index: literal_idx,
});
literal_idx += 1; literal_idx += 1;
} }
} }
@@ -206,8 +214,8 @@ mod tests {
fn pseudo_random(size: usize, seed: u8) -> Vec<u8> { fn pseudo_random(size: usize, seed: u8) -> Vec<u8> {
// Xorshift64 — high entropy, genuinely incompressible output. // Xorshift64 — high entropy, genuinely incompressible output.
let mut s: u64 = 0xDEAD_CAFE_0000_0001u64 let mut s: u64 =
^ (seed as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15); 0xDEAD_CAFE_0000_0001u64 ^ (seed as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15);
(0..size) (0..size)
.map(|_| { .map(|_| {
s ^= s << 13; s ^= s << 13;
@@ -220,10 +228,7 @@ mod tests {
/// Run the full CDC delta pipeline: chunk client file, compute needs, /// Run the full CDC delta pipeline: chunk client file, compute needs,
/// build literal data, reconstruct, verify BLAKE3. /// build literal data, reconstruct, verify BLAKE3.
fn full_roundtrip( fn full_roundtrip(server_data: &[u8], client_data: &[u8]) -> (Vec<u8>, usize) {
server_data: &[u8],
client_data: &[u8],
) -> (Vec<u8>, usize) {
let expected = blake3_hash_large(client_data); let expected = blake3_hash_large(client_data);
let (client_chunks, client_hashes) = chunk_file_for_request(client_data); let (client_chunks, client_hashes) = chunk_file_for_request(client_data);
@@ -252,7 +257,10 @@ mod tests {
fn roundtrip_identical_file_sends_zero_chunk_bytes() { fn roundtrip_identical_file_sends_zero_chunk_bytes() {
let data = pseudo_random(200_000, 1); let data = pseudo_random(200_000, 1);
let (_, literal_bytes) = full_roundtrip(&data, &data); let (_, literal_bytes) = full_roundtrip(&data, &data);
assert_eq!(literal_bytes, 0, "identical file should send 0 literal bytes"); assert_eq!(
literal_bytes, 0,
"identical file should send 0 literal bytes"
);
} }
#[test] #[test]
@@ -316,7 +324,10 @@ mod tests {
let (chunks, _) = chunk_file_for_request(&data); let (chunks, _) = chunk_file_for_request(&data);
let chunk_data_built = build_chunk_data(&data, &chunks, &[0]); let chunk_data_built = build_chunk_data(&data, &chunks, &[0]);
// At least one chunk should be uncompressed (raw) since data is incompressible. // At least one chunk should be uncompressed (raw) since data is incompressible.
assert!(!chunk_data_built[0].compressed, "incompressible chunk should be sent raw"); assert!(
!chunk_data_built[0].compressed,
"incompressible chunk should be sent raw"
);
} }
#[test] #[test]
+36 -14
View File
@@ -39,8 +39,10 @@ impl FileDiff {
/// ///
/// Result is sorted by path for deterministic ordering. /// Result is sorted by path for deterministic ordering.
pub fn diff_manifests(local: &[LocalEntry], remote: &[FsManifestEntry]) -> Vec<FileDiff> { pub fn diff_manifests(local: &[LocalEntry], remote: &[FsManifestEntry]) -> Vec<FileDiff> {
let local_map: HashMap<&str, [u8; 32]> = let local_map: HashMap<&str, [u8; 32]> = local
local.iter().map(|e| (e.rel_path.as_str(), e.blake3)).collect(); .iter()
.map(|e| (e.rel_path.as_str(), e.blake3))
.collect();
let remote_map: HashMap<&str, [u8; 32]> = let remote_map: HashMap<&str, [u8; 32]> =
remote.iter().map(|e| (e.path.as_str(), e.blake3)).collect(); remote.iter().map(|e| (e.path.as_str(), e.blake3)).collect();
@@ -51,10 +53,10 @@ pub fn diff_manifests(local: &[LocalEntry], remote: &[FsManifestEntry]) -> Vec<F
all_paths all_paths
.into_iter() .into_iter()
.map(|path| match (local_map.get(path), remote_map.get(path)) { .map(|path| match (local_map.get(path), remote_map.get(path)) {
(Some(_), None) => FileDiff::Added(path.to_string()), (Some(_), None) => FileDiff::Added(path.to_string()),
(None, Some(_)) => FileDiff::Removed(path.to_string()), (None, Some(_)) => FileDiff::Removed(path.to_string()),
(Some(lh), Some(rh)) if lh != rh => FileDiff::Modified(path.to_string()), (Some(lh), Some(rh)) if lh != rh => FileDiff::Modified(path.to_string()),
_ => FileDiff::Unchanged(path.to_string()), _ => FileDiff::Unchanged(path.to_string()),
}) })
.collect() .collect()
} }
@@ -147,18 +149,38 @@ mod tests {
#[test] #[test]
fn mixed_diff_counts_correct() { fn mixed_diff_counts_correct() {
let l = vec![local("keep.bin", 1), local("new.bin", 3), local("mod.bin", 4)]; let l = vec![
let r = vec![remote("keep.bin", 1), remote("old.bin", 9), remote("mod.bin", 5)]; local("keep.bin", 1),
local("new.bin", 3),
local("mod.bin", 4),
];
let r = vec![
remote("keep.bin", 1),
remote("old.bin", 9),
remote("mod.bin", 5),
];
let diffs = diff_manifests(&l, &r); let diffs = diff_manifests(&l, &r);
let added: Vec<_> = diffs.iter().filter(|d| matches!(d, FileDiff::Added(_))).collect(); let added: Vec<_> = diffs
let modified: Vec<_> = diffs.iter().filter(|d| matches!(d, FileDiff::Modified(_))).collect(); .iter()
let removed: Vec<_> = diffs.iter().filter(|d| matches!(d, FileDiff::Removed(_))).collect(); .filter(|d| matches!(d, FileDiff::Added(_)))
let unchanged: Vec<_> = diffs.iter().filter(|d| matches!(d, FileDiff::Unchanged(_))).collect(); .collect();
let modified: Vec<_> = diffs
.iter()
.filter(|d| matches!(d, FileDiff::Modified(_)))
.collect();
let removed: Vec<_> = diffs
.iter()
.filter(|d| matches!(d, FileDiff::Removed(_)))
.collect();
let unchanged: Vec<_> = diffs
.iter()
.filter(|d| matches!(d, FileDiff::Unchanged(_)))
.collect();
assert_eq!(added.len(), 1); // new.bin assert_eq!(added.len(), 1); // new.bin
assert_eq!(modified.len(), 1); // mod.bin assert_eq!(modified.len(), 1); // mod.bin
assert_eq!(removed.len(), 1); // old.bin assert_eq!(removed.len(), 1); // old.bin
assert_eq!(unchanged.len(), 1); // keep.bin assert_eq!(unchanged.len(), 1); // keep.bin
} }
+24 -10
View File
@@ -51,7 +51,9 @@ impl FsManifest {
let abs = entry.path().to_path_buf(); let abs = entry.path().to_path_buf();
let rel = abs let rel = abs
.strip_prefix(root) .strip_prefix(root)
.map_err(|_| FsSyncError::Path(format!("cannot strip prefix from {}", abs.display())))? .map_err(|_| {
FsSyncError::Path(format!("cannot strip prefix from {}", abs.display()))
})?
.to_string_lossy() .to_string_lossy()
.replace('\\', "/"); // normalise on Windows .replace('\\', "/"); // normalise on Windows
@@ -75,7 +77,13 @@ impl FsManifest {
.and_then(|t| t.duration_since(UNIX_EPOCH).ok()) .and_then(|t| t.duration_since(UNIX_EPOCH).ok())
.map(|d| d.as_secs() as i64) .map(|d| d.as_secs() as i64)
.unwrap_or(0); .unwrap_or(0);
Ok(LocalEntry { rel_path: rel, abs_path: abs, blake3, size, mtime }) Ok(LocalEntry {
rel_path: rel,
abs_path: abs,
blake3,
size,
mtime,
})
}) })
.collect::<Result<Vec<_>, _>>()?; .collect::<Result<Vec<_>, _>>()?;
@@ -88,19 +96,17 @@ impl FsManifest {
self.entries self.entries
.iter() .iter()
.map(|e| FsManifestEntry { .map(|e| FsManifestEntry {
path: e.rel_path.clone(), path: e.rel_path.clone(),
blake3: e.blake3, blake3: e.blake3,
size: e.size, size: e.size,
mtime: e.mtime, mtime: e.mtime,
}) })
.collect() .collect()
} }
/// Look up an entry by its relative path. /// Look up an entry by its relative path.
pub fn get(&self, rel_path: &str) -> Option<&LocalEntry> { pub fn get(&self, rel_path: &str) -> Option<&LocalEntry> {
self.entries self.entries.iter().find(|e| e.rel_path == rel_path)
.iter()
.find(|e| e.rel_path == rel_path)
} }
} }
@@ -162,7 +168,11 @@ mod tests {
fs::write(dir.path().join("sub").join("nested.bin"), b"nested").unwrap(); fs::write(dir.path().join("sub").join("nested.bin"), b"nested").unwrap();
let manifest = FsManifest::build(dir.path(), &empty_excludes()).unwrap(); let manifest = FsManifest::build(dir.path(), &empty_excludes()).unwrap();
let paths: Vec<&str> = manifest.entries.iter().map(|e| e.rel_path.as_str()).collect(); let paths: Vec<&str> = manifest
.entries
.iter()
.map(|e| e.rel_path.as_str())
.collect();
assert!(paths.contains(&"root.bin")); assert!(paths.contains(&"root.bin"));
assert!(paths.contains(&"sub/nested.bin")); assert!(paths.contains(&"sub/nested.bin"));
} }
@@ -175,7 +185,11 @@ mod tests {
fs::write(dir.path().join("m.bin"), b"m").unwrap(); fs::write(dir.path().join("m.bin"), b"m").unwrap();
let manifest = FsManifest::build(dir.path(), &empty_excludes()).unwrap(); let manifest = FsManifest::build(dir.path(), &empty_excludes()).unwrap();
let paths: Vec<&str> = manifest.entries.iter().map(|e| e.rel_path.as_str()).collect(); let paths: Vec<&str> = manifest
.entries
.iter()
.map(|e| e.rel_path.as_str())
.collect();
assert_eq!(paths, vec!["a.bin", "m.bin", "z.bin"]); assert_eq!(paths, vec!["a.bin", "m.bin", "z.bin"]);
} }
} }
+175 -87
View File
@@ -32,9 +32,9 @@ const PIPELINE_WINDOW: usize = 16;
/// Aggregate statistics from one completed sync session. /// Aggregate statistics from one completed sync session.
#[derive(Debug, Default)] #[derive(Debug, Default)]
pub struct SyncStats { pub struct SyncStats {
pub files_added: u32, pub files_added: u32,
pub files_modified: u32, pub files_modified: u32,
pub files_removed: u32, pub files_removed: u32,
pub bytes_transferred: u64, pub bytes_transferred: u64,
} }
@@ -52,7 +52,12 @@ pub struct FsSyncClient {
impl FsSyncClient { impl FsSyncClient {
pub fn new(conn: SyncPeer, local_root: PathBuf, excludes: GlobSet, delete: bool) -> Self { pub fn new(conn: SyncPeer, local_root: PathBuf, excludes: GlobSet, delete: bool) -> Self {
Self { conn, local_root, excludes, _delete: delete } Self {
conn,
local_root,
excludes,
_delete: delete,
}
} }
/// Execute the full directory sync protocol. /// Execute the full directory sync protocol.
@@ -60,9 +65,10 @@ impl FsSyncClient {
// ── RTT 1: build local manifest and send it ────────────────────────── // ── RTT 1: build local manifest and send it ──────────────────────────
let local_root = self.local_root.clone(); let local_root = self.local_root.clone();
let excludes = self.excludes.clone(); let excludes = self.excludes.clone();
let manifest = tokio::task::spawn_blocking(move || FsManifest::build(&local_root, &excludes)) let manifest =
.await tokio::task::spawn_blocking(move || FsManifest::build(&local_root, &excludes))
.map_err(|e| FsSyncError::Join(e.to_string()))??; .await
.map_err(|e| FsSyncError::Join(e.to_string()))??;
self.conn self.conn
.send(&SyncMessage::FsDirManifest { .send(&SyncMessage::FsDirManifest {
@@ -72,17 +78,37 @@ impl FsSyncClient {
.await?; .await?;
let (needed_files, to_delete) = match self.conn.recv().await? { let (needed_files, to_delete) = match self.conn.recv().await? {
SyncMessage::FsDirNeed { needed_files, to_delete } => (needed_files, to_delete), SyncMessage::FsDirNeed {
needed_files,
to_delete,
} => (needed_files, to_delete),
SyncMessage::Error { message } => return Err(FsSyncError::Protocol(message)), SyncMessage::Error { message } => return Err(FsSyncError::Protocol(message)),
other => return Err(FsSyncError::Protocol(format!("expected FsDirNeed, got {other:?}"))), other => {
return Err(FsSyncError::Protocol(format!(
"expected FsDirNeed, got {other:?}"
)));
}
}; };
// Warm no-op: nothing to do on either side. // Warm no-op: nothing to do on either side.
if needed_files.is_empty() && to_delete.is_empty() { if needed_files.is_empty() && to_delete.is_empty() {
let final_stats = match self.conn.recv().await? { let final_stats = match self.conn.recv().await? {
SyncMessage::FsDirComplete { files_added, files_modified, files_removed, bytes_transferred } => SyncMessage::FsDirComplete {
SyncStats { files_added, files_modified, files_removed, bytes_transferred }, files_added,
other => return Err(FsSyncError::Protocol(format!("expected FsDirComplete, got {other:?}"))), files_modified,
files_removed,
bytes_transferred,
} => SyncStats {
files_added,
files_modified,
files_removed,
bytes_transferred,
},
other => {
return Err(FsSyncError::Protocol(format!(
"expected FsDirComplete, got {other:?}"
)));
}
}; };
self.conn.shutdown().await?; self.conn.shutdown().await?;
return Ok(final_stats); return Ok(final_stats);
@@ -112,45 +138,47 @@ impl FsSyncClient {
let sem_writer = semaphore.clone(); let sem_writer = semaphore.clone();
let writer_task: tokio::task::JoinHandle< let writer_task: tokio::task::JoinHandle<Result<PipeWriteHalf, FsSyncError>> =
Result<PipeWriteHalf, FsSyncError>, tokio::spawn(async move {
> = tokio::spawn(async move { for file_need in needed_files {
for file_need in needed_files { let abs_path = path_map.get(&file_need.path).cloned().ok_or_else(|| {
let abs_path = path_map FsSyncError::Path(format!("local path not found: {}", file_need.path))
.get(&file_need.path) })?;
.cloned()
.ok_or_else(|| FsSyncError::Path(format!("local path not found: {}", file_need.path)))?;
sem_writer.acquire().await.expect("semaphore closed").forget(); sem_writer
.acquire()
.await
.expect("semaphore closed")
.forget();
let server_chunks_wire = file_need.server_chunks.clone(); let server_chunks_wire = file_need.server_chunks.clone();
// Read + CDC chunk in a blocking thread. // Read + CDC chunk in a blocking thread.
let (chunk_order, literal_chunks, wire_bytes) = let (chunk_order, literal_chunks, wire_bytes) =
tokio::task::spawn_blocking(move || -> Result<_, FsSyncError> { tokio::task::spawn_blocking(move || -> Result<_, FsSyncError> {
let data = std::fs::read(&abs_path)?; let data = std::fs::read(&abs_path)?;
let (all_chunks, hashes) = chunk_file_for_request(&data); let (all_chunks, hashes) = chunk_file_for_request(&data);
let server_existing: HashSet<u64> = let server_existing: HashSet<u64> =
server_chunks_wire.iter().map(|c| c.hash).collect(); server_chunks_wire.iter().map(|c| c.hash).collect();
let needed_idx = compute_needed_indices(&server_existing, &hashes); let needed_idx = compute_needed_indices(&server_existing, &hashes);
let literals = build_chunk_data(&data, &all_chunks, &needed_idx); let literals = build_chunk_data(&data, &all_chunks, &needed_idx);
let bytes: u64 = literals.iter().map(|c| c.data.len() as u64).sum(); let bytes: u64 = literals.iter().map(|c| c.data.len() as u64).sum();
Ok((hashes, literals, bytes)) Ok((hashes, literals, bytes))
}) })
.await .await
.map_err(|e| FsSyncError::Join(e.to_string()))??; .map_err(|e| FsSyncError::Join(e.to_string()))??;
meta_tx.send(wire_bytes).expect("meta channel closed"); meta_tx.send(wire_bytes).expect("meta channel closed");
write_half write_half
.send(&SyncMessage::FsCdcData { .send(&SyncMessage::FsCdcData {
path: file_need.path, path: file_need.path,
chunk_order, chunk_order,
chunks: literal_chunks, chunks: literal_chunks,
}) })
.await?; .await?;
} }
Ok(write_half) Ok(write_half)
}); });
// Reader: collect acks. // Reader: collect acks.
let mut bytes_transferred = 0u64; let mut bytes_transferred = 0u64;
@@ -161,7 +189,11 @@ impl FsSyncClient {
bytes_transferred += meta_rx.recv().await.expect("meta closed early"); bytes_transferred += meta_rx.recv().await.expect("meta closed early");
} }
SyncMessage::Error { message } => return Err(FsSyncError::Protocol(message)), SyncMessage::Error { message } => return Err(FsSyncError::Protocol(message)),
other => return Err(FsSyncError::Protocol(format!("expected FsFileAck, got {other:?}"))), other => {
return Err(FsSyncError::Protocol(format!(
"expected FsFileAck, got {other:?}"
)));
}
} }
} }
@@ -171,9 +203,22 @@ impl FsSyncClient {
// Receive final FsDirComplete from server. // Receive final FsDirComplete from server.
let stats = match read_half.recv().await? { let stats = match read_half.recv().await? {
SyncMessage::FsDirComplete { files_added, files_modified, files_removed, .. } => SyncMessage::FsDirComplete {
SyncStats { files_added, files_modified, files_removed, bytes_transferred }, files_added,
other => return Err(FsSyncError::Protocol(format!("expected FsDirComplete, got {other:?}"))), files_modified,
files_removed,
..
} => SyncStats {
files_added,
files_modified,
files_removed,
bytes_transferred,
},
other => {
return Err(FsSyncError::Protocol(format!(
"expected FsDirComplete, got {other:?}"
)));
}
}; };
write_half.shutdown().await?; write_half.shutdown().await?;
@@ -194,13 +239,13 @@ pub struct FsSyncServer {
} }
impl FsSyncServer { impl FsSyncServer {
pub fn new( pub fn new(conn: SyncPeer, serve_root: PathBuf, excludes: GlobSet, allow_delete: bool) -> Self {
conn: SyncPeer, Self {
serve_root: PathBuf, conn,
excludes: GlobSet, serve_root,
allow_delete: bool, excludes,
) -> Self { allow_delete,
Self { conn, serve_root, excludes, allow_delete } }
} }
/// Handle one complete client session. /// Handle one complete client session.
@@ -208,7 +253,11 @@ impl FsSyncServer {
// Receive client's directory manifest. // Receive client's directory manifest.
let client_entries = match self.conn.recv().await? { let client_entries = match self.conn.recv().await? {
SyncMessage::FsDirManifest { entries, .. } => entries, SyncMessage::FsDirManifest { entries, .. } => entries,
other => return Err(FsSyncError::Protocol(format!("expected FsDirManifest, got {other:?}"))), other => {
return Err(FsSyncError::Protocol(format!(
"expected FsDirManifest, got {other:?}"
)));
}
}; };
// Build server's own manifest. // Build server's own manifest.
@@ -277,9 +326,9 @@ impl FsSyncServer {
for rel_path in &needed_paths { for rel_path in &needed_paths {
let is_add = is_add_set.contains(*rel_path); let is_add = is_add_set.contains(*rel_path);
let abs_path = self.serve_root.join( let abs_path = self
rel_path.replace('/', std::path::MAIN_SEPARATOR_STR) .serve_root
); .join(rel_path.replace('/', std::path::MAIN_SEPARATOR_STR));
let server_bytes: Vec<u8> = if is_add { let server_bytes: Vec<u8> = if is_add {
vec![] vec![]
@@ -293,7 +342,10 @@ impl FsSyncServer {
let server_chunk_map = build_server_chunk_map(&server_bytes); let server_chunk_map = build_server_chunk_map(&server_bytes);
let server_chunk_hashes: Vec<FsChunkHash> = chunk_data(&server_bytes) let server_chunk_hashes: Vec<FsChunkHash> = chunk_data(&server_bytes)
.into_iter() .into_iter()
.map(|c| FsChunkHash { hash: c.hash, length: c.length as u32 }) .map(|c| FsChunkHash {
hash: c.hash,
length: c.length as u32,
})
.collect(); .collect();
let expected_blake3 = client_entries let expected_blake3 = client_entries
@@ -308,37 +360,50 @@ impl FsSyncServer {
expected_blake3, expected_blake3,
}); });
file_ctx.insert(rel_path.to_string(), FileCtx { file_ctx.insert(
server_bytes, rel_path.to_string(),
server_chunk_map, FileCtx {
expected_blake3, server_bytes,
is_add, server_chunk_map,
}); expected_blake3,
is_add,
},
);
} }
// Send FsDirNeed. // Send FsDirNeed.
self.conn self.conn
.send(&SyncMessage::FsDirNeed { needed_files, to_delete: to_delete.clone() }) .send(&SyncMessage::FsDirNeed {
needed_files,
to_delete: to_delete.clone(),
})
.await?; .await?;
// No-op path (nothing to transfer or delete). // No-op path (nothing to transfer or delete).
if file_ctx.is_empty() && to_delete.is_empty() { if file_ctx.is_empty() && to_delete.is_empty() {
self.conn self.conn
.send(&SyncMessage::FsDirComplete { .send(&SyncMessage::FsDirComplete {
files_added: 0, files_modified: 0, files_removed: 0, bytes_transferred: 0, files_added: 0,
files_modified: 0,
files_removed: 0,
bytes_transferred: 0,
}) })
.await?; .await?;
return Ok(SyncStats::default()); return Ok(SyncStats::default());
} }
// Receive FsCdcData for each needed file (pipelined, any order). // Receive FsCdcData for each needed file (pipelined, any order).
let mut files_added = 0u32; let mut files_added = 0u32;
let mut files_modified = 0u32; let mut files_modified = 0u32;
let mut bytes_transferred = 0u64; let mut bytes_transferred = 0u64;
for _ in 0..file_ctx.len() { for _ in 0..file_ctx.len() {
match self.conn.recv().await? { match self.conn.recv().await? {
SyncMessage::FsCdcData { path, chunk_order, chunks } => { SyncMessage::FsCdcData {
path,
chunk_order,
chunks,
} => {
let ctx = file_ctx.remove(&path).ok_or_else(|| { let ctx = file_ctx.remove(&path).ok_or_else(|| {
FsSyncError::Protocol(format!("FsCdcData for unknown path: {path}")) FsSyncError::Protocol(format!("FsCdcData for unknown path: {path}"))
})?; })?;
@@ -366,9 +431,9 @@ impl FsSyncServer {
bytes_transferred += output.len() as u64; bytes_transferred += output.len() as u64;
// Atomic write: temp file + rename. // Atomic write: temp file + rename.
let abs_path = self.serve_root.join( let abs_path = self
path.replace('/', std::path::MAIN_SEPARATOR_STR) .serve_root
); .join(path.replace('/', std::path::MAIN_SEPARATOR_STR));
// Ensure parent directory exists. // Ensure parent directory exists.
if let Some(parent) = abs_path.parent() { if let Some(parent) = abs_path.parent() {
tokio::task::spawn_blocking({ tokio::task::spawn_blocking({
@@ -391,18 +456,28 @@ impl FsSyncServer {
.await .await
.map_err(|e| FsSyncError::Join(e.to_string()))??; .map_err(|e| FsSyncError::Join(e.to_string()))??;
if is_add { files_added += 1; } else { files_modified += 1; } if is_add {
files_added += 1;
} else {
files_modified += 1;
}
self.conn.send(&SyncMessage::FsFileAck { path }).await?; self.conn.send(&SyncMessage::FsFileAck { path }).await?;
} }
SyncMessage::Error { message } => return Err(FsSyncError::Protocol(message)), SyncMessage::Error { message } => return Err(FsSyncError::Protocol(message)),
other => return Err(FsSyncError::Protocol(format!("expected FsCdcData, got {other:?}"))), other => {
return Err(FsSyncError::Protocol(format!(
"expected FsCdcData, got {other:?}"
)));
}
} }
} }
// Apply deletions after all files reconstructed. // Apply deletions after all files reconstructed.
for rel in &to_delete { for rel in &to_delete {
let abs = self.serve_root.join(rel.replace('/', std::path::MAIN_SEPARATOR_STR)); let abs = self
.serve_root
.join(rel.replace('/', std::path::MAIN_SEPARATOR_STR));
if abs.exists() { if abs.exists() {
let abs_clone = abs.clone(); let abs_clone = abs.clone();
tokio::task::spawn_blocking(move || std::fs::remove_file(&abs_clone)) tokio::task::spawn_blocking(move || std::fs::remove_file(&abs_clone))
@@ -421,7 +496,12 @@ impl FsSyncServer {
}) })
.await?; .await?;
Ok(SyncStats { files_added, files_modified, files_removed, bytes_transferred }) Ok(SyncStats {
files_added,
files_modified,
files_removed,
bytes_transferred,
})
} }
} }
@@ -432,11 +512,11 @@ impl FsSyncServer {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
use clawsync_transport::tcp::{TcpConnection, TcpServer};
use globset::GlobSetBuilder; use globset::GlobSetBuilder;
use std::fs; use std::fs;
use tempfile::TempDir;
use clawsync_transport::tcp::{TcpConnection, TcpServer};
use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use tempfile::TempDir;
fn empty_excludes() -> GlobSet { fn empty_excludes() -> GlobSet {
GlobSetBuilder::new().build().unwrap() GlobSetBuilder::new().build().unwrap()
@@ -463,10 +543,15 @@ mod tests {
}); });
let conn = TcpConnection::connect(addr).await.unwrap(); let conn = TcpConnection::connect(addr).await.unwrap();
let stats = FsSyncClient::new(SyncPeer::Tcp(conn), src_dir.to_path_buf(), empty_excludes(), delete) let stats = FsSyncClient::new(
.run() SyncPeer::Tcp(conn),
.await src_dir.to_path_buf(),
.unwrap(); empty_excludes(),
delete,
)
.run()
.await
.unwrap();
server_task.await.unwrap(); server_task.await.unwrap();
stats stats
@@ -486,7 +571,10 @@ mod tests {
assert_eq!(stats.files_modified, 0); assert_eq!(stats.files_modified, 0);
assert!(dst.path().join("a.bin").exists()); assert!(dst.path().join("a.bin").exists());
assert!(dst.path().join("b.bin").exists()); assert!(dst.path().join("b.bin").exists());
assert_eq!(fs::read(dst.path().join("a.bin")).unwrap(), vec![0x11u8; 4096]); assert_eq!(
fs::read(dst.path().join("a.bin")).unwrap(),
vec![0x11u8; 4096]
);
} }
#[tokio::test] #[tokio::test]
+13 -4
View File
@@ -102,7 +102,10 @@ pub fn diff_manifests(old: &DatasetManifest, new: &DatasetManifest) -> DiffResul
ord.then_with(|| a.path.cmp(&b.path)) ord.then_with(|| a.path.cmp(&b.path))
}); });
DiffResult { patches, unchanged_count } DiffResult {
patches,
unchanged_count,
}
} }
/// Given a set of patches keyed by dataset path, look up data bytes in `src` /// Given a set of patches keyed by dataset path, look up data bytes in `src`
@@ -207,7 +210,7 @@ mod tests {
let new = manifest_from(&[("a", vec![1.0]), ("b", vec![9.9]), ("c", vec![3.0])]); let new = manifest_from(&[("a", vec![1.0]), ("b", vec![9.9]), ("c", vec![3.0])]);
let diff = diff_manifests(&old, &new); let diff = diff_manifests(&old, &new);
assert_eq!(diff.unchanged_count, 2); // a and c assert_eq!(diff.unchanged_count, 2); // a and c
assert_eq!(diff.patches.len(), 1); // only b modified assert_eq!(diff.patches.len(), 1); // only b modified
} }
#[test] #[test]
@@ -269,7 +272,10 @@ mod tests {
let resolved = resolve_patch_data(&diff.patches, &src_file); let resolved = resolve_patch_data(&diff.patches, &src_file);
assert_eq!(resolved.len(), 1); assert_eq!(resolved.len(), 1);
assert!(resolved[0].is_some(), "Added patch should resolve to Some(bytes)"); assert!(
resolved[0].is_some(),
"Added patch should resolve to Some(bytes)"
);
// 3 f64 values = 24 bytes // 3 f64 values = 24 bytes
assert_eq!(resolved[0].as_ref().unwrap().len(), 3 * 8); assert_eq!(resolved[0].as_ref().unwrap().len(), 3 * 8);
} }
@@ -284,7 +290,10 @@ mod tests {
// The Removed patch has kind=Removed, so resolve should return None. // The Removed patch has kind=Removed, so resolve should return None.
let resolved = resolve_patch_data(&diff.patches, &src_file); let resolved = resolve_patch_data(&diff.patches, &src_file);
assert_eq!(resolved.len(), 1); assert_eq!(resolved.len(), 1);
assert!(resolved[0].is_none(), "Removed patch should resolve to None"); assert!(
resolved[0].is_none(),
"Removed patch should resolve to None"
);
} }
#[test] #[test]
+5 -1
View File
@@ -14,7 +14,11 @@ pub enum Hdf5SyncError {
DatasetNotFound(String), DatasetNotFound(String),
#[error("patch type mismatch for dataset '{path}': expected {expected}, got {actual}")] #[error("patch type mismatch for dataset '{path}': expected {expected}, got {actual}")]
TypeMismatch { path: String, expected: String, actual: String }, TypeMismatch {
path: String,
expected: String,
actual: String,
},
#[error("manifest serialization error: {0}")] #[error("manifest serialization error: {0}")]
Serialize(String), Serialize(String),
+3 -2
View File
@@ -24,5 +24,6 @@ pub mod patcher;
pub use differ::{DatasetPatch, DiffResult, PatchKind, diff_manifests}; pub use differ::{DatasetPatch, DiffResult, PatchKind, diff_manifests};
pub use error::Hdf5SyncError; pub use error::Hdf5SyncError;
pub use manifest::{DatasetEntry, DatasetManifest}; pub use manifest::{DatasetEntry, DatasetManifest};
pub use patcher::{PatchStats, ReceivedDataset, apply_patches, apply_patches_to_bytes, pub use patcher::{
apply_received_payloads}; PatchStats, ReceivedDataset, apply_patches, apply_patches_to_bytes, apply_received_payloads,
};
+19 -7
View File
@@ -48,7 +48,10 @@ impl DatasetManifest {
let file = File::open(h5_path)?; let file = File::open(h5_path)?;
let mut datasets = HashMap::new(); let mut datasets = HashMap::new();
collect_datasets(file.root(), "", &mut datasets)?; collect_datasets(file.root(), "", &mut datasets)?;
Ok(Self { file_blake3, datasets }) Ok(Self {
file_blake3,
datasets,
})
} }
/// Build a manifest from already-read bytes (avoids double I/O when the /// Build a manifest from already-read bytes (avoids double I/O when the
@@ -58,7 +61,10 @@ impl DatasetManifest {
let file = File::from_bytes(raw.to_vec())?; let file = File::from_bytes(raw.to_vec())?;
let mut datasets = HashMap::new(); let mut datasets = HashMap::new();
collect_datasets(file.root(), "", &mut datasets)?; collect_datasets(file.root(), "", &mut datasets)?;
Ok(Self { file_blake3, datasets }) Ok(Self {
file_blake3,
datasets,
})
} }
/// Return paths of all datasets in this manifest. /// Return paths of all datasets in this manifest.
@@ -113,7 +119,16 @@ fn collect_datasets(
let blake3 = blake3_bytes(&raw); let blake3 = blake3_bytes(&raw);
let byte_len = raw.len() as u64; let byte_len = raw.len() as u64;
out.insert(path.clone(), DatasetEntry { path, blake3, shape, dtype, byte_len }); out.insert(
path.clone(),
DatasetEntry {
path,
blake3,
shape,
dtype,
byte_len,
},
);
} }
// Recurse into sub-groups // Recurse into sub-groups
@@ -167,10 +182,7 @@ mod tests {
#[test] #[test]
fn manifest_counts_datasets() { fn manifest_counts_datasets() {
let bytes = write_test_h5(&[ let bytes = write_test_h5(&[("alpha", vec![1.0, 2.0]), ("beta", vec![3.0, 4.0, 5.0])]);
("alpha", vec![1.0, 2.0]),
("beta", vec![3.0, 4.0, 5.0]),
]);
let m = DatasetManifest::from_bytes(&bytes).unwrap(); let m = DatasetManifest::from_bytes(&bytes).unwrap();
assert_eq!(m.len(), 2); assert_eq!(m.len(), 2);
} }
+36 -11
View File
@@ -178,7 +178,9 @@ pub fn apply_patches_to_bytes(
for patch in patches { for patch in patches {
match patch.kind { match patch.kind {
PatchKind::Removed => { stats.datasets_removed += 1; } PatchKind::Removed => {
stats.datasets_removed += 1;
}
PatchKind::Added => { PatchKind::Added => {
if let Some(data) = patched.get(&patch.path) { if let Some(data) = patched.get(&patch.path) {
stats.bytes_written += data.raw.len() as u64; stats.bytes_written += data.raw.len() as u64;
@@ -261,7 +263,11 @@ pub fn apply_received_payloads(
// Write received (Added/Modified) datasets. // Write received (Added/Modified) datasets.
for p in payloads { for p in payloads {
let dtype = dtype_from_str(&p.dtype); let dtype = dtype_from_str(&p.dtype);
let raw = DatasetRaw { raw: p.data.clone(), dtype, shape: p.shape.clone() }; let raw = DatasetRaw {
raw: p.data.clone(),
dtype,
shape: p.shape.clone(),
};
write_dataset(&mut builder, p.path.trim_start_matches('/'), &raw); write_dataset(&mut builder, p.path.trim_start_matches('/'), &raw);
// Classify as added or modified based on whether server had it. // Classify as added or modified based on whether server had it.
if existing.contains_key(&p.path) { if existing.contains_key(&p.path) {
@@ -383,31 +389,41 @@ fn write_dataset(builder: &mut FileBuilder, name: &str, data: &DatasetRaw) {
let ds = builder.create_dataset(name); let ds = builder.create_dataset(name);
match &data.dtype { match &data.dtype {
DType::F64 => { DType::F64 => {
let vals: Vec<f64> = data.raw.chunks_exact(8) let vals: Vec<f64> = data
.raw
.chunks_exact(8)
.map(|b| f64::from_le_bytes(b.try_into().unwrap_or([0u8; 8]))) .map(|b| f64::from_le_bytes(b.try_into().unwrap_or([0u8; 8])))
.collect(); .collect();
ds.with_f64_data(&vals); ds.with_f64_data(&vals);
} }
DType::F32 => { DType::F32 => {
let vals: Vec<f32> = data.raw.chunks_exact(4) let vals: Vec<f32> = data
.raw
.chunks_exact(4)
.map(|b| f32::from_le_bytes(b.try_into().unwrap_or([0u8; 4]))) .map(|b| f32::from_le_bytes(b.try_into().unwrap_or([0u8; 4])))
.collect(); .collect();
ds.with_f32_data(&vals); ds.with_f32_data(&vals);
} }
DType::I32 => { DType::I32 => {
let vals: Vec<i32> = data.raw.chunks_exact(4) let vals: Vec<i32> = data
.raw
.chunks_exact(4)
.map(|b| i32::from_le_bytes(b.try_into().unwrap_or([0u8; 4]))) .map(|b| i32::from_le_bytes(b.try_into().unwrap_or([0u8; 4])))
.collect(); .collect();
ds.with_i32_data(&vals); ds.with_i32_data(&vals);
} }
DType::I64 => { DType::I64 => {
let vals: Vec<i64> = data.raw.chunks_exact(8) let vals: Vec<i64> = data
.raw
.chunks_exact(8)
.map(|b| i64::from_le_bytes(b.try_into().unwrap_or([0u8; 8]))) .map(|b| i64::from_le_bytes(b.try_into().unwrap_or([0u8; 8])))
.collect(); .collect();
ds.with_i64_data(&vals); ds.with_i64_data(&vals);
} }
DType::U64 => { DType::U64 => {
let vals: Vec<u64> = data.raw.chunks_exact(8) let vals: Vec<u64> = data
.raw
.chunks_exact(8)
.map(|b| u64::from_le_bytes(b.try_into().unwrap_or([0u8; 8]))) .map(|b| u64::from_le_bytes(b.try_into().unwrap_or([0u8; 8])))
.collect(); .collect();
// Use i64 storage for u64 (bit-preserving) // Use i64 storage for u64 (bit-preserving)
@@ -462,7 +478,10 @@ mod tests {
let result = DatasetManifest::from_bytes(&patched).unwrap(); let result = DatasetManifest::from_bytes(&patched).unwrap();
let src_m = DatasetManifest::from_bytes(&src).unwrap(); let src_m = DatasetManifest::from_bytes(&src).unwrap();
assert_eq!(result.get("/x").unwrap().blake3, src_m.get("/x").unwrap().blake3); assert_eq!(
result.get("/x").unwrap().blake3,
src_m.get("/x").unwrap().blake3
);
} }
#[test] #[test]
@@ -495,8 +514,14 @@ mod tests {
let orig = DatasetManifest::from_bytes(&bytes).unwrap(); let orig = DatasetManifest::from_bytes(&bytes).unwrap();
let result = DatasetManifest::from_bytes(&patched).unwrap(); let result = DatasetManifest::from_bytes(&patched).unwrap();
assert_eq!(orig.get("/a").unwrap().blake3, result.get("/a").unwrap().blake3); assert_eq!(
assert_eq!(orig.get("/b").unwrap().blake3, result.get("/b").unwrap().blake3); orig.get("/a").unwrap().blake3,
result.get("/a").unwrap().blake3
);
assert_eq!(
orig.get("/b").unwrap().blake3,
result.get("/b").unwrap().blake3
);
} }
#[test] #[test]
@@ -511,7 +536,7 @@ mod tests {
let (_, stats) = apply_patches_to_bytes(&diff.patches, &src, &dst).unwrap(); let (_, stats) = apply_patches_to_bytes(&diff.patches, &src, &dst).unwrap();
assert_eq!(stats.datasets_modified, 1); // a assert_eq!(stats.datasets_modified, 1); // a
assert_eq!(stats.datasets_removed, 1); // b assert_eq!(stats.datasets_removed, 1); // b
assert_eq!(stats.datasets_unchanged, 1); // c assert_eq!(stats.datasets_unchanged, 1); // c
assert_eq!(stats.datasets_added, 0); assert_eq!(stats.datasets_added, 0);
} }
+37 -32
View File
@@ -5,7 +5,9 @@
//! creating real HDF5 files in memory and on disk as needed. //! creating real HDF5 files in memory and on disk as needed.
use clawhdf5::{File, FileBuilder}; use clawhdf5::{File, FileBuilder};
use clawsync_hdf5::{DatasetManifest, PatchKind, apply_patches, apply_patches_to_bytes, diff_manifests}; use clawsync_hdf5::{
DatasetManifest, PatchKind, apply_patches, apply_patches_to_bytes, diff_manifests,
};
use tempfile::NamedTempFile; use tempfile::NamedTempFile;
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@@ -37,7 +39,10 @@ fn write_temp_h5(bytes: &[u8]) -> (NamedTempFile, std::path::PathBuf) {
/// A dataset present in source but absent from target is added after patching. /// A dataset present in source but absent from target is added after patching.
#[test] #[test]
fn patch_adds_new_dataset() { fn patch_adds_new_dataset() {
let src = h5_bytes(&[("temperature", &[20.0, 21.5, 22.0]), ("humidity", &[55.0, 57.0])]); let src = h5_bytes(&[
("temperature", &[20.0, 21.5, 22.0]),
("humidity", &[55.0, 57.0]),
]);
let tgt = h5_bytes(&[("temperature", &[20.0, 21.5, 22.0])]); let tgt = h5_bytes(&[("temperature", &[20.0, 21.5, 22.0])]);
let old_m = DatasetManifest::from_bytes(&tgt).unwrap(); let old_m = DatasetManifest::from_bytes(&tgt).unwrap();
@@ -51,7 +56,8 @@ fn patch_adds_new_dataset() {
let result_m = DatasetManifest::from_bytes(&patched).unwrap(); let result_m = DatasetManifest::from_bytes(&patched).unwrap();
// The new dataset must be present with the correct hash. // The new dataset must be present with the correct hash.
let result_entry = result_m.get("/humidity") let result_entry = result_m
.get("/humidity")
.expect("humidity should exist after patching"); .expect("humidity should exist after patching");
let src_entry = new_m.get("/humidity").unwrap(); let src_entry = new_m.get("/humidity").unwrap();
assert_eq!(result_entry.blake3, src_entry.blake3); assert_eq!(result_entry.blake3, src_entry.blake3);
@@ -107,13 +113,22 @@ fn patch_removes_dataset() {
let diff = diff_manifests(&old_m, &new_m); let diff = diff_manifests(&old_m, &new_m);
assert_eq!(diff.of_kind(&PatchKind::Removed).count(), 1); assert_eq!(diff.of_kind(&PatchKind::Removed).count(), 1);
assert_eq!(diff.of_kind(&PatchKind::Removed).next().unwrap().path, "/current"); assert_eq!(
diff.of_kind(&PatchKind::Removed).next().unwrap().path,
"/current"
);
let (patched, _) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap(); let (patched, _) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap();
let result_m = DatasetManifest::from_bytes(&patched).unwrap(); let result_m = DatasetManifest::from_bytes(&patched).unwrap();
assert!(result_m.get("/current").is_none(), "current should have been removed"); assert!(
assert!(result_m.get("/voltage").is_some(), "voltage should be retained"); result_m.get("/current").is_none(),
"current should have been removed"
);
assert!(
result_m.get("/voltage").is_some(),
"voltage should be retained"
);
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@@ -148,13 +163,13 @@ fn patch_noop_when_identical() {
fn patch_multiple_changes() { fn patch_multiple_changes() {
// source: "kept" (same), "updated" (changed), "added" (new) // source: "kept" (same), "updated" (changed), "added" (new)
let src = h5_bytes(&[ let src = h5_bytes(&[
("kept", &[1.0, 2.0]), ("kept", &[1.0, 2.0]),
("updated", &[99.0, 100.0]), ("updated", &[99.0, 100.0]),
("added", &[7.0, 8.0, 9.0]), ("added", &[7.0, 8.0, 9.0]),
]); ]);
// target: "kept" (same), "updated" (old values), "removed" (extra) // target: "kept" (same), "updated" (old values), "removed" (extra)
let tgt = h5_bytes(&[ let tgt = h5_bytes(&[
("kept", &[1.0, 2.0]), ("kept", &[1.0, 2.0]),
("updated", &[0.0, 0.0]), ("updated", &[0.0, 0.0]),
("removed", &[42.0]), ("removed", &[42.0]),
]); ]);
@@ -163,9 +178,9 @@ fn patch_multiple_changes() {
let new_m = DatasetManifest::from_bytes(&src).unwrap(); let new_m = DatasetManifest::from_bytes(&src).unwrap();
let diff = diff_manifests(&old_m, &new_m); let diff = diff_manifests(&old_m, &new_m);
assert_eq!(diff.of_kind(&PatchKind::Added).count(), 1); assert_eq!(diff.of_kind(&PatchKind::Added).count(), 1);
assert_eq!(diff.of_kind(&PatchKind::Modified).count(), 1); assert_eq!(diff.of_kind(&PatchKind::Modified).count(), 1);
assert_eq!(diff.of_kind(&PatchKind::Removed).count(), 1); assert_eq!(diff.of_kind(&PatchKind::Removed).count(), 1);
let (patched, _) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap(); let (patched, _) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap();
let result_m = DatasetManifest::from_bytes(&patched).unwrap(); let result_m = DatasetManifest::from_bytes(&patched).unwrap();
@@ -194,26 +209,18 @@ fn patch_multiple_changes() {
#[test] #[test]
fn stats_counts_correct() { fn stats_counts_correct() {
// source: "a" (modified), "c" (unchanged), "d" (new) // source: "a" (modified), "c" (unchanged), "d" (new)
let src = h5_bytes(&[ let src = h5_bytes(&[("a", &[9.0, 9.0]), ("c", &[3.0]), ("d", &[4.0, 5.0])]);
("a", &[9.0, 9.0]),
("c", &[3.0]),
("d", &[4.0, 5.0]),
]);
// target: "a" (old values), "b" (to be removed), "c" (same as source) // target: "a" (old values), "b" (to be removed), "c" (same as source)
let tgt = h5_bytes(&[ let tgt = h5_bytes(&[("a", &[1.0, 2.0]), ("b", &[0.0]), ("c", &[3.0])]);
("a", &[1.0, 2.0]),
("b", &[0.0]),
("c", &[3.0]),
]);
let old_m = DatasetManifest::from_bytes(&tgt).unwrap(); let old_m = DatasetManifest::from_bytes(&tgt).unwrap();
let new_m = DatasetManifest::from_bytes(&src).unwrap(); let new_m = DatasetManifest::from_bytes(&src).unwrap();
let diff = diff_manifests(&old_m, &new_m); let diff = diff_manifests(&old_m, &new_m);
let (_, stats) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap(); let (_, stats) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap();
assert_eq!(stats.datasets_added, 1, "d was added"); assert_eq!(stats.datasets_added, 1, "d was added");
assert_eq!(stats.datasets_modified, 1, "a was modified"); assert_eq!(stats.datasets_modified, 1, "a was modified");
assert_eq!(stats.datasets_removed, 1, "b was removed"); assert_eq!(stats.datasets_removed, 1, "b was removed");
assert_eq!(stats.datasets_unchanged, 1, "c was unchanged"); assert_eq!(stats.datasets_unchanged, 1, "c was unchanged");
// bytes_written must be positive (added + modified + unchanged datasets). // bytes_written must be positive (added + modified + unchanged datasets).
assert!(stats.bytes_written > 0); assert!(stats.bytes_written > 0);
@@ -227,13 +234,8 @@ fn stats_counts_correct() {
/// manifest read back from disk matches the source manifest. /// manifest read back from disk matches the source manifest.
#[test] #[test]
fn apply_patches_file_path_api() { fn apply_patches_file_path_api() {
let src_bytes = h5_bytes(&[ let src_bytes = h5_bytes(&[("chan_x", &[1.1, 2.2, 3.3]), ("chan_y", &[4.4, 5.5])]);
("chan_x", &[1.1, 2.2, 3.3]), let tgt_bytes = h5_bytes(&[("chan_x", &[0.0, 0.0, 0.0])]);
("chan_y", &[4.4, 5.5]),
]);
let tgt_bytes = h5_bytes(&[
("chan_x", &[0.0, 0.0, 0.0]),
]);
// Write both files to disk using NamedTempFile. // Write both files to disk using NamedTempFile.
let (_src_guard, src_path) = write_temp_h5(&src_bytes); let (_src_guard, src_path) = write_temp_h5(&src_bytes);
@@ -248,8 +250,11 @@ fn apply_patches_file_path_api() {
// Use the file-path API (the one under test). // Use the file-path API (the one under test).
let stats = apply_patches(&diff.patches, &src_path, &tgt_path).unwrap(); let stats = apply_patches(&diff.patches, &src_path, &tgt_path).unwrap();
assert!(stats.datasets_added >= 1, "chan_y should have been added"); assert!(stats.datasets_added >= 1, "chan_y should have been added");
assert!(stats.datasets_modified >= 1, "chan_x should have been modified"); assert!(
stats.datasets_modified >= 1,
"chan_x should have been modified"
);
// Read the patched file back from disk and verify it matches source. // Read the patched file back from disk and verify it matches source.
let result_m = DatasetManifest::from_path(&tgt_path).unwrap(); let result_m = DatasetManifest::from_path(&tgt_path).unwrap();
+21 -7
View File
@@ -9,7 +9,7 @@
//! Run: //! Run:
//! cargo bench -p clawsync-onion -- iblt_bench //! cargo bench -p clawsync-onion -- iblt_bench
use clawsync_onion::iblt::{IbltSketch, DEFAULT_HASH_COUNT, MIN_CELLS}; use clawsync_onion::iblt::{DEFAULT_HASH_COUNT, IbltSketch, MIN_CELLS};
use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main}; use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@@ -28,8 +28,12 @@ fn diff_sketches(n: usize, d: usize) -> (IbltSketch, Vec<u8>) {
let m = ((d * 4).max(MIN_CELLS)).max(IbltSketch::recommended_cells(n)); let m = ((d * 4).max(MIN_CELLS)).max(IbltSketch::recommended_cells(n));
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED); let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED); let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
for &k in &a_keys { a.insert(k); } for &k in &a_keys {
for &k in &b_keys { b.insert(k); } a.insert(k);
}
for &k in &b_keys {
b.insert(k);
}
let b_bytes = b.to_bytes(); let b_bytes = b.to_bytes();
a.subtract(&b); a.subtract(&b);
(a, b_bytes) (a, b_bytes)
@@ -79,8 +83,12 @@ fn bench_iblt(c: &mut Criterion) {
let m = (d * 4).max(MIN_CELLS).max(IbltSketch::recommended_cells(n)); let m = (d * 4).max(MIN_CELLS).max(IbltSketch::recommended_cells(n));
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED); let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
let mut b_sk = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED); let mut b_sk = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
for &k in black_box(ak) { a.insert(k); } for &k in black_box(ak) {
for &k in black_box(bk) { b_sk.insert(k); } a.insert(k);
}
for &k in black_box(bk) {
b_sk.insert(k);
}
a.subtract(&b_sk); a.subtract(&b_sk);
black_box(a.decode()) black_box(a.decode())
}); });
@@ -92,14 +100,20 @@ fn bench_iblt(c: &mut Criterion) {
// ── Wire size table (not timed) ─────────────────────────────────────────── // ── Wire size table (not timed) ───────────────────────────────────────────
println!("\n=== IBLT sketch size vs flat manifest (N × 60 B) ==="); println!("\n=== IBLT sketch size vs flat manifest (N × 60 B) ===");
println!("{:>8} {:>10} {:>12} {:>10}", "N", "IBLT (B)", "Flat (B)", "ratio"); println!(
"{:>8} {:>10} {:>12} {:>10}",
"N", "IBLT (B)", "Flat (B)", "ratio"
);
for &n in &[10usize, 100, 1_000, 10_000, 100_000] { for &n in &[10usize, 100, 1_000, 10_000, 100_000] {
let ks = keys(n); let ks = keys(n);
let sz = IbltSketch::from_keys(&ks, SEED).to_bytes().len(); let sz = IbltSketch::from_keys(&ks, SEED).to_bytes().len();
let flat = n * 60; let flat = n * 60;
println!( println!(
"{:>8} {:>10} {:>12} {:>9.1}×", "{:>8} {:>10} {:>12} {:>9.1}×",
n, sz, flat, flat as f64 / sz as f64 n,
sz,
flat,
flat as f64 / sz as f64
); );
} }
} }
+29 -14
View File
@@ -49,16 +49,21 @@ pub fn diff_revisions(
let pages: Vec<OnionPage> = raw_pages let pages: Vec<OnionPage> = raw_pages
.into_iter() .into_iter()
.map(|(h5_offset, data, codec, orig_size)| OnionPage { h5_offset, data, codec, orig_size }) .map(|(h5_offset, data, codec, orig_size)| OnionPage {
h5_offset,
data,
codec,
orig_size,
})
.collect(); .collect();
// Parse the blake3 hex back to bytes // Parse the blake3 hex back to bytes
let blake3_root = hex_to_bytes32(&summary.blake3_hex) let blake3_root = hex_to_bytes32(&summary.blake3_hex).ok_or_else(|| {
.ok_or_else(|| { SyncOnionError::Onion(clawhdf5_onion::OnionError::Malformed(format!(
SyncOnionError::Onion(clawhdf5_onion::OnionError::Malformed( "invalid blake3_hex for rev {}",
format!("invalid blake3_hex for rev {}", summary.revision), summary.revision
)) )))
})?; })?;
packets.push(OnionLayerPacket { packets.push(OnionLayerPacket {
revision: summary.revision, revision: summary.revision,
@@ -103,13 +108,18 @@ pub fn packets_for_revisions(
let raw_pages = local.revision_pages_raw(rev)?; let raw_pages = local.revision_pages_raw(rev)?;
let pages: Vec<OnionPage> = raw_pages let pages: Vec<OnionPage> = raw_pages
.into_iter() .into_iter()
.map(|(h5_offset, data, codec, orig_size)| OnionPage { h5_offset, data, codec, orig_size }) .map(|(h5_offset, data, codec, orig_size)| OnionPage {
h5_offset,
data,
codec,
orig_size,
})
.collect(); .collect();
let blake3_root = hex_to_bytes32(&summary.blake3_hex).ok_or_else(|| { let blake3_root = hex_to_bytes32(&summary.blake3_hex).ok_or_else(|| {
SyncOnionError::Onion(clawhdf5_onion::OnionError::Malformed( SyncOnionError::Onion(clawhdf5_onion::OnionError::Malformed(format!(
format!("invalid blake3_hex for rev {rev}"), "invalid blake3_hex for rev {rev}"
)) )))
})?; })?;
packets.push(OnionLayerPacket { packets.push(OnionLayerPacket {
@@ -198,7 +208,12 @@ pub fn diff_revisions_merkle(
let raw_pages = local.revision_pages_raw(rev)?; let raw_pages = local.revision_pages_raw(rev)?;
let pages: Vec<OnionPage> = raw_pages let pages: Vec<OnionPage> = raw_pages
.into_iter() .into_iter()
.map(|(h5_offset, data, codec, orig_size)| OnionPage { h5_offset, data, codec, orig_size }) .map(|(h5_offset, data, codec, orig_size)| OnionPage {
h5_offset,
data,
codec,
orig_size,
})
.collect(); .collect();
let blake3_root = hex_to_bytes32(&summary.blake3_hex).ok_or_else(|| { let blake3_root = hex_to_bytes32(&summary.blake3_hex).ok_or_else(|| {
@@ -413,11 +428,11 @@ mod tests {
}; };
let merkle_packets = diff_revisions_merkle(&local, &remote_tree.serialise()).unwrap(); let merkle_packets = diff_revisions_merkle(&local, &remote_tree.serialise()).unwrap();
let flat_packets = diff_revisions(&local, 1 /* remote head = rev 1 */).unwrap(); let flat_packets = diff_revisions(&local, 1 /* remote head = rev 1 */).unwrap();
// Revision numbers should match // Revision numbers should match
let merkle_revs: Vec<u64> = merkle_packets.iter().map(|p| p.revision).collect(); let merkle_revs: Vec<u64> = merkle_packets.iter().map(|p| p.revision).collect();
let flat_revs: Vec<u64> = flat_packets.iter().map(|p| p.revision).collect(); let flat_revs: Vec<u64> = flat_packets.iter().map(|p| p.revision).collect();
assert_eq!(merkle_revs, flat_revs); assert_eq!(merkle_revs, flat_revs);
} }
} }
+92 -32
View File
@@ -139,7 +139,9 @@ impl IbltSketch {
pub fn from_keys(keys: &[u64], seed: u64) -> Self { pub fn from_keys(keys: &[u64], seed: u64) -> Self {
let m = Self::recommended_cells(keys.len()); let m = Self::recommended_cells(keys.len());
let mut sketch = Self::new(m, DEFAULT_HASH_COUNT, seed); let mut sketch = Self::new(m, DEFAULT_HASH_COUNT, seed);
for &k in keys { sketch.insert(k); } for &k in keys {
sketch.insert(k);
}
sketch sketch
} }
@@ -182,12 +184,13 @@ impl IbltSketch {
/// Panics if `self` and `other` have different cell counts. /// Panics if `self` and `other` have different cell counts.
pub fn subtract(&mut self, other: &IbltSketch) { pub fn subtract(&mut self, other: &IbltSketch) {
assert_eq!( assert_eq!(
self.cells.len(), other.cells.len(), self.cells.len(),
other.cells.len(),
"cannot subtract sketches of different sizes" "cannot subtract sketches of different sizes"
); );
for (a, b) in self.cells.iter_mut().zip(&other.cells) { for (a, b) in self.cells.iter_mut().zip(&other.cells) {
a.count -= b.count; a.count -= b.count;
a.id_sum ^= b.id_sum; a.id_sum ^= b.id_sum;
a.hash_sum ^= b.hash_sum; a.hash_sum ^= b.hash_sum;
} }
} }
@@ -213,9 +216,14 @@ impl IbltSketch {
let Some(pos) = pure_pos else { let Some(pos) = pure_pos else {
// No pure cell found. // No pure cell found.
let done = cells.iter().all(|c| c.count == 0 && c.id_sum == 0 && c.hash_sum == 0); let done = cells
.iter()
.all(|c| c.count == 0 && c.id_sum == 0 && c.hash_sum == 0);
return if done { return if done {
let mut diff = IbltDiff { only_in_a, only_in_b }; let mut diff = IbltDiff {
only_in_a,
only_in_b,
};
diff.only_in_a.sort_unstable(); diff.only_in_a.sort_unstable();
diff.only_in_b.sort_unstable(); diff.only_in_b.sort_unstable();
IbltDecodeResult::Complete(diff) IbltDecodeResult::Complete(diff)
@@ -227,15 +235,19 @@ impl IbltSketch {
let key = cells[pos].id_sum; let key = cells[pos].id_sum;
let positive = cells[pos].count == 1; let positive = cells[pos].count == 1;
if positive { only_in_a.push(key); } else { only_in_b.push(key); } if positive {
only_in_a.push(key);
} else {
only_in_b.push(key);
}
// Peel this key from all its mapped cells. // Peel this key from all its mapped cells.
let sign: i32 = if positive { 1 } else { -1 }; let sign: i32 = if positive { 1 } else { -1 };
let h_key = xxh3_key(key); let h_key = xxh3_key(key);
for h in 0..k { for h in 0..k {
let idx = cell_index(key, h, seed, m); let idx = cell_index(key, h, seed, m);
cells[idx].count -= sign; cells[idx].count -= sign;
cells[idx].id_sum ^= key; cells[idx].id_sum ^= key;
cells[idx].hash_sum ^= h_key; cells[idx].hash_sum ^= h_key;
} }
} }
@@ -287,13 +299,21 @@ impl IbltSketch {
let mut cells = Vec::with_capacity(m); let mut cells = Vec::with_capacity(m);
let mut off = 21usize; let mut off = 21usize;
for _ in 0..m { for _ in 0..m {
let count = i32::from_le_bytes(data[off..off + 4].try_into().unwrap()); let count = i32::from_le_bytes(data[off..off + 4].try_into().unwrap());
let id_sum = u64::from_le_bytes(data[off + 4..off + 12].try_into().unwrap()); let id_sum = u64::from_le_bytes(data[off + 4..off + 12].try_into().unwrap());
let hash_sum = u64::from_le_bytes(data[off + 12..off + 20].try_into().unwrap()); let hash_sum = u64::from_le_bytes(data[off + 12..off + 20].try_into().unwrap());
cells.push(IbltCell { count, id_sum, hash_sum }); cells.push(IbltCell {
count,
id_sum,
hash_sum,
});
off += 20; off += 20;
} }
Ok(Self { cells, hash_count: k, seed }) Ok(Self {
cells,
hash_count: k,
seed,
})
} }
// ── Private helpers ─────────────────────────────────────────────────────── // ── Private helpers ───────────────────────────────────────────────────────
@@ -303,8 +323,8 @@ impl IbltSketch {
let m = self.cells.len(); let m = self.cells.len();
for h in 0..self.hash_count { for h in 0..self.hash_count {
let idx = cell_index(key, h, self.seed, m); let idx = cell_index(key, h, self.seed, m);
self.cells[idx].count += delta; self.cells[idx].count += delta;
self.cells[idx].id_sum ^= key; self.cells[idx].id_sum ^= key;
self.cells[idx].hash_sum ^= h_key; self.cells[idx].hash_sum ^= h_key;
} }
} }
@@ -349,12 +369,16 @@ pub fn reconcile(
seed: u64, seed: u64,
) -> Result<(IbltDiff, IbltSketch), IbltError> { ) -> Result<(IbltDiff, IbltSketch), IbltError> {
let b_sketch = IbltSketch::from_bytes(b_sketch_bytes)?; let b_sketch = IbltSketch::from_bytes(b_sketch_bytes)?;
let m_start = b_sketch.cell_count().max(IbltSketch::recommended_cells(a_keys.len())); let m_start = b_sketch
.cell_count()
.max(IbltSketch::recommended_cells(a_keys.len()));
let mut m = m_start; let mut m = m_start;
loop { loop {
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, seed); let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, seed);
for &k in a_keys { a.insert(k); } for &k in a_keys {
a.insert(k);
}
let mut diff_sketch = a.clone(); let mut diff_sketch = a.clone();
let b_padded = pad_or_trim_sketch(&b_sketch, m, seed); let b_padded = pad_or_trim_sketch(&b_sketch, m, seed);
@@ -372,7 +396,11 @@ pub fn reconcile(
fn pad_or_trim_sketch(src: &IbltSketch, target_m: usize, seed: u64) -> IbltSketch { fn pad_or_trim_sketch(src: &IbltSketch, target_m: usize, seed: u64) -> IbltSketch {
let mut cells = src.cells.clone(); let mut cells = src.cells.clone();
cells.resize(target_m, IbltCell::default()); cells.resize(target_m, IbltCell::default());
IbltSketch { cells, hash_count: src.hash_count, seed } IbltSketch {
cells,
hash_count: src.hash_count,
seed,
}
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@@ -397,7 +425,11 @@ mod tests {
let mut s = IbltSketch::new(64, 3, SEED); let mut s = IbltSketch::new(64, 3, SEED);
s.insert(42); s.insert(42);
s.remove(42); s.remove(42);
assert!(s.cells.iter().all(|c| c.count == 0 && c.id_sum == 0 && c.hash_sum == 0)); assert!(
s.cells
.iter()
.all(|c| c.count == 0 && c.id_sum == 0 && c.hash_sum == 0)
);
} }
// ── decode: empty diff ──────────────────────────────────────────────────── // ── decode: empty diff ────────────────────────────────────────────────────
@@ -409,9 +441,13 @@ mod tests {
let b = sketch_of(&keys); let b = sketch_of(&keys);
a.subtract(&b); a.subtract(&b);
let result = a.decode(); let result = a.decode();
assert_eq!(result, IbltDecodeResult::Complete(IbltDiff { assert_eq!(
only_in_a: vec![], only_in_b: vec![], result,
})); IbltDecodeResult::Complete(IbltDiff {
only_in_a: vec![],
only_in_b: vec![],
})
);
} }
// ── decode: one-sided diff ──────────────────────────────────────────────── // ── decode: one-sided diff ────────────────────────────────────────────────
@@ -420,14 +456,22 @@ mod tests {
fn decode_one_sided_diff_a_has_extra() { fn decode_one_sided_diff_a_has_extra() {
let base: Vec<u64> = (0..20).collect(); let base: Vec<u64> = (0..20).collect();
let extra = 9999u64; let extra = 9999u64;
let a_keys: Vec<u64> = base.iter().chain(std::iter::once(&extra)).copied().collect(); let a_keys: Vec<u64> = base
.iter()
.chain(std::iter::once(&extra))
.copied()
.collect();
let b_keys = base.clone(); let b_keys = base.clone();
let m = IbltSketch::recommended_cells(a_keys.len()) * 4; // generous m let m = IbltSketch::recommended_cells(a_keys.len()) * 4; // generous m
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED); let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED); let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
for &k in &a_keys { a.insert(k); } for &k in &a_keys {
for &k in &b_keys { b.insert(k); } a.insert(k);
}
for &k in &b_keys {
b.insert(k);
}
a.subtract(&b); a.subtract(&b);
match a.decode() { match a.decode() {
@@ -444,13 +488,21 @@ mod tests {
let base: Vec<u64> = (0..20).collect(); let base: Vec<u64> = (0..20).collect();
let extra = 8888u64; let extra = 8888u64;
let a_keys = base.clone(); let a_keys = base.clone();
let b_keys: Vec<u64> = base.iter().chain(std::iter::once(&extra)).copied().collect(); let b_keys: Vec<u64> = base
.iter()
.chain(std::iter::once(&extra))
.copied()
.collect();
let m = IbltSketch::recommended_cells(b_keys.len()) * 4; let m = IbltSketch::recommended_cells(b_keys.len()) * 4;
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED); let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED); let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
for &k in &a_keys { a.insert(k); } for &k in &a_keys {
for &k in &b_keys { b.insert(k); } a.insert(k);
}
for &k in &b_keys {
b.insert(k);
}
a.subtract(&b); a.subtract(&b);
match a.decode() { match a.decode() {
@@ -476,8 +528,12 @@ mod tests {
let m = IbltSketch::recommended_cells(a_keys.len().max(b_keys.len())) * 8; let m = IbltSketch::recommended_cells(a_keys.len().max(b_keys.len())) * 8;
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED); let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED); let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
for &k in &a_keys { a.insert(k); } for &k in &a_keys {
for &k in &b_keys { b.insert(k); } a.insert(k);
}
for &k in &b_keys {
b.insert(k);
}
a.subtract(&b); a.subtract(&b);
match a.decode() { match a.decode() {
@@ -500,8 +556,12 @@ mod tests {
let m = 1; // deliberately too small let m = 1; // deliberately too small
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED); let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED); let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
for &k in &a_keys { a.insert(k); } for &k in &a_keys {
for &k in &b_keys { b.insert(k); } a.insert(k);
}
for &k in &b_keys {
b.insert(k);
}
a.subtract(&b); a.subtract(&b);
// Must not panic — returns NeedMoreCells // Must not panic — returns NeedMoreCells
+5 -3
View File
@@ -19,9 +19,11 @@ pub mod merger;
pub mod packet; pub mod packet;
pub mod selector; pub mod selector;
pub use error::SyncOnionError;
pub use iblt::{IbltDecodeResult, IbltDiff, IbltSketch, DEFAULT_HASH_COUNT, IBLT_SYNC_SEED, MIN_CELLS};
pub use manifest::{ClawSyncManifest, IbltManifest};
pub use differ::packets_for_revisions; pub use differ::packets_for_revisions;
pub use error::SyncOnionError;
pub use iblt::{
DEFAULT_HASH_COUNT, IBLT_SYNC_SEED, IbltDecodeResult, IbltDiff, IbltSketch, MIN_CELLS,
};
pub use manifest::{ClawSyncManifest, IbltManifest};
pub use merger::MergeStats; pub use merger::MergeStats;
pub use packet::{OnionLayerPacket, OnionPage}; pub use packet::{OnionLayerPacket, OnionPage};
+21 -21
View File
@@ -10,7 +10,7 @@ use rkyv::{Archive, Deserialize, Serialize};
use clawhdf5_onion::writer::OnionFile; use clawhdf5_onion::writer::OnionFile;
use crate::iblt::{IbltDiff, IbltDecodeResult, IbltSketch, DEFAULT_HASH_COUNT}; use crate::iblt::{DEFAULT_HASH_COUNT, IbltDecodeResult, IbltDiff, IbltSketch};
/// A compact summary of one revision — used in the manifest. /// A compact summary of one revision — used in the manifest.
#[derive(Archive, Deserialize, Serialize, Debug, Clone, PartialEq)] #[derive(Archive, Deserialize, Serialize, Debug, Clone, PartialEq)]
@@ -60,10 +60,7 @@ impl ClawSyncManifest {
}) })
.unwrap_or((0, [0u8; 32])); .unwrap_or((0, [0u8; 32]));
let last_write = summaries let last_write = summaries.last().map(|s| s.timestamp).unwrap_or(0.0);
.last()
.map(|s| s.timestamp)
.unwrap_or(0.0);
let revisions = summaries let revisions = summaries
.into_iter() .into_iter()
@@ -122,16 +119,16 @@ impl ClawSyncManifest {
/// For N=1 000: ~3 KB vs ~60 KB for `ClawSyncManifest`. /// For N=1 000: ~3 KB vs ~60 KB for `ClawSyncManifest`.
#[derive(Archive, Deserialize, Serialize, Debug, Clone, PartialEq)] #[derive(Archive, Deserialize, Serialize, Debug, Clone, PartialEq)]
pub struct IbltManifest { pub struct IbltManifest {
pub agent_id: String, pub agent_id: String,
pub file_blake3: [u8; 32], pub file_blake3: [u8; 32],
pub revision_count: u64, pub revision_count: u64,
pub head_revision: u64, pub head_revision: u64,
pub head_blake3: [u8; 32], pub head_blake3: [u8; 32],
pub last_write: f64, pub last_write: f64,
/// Serialised [`IbltSketch`] bytes. /// Serialised [`IbltSketch`] bytes.
pub sketch: Vec<u8>, pub sketch: Vec<u8>,
/// Number of cells in the sketch (m). /// Number of cells in the sketch (m).
pub sketch_cells: u32, pub sketch_cells: u32,
} }
impl IbltManifest { impl IbltManifest {
@@ -144,7 +141,12 @@ impl IbltManifest {
let (head_revision, head_blake3) = summaries let (head_revision, head_blake3) = summaries
.last() .last()
.map(|s| (s.revision, hex_to_bytes32(&s.blake3_hex).unwrap_or([0u8; 32]))) .map(|s| {
(
s.revision,
hex_to_bytes32(&s.blake3_hex).unwrap_or([0u8; 32]),
)
})
.unwrap_or((0, [0u8; 32])); .unwrap_or((0, [0u8; 32]));
let last_write = summaries.last().map(|s| s.timestamp).unwrap_or(0.0); let last_write = summaries.last().map(|s| s.timestamp).unwrap_or(0.0);
@@ -174,10 +176,7 @@ impl IbltManifest {
/// ///
/// Returns `Ok(IbltDiff)` or `Err` if the sketch cannot be decoded (too /// Returns `Ok(IbltDiff)` or `Err` if the sketch cannot be decoded (too
/// small or malformed). /// small or malformed).
pub fn diff_against( pub fn diff_against(&self, local_keys: &[u64]) -> Result<IbltDiff, &'static str> {
&self,
local_keys: &[u64],
) -> Result<IbltDiff, &'static str> {
let remote_sketch = IbltSketch::from_bytes(&self.sketch) let remote_sketch = IbltSketch::from_bytes(&self.sketch)
.map_err(|_| "cannot deserialise remote IBLT sketch")?; .map_err(|_| "cannot deserialise remote IBLT sketch")?;
@@ -189,7 +188,9 @@ impl IbltManifest {
.max(IbltSketch::recommended_cells(local_keys.len())); .max(IbltSketch::recommended_cells(local_keys.len()));
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, seed); let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, seed);
for &k in local_keys { a.insert(k); } for &k in local_keys {
a.insert(k);
}
// Pad remote to same m if needed. // Pad remote to same m if needed.
let b = if remote_sketch.cell_count() == m { let b = if remote_sketch.cell_count() == m {
@@ -202,7 +203,7 @@ impl IbltManifest {
a.subtract(&b); a.subtract(&b);
match a.decode() { match a.decode() {
IbltDecodeResult::Complete(diff) => Ok(diff), IbltDecodeResult::Complete(diff) => Ok(diff),
IbltDecodeResult::NeedMoreCells => Err("IBLT sketch too small; retry with larger m"), IbltDecodeResult::NeedMoreCells => Err("IBLT sketch too small; retry with larger m"),
} }
} }
@@ -217,8 +218,7 @@ impl IbltManifest {
pub fn from_bytes(bytes: &[u8]) -> Result<Self, String> { pub fn from_bytes(bytes: &[u8]) -> Result<Self, String> {
let mut aligned = rkyv::util::AlignedVec::<16>::with_capacity(bytes.len()); let mut aligned = rkyv::util::AlignedVec::<16>::with_capacity(bytes.len());
aligned.extend_from_slice(bytes); aligned.extend_from_slice(bytes);
rkyv::from_bytes::<IbltManifest, rkyv::rancor::Error>(&aligned) rkyv::from_bytes::<IbltManifest, rkyv::rancor::Error>(&aligned).map_err(|e| e.to_string())
.map_err(|e| e.to_string())
} }
} }
+14 -6
View File
@@ -63,8 +63,7 @@ pub fn merge_packets(
session.record_page(page.h5_offset, &page.data); session.record_page(page.h5_offset, &page.data);
total_bytes += page.data.len() as u64; total_bytes += page.data.len() as u64;
} else { } else {
let codec = Codec::from_u8(page.codec) let codec = Codec::from_u8(page.codec).map_err(SyncOnionError::Onion)?;
.map_err(SyncOnionError::Onion)?;
let raw = decompress_page(&page.data, codec, page.orig_size) let raw = decompress_page(&page.data, codec, page.orig_size)
.map_err(SyncOnionError::Onion)?; .map_err(SyncOnionError::Onion)?;
total_bytes += raw.len() as u64; total_bytes += raw.len() as u64;
@@ -101,8 +100,7 @@ fn verify_packet_hash(packet: &OnionLayerPacket) -> Result<(), SyncOnionError> {
if page.codec == 0 { if page.codec == 0 {
hasher.update(&page.data); hasher.update(&page.data);
} else { } else {
let codec = Codec::from_u8(page.codec) let codec = Codec::from_u8(page.codec).map_err(SyncOnionError::Onion)?;
.map_err(SyncOnionError::Onion)?;
let raw = decompress_page(&page.data, codec, page.orig_size) let raw = decompress_page(&page.data, codec, page.orig_size)
.map_err(SyncOnionError::Onion)?; .map_err(SyncOnionError::Onion)?;
hasher.update(&raw); hasher.update(&raw);
@@ -204,7 +202,12 @@ mod tests {
timestamp: 0.0, timestamp: 0.0,
annotation: None, annotation: None,
blake3_root: [0xDEu8; 32], // wrong hash blake3_root: [0xDEu8; 32], // wrong hash
pages: vec![OnionPage { h5_offset: 0, data: page_data.clone(), codec: 0, orig_size: page_data.len() as u32 }], pages: vec![OnionPage {
h5_offset: 0,
data: page_data.clone(),
codec: 0,
orig_size: page_data.len() as u32,
}],
}; };
bad_packet.blake3_root = [0xDE; 32]; // deliberately wrong bad_packet.blake3_root = [0xDE; 32]; // deliberately wrong
@@ -229,7 +232,12 @@ mod tests {
timestamp: 0.0, timestamp: 0.0,
annotation: None, annotation: None,
blake3_root: hash, blake3_root: hash,
pages: vec![OnionPage { h5_offset: 0, data: page_data.clone(), codec: 0, orig_size: page_data.len() as u32 }], pages: vec![OnionPage {
h5_offset: 0,
data: page_data.clone(),
codec: 0,
orig_size: page_data.len() as u32,
}],
}; };
let (_h5, mut dst) = make_empty_onion(); let (_h5, mut dst) = make_empty_onion();
+20 -5
View File
@@ -113,8 +113,18 @@ mod tests {
annotation: Some(format!("rev {revision}")), annotation: Some(format!("rev {revision}")),
blake3_root: [0xABu8; 32], blake3_root: [0xABu8; 32],
pages: vec![ pages: vec![
OnionPage { h5_offset: 0, data: vec![0xAAu8; 4096], codec: 0, orig_size: 4096 }, OnionPage {
OnionPage { h5_offset: 4096, data: vec![0xBBu8; 4096], codec: 0, orig_size: 4096 }, h5_offset: 0,
data: vec![0xAAu8; 4096],
codec: 0,
orig_size: 4096,
},
OnionPage {
h5_offset: 4096,
data: vec![0xBBu8; 4096],
codec: 0,
orig_size: 4096,
},
], ],
} }
} }
@@ -193,11 +203,16 @@ mod tests {
blake3_root: [0u8; 32], blake3_root: [0u8; 32],
pages: vec![ pages: vec![
// "compressed" to 1600 bytes, original 4096 // "compressed" to 1600 bytes, original 4096
OnionPage { h5_offset: 0, data: vec![0xCCu8; 1600], codec: 1, orig_size: 4096 }, OnionPage {
h5_offset: 0,
data: vec![0xCCu8; 1600],
codec: 1,
orig_size: 4096,
},
], ],
}; };
assert_eq!(pkt.page_data_size(), 4096); // logical (uncompressed) assert_eq!(pkt.page_data_size(), 4096); // logical (uncompressed)
assert_eq!(pkt.page_wire_size(), 1600); // wire (compressed) assert_eq!(pkt.page_wire_size(), 1600); // wire (compressed)
} }
#[test] #[test]
+6 -6
View File
@@ -63,10 +63,7 @@ pub fn filter_packets(
/// Compute the remote HEAD for a given selector on the local `OnionFile`. /// Compute the remote HEAD for a given selector on the local `OnionFile`.
/// ///
/// Returns [`NO_PARENT`] if no revisions match (full sync needed). /// Returns [`NO_PARENT`] if no revisions match (full sync needed).
pub fn remote_head_for_selector( pub fn remote_head_for_selector(_selector: &SyncSelector, remote_revision_count: u64) -> u64 {
_selector: &SyncSelector,
remote_revision_count: u64,
) -> u64 {
// For a simple linear sync, remote HEAD = remote_revision_count - 1 // For a simple linear sync, remote HEAD = remote_revision_count - 1
// (or NO_PARENT if empty). // (or NO_PARENT if empty).
if remote_revision_count == 0 { if remote_revision_count == 0 {
@@ -131,8 +128,11 @@ mod tests {
fn selector_branch_nonexistent_errors() { fn selector_branch_nonexistent_errors() {
let onion = make_onion(2); let onion = make_onion(2);
let packets = diff_revisions(&onion, NO_PARENT).unwrap(); let packets = diff_revisions(&onion, NO_PARENT).unwrap();
let result = let result = filter_packets(
filter_packets(packets, &SyncSelector::Branch("no-such".to_string()), &onion); packets,
&SyncSelector::Branch("no-such".to_string()),
&onion,
);
assert!(matches!(result, Err(SyncOnionError::BranchNotFound(_)))); assert!(matches!(result, Err(SyncOnionError::BranchNotFound(_))));
} }
+12 -7
View File
@@ -16,7 +16,7 @@ use clawhdf5_onion::format::NO_PARENT;
use clawhdf5_onion::writer::OnionFile; use clawhdf5_onion::writer::OnionFile;
use clawsync_onion::differ::diff_revisions; use clawsync_onion::differ::diff_revisions;
use clawsync_onion::merger::merge_packets; use clawsync_onion::merger::merge_packets;
use clawsync_onion::selector::{filter_packets, remote_head_for_selector, SyncSelector}; use clawsync_onion::selector::{SyncSelector, filter_packets, remote_head_for_selector};
use tempfile::NamedTempFile; use tempfile::NamedTempFile;
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@@ -309,7 +309,11 @@ fn merge_auto_registers_unknown_branch() {
assert_eq!(stats.revisions_merged, 4); assert_eq!(stats.revisions_merged, 4);
// The remote should have both branches registered // The remote should have both branches registered
let branches = dst.list_branches(); let branches = dst.list_branches();
assert!(branches.len() >= 2, "expected at least 2 branches, got {:?}", branches.len()); assert!(
branches.len() >= 2,
"expected at least 2 branches, got {:?}",
branches.len()
);
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@@ -327,7 +331,10 @@ fn can_branch_on_remote_after_branch_selective_sync() {
// The remote should now be able to fork its own branch // The remote should now be able to fork its own branch
let result = dst.create_branch("experiment", "main"); let result = dst.create_branch("experiment", "main");
assert!(result.is_ok(), "should be able to create branch after sync: {result:?}"); assert!(
result.is_ok(),
"should be able to create branch after sync: {result:?}"
);
} }
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
@@ -340,14 +347,12 @@ fn branch_filtered_merge_is_idempotent() {
let mut dst = empty_onion(); let mut dst = empty_onion();
let packets1 = diff_revisions(&src, NO_PARENT).unwrap(); let packets1 = diff_revisions(&src, NO_PARENT).unwrap();
let filtered1 = let filtered1 = filter_packets(packets1, &SyncSelector::Branch("main".into()), &src).unwrap();
filter_packets(packets1, &SyncSelector::Branch("main".into()), &src).unwrap();
merge_packets(&mut dst, filtered1, false).unwrap(); merge_packets(&mut dst, filtered1, false).unwrap();
// Apply again — should skip all // Apply again — should skip all
let packets2 = diff_revisions(&src, NO_PARENT).unwrap(); let packets2 = diff_revisions(&src, NO_PARENT).unwrap();
let filtered2 = let filtered2 = filter_packets(packets2, &SyncSelector::Branch("main".into()), &src).unwrap();
filter_packets(packets2, &SyncSelector::Branch("main".into()), &src).unwrap();
let stats2 = merge_packets(&mut dst, filtered2, false).unwrap(); let stats2 = merge_packets(&mut dst, filtered2, false).unwrap();
assert_eq!(stats2.revisions_merged, 0); assert_eq!(stats2.revisions_merged, 0);
+2 -2
View File
@@ -16,8 +16,8 @@ pub mod quic;
pub mod tcp; pub mod tcp;
pub use error::TransportError; pub use error::TransportError;
pub use mmap::{MmapChannel, DEFAULT_CAPACITY}; pub use mmap::{DEFAULT_CAPACITY, MmapChannel};
pub use peer::{PipeReadHalf, PipeWriteHalf, SyncPeer}; pub use peer::{PipeReadHalf, PipeWriteHalf, SyncPeer};
pub use protocol::SyncMessage; pub use protocol::SyncMessage;
pub use tcp::{TcpConnection, TcpServer};
pub use quic::{QuicConfig, QuicConnection, QuicServer, quic_connect}; pub use quic::{QuicConfig, QuicConnection, QuicServer, quic_connect};
pub use tcp::{TcpConnection, TcpServer};
+33 -12
View File
@@ -26,12 +26,12 @@
//! - Capacity is fixed at creation time. //! - Capacity is fixed at creation time.
//! - No cross-machine support (file must be on a shared filesystem). //! - No cross-machine support (file must be on a shared filesystem).
use memmap2::MmapMut;
use std::fs::OpenOptions; use std::fs::OpenOptions;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
use memmap2::MmapMut;
use crate::error::TransportError; use crate::error::TransportError;
use crate::protocol::{SyncMessage, MAX_FRAME_SIZE}; use crate::protocol::{MAX_FRAME_SIZE, SyncMessage};
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
// Constants // Constants
@@ -92,7 +92,11 @@ impl MmapChannel {
write_u64(&mut mmap, 24, capacity as u64); // capacity write_u64(&mut mmap, 24, capacity as u64); // capacity
mmap.flush().map_err(TransportError::Io)?; mmap.flush().map_err(TransportError::Io)?;
Ok(Self { path: path.to_owned(), mmap, capacity }) Ok(Self {
path: path.to_owned(),
mmap,
capacity,
})
} }
/// Open an existing channel file. /// Open an existing channel file.
@@ -105,10 +109,16 @@ impl MmapChannel {
let mmap = unsafe { MmapMut::map_mut(&file).map_err(TransportError::Io)? }; let mmap = unsafe { MmapMut::map_mut(&file).map_err(TransportError::Io)? };
if &mmap[..8] != MAGIC { if &mmap[..8] != MAGIC {
return Err(TransportError::Protocol("invalid mmap channel magic".into())); return Err(TransportError::Protocol(
"invalid mmap channel magic".into(),
));
} }
let capacity = read_u64(&mmap, 24) as usize; let capacity = read_u64(&mmap, 24) as usize;
Ok(Self { path: path.to_owned(), mmap, capacity }) Ok(Self {
path: path.to_owned(),
mmap,
capacity,
})
} }
/// Send a message into the ring buffer. /// Send a message into the ring buffer.
@@ -194,8 +204,7 @@ impl MmapChannel {
write_u64(&mut self.mmap, 16, new_rh); write_u64(&mut self.mmap, 16, new_rh);
self.mmap.flush_range(0, 32).map_err(TransportError::Io)?; self.mmap.flush_range(0, 32).map_err(TransportError::Io)?;
SyncMessage::from_bytes(&body) SyncMessage::from_bytes(&body).map_err(TransportError::Protocol)
.map_err(TransportError::Protocol)
} }
/// Path to the backing file. /// Path to the backing file.
@@ -231,14 +240,20 @@ impl MmapChannel {
impl MmapSender { impl MmapSender {
pub fn open(path: &Path) -> Result<Self, TransportError> { pub fn open(path: &Path) -> Result<Self, TransportError> {
let ch = MmapChannel::open(path)?; let ch = MmapChannel::open(path)?;
Ok(Self { _path: path.to_owned(), _capacity: ch.capacity }) Ok(Self {
_path: path.to_owned(),
_capacity: ch.capacity,
})
} }
} }
impl MmapReceiver { impl MmapReceiver {
pub fn open(path: &Path) -> Result<Self, TransportError> { pub fn open(path: &Path) -> Result<Self, TransportError> {
let ch = MmapChannel::open(path)?; let ch = MmapChannel::open(path)?;
Ok(Self { _path: path.to_owned(), _capacity: ch.capacity }) Ok(Self {
_path: path.to_owned(),
_capacity: ch.capacity,
})
} }
} }
@@ -310,7 +325,9 @@ mod tests {
let (_f, path) = tmp_path(); let (_f, path) = tmp_path();
let mut ch = MmapChannel::create(&path, DEFAULT_CAPACITY).unwrap(); let mut ch = MmapChannel::create(&path, DEFAULT_CAPACITY).unwrap();
let msg = SyncMessage::Error { message: "oops".into() }; let msg = SyncMessage::Error {
message: "oops".into(),
};
ch.send(&msg).unwrap(); ch.send(&msg).unwrap();
let recv = ch.recv().unwrap(); let recv = ch.recv().unwrap();
assert!(matches!(recv, SyncMessage::Error { .. })); assert!(matches!(recv, SyncMessage::Error { .. }));
@@ -324,9 +341,13 @@ mod tests {
ch.send(&SyncMessage::SyncComplete { ch.send(&SyncMessage::SyncComplete {
revisions_transferred: 100, revisions_transferred: 100,
bytes_transferred: 4096 * 100, bytes_transferred: 4096 * 100,
}).unwrap(); })
.unwrap();
match ch.recv().unwrap() { match ch.recv().unwrap() {
SyncMessage::SyncComplete { revisions_transferred, .. } => { SyncMessage::SyncComplete {
revisions_transferred,
..
} => {
assert_eq!(revisions_transferred, 100); assert_eq!(revisions_transferred, 100);
} }
other => panic!("unexpected: {other:?}"), other => panic!("unexpected: {other:?}"),
+1 -3
View File
@@ -63,9 +63,7 @@ impl SyncPeer {
let (r, w) = conn.into_split(); let (r, w) = conn.into_split();
(PipeReadHalf::Tcp(r), PipeWriteHalf::Tcp(w)) (PipeReadHalf::Tcp(r), PipeWriteHalf::Tcp(w))
} }
SyncPeer::Quic(arc) => { SyncPeer::Quic(arc) => (PipeReadHalf::Quic(arc.clone()), PipeWriteHalf::Quic(arc)),
(PipeReadHalf::Quic(arc.clone()), PipeWriteHalf::Quic(arc))
}
} }
} }
} }
+134 -61
View File
@@ -156,8 +156,8 @@ pub enum SyncMessage {
/// lacks (what to push). The client decodes `sketch` against its own /// lacks (what to push). The client decodes `sketch` against its own
/// sketch to find revisions it needs to pull. /// sketch to find revisions it needs to pull.
IbltResponse { IbltResponse {
sketch: IbltManifest, sketch: IbltManifest,
missing_from_remote: Vec<u64>, missing_from_remote: Vec<u64>,
}, },
/// An error occurred; connection will be closed after this message. /// An error occurred; connection will be closed after this message.
@@ -167,7 +167,6 @@ pub enum SyncMessage {
// //
// IMPORTANT: rkyv assigns discriminants positionally. These variants MUST // IMPORTANT: rkyv assigns discriminants positionally. These variants MUST
// remain appended after `Error` (index 8). Never insert or reorder. // remain appended after `Error` (index 8). Never insert or reorder.
/// Client announces it wants to sync a single file using CDC delta. /// Client announces it wants to sync a single file using CDC delta.
/// ///
/// Carries the ordered CDC chunk descriptors of the client's version of /// Carries the ordered CDC chunk descriptors of the client's version of
@@ -205,7 +204,6 @@ pub enum SyncMessage {
FsFileAck { path: String }, FsFileAck { path: String },
// ── FS directory tree sync (variants 1316) ─────────────────────────────── // ── FS directory tree sync (variants 1316) ───────────────────────────────
/// Client sends a per-file manifest for an entire directory tree. /// Client sends a per-file manifest for an entire directory tree.
FsDirManifest { FsDirManifest {
/// Root path on the client (informational). /// Root path on the client (informational).
@@ -227,15 +225,15 @@ pub enum SyncMessage {
/// Server reports that all changes have been applied. /// Server reports that all changes have been applied.
FsDirComplete { FsDirComplete {
files_added: u32, files_added: u32,
files_modified: u32, files_modified: u32,
files_removed: u32, files_removed: u32,
bytes_transferred: u64, bytes_transferred: u64,
}, },
/// Dry-run: reports what *would* change without applying anything. /// Dry-run: reports what *would* change without applying anything.
FsDirDryRun { FsDirDryRun {
would_add: Vec<String>, would_add: Vec<String>,
would_modify: Vec<String>, would_modify: Vec<String>,
would_remove: Vec<String>, would_remove: Vec<String>,
}, },
@@ -244,7 +242,6 @@ pub enum SyncMessage {
// //
// IMPORTANT: rkyv assigns discriminants positionally. These variants MUST // IMPORTANT: rkyv assigns discriminants positionally. These variants MUST
// remain appended after `FsDirDryRun` (index 16). Never insert or reorder. // remain appended after `FsDirDryRun` (index 16). Never insert or reorder.
/// Client sends its per-dataset manifest and the remote h5 path to sync. /// Client sends its per-dataset manifest and the remote h5 path to sync.
/// ///
/// The server compares this manifest against its own to find which datasets /// The server compares this manifest against its own to find which datasets
@@ -288,9 +285,9 @@ pub enum SyncMessage {
/// Server reports that the HDF5 sync is complete. /// Server reports that the HDF5 sync is complete.
Hdf5SyncComplete { Hdf5SyncComplete {
datasets_added: u32, datasets_added: u32,
datasets_modified: u32, datasets_modified: u32,
datasets_removed: u32, datasets_removed: u32,
bytes_transferred: u64, bytes_transferred: u64,
}, },
} }
@@ -310,8 +307,7 @@ impl SyncMessage {
pub fn from_bytes(bytes: &[u8]) -> Result<Self, String> { pub fn from_bytes(bytes: &[u8]) -> Result<Self, String> {
let mut aligned = rkyv::util::AlignedVec::<16>::with_capacity(bytes.len()); let mut aligned = rkyv::util::AlignedVec::<16>::with_capacity(bytes.len());
aligned.extend_from_slice(bytes); aligned.extend_from_slice(bytes);
rkyv::from_bytes::<SyncMessage, rkyv::rancor::Error>(&aligned) rkyv::from_bytes::<SyncMessage, rkyv::rancor::Error>(&aligned).map_err(|e| e.to_string())
.map_err(|e| e.to_string())
} }
/// Encode to a length-prefixed frame: `[u32 LE length][message bytes]`. /// Encode to a length-prefixed frame: `[u32 LE length][message bytes]`.
@@ -333,10 +329,16 @@ impl SyncMessage {
} }
let len = u32::from_le_bytes(data[..4].try_into().unwrap()) as usize; let len = u32::from_le_bytes(data[..4].try_into().unwrap()) as usize;
if len > MAX_FRAME_SIZE { if len > MAX_FRAME_SIZE {
return Err(format!("frame too large: {len} bytes (max {MAX_FRAME_SIZE})")); return Err(format!(
"frame too large: {len} bytes (max {MAX_FRAME_SIZE})"
));
} }
if data.len() < 4 + len { if data.len() < 4 + len {
return Err(format!("incomplete frame: have {} bytes, need {}", data.len(), 4 + len)); return Err(format!(
"incomplete frame: have {} bytes, need {}",
data.len(),
4 + len
));
} }
let msg = Self::from_bytes(&data[4..4 + len])?; let msg = Self::from_bytes(&data[4..4 + len])?;
Ok((msg, 4 + len)) Ok((msg, 4 + len))
@@ -377,7 +379,11 @@ mod tests {
let bytes = msg.to_bytes().unwrap(); let bytes = msg.to_bytes().unwrap();
let recovered = SyncMessage::from_bytes(&bytes).unwrap(); let recovered = SyncMessage::from_bytes(&bytes).unwrap();
match recovered { match recovered {
SyncMessage::ManifestRequest { agent_id, head_revision, revision_count } => { SyncMessage::ManifestRequest {
agent_id,
head_revision,
revision_count,
} => {
assert_eq!(agent_id, "test-agent"); assert_eq!(agent_id, "test-agent");
assert_eq!(head_revision, 5); assert_eq!(head_revision, 5);
assert_eq!(revision_count, 6); assert_eq!(revision_count, 6);
@@ -419,7 +425,10 @@ mod tests {
let bytes = msg.to_bytes().unwrap(); let bytes = msg.to_bytes().unwrap();
let recovered = SyncMessage::from_bytes(&bytes).unwrap(); let recovered = SyncMessage::from_bytes(&bytes).unwrap();
match recovered { match recovered {
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => { SyncMessage::SyncComplete {
revisions_transferred,
bytes_transferred,
} => {
assert_eq!(revisions_transferred, 10); assert_eq!(revisions_transferred, 10);
assert_eq!(bytes_transferred, 40960); assert_eq!(bytes_transferred, 40960);
} }
@@ -429,7 +438,9 @@ mod tests {
#[test] #[test]
fn error_message_roundtrip() { fn error_message_roundtrip() {
let msg = SyncMessage::Error { message: "something went wrong".to_string() }; let msg = SyncMessage::Error {
message: "something went wrong".to_string(),
};
let bytes = msg.to_bytes().unwrap(); let bytes = msg.to_bytes().unwrap();
let recovered = SyncMessage::from_bytes(&bytes).unwrap(); let recovered = SyncMessage::from_bytes(&bytes).unwrap();
match recovered { match recovered {
@@ -440,21 +451,23 @@ mod tests {
#[test] #[test]
fn iblt_request_roundtrip() { fn iblt_request_roundtrip() {
use clawsync_onion::manifest::IbltManifest;
use clawsync_onion::iblt::IbltSketch; use clawsync_onion::iblt::IbltSketch;
use clawsync_onion::manifest::IbltManifest;
// Build a minimal IbltManifest // Build a minimal IbltManifest
let sketch = IbltSketch::from_keys(&[0u64, 1, 2], 0xABCD); let sketch = IbltSketch::from_keys(&[0u64, 1, 2], 0xABCD);
let iblt_manifest = IbltManifest { let iblt_manifest = IbltManifest {
agent_id: "test".to_string(), agent_id: "test".to_string(),
file_blake3: [0u8; 32], file_blake3: [0u8; 32],
revision_count: 3, revision_count: 3,
head_revision: 2, head_revision: 2,
head_blake3: [0u8; 32], head_blake3: [0u8; 32],
last_write: 0.0, last_write: 0.0,
sketch: sketch.to_bytes(), sketch: sketch.to_bytes(),
sketch_cells: sketch.cell_count() as u32, sketch_cells: sketch.cell_count() as u32,
};
let msg = SyncMessage::IbltRequest {
sketch: iblt_manifest,
}; };
let msg = SyncMessage::IbltRequest { sketch: iblt_manifest };
let bytes = msg.to_bytes().unwrap(); let bytes = msg.to_bytes().unwrap();
let recovered = SyncMessage::from_bytes(&bytes).unwrap(); let recovered = SyncMessage::from_bytes(&bytes).unwrap();
assert!(matches!(recovered, SyncMessage::IbltRequest { .. })); assert!(matches!(recovered, SyncMessage::IbltRequest { .. }));
@@ -462,18 +475,18 @@ mod tests {
#[test] #[test]
fn iblt_response_roundtrip() { fn iblt_response_roundtrip() {
use clawsync_onion::manifest::IbltManifest;
use clawsync_onion::iblt::IbltSketch; use clawsync_onion::iblt::IbltSketch;
use clawsync_onion::manifest::IbltManifest;
let sketch = IbltSketch::from_keys(&[5u64, 6], 0xDEAD); let sketch = IbltSketch::from_keys(&[5u64, 6], 0xDEAD);
let iblt_manifest = IbltManifest { let iblt_manifest = IbltManifest {
agent_id: "srv".to_string(), agent_id: "srv".to_string(),
file_blake3: [1u8; 32], file_blake3: [1u8; 32],
revision_count: 2, revision_count: 2,
head_revision: 6, head_revision: 6,
head_blake3: [2u8; 32], head_blake3: [2u8; 32],
last_write: 1.0, last_write: 1.0,
sketch: sketch.to_bytes(), sketch: sketch.to_bytes(),
sketch_cells: sketch.cell_count() as u32, sketch_cells: sketch.cell_count() as u32,
}; };
let msg = SyncMessage::IbltResponse { let msg = SyncMessage::IbltResponse {
sketch: iblt_manifest, sketch: iblt_manifest,
@@ -482,7 +495,10 @@ mod tests {
let bytes = msg.to_bytes().unwrap(); let bytes = msg.to_bytes().unwrap();
let recovered = SyncMessage::from_bytes(&bytes).unwrap(); let recovered = SyncMessage::from_bytes(&bytes).unwrap();
match recovered { match recovered {
SyncMessage::IbltResponse { missing_from_remote, .. } => { SyncMessage::IbltResponse {
missing_from_remote,
..
} => {
assert_eq!(missing_from_remote, vec![7, 8, 9]); assert_eq!(missing_from_remote, vec![7, 8, 9]);
} }
_ => panic!("wrong variant"), _ => panic!("wrong variant"),
@@ -496,9 +512,18 @@ mod tests {
let msg = SyncMessage::FsCdcRequest { let msg = SyncMessage::FsCdcRequest {
path: "data/weights.bin".to_string(), path: "data/weights.bin".to_string(),
chunk_hashes: vec![ chunk_hashes: vec![
FsChunkHash { hash: 0xDEAD, length: 65536 }, FsChunkHash {
FsChunkHash { hash: 0xBEEF, length: 32768 }, hash: 0xDEAD,
FsChunkHash { hash: 0xCAFE, length: 65536 }, length: 65536,
},
FsChunkHash {
hash: 0xBEEF,
length: 32768,
},
FsChunkHash {
hash: 0xCAFE,
length: 65536,
},
], ],
}; };
let bytes = msg.to_bytes().unwrap(); let bytes = msg.to_bytes().unwrap();
@@ -522,7 +547,10 @@ mod tests {
let bytes = msg.to_bytes().unwrap(); let bytes = msg.to_bytes().unwrap();
let recovered = SyncMessage::from_bytes(&bytes).unwrap(); let recovered = SyncMessage::from_bytes(&bytes).unwrap();
match recovered { match recovered {
SyncMessage::FsCdcNeed { path, needed_indices } => { SyncMessage::FsCdcNeed {
path,
needed_indices,
} => {
assert_eq!(path, "model.bin"); assert_eq!(path, "model.bin");
assert_eq!(needed_indices, vec![0, 2]); assert_eq!(needed_indices, vec![0, 2]);
} }
@@ -535,18 +563,36 @@ mod tests {
let msg = SyncMessage::FsCdcData { let msg = SyncMessage::FsCdcData {
path: "cfg.bin".to_string(), path: "cfg.bin".to_string(),
chunk_order: vec![ chunk_order: vec![
FsChunkHash { hash: 0x11, length: 65536 }, FsChunkHash {
FsChunkHash { hash: 0x22, length: 65536 }, hash: 0x11,
length: 65536,
},
FsChunkHash {
hash: 0x22,
length: 65536,
},
], ],
chunks: vec![ chunks: vec![
FsChunk { hash: 0x11, data: vec![0xAAu8; 100], compressed: true }, FsChunk {
FsChunk { hash: 0x22, data: vec![0xBBu8; 200], compressed: false }, hash: 0x11,
data: vec![0xAAu8; 100],
compressed: true,
},
FsChunk {
hash: 0x22,
data: vec![0xBBu8; 200],
compressed: false,
},
], ],
}; };
let bytes = msg.to_bytes().unwrap(); let bytes = msg.to_bytes().unwrap();
let recovered = SyncMessage::from_bytes(&bytes).unwrap(); let recovered = SyncMessage::from_bytes(&bytes).unwrap();
match recovered { match recovered {
SyncMessage::FsCdcData { path, chunk_order, chunks } => { SyncMessage::FsCdcData {
path,
chunk_order,
chunks,
} => {
assert_eq!(path, "cfg.bin"); assert_eq!(path, "cfg.bin");
assert_eq!(chunk_order.len(), 2); assert_eq!(chunk_order.len(), 2);
assert_eq!(chunks.len(), 2); assert_eq!(chunks.len(), 2);
@@ -560,10 +606,14 @@ mod tests {
#[test] #[test]
fn fs_file_ack_roundtrip() { fn fs_file_ack_roundtrip() {
let msg = SyncMessage::FsFileAck { path: "embeddings.bin".to_string() }; let msg = SyncMessage::FsFileAck {
path: "embeddings.bin".to_string(),
};
let bytes = msg.to_bytes().unwrap(); let bytes = msg.to_bytes().unwrap();
let recovered = SyncMessage::from_bytes(&bytes).unwrap(); let recovered = SyncMessage::from_bytes(&bytes).unwrap();
assert!(matches!(recovered, SyncMessage::FsFileAck { ref path } if path == "embeddings.bin")); assert!(
matches!(recovered, SyncMessage::FsFileAck { ref path } if path == "embeddings.bin")
);
} }
#[test] #[test]
@@ -571,8 +621,18 @@ mod tests {
let msg = SyncMessage::FsDirManifest { let msg = SyncMessage::FsDirManifest {
local_root: "/home/user/data".to_string(), local_root: "/home/user/data".to_string(),
entries: vec![ entries: vec![
FsManifestEntry { path: "a.bin".to_string(), blake3: [1u8; 32], size: 4096, mtime: 1000 }, FsManifestEntry {
FsManifestEntry { path: "b.bin".to_string(), blake3: [2u8; 32], size: 8192, mtime: 2000 }, path: "a.bin".to_string(),
blake3: [1u8; 32],
size: 4096,
mtime: 1000,
},
FsManifestEntry {
path: "b.bin".to_string(),
blake3: [2u8; 32],
size: 8192,
mtime: 2000,
},
], ],
}; };
let bytes = msg.to_bytes().unwrap(); let bytes = msg.to_bytes().unwrap();
@@ -590,19 +650,23 @@ mod tests {
#[test] #[test]
fn fs_dir_need_roundtrip() { fn fs_dir_need_roundtrip() {
let msg = SyncMessage::FsDirNeed { let msg = SyncMessage::FsDirNeed {
needed_files: vec![ needed_files: vec![FsFileNeed {
FsFileNeed { path: "weights.bin".to_string(),
path: "weights.bin".to_string(), server_chunks: vec![FsChunkHash {
server_chunks: vec![FsChunkHash { hash: 0xAA, length: 65536 }], hash: 0xAA,
expected_blake3: [0xABu8; 32], length: 65536,
}, }],
], expected_blake3: [0xABu8; 32],
}],
to_delete: vec!["old_model.bin".to_string()], to_delete: vec!["old_model.bin".to_string()],
}; };
let bytes = msg.to_bytes().unwrap(); let bytes = msg.to_bytes().unwrap();
let recovered = SyncMessage::from_bytes(&bytes).unwrap(); let recovered = SyncMessage::from_bytes(&bytes).unwrap();
match recovered { match recovered {
SyncMessage::FsDirNeed { needed_files, to_delete } => { SyncMessage::FsDirNeed {
needed_files,
to_delete,
} => {
assert_eq!(needed_files.len(), 1); assert_eq!(needed_files.len(), 1);
assert_eq!(needed_files[0].path, "weights.bin"); assert_eq!(needed_files[0].path, "weights.bin");
assert_eq!(to_delete, vec!["old_model.bin"]); assert_eq!(to_delete, vec!["old_model.bin"]);
@@ -614,12 +678,19 @@ mod tests {
#[test] #[test]
fn fs_dir_complete_roundtrip() { fn fs_dir_complete_roundtrip() {
let msg = SyncMessage::FsDirComplete { let msg = SyncMessage::FsDirComplete {
files_added: 2, files_modified: 3, files_removed: 1, bytes_transferred: 1_048_576, files_added: 2,
files_modified: 3,
files_removed: 1,
bytes_transferred: 1_048_576,
}; };
let bytes = msg.to_bytes().unwrap(); let bytes = msg.to_bytes().unwrap();
let recovered = SyncMessage::from_bytes(&bytes).unwrap(); let recovered = SyncMessage::from_bytes(&bytes).unwrap();
match recovered { match recovered {
SyncMessage::FsDirComplete { files_added, bytes_transferred, .. } => { SyncMessage::FsDirComplete {
files_added,
bytes_transferred,
..
} => {
assert_eq!(files_added, 2); assert_eq!(files_added, 2);
assert_eq!(bytes_transferred, 1_048_576); assert_eq!(bytes_transferred, 1_048_576);
} }
@@ -637,8 +708,10 @@ mod tests {
// We just verify the roundtrip still produces the exact same ManifestRequest — // We just verify the roundtrip still produces the exact same ManifestRequest —
// any reordering would cause a deserialization error here. // any reordering would cause a deserialization error here.
let recovered = SyncMessage::from_bytes(&bytes).unwrap(); let recovered = SyncMessage::from_bytes(&bytes).unwrap();
assert!(matches!(recovered, SyncMessage::ManifestRequest { .. }), assert!(
"ManifestRequest discriminant changed — rkyv variant ordering was modified"); matches!(recovered, SyncMessage::ManifestRequest { .. }),
"ManifestRequest discriminant changed — rkyv variant ordering was modified"
);
} }
#[test] #[test]
+25 -19
View File
@@ -49,11 +49,8 @@ impl QuicConfig {
.map_err(|e| TransportError::Tls(e.to_string()))?; .map_err(|e| TransportError::Tls(e.to_string()))?;
// Server config // Server config
let server_config = ServerConfig::with_single_cert( let server_config = ServerConfig::with_single_cert(vec![cert_der.clone()], key_der)
vec![cert_der.clone()], .map_err(|e| TransportError::Tls(e.to_string()))?;
key_der,
)
.map_err(|e| TransportError::Tls(e.to_string()))?;
// Client config: trust only our self-signed cert // Client config: trust only our self-signed cert
let mut roots = rustls::RootCertStore::empty(); let mut roots = rustls::RootCertStore::empty();
@@ -63,7 +60,10 @@ impl QuicConfig {
let client_config = ClientConfig::with_root_certificates(Arc::new(roots)) let client_config = ClientConfig::with_root_certificates(Arc::new(roots))
.map_err(|e| TransportError::Tls(e.to_string()))?; .map_err(|e| TransportError::Tls(e.to_string()))?;
Ok(Self { server_config, client_config }) Ok(Self {
server_config,
client_config,
})
} }
} }
@@ -83,9 +83,7 @@ impl QuicConnection {
/// Send a `SyncMessage` on a new unidirectional QUIC stream. /// Send a `SyncMessage` on a new unidirectional QUIC stream.
pub async fn send(&self, msg: &SyncMessage) -> Result<(), TransportError> { pub async fn send(&self, msg: &SyncMessage) -> Result<(), TransportError> {
let body = msg let body = msg.to_bytes().map_err(TransportError::Serialization)?;
.to_bytes()
.map_err(TransportError::Serialization)?;
let mut send = self let mut send = self
.conn .conn
@@ -101,7 +99,8 @@ impl QuicConnection {
send.write_all(&body) send.write_all(&body)
.await .await
.map_err(|e| TransportError::Io(e.into()))?; .map_err(|e| TransportError::Io(e.into()))?;
send.finish().map_err(|e| TransportError::Quic(e.to_string()))?; send.finish()
.map_err(|e| TransportError::Quic(e.to_string()))?;
Ok(()) Ok(())
} }
@@ -147,10 +146,12 @@ pub struct QuicServer {
impl QuicServer { impl QuicServer {
/// Bind a QUIC server endpoint. /// Bind a QUIC server endpoint.
pub async fn bind(addr: SocketAddr, config: QuicConfig) -> Result<Self, TransportError> { pub async fn bind(addr: SocketAddr, config: QuicConfig) -> Result<Self, TransportError> {
let endpoint = Endpoint::server(config.server_config, addr) let endpoint = Endpoint::server(config.server_config, addr).map_err(TransportError::Io)?;
.map_err(TransportError::Io)?;
let local_addr = endpoint.local_addr()?; let local_addr = endpoint.local_addr()?;
Ok(Self { endpoint, local_addr }) Ok(Self {
endpoint,
local_addr,
})
} }
/// Accept the next incoming QUIC connection. /// Accept the next incoming QUIC connection.
@@ -177,8 +178,8 @@ pub async fn quic_connect(
server_name: &str, server_name: &str,
config: QuicConfig, config: QuicConfig,
) -> Result<QuicConnection, TransportError> { ) -> Result<QuicConnection, TransportError> {
let mut endpoint = Endpoint::client("0.0.0.0:0".parse().unwrap()) let mut endpoint =
.map_err(TransportError::Io)?; Endpoint::client("0.0.0.0:0".parse().unwrap()).map_err(TransportError::Io)?;
endpoint.set_default_client_config(config.client_config); endpoint.set_default_client_config(config.client_config);
let conn = endpoint let conn = endpoint
@@ -217,10 +218,15 @@ mod tests {
roots.add(cert_der).unwrap(); roots.add(cert_der).unwrap();
let cli_cfg = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap(); let cli_cfg = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap();
let server = QuicServer::bind(any_addr(), QuicConfig { let server = QuicServer::bind(
server_config: srv_cfg, any_addr(),
client_config: cli_cfg.clone(), QuicConfig {
}).await.unwrap(); server_config: srv_cfg,
client_config: cli_cfg.clone(),
},
)
.await
.unwrap();
let addr = server.local_addr; let addr = server.local_addr;
let server_task = tokio::spawn(async move { let server_task = tokio::spawn(async move {
+33 -14
View File
@@ -7,11 +7,11 @@
use std::net::SocketAddr; use std::net::SocketAddr;
use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf}; use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf};
use tokio::net::{TcpListener, TcpStream};
use crate::error::TransportError; use crate::error::TransportError;
use crate::protocol::{SyncMessage, MAX_FRAME_SIZE}; use crate::protocol::{MAX_FRAME_SIZE, SyncMessage};
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
// TcpConnection — async read/write of SyncMessages // TcpConnection — async read/write of SyncMessages
@@ -37,9 +37,7 @@ impl TcpConnection {
/// Send a `SyncMessage` over the connection. /// Send a `SyncMessage` over the connection.
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> { pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
let body = msg let body = msg.to_bytes().map_err(TransportError::Serialization)?;
.to_bytes()
.map_err(TransportError::Serialization)?;
let len = body.len() as u32; let len = body.len() as u32;
self.stream.write_all(&len.to_le_bytes()).await?; self.stream.write_all(&len.to_le_bytes()).await?;
self.stream.write_all(&body).await?; self.stream.write_all(&body).await?;
@@ -65,8 +63,7 @@ impl TcpConnection {
let mut body = vec![0u8; len]; let mut body = vec![0u8; len];
self.stream.read_exact(&mut body).await?; self.stream.read_exact(&mut body).await?;
SyncMessage::from_bytes(&body) SyncMessage::from_bytes(&body).map_err(TransportError::Deserialization)
.map_err(TransportError::Deserialization)
} }
/// Gracefully shut down the write half of the connection. /// Gracefully shut down the write half of the connection.
@@ -81,7 +78,10 @@ impl TcpConnection {
/// pipelined send/receive (sliding window) without holding a single lock. /// pipelined send/receive (sliding window) without holding a single lock.
pub fn into_split(self) -> (TcpReadHalf, TcpWriteHalf) { pub fn into_split(self) -> (TcpReadHalf, TcpWriteHalf) {
let (read_half, write_half) = self.stream.into_split(); let (read_half, write_half) = self.stream.into_split();
(TcpReadHalf { half: read_half }, TcpWriteHalf { half: write_half }) (
TcpReadHalf { half: read_half },
TcpWriteHalf { half: write_half },
)
} }
} }
@@ -102,7 +102,10 @@ impl TcpReadHalf {
let len = u32::from_le_bytes(len_buf) as usize; let len = u32::from_le_bytes(len_buf) as usize;
if len > MAX_FRAME_SIZE { if len > MAX_FRAME_SIZE {
return Err(TransportError::FrameTooLarge { size: len, max: MAX_FRAME_SIZE }); return Err(TransportError::FrameTooLarge {
size: len,
max: MAX_FRAME_SIZE,
});
} }
let mut body = vec![0u8; len]; let mut body = vec![0u8; len];
@@ -149,7 +152,10 @@ impl TcpServer {
pub async fn bind(addr: SocketAddr) -> Result<Self, TransportError> { pub async fn bind(addr: SocketAddr) -> Result<Self, TransportError> {
let listener = TcpListener::bind(addr).await?; let listener = TcpListener::bind(addr).await?;
let local_addr = listener.local_addr()?; let local_addr = listener.local_addr()?;
Ok(Self { listener, local_addr }) Ok(Self {
listener,
local_addr,
})
} }
/// Accept the next incoming connection. /// Accept the next incoming connection.
@@ -186,7 +192,10 @@ mod tests {
}); });
let mut client = TcpConnection::connect(addr).await.unwrap(); let mut client = TcpConnection::connect(addr).await.unwrap();
client.send(&SyncMessage::Ack { revision: 42 }).await.unwrap(); client
.send(&SyncMessage::Ack { revision: 42 })
.await
.unwrap();
let reply = client.recv().await.unwrap(); let reply = client.recv().await.unwrap();
assert!(matches!(reply, SyncMessage::Ack { revision: 42 })); assert!(matches!(reply, SyncMessage::Ack { revision: 42 }));
@@ -208,7 +217,10 @@ mod tests {
let mut client = TcpConnection::connect(addr).await.unwrap(); let mut client = TcpConnection::connect(addr).await.unwrap();
for rev in 0u64..5 { for rev in 0u64..5 {
client.send(&SyncMessage::Ack { revision: rev }).await.unwrap(); client
.send(&SyncMessage::Ack { revision: rev })
.await
.unwrap();
} }
server_task.await.unwrap(); server_task.await.unwrap();
} }
@@ -233,7 +245,10 @@ mod tests {
client.send(&msg).await.unwrap(); client.send(&msg).await.unwrap();
let reply = client.recv().await.unwrap(); let reply = client.recv().await.unwrap();
match reply { match reply {
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => { SyncMessage::SyncComplete {
revisions_transferred,
bytes_transferred,
} => {
assert_eq!(revisions_transferred, 7); assert_eq!(revisions_transferred, 7);
assert_eq!(bytes_transferred, 28672); assert_eq!(bytes_transferred, 28672);
} }
@@ -263,7 +278,11 @@ mod tests {
client.send(&msg).await.unwrap(); client.send(&msg).await.unwrap();
let reply = client.recv().await.unwrap(); let reply = client.recv().await.unwrap();
match reply { match reply {
SyncMessage::ManifestRequest { agent_id, head_revision, .. } => { SyncMessage::ManifestRequest {
agent_id,
head_revision,
..
} => {
assert_eq!(agent_id, "my-agent"); assert_eq!(agent_id, "my-agent");
assert_eq!(head_revision, 3); assert_eq!(head_revision, 3);
} }
@@ -12,7 +12,7 @@ use clawhdf5_onion::writer::OnionFile;
use clawsync_onion::{differ::diff_revisions, manifest::ClawSyncManifest, merger::merge_packets}; use clawsync_onion::{differ::diff_revisions, manifest::ClawSyncManifest, merger::merge_packets};
use clawsync_transport::{ use clawsync_transport::{
protocol::SyncMessage, protocol::SyncMessage,
quic::{quic_connect, QuicConfig, QuicServer}, quic::{QuicConfig, QuicServer, quic_connect},
}; };
use tempfile::NamedTempFile; use tempfile::NamedTempFile;
@@ -62,8 +62,7 @@ fn make_quic_configs() -> (QuicConfig, QuicConfig) {
let mut roots = rustls::RootCertStore::empty(); let mut roots = rustls::RootCertStore::empty();
roots.add(srv_cert_der).unwrap(); roots.add(srv_cert_der).unwrap();
let client_config = let client_config = quinn::ClientConfig::with_root_certificates(Arc::new(roots)).unwrap();
quinn::ClientConfig::with_root_certificates(Arc::new(roots)).unwrap();
// Client-side QuicConfig requires a server_config field; generate a fresh // Client-side QuicConfig requires a server_config field; generate a fresh
// cert+key pair so there is no key/cert mismatch. // cert+key pair so there is no key/cert mismatch.
@@ -71,8 +70,14 @@ fn make_quic_configs() -> (QuicConfig, QuicConfig) {
let cli_server_config = let cli_server_config =
quinn::ServerConfig::with_single_cert(vec![cli_cert_der], cli_key_der).unwrap(); quinn::ServerConfig::with_single_cert(vec![cli_cert_der], cli_key_der).unwrap();
let srv_cfg = QuicConfig { server_config: srv_server_config, client_config: client_config.clone() }; let srv_cfg = QuicConfig {
let cli_cfg = QuicConfig { server_config: cli_server_config, client_config }; server_config: srv_server_config,
client_config: client_config.clone(),
};
let cli_cfg = QuicConfig {
server_config: cli_server_config,
client_config,
};
(srv_cfg, cli_cfg) (srv_cfg, cli_cfg)
} }
@@ -99,16 +104,20 @@ async fn run_quic_push(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
match msg { match msg {
SyncMessage::ManifestRequest { .. } => { SyncMessage::ManifestRequest { .. } => {
let dst_manifest = ClawSyncManifest::from_onion("dst", &dst, H5_BASE); let dst_manifest = ClawSyncManifest::from_onion("dst", &dst, H5_BASE);
conn.send(&SyncMessage::ManifestResponse { manifest: dst_manifest }) conn.send(&SyncMessage::ManifestResponse {
.await manifest: dst_manifest,
.unwrap(); })
.await
.unwrap();
} }
SyncMessage::LayerPacket { packet } => { SyncMessage::LayerPacket { packet } => {
let rev = packet.revision; let rev = packet.revision;
merge_packets(&mut dst, vec![packet], false).unwrap(); merge_packets(&mut dst, vec![packet], false).unwrap();
packets_received += 1; packets_received += 1;
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap(); conn.send(&SyncMessage::Ack { revision: rev })
.await
.unwrap();
} }
SyncMessage::SyncComplete { .. } => { SyncMessage::SyncComplete { .. } => {
@@ -125,7 +134,9 @@ async fn run_quic_push(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
}); });
// ── Client logic ───────────────────────────────────────────────────────── // ── Client logic ─────────────────────────────────────────────────────────
let client = quic_connect(server_addr, "localhost", cli_cfg).await.unwrap(); let client = quic_connect(server_addr, "localhost", cli_cfg)
.await
.unwrap();
client client
.send(&SyncMessage::ManifestRequest { .send(&SyncMessage::ManifestRequest {
@@ -152,7 +163,10 @@ async fn run_quic_push(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
for packet in packets { for packet in packets {
let rev = packet.revision; let rev = packet.revision;
client.send(&SyncMessage::LayerPacket { packet }).await.unwrap(); client
.send(&SyncMessage::LayerPacket { packet })
.await
.unwrap();
match client.recv().await.unwrap() { match client.recv().await.unwrap() {
SyncMessage::Ack { revision } => assert_eq!(revision, rev), SyncMessage::Ack { revision } => assert_eq!(revision, rev),
other => panic!("client: expected Ack, got {other:?}"), other => panic!("client: expected Ack, got {other:?}"),
@@ -228,7 +242,11 @@ async fn quic_revision_hashes_match_after_sync() {
assert_eq!(src_revisions.len(), dst_revisions.len()); assert_eq!(src_revisions.len(), dst_revisions.len());
for (s, d) in src_revisions.iter().zip(dst_revisions.iter()) { for (s, d) in src_revisions.iter().zip(dst_revisions.iter()) {
assert_eq!(s.blake3_hex, d.blake3_hex, "rev {} hash mismatch", s.revision); assert_eq!(
s.blake3_hex, d.blake3_hex,
"rev {} hash mismatch",
s.revision
);
assert_eq!(s.annotation, d.annotation); assert_eq!(s.annotation, d.annotation);
} }
} }
@@ -269,9 +287,11 @@ async fn run_quic_pull(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
}; };
let srv_manifest = ClawSyncManifest::from_onion("srv", &src, H5_BASE); let srv_manifest = ClawSyncManifest::from_onion("srv", &src, H5_BASE);
conn.send(&SyncMessage::ManifestResponse { manifest: srv_manifest }) conn.send(&SyncMessage::ManifestResponse {
.await manifest: srv_manifest,
.unwrap(); })
.await
.unwrap();
if src.revision_count() > client_rev_count { if src.revision_count() > client_rev_count {
let remote_head = if client_rev_count == 0 { let remote_head = if client_rev_count == 0 {
@@ -284,7 +304,9 @@ async fn run_quic_pull(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
let mut bytes_sent = 0u64; let mut bytes_sent = 0u64;
for packet in packets { for packet in packets {
bytes_sent += packet.page_data_size() as u64; bytes_sent += packet.page_data_size() as u64;
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap(); conn.send(&SyncMessage::LayerPacket { packet })
.await
.unwrap();
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::Ack { .. } => {} SyncMessage::Ack { .. } => {}
other => panic!("server: expected Ack, got {other:?}"), other => panic!("server: expected Ack, got {other:?}"),
@@ -310,7 +332,9 @@ async fn run_quic_pull(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
}); });
// ── Client logic ───────────────────────────────────────────────────────── // ── Client logic ─────────────────────────────────────────────────────────
let client = quic_connect(server_addr, "localhost", cli_cfg).await.unwrap(); let client = quic_connect(server_addr, "localhost", cli_cfg)
.await
.unwrap();
let local_rev_count = dst.revision_count(); let local_rev_count = dst.revision_count();
client client
@@ -344,10 +368,16 @@ async fn run_quic_pull(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
match client.recv().await.unwrap() { match client.recv().await.unwrap() {
SyncMessage::LayerPacket { packet } => { SyncMessage::LayerPacket { packet } => {
let rev = packet.revision; let rev = packet.revision;
client.send(&SyncMessage::Ack { revision: rev }).await.unwrap(); client
.send(&SyncMessage::Ack { revision: rev })
.await
.unwrap();
packets.push(packet); packets.push(packet);
} }
SyncMessage::SyncComplete { revisions_transferred, .. } => { SyncMessage::SyncComplete {
revisions_transferred,
..
} => {
merge_packets(&mut dst, packets, true).unwrap(); merge_packets(&mut dst, packets, true).unwrap();
server_handle.await.unwrap(); server_handle.await.unwrap();
return (revisions_transferred, dst); return (revisions_transferred, dst);
@@ -412,7 +442,11 @@ async fn quic_pull_revision_hashes_match() {
assert_eq!(srv_revisions.len(), cli_revisions.len()); assert_eq!(srv_revisions.len(), cli_revisions.len());
for (s, c) in srv_revisions.iter().zip(cli_revisions.iter()) { for (s, c) in srv_revisions.iter().zip(cli_revisions.iter()) {
assert_eq!(s.blake3_hex, c.blake3_hex, "rev {} hash mismatch", s.revision); assert_eq!(
s.blake3_hex, c.blake3_hex,
"rev {} hash mismatch",
s.revision
);
assert_eq!(s.annotation, c.annotation); assert_eq!(s.annotation, c.annotation);
} }
} }
+30 -23
View File
@@ -18,11 +18,7 @@
use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use clawhdf5_onion::writer::OnionFile; use clawhdf5_onion::writer::OnionFile;
use clawsync_onion::{ use clawsync_onion::{differ::diff_revisions, manifest::ClawSyncManifest, merger::merge_packets};
differ::diff_revisions,
manifest::ClawSyncManifest,
merger::merge_packets,
};
use clawsync_transport::{ use clawsync_transport::{
protocol::SyncMessage, protocol::SyncMessage,
tcp::{TcpConnection, TcpServer}, tcp::{TcpConnection, TcpServer},
@@ -45,9 +41,7 @@ fn make_onion(n: u8) -> (NamedTempFile, OnionFile) {
for i in 0..n { for i in 0..n {
let mut s = onion.begin_session(None).unwrap(); let mut s = onion.begin_session(None).unwrap();
s.record_page(0, &vec![i; 4096]); s.record_page(0, &vec![i; 4096]);
onion onion.commit_session(s, Some(&format!("rev {i}"))).unwrap();
.commit_session(s, Some(&format!("rev {i}")))
.unwrap();
} }
(tmp, onion) (tmp, onion)
} }
@@ -57,10 +51,7 @@ fn make_onion(n: u8) -> (NamedTempFile, OnionFile) {
// number of LayerPackets actually transferred. // number of LayerPackets actually transferred.
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
async fn run_push( async fn run_push(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
src: OnionFile,
mut dst: OnionFile,
) -> (u64, OnionFile) {
// Bind the server; the OS picks a free port. // Bind the server; the OS picks a free port.
let server = TcpServer::bind(localhost_any()).await.unwrap(); let server = TcpServer::bind(localhost_any()).await.unwrap();
let server_addr = server.local_addr; let server_addr = server.local_addr;
@@ -79,8 +70,7 @@ async fn run_push(
match msg { match msg {
SyncMessage::ManifestRequest { head_revision, .. } => { SyncMessage::ManifestRequest { head_revision, .. } => {
// Build our local manifest and send it back. // Build our local manifest and send it back.
let dst_manifest = let dst_manifest = ClawSyncManifest::from_onion("dst", &dst, H5_BASE);
ClawSyncManifest::from_onion("dst", &dst, H5_BASE);
conn.send(&SyncMessage::ManifestResponse { conn.send(&SyncMessage::ManifestResponse {
manifest: dst_manifest, manifest: dst_manifest,
}) })
@@ -163,7 +153,10 @@ async fn run_push(
.unwrap(); .unwrap();
let (server_received, dst_final) = server_handle.await.unwrap(); let (server_received, dst_final) = server_handle.await.unwrap();
assert_eq!(server_received, total_sent, "server count must match client send count"); assert_eq!(
server_received, total_sent,
"server count must match client send count"
);
(total_sent, dst_final) (total_sent, dst_final)
} }
@@ -287,7 +280,7 @@ async fn large_push_correct_packet_count() {
// ───────────────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────────────
async fn run_pull( async fn run_pull(
src: OnionFile, // server-side onion (richer) src: OnionFile, // server-side onion (richer)
mut dst: OnionFile, // client-side onion (missing revisions) mut dst: OnionFile, // client-side onion (missing revisions)
) -> (u64, OnionFile) { ) -> (u64, OnionFile) {
let server = TcpServer::bind(localhost_any()).await.unwrap(); let server = TcpServer::bind(localhost_any()).await.unwrap();
@@ -305,9 +298,11 @@ async fn run_pull(
// 2. Send back the server manifest. // 2. Send back the server manifest.
let srv_manifest = ClawSyncManifest::from_onion("srv", &src, H5_BASE); let srv_manifest = ClawSyncManifest::from_onion("srv", &src, H5_BASE);
conn.send(&SyncMessage::ManifestResponse { manifest: srv_manifest }) conn.send(&SyncMessage::ManifestResponse {
.await manifest: srv_manifest,
.unwrap(); })
.await
.unwrap();
// 3. Server has more revisions — push the delta to the client. // 3. Server has more revisions — push the delta to the client.
if src.revision_count() > client_rev_count { if src.revision_count() > client_rev_count {
@@ -321,7 +316,9 @@ async fn run_pull(
let mut bytes_sent = 0u64; let mut bytes_sent = 0u64;
for packet in packets { for packet in packets {
bytes_sent += packet.page_data_size() as u64; bytes_sent += packet.page_data_size() as u64;
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap(); conn.send(&SyncMessage::LayerPacket { packet })
.await
.unwrap();
match conn.recv().await.unwrap() { match conn.recv().await.unwrap() {
SyncMessage::Ack { .. } => {} SyncMessage::Ack { .. } => {}
other => panic!("server: expected Ack, got {other:?}"), other => panic!("server: expected Ack, got {other:?}"),
@@ -381,10 +378,16 @@ async fn run_pull(
match client.recv().await.unwrap() { match client.recv().await.unwrap() {
SyncMessage::LayerPacket { packet } => { SyncMessage::LayerPacket { packet } => {
let rev = packet.revision; let rev = packet.revision;
client.send(&SyncMessage::Ack { revision: rev }).await.unwrap(); client
.send(&SyncMessage::Ack { revision: rev })
.await
.unwrap();
packets.push(packet); packets.push(packet);
} }
SyncMessage::SyncComplete { revisions_transferred, .. } => { SyncMessage::SyncComplete {
revisions_transferred,
..
} => {
merge_packets(&mut dst, packets, true).unwrap(); merge_packets(&mut dst, packets, true).unwrap();
server_handle.await.unwrap(); server_handle.await.unwrap();
return (revisions_transferred, dst); return (revisions_transferred, dst);
@@ -449,7 +452,11 @@ async fn pull_revision_hashes_match() {
assert_eq!(srv_revisions.len(), cli_revisions.len()); assert_eq!(srv_revisions.len(), cli_revisions.len());
for (s, c) in srv_revisions.iter().zip(cli_revisions.iter()) { for (s, c) in srv_revisions.iter().zip(cli_revisions.iter()) {
assert_eq!(s.blake3_hex, c.blake3_hex, "rev {} hash mismatch", s.revision); assert_eq!(
s.blake3_hex, c.blake3_hex,
"rev {} hash mismatch",
s.revision
);
assert_eq!(s.annotation, c.annotation); assert_eq!(s.annotation, c.annotation);
} }
} }