Apply rustfmt to entire workspace
Runs cargo fmt --all; all 573 tests still passing, clippy still clean. No logic changes — formatting only. Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
3d524e2d63
commit
1c107fe58a
@@ -46,7 +46,9 @@ fn bench_epoch_gc(c: &mut Criterion) {
|
||||
b.iter_with_setup(
|
||||
|| make_onion(n),
|
||||
|(_tmp, _h5, mut onion)| {
|
||||
let _ = onion.gc(black_box(GcPolicy::KeepLastN((n / 2) as u64))).unwrap();
|
||||
let _ = onion
|
||||
.gc(black_box(GcPolicy::KeepLastN((n / 2) as u64)))
|
||||
.unwrap();
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -60,9 +62,11 @@ fn bench_epoch_gc(c: &mut Criterion) {
|
||||
b.iter_with_setup(
|
||||
|| make_onion(n),
|
||||
|(_tmp, _h5, mut onion)| {
|
||||
let _ = onion.gc(black_box(GcPolicy::EpochFlip(Box::new(
|
||||
let _ = onion
|
||||
.gc(black_box(GcPolicy::EpochFlip(Box::new(
|
||||
GcPolicy::KeepLastN((n / 2) as u64),
|
||||
)))).unwrap();
|
||||
))))
|
||||
.unwrap();
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -76,9 +80,11 @@ fn bench_epoch_gc(c: &mut Criterion) {
|
||||
b.iter_with_setup(
|
||||
|| {
|
||||
let (tmp, h5, mut onion) = make_onion(n);
|
||||
onion.gc(GcPolicy::EpochFlip(Box::new(
|
||||
GcPolicy::KeepLastN((n / 2) as u64),
|
||||
))).unwrap();
|
||||
onion
|
||||
.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(
|
||||
(n / 2) as u64,
|
||||
))))
|
||||
.unwrap();
|
||||
(tmp, h5, onion)
|
||||
},
|
||||
|(_tmp, _h5, mut onion)| {
|
||||
|
||||
@@ -51,21 +51,31 @@ fn bench_merkle(c: &mut Criterion) {
|
||||
for &n in &ns {
|
||||
let full = RevisionMerkleTree::build(&make_entries(n));
|
||||
let minus1 = RevisionMerkleTree::build(&make_entries(n - 1));
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, minus1), |b, (f, m)| {
|
||||
group.bench_with_input(
|
||||
BenchmarkId::from_parameter(n),
|
||||
&(full, minus1),
|
||||
|b, (f, m)| {
|
||||
b.iter(|| f.diff_missing_revisions(black_box(m)));
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
group.finish();
|
||||
|
||||
// ── Diff walk: D=10 ──────────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("merkle_index/diff_walk_d10");
|
||||
for &n in &ns {
|
||||
if n < 20 { continue; }
|
||||
if n < 20 {
|
||||
continue;
|
||||
}
|
||||
let full = RevisionMerkleTree::build(&make_entries(n));
|
||||
let base = RevisionMerkleTree::build(&make_entries(n - 10));
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, base), |b, (f, base)| {
|
||||
group.bench_with_input(
|
||||
BenchmarkId::from_parameter(n),
|
||||
&(full, base),
|
||||
|b, (f, base)| {
|
||||
b.iter(|| f.diff_missing_revisions(black_box(base)));
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
group.finish();
|
||||
|
||||
@@ -74,9 +84,13 @@ fn bench_merkle(c: &mut Criterion) {
|
||||
for &n in &[1_000usize, 10_000] {
|
||||
let full = RevisionMerkleTree::build(&make_entries(n));
|
||||
let base = RevisionMerkleTree::build(&make_entries(n - 100));
|
||||
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, base), |b, (f, base)| {
|
||||
group.bench_with_input(
|
||||
BenchmarkId::from_parameter(n),
|
||||
&(full, base),
|
||||
|b, (f, base)| {
|
||||
b.iter(|| f.diff_missing_revisions(black_box(base)));
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
group.finish();
|
||||
|
||||
@@ -102,14 +116,19 @@ fn bench_merkle(c: &mut Criterion) {
|
||||
|
||||
// ── Serialised size report (not timed) ───────────────────────────────────
|
||||
println!("\n=== Serialised size vs flat manifest (N × 60 B) ===");
|
||||
println!("{:>8} {:>12} {:>12} {:>10}", "N", "Merkle (B)", "Flat (B)", "ratio");
|
||||
println!(
|
||||
"{:>8} {:>12} {:>12} {:>10}",
|
||||
"N", "Merkle (B)", "Flat (B)", "ratio"
|
||||
);
|
||||
for &n in &[10usize, 100, 1_000, 10_000] {
|
||||
let tree = RevisionMerkleTree::build(&make_entries(n));
|
||||
let merkle_sz = tree.serialise().len();
|
||||
let flat_sz = n * 60;
|
||||
println!(
|
||||
"{:>8} {:>12} {:>12} {:>10.2}x",
|
||||
n, merkle_sz, flat_sz,
|
||||
n,
|
||||
merkle_sz,
|
||||
flat_sz,
|
||||
flat_sz as f64 / merkle_sz as f64
|
||||
);
|
||||
}
|
||||
|
||||
@@ -66,7 +66,10 @@ fn bench_write_one_revision(c: &mut Criterion) {
|
||||
b.iter(|| {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
for i in 0u64..4 {
|
||||
s.record_page(i * PAGE_SIZE as u64, black_box(&vec![0xCDu8; PAGE_SIZE as usize]));
|
||||
s.record_page(
|
||||
i * PAGE_SIZE as u64,
|
||||
black_box(&vec![0xCDu8; PAGE_SIZE as usize]),
|
||||
);
|
||||
}
|
||||
black_box(onion.commit_session(s, None).unwrap());
|
||||
});
|
||||
@@ -83,15 +86,15 @@ fn bench_reconstruct_revision(c: &mut Criterion) {
|
||||
let (_f, _h5, onion, base) = tmp_onion_with_n(depth);
|
||||
let target_rev = (depth - 1) as u64;
|
||||
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new("no_snapshot", depth),
|
||||
&depth,
|
||||
|b, _| {
|
||||
group.bench_with_input(BenchmarkId::new("no_snapshot", depth), &depth, |b, _| {
|
||||
b.iter(|| {
|
||||
black_box(onion.reconstruct_revision(target_rev, black_box(&base)).unwrap());
|
||||
});
|
||||
},
|
||||
black_box(
|
||||
onion
|
||||
.reconstruct_revision(target_rev, black_box(&base))
|
||||
.unwrap(),
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// With snapshot: create a real snapshot at midpoint via create_snapshot()
|
||||
@@ -107,15 +110,15 @@ fn bench_reconstruct_revision(c: &mut Criterion) {
|
||||
}
|
||||
let target_rev = onion.revision_count() - 1;
|
||||
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new("with_snapshot", depth),
|
||||
&depth,
|
||||
|b, _| {
|
||||
group.bench_with_input(BenchmarkId::new("with_snapshot", depth), &depth, |b, _| {
|
||||
b.iter(|| {
|
||||
black_box(onion.reconstruct_revision(target_rev, black_box(&base)).unwrap());
|
||||
});
|
||||
},
|
||||
black_box(
|
||||
onion
|
||||
.reconstruct_revision(target_rev, black_box(&base))
|
||||
.unwrap(),
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
group.finish();
|
||||
|
||||
@@ -35,7 +35,9 @@ fn make_f32_random(n_bytes: usize) -> Vec<u8> {
|
||||
let mut state = 0x_dead_beef_u64;
|
||||
let mut out = Vec::with_capacity(n_bytes);
|
||||
while out.len() < n_bytes {
|
||||
state = state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
||||
state = state
|
||||
.wrapping_mul(6364136223846793005)
|
||||
.wrapping_add(1442695040888963407);
|
||||
out.extend_from_slice(&(state as u32).to_le_bytes());
|
||||
}
|
||||
out.truncate(n_bytes);
|
||||
@@ -78,10 +80,7 @@ fn bench_tdt_compress(c: &mut Criterion) {
|
||||
("int8_seq", make_int8_seq, 1),
|
||||
];
|
||||
|
||||
let codecs = [
|
||||
("zstd", Codec::Zstd),
|
||||
("zstd_tdt", Codec::ZstdTdt),
|
||||
];
|
||||
let codecs = [("zstd", Codec::Zstd), ("zstd_tdt", Codec::ZstdTdt)];
|
||||
|
||||
// ── Throughput benchmark ──────────────────────────────────────────────────
|
||||
let mut group = c.benchmark_group("tdt_compress/throughput");
|
||||
@@ -90,10 +89,7 @@ fn bench_tdt_compress(c: &mut Criterion) {
|
||||
let data = make(size);
|
||||
group.throughput(Throughput::Bytes(size as u64));
|
||||
for &(codec_name, codec) in &codecs {
|
||||
let id = BenchmarkId::new(
|
||||
format!("{codec_name}/{dtype}"),
|
||||
format!("{size}B"),
|
||||
);
|
||||
let id = BenchmarkId::new(format!("{codec_name}/{dtype}"), format!("{size}B"));
|
||||
group.bench_with_input(id, &data, |b, d| {
|
||||
b.iter(|| bench_compress(d, codec));
|
||||
});
|
||||
@@ -126,16 +122,24 @@ fn bench_tdt_compress(c: &mut Criterion) {
|
||||
// ── Compression ratio summary (printed, not timed) ───────────────────────
|
||||
// Run once outside Criterion to print ratio comparison.
|
||||
println!("\n─── TDT compression ratio summary ───");
|
||||
println!("{:<20} {:>8} {:>10} {:>10} {:>8}",
|
||||
"dataset/size", "orig", "zstd", "zstd_tdt", "savings");
|
||||
println!(
|
||||
"{:<20} {:>8} {:>10} {:>10} {:>8}",
|
||||
"dataset/size", "orig", "zstd", "zstd_tdt", "savings"
|
||||
);
|
||||
for &size in &sizes {
|
||||
for &(dtype, make, _w) in datasets {
|
||||
let data = make(size);
|
||||
let zstd_size = compress_page(&data, Codec::Zstd).unwrap().len();
|
||||
let tdt_size = compress_page(&data, Codec::ZstdTdt).unwrap().len();
|
||||
let savings_pct = 100.0 * (1.0 - tdt_size as f64 / zstd_size as f64);
|
||||
println!("{:<20} {:>8} {:>10} {:>10} {:>7.1}%",
|
||||
format!("{dtype}/{size}B"), size, zstd_size, tdt_size, savings_pct);
|
||||
println!(
|
||||
"{:<20} {:>8} {:>10} {:>10} {:>7.1}%",
|
||||
format!("{dtype}/{size}B"),
|
||||
size,
|
||||
zstd_size,
|
||||
tdt_size,
|
||||
savings_pct
|
||||
);
|
||||
}
|
||||
}
|
||||
println!();
|
||||
|
||||
@@ -37,10 +37,7 @@ use crate::writer::OnionFile;
|
||||
/// let (bytes, onion) = open_revision("agent.h5", 7)?;
|
||||
/// let file = clawhdf5::File::from_bytes(bytes)?;
|
||||
/// ```
|
||||
pub fn open_revision(
|
||||
h5_path: &Path,
|
||||
revision: u64,
|
||||
) -> Result<(Vec<u8>, OnionFile), OnionError> {
|
||||
pub fn open_revision(h5_path: &Path, revision: u64) -> Result<(Vec<u8>, OnionFile), OnionError> {
|
||||
let h5_base = std::fs::read(h5_path)?;
|
||||
let onion = OnionFile::open(h5_path)?;
|
||||
let reconstructed = onion.reconstruct_revision(revision, &h5_base)?;
|
||||
@@ -48,10 +45,7 @@ pub fn open_revision(
|
||||
}
|
||||
|
||||
/// Open the HEAD of a named branch.
|
||||
pub fn open_branch(
|
||||
h5_path: &Path,
|
||||
branch: &str,
|
||||
) -> Result<(Vec<u8>, OnionFile), OnionError> {
|
||||
pub fn open_branch(h5_path: &Path, branch: &str) -> Result<(Vec<u8>, OnionFile), OnionError> {
|
||||
let h5_base = std::fs::read(h5_path)?;
|
||||
let onion = OnionFile::open(h5_path)?;
|
||||
let reconstructed = onion.open_rev(OpenRevision::Branch(branch.to_owned()), &h5_base)?;
|
||||
@@ -66,8 +60,10 @@ pub fn open_branch_at(
|
||||
) -> Result<(Vec<u8>, OnionFile), OnionError> {
|
||||
let h5_base = std::fs::read(h5_path)?;
|
||||
let onion = OnionFile::open(h5_path)?;
|
||||
let reconstructed =
|
||||
onion.open_rev(OpenRevision::BranchAt(branch.to_owned(), revision), &h5_base)?;
|
||||
let reconstructed = onion.open_rev(
|
||||
OpenRevision::BranchAt(branch.to_owned(), revision),
|
||||
&h5_base,
|
||||
)?;
|
||||
Ok((reconstructed, onion))
|
||||
}
|
||||
|
||||
|
||||
@@ -41,11 +41,7 @@ impl OnionFile {
|
||||
/// Create a new named branch forked from the current HEAD of `source_branch`.
|
||||
///
|
||||
/// Returns the new branch ID.
|
||||
pub fn create_branch(
|
||||
&mut self,
|
||||
name: &str,
|
||||
source_branch: &str,
|
||||
) -> Result<u32, OnionError> {
|
||||
pub fn create_branch(&mut self, name: &str, source_branch: &str) -> Result<u32, OnionError> {
|
||||
if self.branch_by_name(name).is_some() {
|
||||
return Err(OnionError::BranchExists(name.to_string()));
|
||||
}
|
||||
@@ -133,11 +129,7 @@ impl OnionFile {
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_latest_wins(
|
||||
&mut self,
|
||||
source_id: u32,
|
||||
target_id: u32,
|
||||
) -> Result<u64, OnionError> {
|
||||
fn merge_latest_wins(&mut self, source_id: u32, target_id: u32) -> Result<u64, OnionError> {
|
||||
let fork_rev = self
|
||||
.branches
|
||||
.iter()
|
||||
@@ -231,21 +223,21 @@ impl OnionFile {
|
||||
self.commit_session(session, Some(&annotation))
|
||||
}
|
||||
|
||||
fn merge_three_way(
|
||||
&mut self,
|
||||
source_id: u32,
|
||||
target_id: u32,
|
||||
) -> Result<u64, OnionError> {
|
||||
fn merge_three_way(&mut self, source_id: u32, target_id: u32) -> Result<u64, OnionError> {
|
||||
let source_head = self
|
||||
.index
|
||||
.branch_head(source_id)
|
||||
.ok_or_else(|| OnionError::Malformed(format!("source branch {source_id} has no revisions")))?
|
||||
.ok_or_else(|| {
|
||||
OnionError::Malformed(format!("source branch {source_id} has no revisions"))
|
||||
})?
|
||||
.revision;
|
||||
|
||||
let target_head = self
|
||||
.index
|
||||
.branch_head(target_id)
|
||||
.ok_or_else(|| OnionError::Malformed(format!("target branch {target_id} has no revisions")))?
|
||||
.ok_or_else(|| {
|
||||
OnionError::Malformed(format!("target branch {target_id} has no revisions"))
|
||||
})?
|
||||
.revision;
|
||||
|
||||
// Find common ancestor of the two branch HEADs
|
||||
@@ -253,13 +245,24 @@ impl OnionFile {
|
||||
.index
|
||||
.common_ancestor(source_head, target_head)
|
||||
.ok_or_else(|| {
|
||||
let src_name = self.branches.iter().find(|b| b.id == source_id)
|
||||
let src_name = self
|
||||
.branches
|
||||
.iter()
|
||||
.find(|b| b.id == source_id)
|
||||
.and_then(|b| self.annotations.get(b.name_off))
|
||||
.unwrap_or("?").to_owned();
|
||||
let tgt_name = self.branches.iter().find(|b| b.id == target_id)
|
||||
.unwrap_or("?")
|
||||
.to_owned();
|
||||
let tgt_name = self
|
||||
.branches
|
||||
.iter()
|
||||
.find(|b| b.id == target_id)
|
||||
.and_then(|b| self.annotations.get(b.name_off))
|
||||
.unwrap_or("?").to_owned();
|
||||
OnionError::NoCommonAncestor { a: src_name, b: tgt_name }
|
||||
.unwrap_or("?")
|
||||
.to_owned();
|
||||
OnionError::NoCommonAncestor {
|
||||
a: src_name,
|
||||
b: tgt_name,
|
||||
}
|
||||
})?;
|
||||
|
||||
// Collect source delta since ancestor (latest write per offset)
|
||||
@@ -321,7 +324,8 @@ impl OnionFile {
|
||||
|
||||
if merged.is_empty() {
|
||||
return Err(OnionError::Malformed(
|
||||
"three-way merge: no source changes to apply (target already has all changes)".to_string(),
|
||||
"three-way merge: no source changes to apply (target already has all changes)"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
@@ -329,9 +333,8 @@ impl OnionFile {
|
||||
for (h5_off, bytes) in &merged {
|
||||
session.record_page(*h5_off, bytes);
|
||||
}
|
||||
let annotation = format!(
|
||||
"3-way merge {source_id} → {target_id} [ancestor rev {ancestor_rev}]"
|
||||
);
|
||||
let annotation =
|
||||
format!("3-way merge {source_id} → {target_id} [ancestor rev {ancestor_rev}]");
|
||||
self.commit_session(session, Some(&annotation))
|
||||
}
|
||||
|
||||
@@ -371,11 +374,7 @@ impl OnionFile {
|
||||
.iter()
|
||||
.map(|b| BranchInfo {
|
||||
id: b.id,
|
||||
name: self
|
||||
.annotations
|
||||
.get(b.name_off)
|
||||
.unwrap_or("?")
|
||||
.to_owned(),
|
||||
name: self.annotations.get(b.name_off).unwrap_or("?").to_owned(),
|
||||
head_rev: b.head_rev,
|
||||
fork_rev: b.fork_rev,
|
||||
})
|
||||
@@ -383,7 +382,10 @@ impl OnionFile {
|
||||
}
|
||||
|
||||
/// Return revision entries for a named branch in chronological order.
|
||||
pub fn branch_history(&self, name: &str) -> Result<Vec<&crate::format::RevisionEntry>, OnionError> {
|
||||
pub fn branch_history(
|
||||
&self,
|
||||
name: &str,
|
||||
) -> Result<Vec<&crate::format::RevisionEntry>, OnionError> {
|
||||
let branch = self
|
||||
.branch_by_name(name)
|
||||
.ok_or_else(|| OnionError::BranchNotFound(name.to_string()))?;
|
||||
@@ -414,7 +416,9 @@ impl OnionFile {
|
||||
.branch_by_name(name)
|
||||
.ok_or_else(|| OnionError::BranchNotFound(name.to_string()))?;
|
||||
if branch.id == crate::format::BRANCH_MAIN {
|
||||
return Err(OnionError::Malformed("cannot delete the main branch".into()));
|
||||
return Err(OnionError::Malformed(
|
||||
"cannot delete the main branch".into(),
|
||||
));
|
||||
}
|
||||
let id = branch.id;
|
||||
self.branches.retain(|b| b.id != id);
|
||||
@@ -605,14 +609,18 @@ mod tests {
|
||||
fn merge_latest_wins_produces_new_revision() {
|
||||
let (_h5, mut onion, _feat_id) = make_fork_scenario();
|
||||
let pre_count = onion.revision_count();
|
||||
onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap();
|
||||
onion
|
||||
.merge_into("feat", "main", MergeStrategy::LatestWins)
|
||||
.unwrap();
|
||||
assert_eq!(onion.revision_count(), pre_count + 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_latest_wins_page_content_correct() {
|
||||
let (h5, mut onion, _feat_id) = make_fork_scenario();
|
||||
onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap();
|
||||
onion
|
||||
.merge_into("feat", "main", MergeStrategy::LatestWins)
|
||||
.unwrap();
|
||||
onion.flush().unwrap();
|
||||
|
||||
// Reload from disk and reconstruct main HEAD
|
||||
@@ -621,8 +629,10 @@ mod tests {
|
||||
let base = std::fs::read(&h5).unwrap();
|
||||
let state = onion2.reconstruct_revision(main_head, &base).unwrap();
|
||||
// Page 0 should now be BB (from feat)
|
||||
assert!(state[0..4096].iter().all(|&b| b == 0xBB),
|
||||
"page 0 should be BB after latest-wins merge");
|
||||
assert!(
|
||||
state[0..4096].iter().all(|&b| b == 0xBB),
|
||||
"page 0 should be BB after latest-wins merge"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -646,15 +656,19 @@ mod tests {
|
||||
s2.record_page(0, &vec![0x22u8; 4096]);
|
||||
onion.commit_session(s2, None).unwrap();
|
||||
|
||||
onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap();
|
||||
onion
|
||||
.merge_into("feat", "main", MergeStrategy::LatestWins)
|
||||
.unwrap();
|
||||
onion.flush().unwrap();
|
||||
|
||||
let onion2 = OnionFile::open(&h5).unwrap();
|
||||
let head = onion2.revision_count() - 1;
|
||||
let base = std::fs::read(&h5).unwrap();
|
||||
let state = onion2.reconstruct_revision(head, &base).unwrap();
|
||||
assert!(state[0..4096].iter().all(|&b| b == 0x22),
|
||||
"latest write (0x22) must win in LatestWins merge");
|
||||
assert!(
|
||||
state[0..4096].iter().all(|&b| b == 0x22),
|
||||
"latest write (0x22) must win in LatestWins merge"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -666,7 +680,9 @@ mod tests {
|
||||
onion.commit_session(s, None).unwrap();
|
||||
// fork but make no commits on feat
|
||||
onion.create_branch("feat", "main").unwrap();
|
||||
let err = onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap_err();
|
||||
let err = onion
|
||||
.merge_into("feat", "main", MergeStrategy::LatestWins)
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, OnionError::Malformed(_)));
|
||||
}
|
||||
|
||||
@@ -682,20 +698,30 @@ mod tests {
|
||||
source.to_vec() // just return source
|
||||
};
|
||||
onion
|
||||
.merge_into("feat", "main", MergeStrategy::DatasetLevel(Box::new(resolver)))
|
||||
.merge_into(
|
||||
"feat",
|
||||
"main",
|
||||
MergeStrategy::DatasetLevel(Box::new(resolver)),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(called.load(std::sync::atomic::Ordering::SeqCst), "resolver must be called");
|
||||
assert!(
|
||||
called.load(std::sync::atomic::Ordering::SeqCst),
|
||||
"resolver must be called"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_dataset_level_resolver_controls_output() {
|
||||
let (h5, mut onion, _feat_id) = make_fork_scenario();
|
||||
// Resolver always returns 0xCC regardless of inputs
|
||||
let resolver = |_path: &str, _target: &[u8], _source: &[u8]| -> Vec<u8> {
|
||||
vec![0xCCu8; 4096]
|
||||
};
|
||||
let resolver =
|
||||
|_path: &str, _target: &[u8], _source: &[u8]| -> Vec<u8> { vec![0xCCu8; 4096] };
|
||||
onion
|
||||
.merge_into("feat", "main", MergeStrategy::DatasetLevel(Box::new(resolver)))
|
||||
.merge_into(
|
||||
"feat",
|
||||
"main",
|
||||
MergeStrategy::DatasetLevel(Box::new(resolver)),
|
||||
)
|
||||
.unwrap();
|
||||
onion.flush().unwrap();
|
||||
|
||||
@@ -703,8 +729,10 @@ mod tests {
|
||||
let head = onion2.revision_count() - 1;
|
||||
let base = std::fs::read(&h5).unwrap();
|
||||
let state = onion2.reconstruct_revision(head, &base).unwrap();
|
||||
assert!(state[0..4096].iter().all(|&b| b == 0xCC),
|
||||
"resolver output 0xCC should be in merged state");
|
||||
assert!(
|
||||
state[0..4096].iter().all(|&b| b == 0xCC),
|
||||
"resolver output 0xCC should be in merged state"
|
||||
);
|
||||
}
|
||||
|
||||
// ── Merge: ThreeWay ──────────────────────────────────────────────────────
|
||||
@@ -736,7 +764,9 @@ mod tests {
|
||||
sf.record_page(0, &vec![0xBBu8; 4096]);
|
||||
onion.commit_session(sf, None).unwrap();
|
||||
|
||||
onion.merge_into("feat", "main", MergeStrategy::ThreeWay).unwrap();
|
||||
onion
|
||||
.merge_into("feat", "main", MergeStrategy::ThreeWay)
|
||||
.unwrap();
|
||||
onion.flush().unwrap();
|
||||
|
||||
let onion2 = OnionFile::open(&h5).unwrap();
|
||||
@@ -744,10 +774,14 @@ mod tests {
|
||||
let base = std::fs::read(&h5).unwrap();
|
||||
let state = onion2.reconstruct_revision(head, &base).unwrap();
|
||||
|
||||
assert!(state[0..4096].iter().all(|&b| b == 0xBB),
|
||||
"page 0 should be BB (from feat)");
|
||||
assert!(state[4096..8192].iter().all(|&b| b == 0xCC),
|
||||
"page 4096 should stay CC (from main — not overwritten by 3-way)");
|
||||
assert!(
|
||||
state[0..4096].iter().all(|&b| b == 0xBB),
|
||||
"page 0 should be BB (from feat)"
|
||||
);
|
||||
assert!(
|
||||
state[4096..8192].iter().all(|&b| b == 0xCC),
|
||||
"page 4096 should stay CC (from main — not overwritten by 3-way)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -772,22 +806,28 @@ mod tests {
|
||||
sf.record_page(0, &vec![0xCCu8; 4096]);
|
||||
onion.commit_session(sf, None).unwrap();
|
||||
|
||||
onion.merge_into("feat", "main", MergeStrategy::ThreeWay).unwrap();
|
||||
onion
|
||||
.merge_into("feat", "main", MergeStrategy::ThreeWay)
|
||||
.unwrap();
|
||||
onion.flush().unwrap();
|
||||
|
||||
let onion2 = OnionFile::open(&h5).unwrap();
|
||||
let head = onion2.revision_count() - 1;
|
||||
let base = std::fs::read(&h5).unwrap();
|
||||
let state = onion2.reconstruct_revision(head, &base).unwrap();
|
||||
assert!(state[0..4096].iter().all(|&b| b == 0xCC),
|
||||
"on conflict, source (CC) should win");
|
||||
assert!(
|
||||
state[0..4096].iter().all(|&b| b == 0xCC),
|
||||
"on conflict, source (CC) should win"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_nonexistent_source_errors() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
let err = onion.merge_into("no-such", "main", MergeStrategy::LatestWins).unwrap_err();
|
||||
let err = onion
|
||||
.merge_into("no-such", "main", MergeStrategy::LatestWins)
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
||||
}
|
||||
|
||||
@@ -799,7 +839,9 @@ mod tests {
|
||||
s.record_page(0, &vec![0u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
onion.create_branch("feat", "main").unwrap();
|
||||
let err = onion.merge_into("feat", "no-target", MergeStrategy::LatestWins).unwrap_err();
|
||||
let err = onion
|
||||
.merge_into("feat", "no-target", MergeStrategy::LatestWins)
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,26 +13,22 @@ use crate::tdt;
|
||||
pub fn compress_page(data: &[u8], codec: Codec) -> Result<Vec<u8>, OnionError> {
|
||||
match codec {
|
||||
Codec::None => Ok(data.to_vec()),
|
||||
Codec::Zstd => zstd::bulk::compress(data, 3)
|
||||
.map_err(|e| OnionError::Compress(e.to_string())),
|
||||
Codec::Zstd => {
|
||||
zstd::bulk::compress(data, 3).map_err(|e| OnionError::Compress(e.to_string()))
|
||||
}
|
||||
Codec::Lz4 => Ok(lz4_flex::compress_prepend_size(data)),
|
||||
Codec::Brotli => Err(OnionError::Compress(
|
||||
"brotli not yet implemented".to_string(),
|
||||
)),
|
||||
Codec::ZstdTdt => {
|
||||
let interleaved = tdt::encode(data, 4);
|
||||
zstd::bulk::compress(&interleaved, 3)
|
||||
.map_err(|e| OnionError::Compress(e.to_string()))
|
||||
zstd::bulk::compress(&interleaved, 3).map_err(|e| OnionError::Compress(e.to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decompress `data` using the given codec back to `orig_size` bytes.
|
||||
pub fn decompress_page(
|
||||
data: &[u8],
|
||||
codec: Codec,
|
||||
orig_size: u32,
|
||||
) -> Result<Vec<u8>, OnionError> {
|
||||
pub fn decompress_page(data: &[u8], codec: Codec, orig_size: u32) -> Result<Vec<u8>, OnionError> {
|
||||
match codec {
|
||||
Codec::None => {
|
||||
if data.len() != orig_size as usize {
|
||||
|
||||
@@ -94,7 +94,10 @@ mod tests {
|
||||
.unwrap();
|
||||
|
||||
let f = vf.current().unwrap();
|
||||
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![1.0, 2.0, 3.0]);
|
||||
assert_eq!(
|
||||
f.dataset("data").unwrap().read_f64().unwrap(),
|
||||
vec![1.0, 2.0, 3.0]
|
||||
);
|
||||
assert_eq!(vf.onion().revision_count(), 0);
|
||||
}
|
||||
|
||||
@@ -106,7 +109,10 @@ mod tests {
|
||||
let _ = make_builder(&[1.0]).with_onion(&h5_path, 4096).unwrap();
|
||||
|
||||
let onion_path = PathBuf::from(format!("{}.onion", h5_path.display()));
|
||||
assert!(onion_path.exists(), "sidecar should be flushed by with_onion");
|
||||
assert!(
|
||||
onion_path.exists(),
|
||||
"sidecar should be flushed by with_onion"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -140,6 +146,9 @@ mod tests {
|
||||
vf.page_size().is_power_of_two(),
|
||||
"auto page size must be a power of two"
|
||||
);
|
||||
assert!(vf.page_size() >= 4096, "auto page size must be at least 4 KiB");
|
||||
assert!(
|
||||
vf.page_size() >= 4096,
|
||||
"auto page size must be at least 4 KiB"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -321,7 +321,10 @@ mod tests {
|
||||
fn header_unknown_version_rejected() {
|
||||
let mut hdr = OnionHeader::new(4096, DEFAULT_FEATURE_FLAGS, 0.0);
|
||||
hdr.format_version = 42;
|
||||
assert!(matches!(hdr.validate(), Err(OnionError::UnknownVersion(42))));
|
||||
assert!(matches!(
|
||||
hdr.validate(),
|
||||
Err(OnionError::UnknownVersion(42))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -488,7 +491,10 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn codec_unknown_returns_error() {
|
||||
assert!(matches!(Codec::from_u8(99), Err(OnionError::UnknownCodec(99))));
|
||||
assert!(matches!(
|
||||
Codec::from_u8(99),
|
||||
Err(OnionError::UnknownCodec(99))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -69,7 +69,10 @@ impl OnionFile {
|
||||
entry.set_epoch(EPOCH_DEAD);
|
||||
}
|
||||
}
|
||||
Ok(GcStats { revisions_removed: count, bytes_reclaimed: 0 })
|
||||
Ok(GcStats {
|
||||
revisions_removed: count,
|
||||
bytes_reclaimed: 0,
|
||||
})
|
||||
}
|
||||
_ => {
|
||||
// ── Immediate path: compact now ───────────────────────────
|
||||
@@ -100,11 +103,7 @@ impl OnionFile {
|
||||
GcPolicy::KeepSince(cutoff) => all_revs
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|&rev| {
|
||||
self.index
|
||||
.get(rev)
|
||||
.is_some_and(|e| e.timestamp >= *cutoff)
|
||||
})
|
||||
.filter(|&rev| self.index.get(rev).is_some_and(|e| e.timestamp >= *cutoff))
|
||||
.collect(),
|
||||
GcPolicy::KeepRevisions(explicit) => {
|
||||
let mut set: HashSet<u64> = explicit.iter().copied().collect();
|
||||
@@ -118,7 +117,11 @@ impl OnionFile {
|
||||
GcPolicy::EpochFlip(inner) => return self.compute_to_remove(inner),
|
||||
};
|
||||
|
||||
all_revs.iter().copied().filter(|rev| !keep.contains(rev)).collect()
|
||||
all_revs
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|rev| !keep.contains(rev))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Consolidate + compact the given revision set immediately.
|
||||
@@ -204,8 +207,7 @@ impl OnionFile {
|
||||
}
|
||||
|
||||
// Compact page_data blob.
|
||||
let surviving_revs: Vec<u64> =
|
||||
self.index.entries().iter().map(|e| e.revision).collect();
|
||||
let surviving_revs: Vec<u64> = self.index.entries().iter().map(|e| e.revision).collect();
|
||||
let mut new_page_data: Vec<u8> = Vec::new();
|
||||
for &rev in &surviving_revs {
|
||||
if let Some(table) = self.page_tables.get_mut(rev as usize) {
|
||||
@@ -222,7 +224,10 @@ impl OnionFile {
|
||||
}
|
||||
self.page_data = new_page_data;
|
||||
|
||||
Ok(GcStats { revisions_removed, bytes_reclaimed })
|
||||
Ok(GcStats {
|
||||
revisions_removed,
|
||||
bytes_reclaimed,
|
||||
})
|
||||
}
|
||||
|
||||
/// Find all entries marked [`EPOCH_DEAD`] and compact them.
|
||||
@@ -270,9 +275,7 @@ mod tests {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
onion
|
||||
.commit_session(s, annotation.as_deref())
|
||||
.unwrap();
|
||||
onion.commit_session(s, annotation.as_deref()).unwrap();
|
||||
}
|
||||
onion
|
||||
}
|
||||
@@ -369,7 +372,10 @@ mod tests {
|
||||
assert_eq!(stats.revisions_removed, 4);
|
||||
let after = onion.page_data.len();
|
||||
// page_data must be smaller (or at most equal for all-same pages that compress to 0)
|
||||
assert!(after <= before, "page_data must shrink after GC: {before} -> {after}");
|
||||
assert!(
|
||||
after <= before,
|
||||
"page_data must shrink after GC: {before} -> {after}"
|
||||
);
|
||||
}
|
||||
|
||||
/// After GC + flush, the on-disk file is smaller than before.
|
||||
@@ -421,7 +427,10 @@ mod tests {
|
||||
// The three surviving revisions must still reconstruct without error.
|
||||
for rev in 3..6u64 {
|
||||
let bytes = onion.reconstruct_revision(rev, &h5_base).unwrap();
|
||||
assert!(!bytes.is_empty(), "rev {rev} should produce non-empty bytes");
|
||||
assert!(
|
||||
!bytes.is_empty(),
|
||||
"rev {rev} should produce non-empty bytes"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -456,7 +465,11 @@ mod tests {
|
||||
let before_gc = onion.reconstruct_revision(2, &h5_base).unwrap();
|
||||
assert_eq!(&before_gc[0..4096], &vec![0xAA_u8; 4096], "pre-GC page0");
|
||||
assert_eq!(&before_gc[4096..8192], &vec![0xBB_u8; 4096], "pre-GC page1");
|
||||
assert_eq!(&before_gc[8192..12288], &vec![0xCC_u8; 4096], "pre-GC page2");
|
||||
assert_eq!(
|
||||
&before_gc[8192..12288],
|
||||
&vec![0xCC_u8; 4096],
|
||||
"pre-GC page2"
|
||||
);
|
||||
|
||||
// GC: keep only rev 2 — revs 0 and 1 are ancestors that will be pruned.
|
||||
onion.gc(GcPolicy::KeepLastN(1)).unwrap();
|
||||
@@ -490,15 +503,20 @@ mod tests {
|
||||
let mut onion = make_onion_with_n_revisions(10);
|
||||
let page_data_len_before = onion.page_data.len();
|
||||
|
||||
let stats = onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(3)))).unwrap();
|
||||
let stats = onion
|
||||
.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(3))))
|
||||
.unwrap();
|
||||
|
||||
// Reports the count that will be removed.
|
||||
assert_eq!(stats.revisions_removed, 7);
|
||||
// bytes_reclaimed is 0 until flush compacts.
|
||||
assert_eq!(stats.bytes_reclaimed, 0);
|
||||
// page_data must NOT have changed yet.
|
||||
assert_eq!(onion.page_data.len(), page_data_len_before,
|
||||
"epoch flip must not compact page_data immediately");
|
||||
assert_eq!(
|
||||
onion.page_data.len(),
|
||||
page_data_len_before,
|
||||
"epoch flip must not compact page_data immediately"
|
||||
);
|
||||
// Index still has all 10 entries (removal deferred).
|
||||
assert_eq!(onion.revision_count(), 10);
|
||||
// Entries 0..6 should be marked EPOCH_DEAD.
|
||||
@@ -525,7 +543,9 @@ mod tests {
|
||||
}
|
||||
|
||||
// Epoch flip: defer compaction.
|
||||
onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(2)))).unwrap();
|
||||
onion
|
||||
.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(2))))
|
||||
.unwrap();
|
||||
assert_eq!(onion.revision_count(), 5, "not compacted yet");
|
||||
|
||||
// After flush, deferred compaction runs.
|
||||
@@ -533,7 +553,11 @@ mod tests {
|
||||
|
||||
// Reload and verify.
|
||||
let reloaded = OnionFile::open(&h5).unwrap();
|
||||
assert_eq!(reloaded.revision_count(), 2, "2 revisions survive after flush");
|
||||
assert_eq!(
|
||||
reloaded.revision_count(),
|
||||
2,
|
||||
"2 revisions survive after flush"
|
||||
);
|
||||
|
||||
// Surviving revisions are still reconstructable.
|
||||
for rev in 3u64..5 {
|
||||
@@ -559,16 +583,25 @@ mod tests {
|
||||
#[test]
|
||||
fn gc_epoch_flip_nested_policy_keep_tagged() {
|
||||
let mut onion = make_onion_with_n_revisions(9); // tagged at 0,3,6
|
||||
let stats = onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepTagged))).unwrap();
|
||||
let stats = onion
|
||||
.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepTagged)))
|
||||
.unwrap();
|
||||
assert_eq!(stats.revisions_removed, 6); // keeps 0,3,6
|
||||
assert_eq!(onion.revision_count(), 9, "deferred — index intact");
|
||||
// 0,3,6 are live; rest are dead.
|
||||
for rev in [0u64, 3, 6] {
|
||||
assert_eq!(onion.index.get(rev).unwrap().epoch(), 0, "tagged rev {rev} must be live");
|
||||
assert_eq!(
|
||||
onion.index.get(rev).unwrap().epoch(),
|
||||
0,
|
||||
"tagged rev {rev} must be live"
|
||||
);
|
||||
}
|
||||
for rev in [1u64, 2, 4, 5, 7, 8] {
|
||||
assert_eq!(onion.index.get(rev).unwrap().epoch(), EPOCH_DEAD,
|
||||
"untagged rev {rev} must be EPOCH_DEAD");
|
||||
assert_eq!(
|
||||
onion.index.get(rev).unwrap().epoch(),
|
||||
EPOCH_DEAD,
|
||||
"untagged rev {rev} must be EPOCH_DEAD"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! RevisionIndex: O(1) lookup + branch-filtered iteration.
|
||||
|
||||
use crate::format::{RevisionEntry, NO_PARENT};
|
||||
use crate::format::{NO_PARENT, RevisionEntry};
|
||||
|
||||
/// In-memory index of all revision entries.
|
||||
///
|
||||
@@ -53,7 +53,9 @@ impl RevisionIndex {
|
||||
/// monotonically greater than the current maximum.
|
||||
pub fn append(&mut self, entry: RevisionEntry) {
|
||||
debug_assert!(
|
||||
self.entries.last().is_none_or(|e| e.revision < entry.revision),
|
||||
self.entries
|
||||
.last()
|
||||
.is_none_or(|e| e.revision < entry.revision),
|
||||
"revision numbers must be monotonically increasing"
|
||||
);
|
||||
self.entries.push(entry);
|
||||
@@ -61,15 +63,14 @@ impl RevisionIndex {
|
||||
|
||||
/// Iterate over all revisions on a specific branch (by `branch_id`).
|
||||
pub fn branch_revisions(&self, branch_id: u32) -> impl Iterator<Item = &RevisionEntry> {
|
||||
self.entries.iter().filter(move |e| e.branch_id == branch_id)
|
||||
self.entries
|
||||
.iter()
|
||||
.filter(move |e| e.branch_id == branch_id)
|
||||
}
|
||||
|
||||
/// Return the HEAD revision entry for a branch (highest revision number).
|
||||
pub fn branch_head(&self, branch_id: u32) -> Option<&RevisionEntry> {
|
||||
self.entries
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|e| e.branch_id == branch_id)
|
||||
self.entries.iter().rev().find(|e| e.branch_id == branch_id)
|
||||
}
|
||||
|
||||
/// Walk the DAG from `start_rev` to the root, following `parent_rev`.
|
||||
@@ -191,8 +192,7 @@ mod tests {
|
||||
.collect();
|
||||
assert_eq!(main_revs, vec![0, 1, 4]);
|
||||
|
||||
let branch1_revs: Vec<u64> =
|
||||
idx.branch_revisions(1).map(|e| e.revision).collect();
|
||||
let branch1_revs: Vec<u64> = idx.branch_revisions(1).map(|e| e.revision).collect();
|
||||
assert_eq!(branch1_revs, vec![2, 3]);
|
||||
}
|
||||
|
||||
@@ -252,7 +252,11 @@ mod tests {
|
||||
fn remove_revisions() {
|
||||
let mut idx = RevisionIndex::new();
|
||||
for i in 0u64..5 {
|
||||
idx.append(make_entry(i, BRANCH_MAIN, if i == 0 { NO_PARENT } else { i - 1 }));
|
||||
idx.append(make_entry(
|
||||
i,
|
||||
BRANCH_MAIN,
|
||||
if i == 0 { NO_PARENT } else { i - 1 },
|
||||
));
|
||||
}
|
||||
let to_remove = [1u64, 2].into_iter().collect();
|
||||
idx.remove_revisions(&to_remove);
|
||||
|
||||
@@ -36,19 +36,18 @@ pub mod tdt;
|
||||
pub mod versioned_file;
|
||||
pub mod writer;
|
||||
|
||||
pub use error::OnionError;
|
||||
pub use format::{
|
||||
BranchEntry, Codec, OnionHeader, PageTableEntry, RevisionEntry,
|
||||
BRANCH_MAIN, DEFAULT_FEATURE_FLAGS, EPOCH_DEAD, FORMAT_VERSION, HEADER_SIZE, MAGIC, NO_PARENT,
|
||||
REV_FLAG_SNAPSHOT, feature_flags,
|
||||
};
|
||||
pub use api::{
|
||||
open_revision, open_branch, open_branch_at,
|
||||
list_revisions, rollback,
|
||||
list_branches, create_branch, delete_branch, rename_branch,
|
||||
create_branch, delete_branch, list_branches, list_revisions, open_branch, open_branch_at,
|
||||
open_revision, rename_branch, rollback,
|
||||
};
|
||||
pub use branch::{BranchInfo, DatasetResolver, MergeStrategy};
|
||||
pub use merkle::{MerkleError, MerkleNode, RevisionMerkleTree, WalkStep};
|
||||
pub use error::OnionError;
|
||||
pub use ext::FileBuilderExt;
|
||||
pub use versioned_file::{VersionedFile, open_at_revision, open_at_branch};
|
||||
pub use format::{
|
||||
BRANCH_MAIN, BranchEntry, Codec, DEFAULT_FEATURE_FLAGS, EPOCH_DEAD, FORMAT_VERSION,
|
||||
HEADER_SIZE, MAGIC, NO_PARENT, OnionHeader, PageTableEntry, REV_FLAG_SNAPSHOT, RevisionEntry,
|
||||
feature_flags,
|
||||
};
|
||||
pub use merkle::{MerkleError, MerkleNode, RevisionMerkleTree, WalkStep};
|
||||
pub use versioned_file::{VersionedFile, open_at_branch, open_at_revision};
|
||||
pub use writer::eof_to_page_size;
|
||||
|
||||
@@ -97,7 +97,11 @@ impl RevisionMerkleTree {
|
||||
/// The slice **must** be sorted by `revision` in ascending order.
|
||||
pub fn build(entries: &[(u64, [u8; 32])]) -> Self {
|
||||
let leaf_count = entries.len();
|
||||
let capacity = if leaf_count == 0 { 1 } else { leaf_count.next_power_of_two() };
|
||||
let capacity = if leaf_count == 0 {
|
||||
1
|
||||
} else {
|
||||
leaf_count.next_power_of_two()
|
||||
};
|
||||
|
||||
// 1-indexed BFS array; index 0 unused.
|
||||
let mut hashes = vec![[0u8; 32]; 2 * capacity + 1];
|
||||
@@ -113,7 +117,12 @@ impl RevisionMerkleTree {
|
||||
// Build internal nodes bottom-up
|
||||
build_internal(&mut hashes, capacity);
|
||||
|
||||
RevisionMerkleTree { hashes, leaf_revisions, leaf_count, capacity }
|
||||
RevisionMerkleTree {
|
||||
hashes,
|
||||
leaf_revisions,
|
||||
leaf_count,
|
||||
capacity,
|
||||
}
|
||||
}
|
||||
|
||||
// ── Queries ───────────────────────────────────────────────────────────────
|
||||
@@ -484,7 +493,10 @@ mod tests {
|
||||
let header_only = &full[..13]; // 5 + 8 bytes
|
||||
assert_eq!(
|
||||
RevisionMerkleTree::deserialise(header_only),
|
||||
Err(MerkleError::TruncatedPayload { expected: 5 + 8 + 5 * 40, got: 13 })
|
||||
Err(MerkleError::TruncatedPayload {
|
||||
expected: 5 + 8 + 5 * 40,
|
||||
got: 13
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -96,7 +96,10 @@ mod tests {
|
||||
// Same pages, different insertion order
|
||||
let h1 = hash_pages(&[(0, a.as_ref()), (4096, b.as_ref())]);
|
||||
let h2 = hash_pages(&[(4096, b.as_ref()), (0, a.as_ref())]);
|
||||
assert_eq!(h1, h2, "hash must be order-independent (sorted by h5_offset)");
|
||||
assert_eq!(
|
||||
h1, h2,
|
||||
"hash must be order-independent (sorted by h5_offset)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -3,11 +3,11 @@
|
||||
//! Opening a historical revision reads only the `.onion` sidecar —
|
||||
//! the primary `.h5` file is always at the latest committed state.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use crate::compress::decompress_page;
|
||||
use crate::error::OnionError;
|
||||
use crate::format::Codec;
|
||||
use crate::writer::OnionFile;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// `(h5_offset, compressed_data, codec_byte, orig_size)` — raw page as stored.
|
||||
pub type RawPage = (u64, Vec<u8>, u8, u32);
|
||||
@@ -41,7 +41,11 @@ impl OnionFile {
|
||||
|
||||
// Collect all ancestor revisions in order from oldest → newest
|
||||
let ancestors: Vec<u64> = {
|
||||
let mut chain = self.index.ancestors(rev).map(|e| e.revision).collect::<Vec<_>>();
|
||||
let mut chain = self
|
||||
.index
|
||||
.ancestors(rev)
|
||||
.map(|e| e.revision)
|
||||
.collect::<Vec<_>>();
|
||||
chain.reverse(); // oldest first
|
||||
chain
|
||||
};
|
||||
@@ -53,9 +57,7 @@ impl OnionFile {
|
||||
// Optimisation: find the newest snapshot in the ancestor chain and
|
||||
// start from there instead of from `h5_base`. This bounds
|
||||
// reconstruction depth to O(N_since_snapshot · P).
|
||||
let snapshot_start_idx = ancestors
|
||||
.iter()
|
||||
.rposition(|&r| {
|
||||
let snapshot_start_idx = ancestors.iter().rposition(|&r| {
|
||||
self.index
|
||||
.get(r)
|
||||
.is_some_and(|e| e.flags & REV_FLAG_SNAPSHOT != 0)
|
||||
@@ -76,7 +78,9 @@ impl OnionFile {
|
||||
let table = self
|
||||
.page_tables
|
||||
.get(*ancestor_rev as usize)
|
||||
.ok_or_else(|| OnionError::Malformed(format!("missing page table for rev {ancestor_rev}")))?;
|
||||
.ok_or_else(|| {
|
||||
OnionError::Malformed(format!("missing page table for rev {ancestor_rev}"))
|
||||
})?;
|
||||
|
||||
for pt_entry in table {
|
||||
let codec = Codec::from_u8(pt_entry.codec)?;
|
||||
@@ -111,11 +115,7 @@ impl OnionFile {
|
||||
}
|
||||
|
||||
/// Open a specific revision using an [`OpenRevision`] selector.
|
||||
pub fn open_rev(
|
||||
&self,
|
||||
selector: OpenRevision,
|
||||
h5_base: &[u8],
|
||||
) -> Result<Vec<u8>, OnionError> {
|
||||
pub fn open_rev(&self, selector: OpenRevision, h5_base: &[u8]) -> Result<Vec<u8>, OnionError> {
|
||||
let rev = self.resolve_selector(selector)?;
|
||||
self.reconstruct_revision(rev, h5_base)
|
||||
}
|
||||
@@ -128,12 +128,11 @@ impl OnionFile {
|
||||
self.branch_head_rev(crate::format::BRANCH_MAIN)
|
||||
.ok_or(OnionError::RevisionNotFound(0))
|
||||
}
|
||||
OpenRevision::At(rev) => {
|
||||
self.index
|
||||
OpenRevision::At(rev) => self
|
||||
.index
|
||||
.get(rev)
|
||||
.map(|_| rev)
|
||||
.ok_or(OnionError::RevisionNotFound(rev))
|
||||
}
|
||||
.ok_or(OnionError::RevisionNotFound(rev)),
|
||||
OpenRevision::Branch(name) => {
|
||||
let branch = self
|
||||
.branch_by_name(&name)
|
||||
@@ -201,10 +200,7 @@ impl OnionFile {
|
||||
/// compressed bytes directly and let the receiver decompress.
|
||||
///
|
||||
/// Returns `(h5_offset, compressed_data, codec_byte, orig_size)`.
|
||||
pub fn revision_pages_raw(
|
||||
&self,
|
||||
rev: u64,
|
||||
) -> Result<Vec<RawPage>, OnionError> {
|
||||
pub fn revision_pages_raw(&self, rev: u64) -> Result<Vec<RawPage>, OnionError> {
|
||||
let table = self
|
||||
.page_tables
|
||||
.get(rev as usize)
|
||||
|
||||
@@ -117,7 +117,10 @@ mod tests {
|
||||
// byte_pos=1: elements 1,5,9,13
|
||||
// byte_pos=2: elements 2,6,10,14
|
||||
// byte_pos=3: elements 3,7,11,15
|
||||
assert_eq!(enc, vec![0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15]);
|
||||
assert_eq!(
|
||||
enc,
|
||||
vec![0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15]
|
||||
);
|
||||
let dec = decode(&enc, 4, 16);
|
||||
assert_eq!(dec, data);
|
||||
}
|
||||
@@ -180,7 +183,8 @@ mod tests {
|
||||
assert!(
|
||||
tdt_compressed.len() <= raw_compressed.len(),
|
||||
"TDT+zstd ({} bytes) should be no worse than raw zstd ({} bytes) on structured float data",
|
||||
tdt_compressed.len(), raw_compressed.len()
|
||||
tdt_compressed.len(),
|
||||
raw_compressed.len()
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -105,7 +105,14 @@ impl VersionedFile {
|
||||
.cloned()
|
||||
.unwrap_or_else(|| h5_base.clone());
|
||||
|
||||
Ok(Self { onion, h5_path, h5_base, current_h5, branch_state, page_size })
|
||||
Ok(Self {
|
||||
onion,
|
||||
h5_path,
|
||||
h5_base,
|
||||
current_h5,
|
||||
branch_state,
|
||||
page_size,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a new versioned HDF5 file from a [`clawhdf5::FileBuilder`].
|
||||
@@ -144,7 +151,14 @@ impl VersionedFile {
|
||||
let h5_base = std::fs::read(&h5_path)?;
|
||||
let current_h5 = h5_base.clone();
|
||||
let onion = OnionFile::create(&h5_path, page_size)?;
|
||||
Ok(Self { onion, h5_path, h5_base, current_h5, branch_state: HashMap::new(), page_size })
|
||||
Ok(Self {
|
||||
onion,
|
||||
h5_path,
|
||||
h5_base,
|
||||
current_h5,
|
||||
branch_state: HashMap::new(),
|
||||
page_size,
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a new versioned HDF5 file, automatically choosing the page size.
|
||||
@@ -163,7 +177,14 @@ impl VersionedFile {
|
||||
let current_h5 = h5_base.clone();
|
||||
let onion = OnionFile::create_auto(&h5_path)?;
|
||||
let page_size = onion.page_size();
|
||||
Ok(Self { onion, h5_path, h5_base, current_h5, branch_state: HashMap::new(), page_size })
|
||||
Ok(Self {
|
||||
onion,
|
||||
h5_path,
|
||||
h5_base,
|
||||
current_h5,
|
||||
branch_state: HashMap::new(),
|
||||
page_size,
|
||||
})
|
||||
}
|
||||
|
||||
// ── Read API ──────────────────────────────────────────────────────────────
|
||||
@@ -352,7 +373,9 @@ impl VersionedFile {
|
||||
/// vf.auto_snapshot(500)?; // snapshot every 500 revisions
|
||||
/// ```
|
||||
pub fn auto_snapshot(&mut self, interval: u64) -> Result<Option<u64>, OnionError> {
|
||||
let result = self.onion.auto_snapshot_if_needed(&self.current_h5, interval)?;
|
||||
let result = self
|
||||
.onion
|
||||
.auto_snapshot_if_needed(&self.current_h5, interval)?;
|
||||
if result.is_some() {
|
||||
self.onion.flush()?;
|
||||
}
|
||||
@@ -383,7 +406,8 @@ impl VersionedFile {
|
||||
// Refresh the target branch's cached state so future commits on it
|
||||
// use the merged content as their diff baseline.
|
||||
let (target_id, target_head) = {
|
||||
let entry = self.onion
|
||||
let entry = self
|
||||
.onion
|
||||
.branch_by_name(target)
|
||||
.ok_or_else(|| OnionError::BranchNotFound(target.to_string()))?;
|
||||
(entry.id, entry.head_rev)
|
||||
@@ -435,8 +459,7 @@ impl VersionedFile {
|
||||
// ── Internal ─────────────────────────────────────────────────────────────
|
||||
|
||||
fn hdf5_from_bytes(&self, bytes: Vec<u8>) -> Result<clawhdf5::File, OnionError> {
|
||||
clawhdf5::File::from_bytes(bytes)
|
||||
.map_err(|e| OnionError::Hdf5(e.to_string()))
|
||||
clawhdf5::File::from_bytes(bytes).map_err(|e| OnionError::Hdf5(e.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -510,11 +533,17 @@ mod tests {
|
||||
|
||||
// Read back rev 0 — should still be [1.0, 2.0]
|
||||
let f0 = vf.revision(0).unwrap();
|
||||
assert_eq!(f0.dataset("data").unwrap().read_f64().unwrap(), vec![1.0, 2.0]);
|
||||
assert_eq!(
|
||||
f0.dataset("data").unwrap().read_f64().unwrap(),
|
||||
vec![1.0, 2.0]
|
||||
);
|
||||
|
||||
// Read back rev 1 — should be [3.0, 4.0]
|
||||
let f1 = vf.revision(1).unwrap();
|
||||
assert_eq!(f1.dataset("data").unwrap().read_f64().unwrap(), vec![3.0, 4.0]);
|
||||
assert_eq!(
|
||||
f1.dataset("data").unwrap().read_f64().unwrap(),
|
||||
vec![3.0, 4.0]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -580,9 +609,15 @@ mod tests {
|
||||
let vf2 = VersionedFile::open(&h5_path).unwrap();
|
||||
assert_eq!(vf2.onion().revision_count(), 2);
|
||||
let f0 = vf2.revision(0).unwrap();
|
||||
assert_eq!(f0.dataset("data").unwrap().read_f64().unwrap(), vec![10.0, 20.0]);
|
||||
assert_eq!(
|
||||
f0.dataset("data").unwrap().read_f64().unwrap(),
|
||||
vec![10.0, 20.0]
|
||||
);
|
||||
let f1 = vf2.revision(1).unwrap();
|
||||
assert_eq!(f1.dataset("data").unwrap().read_f64().unwrap(), vec![30.0, 40.0]);
|
||||
assert_eq!(
|
||||
f1.dataset("data").unwrap().read_f64().unwrap(),
|
||||
vec![30.0, 40.0]
|
||||
);
|
||||
}
|
||||
|
||||
// ── merge ────────────────────────────────────────────────────────────────
|
||||
@@ -598,7 +633,8 @@ mod tests {
|
||||
// Fork feat
|
||||
let feat_id = vf.onion_mut().create_branch("feat", "main").unwrap();
|
||||
// Rev 1: feat writes [3.0, 4.0]
|
||||
vf.commit_on(Some("feat"), make_h5(&[3.0, 4.0]), Some("feat-v1")).unwrap();
|
||||
vf.commit_on(Some("feat"), make_h5(&[3.0, 4.0]), Some("feat-v1"))
|
||||
.unwrap();
|
||||
|
||||
// Merge feat → main
|
||||
let merge_rev = vf.merge("feat", "main", MergeStrategy::LatestWins).unwrap();
|
||||
@@ -610,7 +646,10 @@ mod tests {
|
||||
|
||||
// Content at merge revision reflects feat's state (latest wins)
|
||||
let f = vf.revision(merge_rev).unwrap();
|
||||
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![3.0, 4.0]);
|
||||
assert_eq!(
|
||||
f.dataset("data").unwrap().read_f64().unwrap(),
|
||||
vec![3.0, 4.0]
|
||||
);
|
||||
|
||||
let _ = feat_id;
|
||||
}
|
||||
@@ -622,7 +661,8 @@ mod tests {
|
||||
|
||||
vf.commit(make_h5(&[1.0]), Some("main-v1")).unwrap();
|
||||
vf.onion_mut().create_branch("feat", "main").unwrap();
|
||||
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1")).unwrap();
|
||||
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1"))
|
||||
.unwrap();
|
||||
|
||||
vf.merge("feat", "main", MergeStrategy::LatestWins).unwrap();
|
||||
|
||||
@@ -640,7 +680,9 @@ mod tests {
|
||||
let mut vf = VersionedFile::create(&h5_path, 4096).unwrap();
|
||||
vf.commit(make_h5(&[1.0]), None).unwrap();
|
||||
|
||||
let err = vf.merge("ghost", "main", MergeStrategy::LatestWins).unwrap_err();
|
||||
let err = vf
|
||||
.merge("ghost", "main", MergeStrategy::LatestWins)
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
||||
}
|
||||
|
||||
@@ -653,7 +695,8 @@ mod tests {
|
||||
|
||||
vf.commit(make_h5(&[1.0]), Some("main-v1")).unwrap();
|
||||
let feat_id = vf.onion_mut().create_branch("feat", "main").unwrap();
|
||||
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1")).unwrap();
|
||||
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1"))
|
||||
.unwrap();
|
||||
|
||||
let revs = vf.onion().list_revisions();
|
||||
assert_eq!(revs.len(), 2);
|
||||
@@ -679,7 +722,9 @@ mod tests {
|
||||
let (_tmp, h5_path) = tmp_h5(&[1.0]);
|
||||
let mut vf = VersionedFile::create(&h5_path, 4096).unwrap();
|
||||
|
||||
let err = vf.commit_on(Some("ghost"), make_h5(&[2.0]), None).unwrap_err();
|
||||
let err = vf
|
||||
.commit_on(Some("ghost"), make_h5(&[2.0]), None)
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
||||
}
|
||||
|
||||
@@ -698,7 +743,8 @@ mod tests {
|
||||
vf.onion_mut().create_branch("feat", "main").unwrap();
|
||||
|
||||
// Commit v2 on feat (rev 1: [2.0])
|
||||
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1")).unwrap();
|
||||
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1"))
|
||||
.unwrap();
|
||||
|
||||
// Commit v2 on main (rev 2: [3.0]) — diff must be against main's last
|
||||
// commit ([1.0]), not feat's last commit ([2.0]).
|
||||
@@ -706,16 +752,25 @@ mod tests {
|
||||
|
||||
// Reconstruct each revision and verify round-trip fidelity
|
||||
let f0 = vf.revision(0).unwrap();
|
||||
assert_eq!(f0.dataset("data").unwrap().read_f64().unwrap(), vec![1.0],
|
||||
"rev 0 (main-v1)");
|
||||
assert_eq!(
|
||||
f0.dataset("data").unwrap().read_f64().unwrap(),
|
||||
vec![1.0],
|
||||
"rev 0 (main-v1)"
|
||||
);
|
||||
|
||||
let f1 = vf.revision(1).unwrap();
|
||||
assert_eq!(f1.dataset("data").unwrap().read_f64().unwrap(), vec![2.0],
|
||||
"rev 1 (feat-v1)");
|
||||
assert_eq!(
|
||||
f1.dataset("data").unwrap().read_f64().unwrap(),
|
||||
vec![2.0],
|
||||
"rev 1 (feat-v1)"
|
||||
);
|
||||
|
||||
let f2 = vf.revision(2).unwrap();
|
||||
assert_eq!(f2.dataset("data").unwrap().read_f64().unwrap(), vec![3.0],
|
||||
"rev 2 (main-v2) — would be wrong if diffed against feat state");
|
||||
assert_eq!(
|
||||
f2.dataset("data").unwrap().read_f64().unwrap(),
|
||||
vec![3.0],
|
||||
"rev 2 (main-v2) — would be wrong if diffed against feat state"
|
||||
);
|
||||
}
|
||||
|
||||
/// After a reload, `open()` must restore per-branch state from the sidecar
|
||||
@@ -730,13 +785,21 @@ mod tests {
|
||||
let mut s = vf.onion_mut().begin_session(Some(feat_id)).unwrap();
|
||||
let feat_bytes = make_h5(&[9.0]);
|
||||
let ps = 4096usize;
|
||||
let base = vf.onion().reconstruct_revision(0,
|
||||
&std::fs::read(&h5_path).unwrap()).unwrap();
|
||||
let base = vf
|
||||
.onion()
|
||||
.reconstruct_revision(0, &std::fs::read(&h5_path).unwrap())
|
||||
.unwrap();
|
||||
for i in 0..(feat_bytes.len().max(base.len())).div_ceil(ps) {
|
||||
let start = i * ps;
|
||||
if start >= feat_bytes.len() { break; }
|
||||
if start >= feat_bytes.len() {
|
||||
break;
|
||||
}
|
||||
let ns = &feat_bytes[start..feat_bytes.len().min(start + ps)];
|
||||
let os: &[u8] = if start < base.len() { &base[start..base.len().min(start+ps)] } else { &[] };
|
||||
let os: &[u8] = if start < base.len() {
|
||||
&base[start..base.len().min(start + ps)]
|
||||
} else {
|
||||
&[]
|
||||
};
|
||||
if ns != os {
|
||||
let mut pad = vec![0u8; ps];
|
||||
pad[..ns.len()].copy_from_slice(ns);
|
||||
@@ -752,8 +815,11 @@ mod tests {
|
||||
vf2.commit(make_h5(&[2.0]), Some("main-v2")).unwrap();
|
||||
|
||||
let f_main = vf2.revision(2).unwrap();
|
||||
assert_eq!(f_main.dataset("data").unwrap().read_f64().unwrap(), vec![2.0],
|
||||
"main-v2 content must be [2.0] after reload");
|
||||
assert_eq!(
|
||||
f_main.dataset("data").unwrap().read_f64().unwrap(),
|
||||
vec![2.0],
|
||||
"main-v2 content must be [2.0] after reload"
|
||||
);
|
||||
}
|
||||
|
||||
// ── create_auto ──────────────────────────────────────────────────────────
|
||||
@@ -767,7 +833,10 @@ mod tests {
|
||||
"auto page size must be a power of two, got {}",
|
||||
vf.page_size()
|
||||
);
|
||||
assert!(vf.page_size() >= 4096, "auto page size must be at least 4 KiB");
|
||||
assert!(
|
||||
vf.page_size() >= 4096,
|
||||
"auto page size must be at least 4 KiB"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -778,7 +847,10 @@ mod tests {
|
||||
vf.commit(make_h5(&[7.0, 8.0]), Some("auto-v1")).unwrap();
|
||||
|
||||
let f = vf.revision(0).unwrap();
|
||||
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![7.0, 8.0]);
|
||||
assert_eq!(
|
||||
f.dataset("data").unwrap().read_f64().unwrap(),
|
||||
vec![7.0, 8.0]
|
||||
);
|
||||
}
|
||||
|
||||
// ── snapshot / auto_snapshot ─────────────────────────────────────────────
|
||||
@@ -797,7 +869,10 @@ mod tests {
|
||||
// The snapshot revision must be flagged as a snapshot.
|
||||
let revs = vf.onion().list_revisions();
|
||||
let snap = revs.iter().find(|r| r.revision == snap_rev).unwrap();
|
||||
assert!(snap.is_snapshot, "snapshot revision must have is_snapshot == true");
|
||||
assert!(
|
||||
snap.is_snapshot,
|
||||
"snapshot revision must have is_snapshot == true"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -843,7 +918,10 @@ mod tests {
|
||||
vf.commit(make_h5(&[i as f64]), None).unwrap();
|
||||
}
|
||||
let result = vf.auto_snapshot(5).unwrap();
|
||||
assert!(result.is_some(), "auto_snapshot should fire when count % interval == 0");
|
||||
assert!(
|
||||
result.is_some(),
|
||||
"auto_snapshot should fire when count % interval == 0"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -855,7 +933,10 @@ mod tests {
|
||||
vf.commit(make_h5(&[2.0]), None).unwrap();
|
||||
// 2 revisions, interval 5 — should not trigger
|
||||
let result = vf.auto_snapshot(5).unwrap();
|
||||
assert!(result.is_none(), "auto_snapshot should be a no-op when count % interval != 0");
|
||||
assert!(
|
||||
result.is_none(),
|
||||
"auto_snapshot should be a no-op when count % interval != 0"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -908,12 +989,21 @@ mod tests {
|
||||
// Record a changed page for the feat branch rev
|
||||
let feat_bytes = make_h5(&[3.0]);
|
||||
let ps = 4096usize;
|
||||
let current = vf.onion().reconstruct_revision(1, &std::fs::read(&h5_path).unwrap()).unwrap();
|
||||
let current = vf
|
||||
.onion()
|
||||
.reconstruct_revision(1, &std::fs::read(&h5_path).unwrap())
|
||||
.unwrap();
|
||||
for i in 0..(feat_bytes.len().max(current.len())).div_ceil(ps) {
|
||||
let start = i * ps;
|
||||
if start >= feat_bytes.len() { break; }
|
||||
if start >= feat_bytes.len() {
|
||||
break;
|
||||
}
|
||||
let new_sl = &feat_bytes[start..feat_bytes.len().min(start + ps)];
|
||||
let old_sl: &[u8] = if start < current.len() { ¤t[start..current.len().min(start + ps)] } else { &[] };
|
||||
let old_sl: &[u8] = if start < current.len() {
|
||||
¤t[start..current.len().min(start + ps)]
|
||||
} else {
|
||||
&[]
|
||||
};
|
||||
if new_sl != old_sl {
|
||||
let mut pad = vec![0u8; ps];
|
||||
pad[..new_sl.len()].copy_from_slice(new_sl);
|
||||
@@ -1009,7 +1099,10 @@ mod tests {
|
||||
p.set_extension(&new_ext);
|
||||
p
|
||||
};
|
||||
assert!(onion_path.exists(), ".onion sidecar should exist after from_builder: checked {onion_path:?}");
|
||||
assert!(
|
||||
onion_path.exists(),
|
||||
".onion sidecar should exist after from_builder: checked {onion_path:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1026,7 +1119,10 @@ mod tests {
|
||||
assert_eq!(rev, 0);
|
||||
|
||||
let f = vf.revision(0).unwrap();
|
||||
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![3.0, 4.0]);
|
||||
assert_eq!(
|
||||
f.dataset("data").unwrap().read_f64().unwrap(),
|
||||
vec![3.0, 4.0]
|
||||
);
|
||||
}
|
||||
|
||||
// ── export_revision ───────────────────────────────────────────────────────
|
||||
@@ -1046,7 +1142,10 @@ mod tests {
|
||||
// Re-open the exported file as a raw clawhdf5::File
|
||||
let bytes = std::fs::read(&export_path).unwrap();
|
||||
let f = clawhdf5::File::from_bytes(bytes).unwrap();
|
||||
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![1.0, 2.0]);
|
||||
assert_eq!(
|
||||
f.dataset("data").unwrap().read_f64().unwrap(),
|
||||
vec![1.0, 2.0]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1075,7 +1174,10 @@ mod tests {
|
||||
|
||||
let bytes = std::fs::read(&export_path).unwrap();
|
||||
let f = clawhdf5::File::from_bytes(bytes).unwrap();
|
||||
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![8.0, 9.0]);
|
||||
assert_eq!(
|
||||
f.dataset("data").unwrap().read_f64().unwrap(),
|
||||
vec![8.0, 9.0]
|
||||
);
|
||||
}
|
||||
|
||||
// ── multiple datasets ─────────────────────────────────────────────────────
|
||||
@@ -1098,11 +1200,23 @@ mod tests {
|
||||
vf.commit(bytes_v2, Some("update x only")).unwrap();
|
||||
|
||||
let f0 = vf.revision(0).unwrap();
|
||||
assert_eq!(f0.dataset("x").unwrap().read_f64().unwrap(), vec![1.0, 2.0, 3.0]);
|
||||
assert_eq!(f0.dataset("y").unwrap().read_f64().unwrap(), vec![4.0, 5.0, 6.0]);
|
||||
assert_eq!(
|
||||
f0.dataset("x").unwrap().read_f64().unwrap(),
|
||||
vec![1.0, 2.0, 3.0]
|
||||
);
|
||||
assert_eq!(
|
||||
f0.dataset("y").unwrap().read_f64().unwrap(),
|
||||
vec![4.0, 5.0, 6.0]
|
||||
);
|
||||
|
||||
let f1 = vf.revision(1).unwrap();
|
||||
assert_eq!(f1.dataset("x").unwrap().read_f64().unwrap(), vec![10.0, 20.0, 30.0]);
|
||||
assert_eq!(f1.dataset("y").unwrap().read_f64().unwrap(), vec![4.0, 5.0, 6.0]);
|
||||
assert_eq!(
|
||||
f1.dataset("x").unwrap().read_f64().unwrap(),
|
||||
vec![10.0, 20.0, 30.0]
|
||||
);
|
||||
assert_eq!(
|
||||
f1.dataset("y").unwrap().read_f64().unwrap(),
|
||||
vec![4.0, 5.0, 6.0]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,8 +18,8 @@ use crate::annotation::AnnotationHeap;
|
||||
use crate::compress::compress_page;
|
||||
use crate::error::OnionError;
|
||||
use crate::format::{
|
||||
BranchEntry, Codec, OnionHeader, PageTableEntry, RevisionEntry,
|
||||
BRANCH_MAIN, DEFAULT_FEATURE_FLAGS, HEADER_SIZE, NO_PARENT,
|
||||
BRANCH_MAIN, BranchEntry, Codec, DEFAULT_FEATURE_FLAGS, HEADER_SIZE, NO_PARENT, OnionHeader,
|
||||
PageTableEntry, RevisionEntry,
|
||||
};
|
||||
use crate::index::RevisionIndex;
|
||||
use crate::provenance::{SessionId, hash_pages};
|
||||
@@ -204,9 +204,7 @@ impl OnionFile {
|
||||
annotation: Option<&str>,
|
||||
) -> Result<u64, OnionError> {
|
||||
let revision = self.index.len() as u64;
|
||||
let parent_rev = self
|
||||
.branch_head_rev(session.branch_id)
|
||||
.unwrap_or(NO_PARENT);
|
||||
let parent_rev = self.branch_head_rev(session.branch_id).unwrap_or(NO_PARENT);
|
||||
|
||||
// Compress pages and build BLAKE3 input
|
||||
let mut pages: Vec<(u64, Vec<u8>, u32, Codec)> = Vec::new();
|
||||
@@ -226,9 +224,7 @@ impl OnionFile {
|
||||
);
|
||||
|
||||
// Annotation
|
||||
let annotation_off = annotation
|
||||
.map(|a| self.annotations.push(a))
|
||||
.unwrap_or(0);
|
||||
let annotation_off = annotation.map(|a| self.annotations.push(a)).unwrap_or(0);
|
||||
|
||||
// Append page table entries
|
||||
let mut table: Vec<PageTableEntry> = Vec::new();
|
||||
@@ -451,9 +447,9 @@ impl OnionFile {
|
||||
}
|
||||
|
||||
pub fn branch_by_name(&self, name: &str) -> Option<&BranchEntry> {
|
||||
self.branches.iter().find(|b| {
|
||||
self.annotations.get(b.name_off) == Some(name)
|
||||
})
|
||||
self.branches
|
||||
.iter()
|
||||
.find(|b| self.annotations.get(b.name_off) == Some(name))
|
||||
}
|
||||
|
||||
/// Serialise the entire `.onion` file to bytes.
|
||||
@@ -480,8 +476,7 @@ impl OnionFile {
|
||||
let branch_offset = index_offset + (n_revisions * rev_entry_size) as u64;
|
||||
|
||||
// Calculate PageTable section start
|
||||
let page_table_section_start =
|
||||
branch_offset + (n_branches * branch_entry_size) as u64;
|
||||
let page_table_section_start = branch_offset + (n_branches * branch_entry_size) as u64;
|
||||
|
||||
// Calculate per-revision page_table_off values and total page table size
|
||||
let mut page_table_offsets: Vec<u64> = Vec::with_capacity(n_revisions);
|
||||
@@ -599,7 +594,11 @@ impl OnionFile {
|
||||
// 0-4). We extend the Vec with empty entries so that
|
||||
// page_tables[rev_number] is always valid.
|
||||
let max_rev = entries.iter().map(|e| e.revision).max().unwrap_or(0);
|
||||
let table_len = if entries.is_empty() { 0 } else { max_rev as usize + 1 };
|
||||
let table_len = if entries.is_empty() {
|
||||
0
|
||||
} else {
|
||||
max_rev as usize + 1
|
||||
};
|
||||
let mut page_tables: Vec<Vec<PageTableEntry>> = vec![Vec::new(); table_len];
|
||||
for entry in &entries {
|
||||
let pt_start = entry.page_table_off as usize;
|
||||
@@ -792,7 +791,10 @@ mod tests {
|
||||
fn create_accepts_valid_page_sizes() {
|
||||
for &size in &[512u32, 1024, 4096, 65536] {
|
||||
let path = Path::new("/tmp/dummy.h5");
|
||||
assert!(OnionFile::create(path, size).is_ok(), "size {size} rejected");
|
||||
assert!(
|
||||
OnionFile::create(path, size).is_ok(),
|
||||
"size {size} rejected"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -973,7 +975,9 @@ mod tests {
|
||||
.list_revisions()
|
||||
.into_iter()
|
||||
.filter(|r| {
|
||||
onion.index.get(r.revision)
|
||||
onion
|
||||
.index
|
||||
.get(r.revision)
|
||||
.map_or(false, |e| e.flags & crate::format::REV_FLAG_SNAPSHOT != 0)
|
||||
})
|
||||
.collect();
|
||||
@@ -1041,7 +1045,10 @@ mod tests {
|
||||
fn eof_to_page_size_always_power_of_two() {
|
||||
for eof in [1, 100, 1_000, 10_000, 100_000, 1_000_000, 100_000_000u64] {
|
||||
let ps = eof_to_page_size(eof);
|
||||
assert!(ps.is_power_of_two(), "page_size {ps} is not a power of two for eof={eof}");
|
||||
assert!(
|
||||
ps.is_power_of_two(),
|
||||
"page_size {ps} is not a power of two for eof={eof}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -35,8 +35,12 @@ const PD_OFFSET: usize = PT_OFFSET + PAGE_ENTRY_SIZE; // 312
|
||||
// Helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
fn le64(v: u64) -> [u8; 8] { v.to_le_bytes() }
|
||||
fn le32(v: u32) -> [u8; 4] { v.to_le_bytes() }
|
||||
fn le64(v: u64) -> [u8; 8] {
|
||||
v.to_le_bytes()
|
||||
}
|
||||
fn le32(v: u32) -> [u8; 4] {
|
||||
v.to_le_bytes()
|
||||
}
|
||||
|
||||
/// Build the expected BLAKE3 for one 4096-byte page of 0xAB at h5_offset = 0.
|
||||
///
|
||||
@@ -82,7 +86,8 @@ fn golden_minimal_onion_byte_layout() {
|
||||
let ann_heap: &[u8] = &[0x00, 0x04, 0x00, 0x00, 0x00, b'm', b'a', b'i', b'n'];
|
||||
let expected_total = PD_OFFSET + PAGE_SIZE as usize + ann_heap.len(); // 4417
|
||||
assert_eq!(
|
||||
bytes.len(), expected_total,
|
||||
bytes.len(),
|
||||
expected_total,
|
||||
"total file size mismatch (got {}, expected {expected_total})",
|
||||
bytes.len()
|
||||
);
|
||||
@@ -127,10 +132,17 @@ fn golden_minimal_onion_byte_layout() {
|
||||
// [64..72] created_at — f64, time-dependent; skip exact value check,
|
||||
// but assert it's non-zero (a real Unix timestamp).
|
||||
let created_at = f64::from_le_bytes(bytes[64..72].try_into().unwrap());
|
||||
assert!(created_at > 0.0, "created_at should be a positive Unix timestamp");
|
||||
assert!(
|
||||
created_at > 0.0,
|
||||
"created_at should be a positive Unix timestamp"
|
||||
);
|
||||
|
||||
// [72..128] reserved — 56 bytes, must all be zero
|
||||
assert_eq!(&bytes[72..128], &[0u8; 56], "reserved header bytes must be zero");
|
||||
assert_eq!(
|
||||
&bytes[72..128],
|
||||
&[0u8; 56],
|
||||
"reserved header bytes must be zero"
|
||||
);
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════
|
||||
// §3.2 RevisionEntry — 112 bytes @ offset 128
|
||||
@@ -148,7 +160,11 @@ fn golden_minimal_onion_byte_layout() {
|
||||
assert_eq!(&bytes[re + 12..re + 16], &[0u8; 4], "_pad_bi must be zero");
|
||||
|
||||
// [re+16..re+24] parent_rev = u64::MAX (root has no parent)
|
||||
assert_eq!(&bytes[re+16..re+24], &le64(u64::MAX), "parent_rev (NO_PARENT sentinel)");
|
||||
assert_eq!(
|
||||
&bytes[re + 16..re + 24],
|
||||
&le64(u64::MAX),
|
||||
"parent_rev (NO_PARENT sentinel)"
|
||||
);
|
||||
|
||||
// [re+24..re+28] page_count = 1
|
||||
assert_eq!(&bytes[re + 24..re + 28], &le32(1), "page_count");
|
||||
@@ -157,7 +173,11 @@ fn golden_minimal_onion_byte_layout() {
|
||||
assert_eq!(&bytes[re + 28..re + 32], &[0u8; 4], "_pad_pc must be zero");
|
||||
|
||||
// [re+32..re+40] page_table_off = 280 (PT_OFFSET)
|
||||
assert_eq!(&bytes[re+32..re+40], &le64(PT_OFFSET as u64), "page_table_off");
|
||||
assert_eq!(
|
||||
&bytes[re + 32..re + 40],
|
||||
&le64(PT_OFFSET as u64),
|
||||
"page_table_off"
|
||||
);
|
||||
|
||||
// [re+40..re+48] timestamp — f64, time-dependent; assert positive
|
||||
let ts = f64::from_le_bytes(bytes[re + 40..re + 48].try_into().unwrap());
|
||||
@@ -165,19 +185,35 @@ fn golden_minimal_onion_byte_layout() {
|
||||
|
||||
// [re+48..re+80] blake3 — verify against spec §5 algorithm
|
||||
let computed_blake3 = expected_blake3(&page_bytes);
|
||||
assert_eq!(&bytes[re+48..re+80], &computed_blake3, "BLAKE3 hash mismatch");
|
||||
assert_eq!(
|
||||
&bytes[re + 48..re + 80],
|
||||
&computed_blake3,
|
||||
"BLAKE3 hash mismatch"
|
||||
);
|
||||
|
||||
// [re+80..re+96] session_uuid — 16 bytes UUIDv7; must be non-zero
|
||||
assert_ne!(&bytes[re+80..re+96], &[0u8; 16], "session_uuid must not be all zeros");
|
||||
assert_ne!(
|
||||
&bytes[re + 80..re + 96],
|
||||
&[0u8; 16],
|
||||
"session_uuid must not be all zeros"
|
||||
);
|
||||
|
||||
// [re+96..re+104] annotation_off = 0 (no annotation)
|
||||
assert_eq!(&bytes[re+96..re+104], &le64(0), "annotation_off must be 0 (no annotation)");
|
||||
assert_eq!(
|
||||
&bytes[re + 96..re + 104],
|
||||
&le64(0),
|
||||
"annotation_off must be 0 (no annotation)"
|
||||
);
|
||||
|
||||
// [re+104] flags = 0 (not a snapshot)
|
||||
assert_eq!(bytes[re + 104], 0x00, "revision flags");
|
||||
|
||||
// [re+105..re+112] _pad_flags — must be zero
|
||||
assert_eq!(&bytes[re+105..re+112], &[0u8; 7], "_pad_flags must be zero");
|
||||
assert_eq!(
|
||||
&bytes[re + 105..re + 112],
|
||||
&[0u8; 7],
|
||||
"_pad_flags must be zero"
|
||||
);
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════
|
||||
// §3.3 BranchEntry — 40 bytes @ offset 240
|
||||
@@ -192,13 +228,25 @@ fn golden_minimal_onion_byte_layout() {
|
||||
assert_eq!(&bytes[be + 4..be + 8], &[0u8; 4], "_pad_id must be zero");
|
||||
|
||||
// [be+8..be+16] name_off = 1 (heap-relative; first string after reserved null)
|
||||
assert_eq!(&bytes[be+8..be+16], &le64(1), "name_off for main branch");
|
||||
assert_eq!(
|
||||
&bytes[be + 8..be + 16],
|
||||
&le64(1),
|
||||
"name_off for main branch"
|
||||
);
|
||||
|
||||
// [be+16..be+24] head_rev = 0 (updated after commit)
|
||||
assert_eq!(&bytes[be+16..be+24], &le64(0), "head_rev after first commit");
|
||||
assert_eq!(
|
||||
&bytes[be + 16..be + 24],
|
||||
&le64(0),
|
||||
"head_rev after first commit"
|
||||
);
|
||||
|
||||
// [be+24..be+32] fork_rev = u64::MAX (main is not forked)
|
||||
assert_eq!(&bytes[be+24..be+32], &le64(u64::MAX), "fork_rev (NO_PARENT for main)");
|
||||
assert_eq!(
|
||||
&bytes[be + 24..be + 32],
|
||||
&le64(u64::MAX),
|
||||
"fork_rev (NO_PARENT for main)"
|
||||
);
|
||||
|
||||
// [be+32..be+40] created_at — f64, time-dependent; assert positive
|
||||
let branch_ts = f64::from_le_bytes(bytes[be + 32..be + 40].try_into().unwrap());
|
||||
@@ -214,13 +262,21 @@ fn golden_minimal_onion_byte_layout() {
|
||||
assert_eq!(&bytes[pt..pt + 8], &le64(0), "h5_offset");
|
||||
|
||||
// [pt+8..pt+16] data_offset = 312 (PD_OFFSET, absolute file position)
|
||||
assert_eq!(&bytes[pt+8..pt+16], &le64(PD_OFFSET as u64), "data_offset (absolute)");
|
||||
assert_eq!(
|
||||
&bytes[pt + 8..pt + 16],
|
||||
&le64(PD_OFFSET as u64),
|
||||
"data_offset (absolute)"
|
||||
);
|
||||
|
||||
// [pt+16..pt+20] orig_size = 4096
|
||||
assert_eq!(&bytes[pt + 16..pt + 20], &le32(4096), "orig_size");
|
||||
|
||||
// [pt+20..pt+24] data_size = 4096 (codec=None → no compression)
|
||||
assert_eq!(&bytes[pt+20..pt+24], &le32(4096), "data_size (codec=None, uncompressed)");
|
||||
assert_eq!(
|
||||
&bytes[pt + 20..pt + 24],
|
||||
&le32(4096),
|
||||
"data_size (codec=None, uncompressed)"
|
||||
);
|
||||
|
||||
// [pt+24] codec = 0 (None)
|
||||
assert_eq!(bytes[pt + 24], 0x00, "codec = None (0)");
|
||||
@@ -232,15 +288,22 @@ fn golden_minimal_onion_byte_layout() {
|
||||
// PageData — 4096 bytes @ offset 312
|
||||
// ══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
assert_eq!(&bytes[PD_OFFSET..PD_OFFSET + 4096], page_bytes.as_slice(),
|
||||
"page data must equal the uncompressed original bytes");
|
||||
assert_eq!(
|
||||
&bytes[PD_OFFSET..PD_OFFSET + 4096],
|
||||
page_bytes.as_slice(),
|
||||
"page data must equal the uncompressed original bytes"
|
||||
);
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════
|
||||
// §3.5 AnnotationHeap — tail of file
|
||||
// ══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
// Last 9 bytes: [0x00 reserved] [u32 LE length=4] ["main"]
|
||||
assert_eq!(&bytes[bytes.len()-9..], ann_heap, "annotation heap content");
|
||||
assert_eq!(
|
||||
&bytes[bytes.len() - 9..],
|
||||
ann_heap,
|
||||
"annotation heap content"
|
||||
);
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════
|
||||
// §7 Reconstruction — verify round-trip
|
||||
@@ -360,9 +423,13 @@ fn byte_exact_format_freeze() {
|
||||
actual[be + 32..be + 40].fill(0);
|
||||
|
||||
// ── Compare byte-for-byte ────────────────────────────────────────────────
|
||||
assert_eq!(actual.len(), expected.len(),
|
||||
assert_eq!(
|
||||
actual.len(),
|
||||
expected.len(),
|
||||
"total file size mismatch: actual={} expected={}",
|
||||
actual.len(), expected.len());
|
||||
actual.len(),
|
||||
expected.len()
|
||||
);
|
||||
|
||||
// Find the first differing byte for a useful failure message.
|
||||
if actual != expected {
|
||||
@@ -387,13 +454,29 @@ fn byte_exact_format_freeze() {
|
||||
|
||||
#[test]
|
||||
fn struct_sizes_match_spec() {
|
||||
use clawhdf5_onion::format::{BranchEntry, OnionHeader, PageTableEntry, RevisionEntry};
|
||||
use std::mem::size_of;
|
||||
use clawhdf5_onion::format::{OnionHeader, RevisionEntry, BranchEntry, PageTableEntry};
|
||||
|
||||
assert_eq!(size_of::<OnionHeader>(), 128, "OnionHeader must be 128 bytes (§3.1)");
|
||||
assert_eq!(size_of::<RevisionEntry>(), 112, "RevisionEntry must be 112 bytes (§3.2)");
|
||||
assert_eq!(size_of::<BranchEntry>(), 40, "BranchEntry must be 40 bytes (§3.3)");
|
||||
assert_eq!(size_of::<PageTableEntry>(), 32, "PageTableEntry must be 32 bytes (§3.4)");
|
||||
assert_eq!(
|
||||
size_of::<OnionHeader>(),
|
||||
128,
|
||||
"OnionHeader must be 128 bytes (§3.1)"
|
||||
);
|
||||
assert_eq!(
|
||||
size_of::<RevisionEntry>(),
|
||||
112,
|
||||
"RevisionEntry must be 112 bytes (§3.2)"
|
||||
);
|
||||
assert_eq!(
|
||||
size_of::<BranchEntry>(),
|
||||
40,
|
||||
"BranchEntry must be 40 bytes (§3.3)"
|
||||
);
|
||||
assert_eq!(
|
||||
size_of::<PageTableEntry>(),
|
||||
32,
|
||||
"PageTableEntry must be 32 bytes (§3.4)"
|
||||
);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -258,8 +258,13 @@ fn assert_gc_invariants(onion: &OnionFile) {
|
||||
|
||||
// 5. blake3_hex has correct length (64 chars)
|
||||
for rev in &revisions {
|
||||
assert_eq!(rev.blake3_hex.len(), 64,
|
||||
"revision {} has blake3_hex of wrong length {}", rev.revision, rev.blake3_hex.len());
|
||||
assert_eq!(
|
||||
rev.blake3_hex.len(),
|
||||
64,
|
||||
"revision {} has blake3_hex of wrong length {}",
|
||||
rev.revision,
|
||||
rev.blake3_hex.len()
|
||||
);
|
||||
}
|
||||
|
||||
// 6. Pages are accessible for every revision
|
||||
|
||||
@@ -48,7 +48,11 @@ struct Round {
|
||||
|
||||
fn arb_round() -> impl Strategy<Value = Round> {
|
||||
(any::<bool>(), 0usize..8usize, 1u8..=255u8).prop_map(|(create_branch, branch_idx, fill)| {
|
||||
Round { create_branch, branch_idx, fill }
|
||||
Round {
|
||||
create_branch,
|
||||
branch_idx,
|
||||
fill,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -72,10 +76,7 @@ fn make_vf() -> (TempDir, std::path::PathBuf, VersionedFile) {
|
||||
/// Apply `rounds` to `vf`, returning a Vec of `(revision_number, expected_page_bytes)`.
|
||||
///
|
||||
/// `branches` starts as `["main"]` and grows as new branches are created.
|
||||
fn apply_rounds(
|
||||
vf: &mut VersionedFile,
|
||||
rounds: &[Round],
|
||||
) -> Vec<(u64, Vec<u8>)> {
|
||||
fn apply_rounds(vf: &mut VersionedFile, rounds: &[Round]) -> Vec<(u64, Vec<u8>)> {
|
||||
let mut branches: Vec<String> = vec!["main".to_string()];
|
||||
let mut expectations: Vec<(u64, Vec<u8>)> = Vec::new();
|
||||
let mut branch_counter: usize = 0;
|
||||
@@ -124,7 +125,8 @@ fn verify_expectations(
|
||||
prop_assert_eq!(
|
||||
&actual[..page_end],
|
||||
expected.as_slice(),
|
||||
"revision {}: page 0 mismatch", rev
|
||||
"revision {}: page 0 mismatch",
|
||||
rev
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
|
||||
@@ -78,17 +78,25 @@ async fn serve_one(server: TcpServer, server_onion_path: PathBuf) {
|
||||
let server_onion = OnionFile::open(&server_onion_path).unwrap();
|
||||
let server_rev_count = server_onion.revision_count();
|
||||
let manifest = ClawSyncManifest::from_onion("bench-server", &server_onion, H5_MAGIC);
|
||||
conn.send(&SyncMessage::ManifestResponse { manifest }).await.unwrap();
|
||||
conn.send(&SyncMessage::ManifestResponse { manifest })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
if server_rev_count > client_rev_count {
|
||||
// Server is ahead — push its extra revisions to client
|
||||
let client_head = if client_rev_count == 0 { NO_PARENT } else { client_rev_count - 1 };
|
||||
let client_head = if client_rev_count == 0 {
|
||||
NO_PARENT
|
||||
} else {
|
||||
client_rev_count - 1
|
||||
};
|
||||
let packets = diff_revisions(&server_onion, client_head).unwrap();
|
||||
let total = packets.len() as u64;
|
||||
let mut bytes_sent = 0u64;
|
||||
for packet in packets {
|
||||
bytes_sent += packet.page_data_size() as u64;
|
||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
||||
conn.send(&SyncMessage::LayerPacket { packet })
|
||||
.await
|
||||
.unwrap();
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::Ack { .. } => {}
|
||||
other => panic!("server: expected Ack, got {other:?}"),
|
||||
@@ -97,7 +105,9 @@ async fn serve_one(server: TcpServer, server_onion_path: PathBuf) {
|
||||
conn.send(&SyncMessage::SyncComplete {
|
||||
revisions_transferred: total,
|
||||
bytes_transferred: bytes_sent,
|
||||
}).await.unwrap();
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
} else {
|
||||
// Server is behind or equal — receive packets from client
|
||||
let mut server_onion_rw = OnionFile::open(&server_onion_path).unwrap();
|
||||
@@ -106,7 +116,9 @@ async fn serve_one(server: TcpServer, server_onion_path: PathBuf) {
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::LayerPacket { packet } => {
|
||||
let rev = packet.revision;
|
||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
||||
conn.send(&SyncMessage::Ack { revision: rev })
|
||||
.await
|
||||
.unwrap();
|
||||
packets.push(packet);
|
||||
}
|
||||
SyncMessage::SyncComplete { .. } => {
|
||||
@@ -157,7 +169,10 @@ fn bench_e2e(c: &mut Criterion) {
|
||||
rt.block_on(async {
|
||||
let server_task = tokio::spawn(serve_one(server, dst_h5));
|
||||
let backend = TcpSyncBackend::new(addr, "bench-agent");
|
||||
backend.push(Path::new(&src_h5), &SyncSelector::All).await.unwrap();
|
||||
backend
|
||||
.push(Path::new(&src_h5), &SyncSelector::All)
|
||||
.await
|
||||
.unwrap();
|
||||
server_task.await.unwrap();
|
||||
});
|
||||
drop(src_dir);
|
||||
@@ -275,7 +290,10 @@ fn bench_e2e(c: &mut Criterion) {
|
||||
let addr = server.local_addr;
|
||||
let server_task = tokio::spawn(serve_one(server, dst_h5));
|
||||
let backend = TcpSyncBackend::new(addr, "bench-agent");
|
||||
let stats = backend.push(Path::new(&src_h5), &SyncSelector::All).await.unwrap();
|
||||
let stats = backend
|
||||
.push(Path::new(&src_h5), &SyncSelector::All)
|
||||
.await
|
||||
.unwrap();
|
||||
server_task.await.unwrap();
|
||||
stats.revisions_transferred
|
||||
});
|
||||
|
||||
@@ -98,7 +98,9 @@ async fn serve_with_latency(server: TcpServer, rtt_ms: f64) {
|
||||
let rev = packet.revision;
|
||||
// Inject simulated WAN RTT before sending Ack.
|
||||
tokio::time::sleep(delay).await;
|
||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
||||
conn.send(&SyncMessage::Ack { revision: rev })
|
||||
.await
|
||||
.unwrap();
|
||||
received += 1;
|
||||
}
|
||||
SyncMessage::SyncComplete { .. } => break,
|
||||
@@ -223,8 +225,7 @@ fn bench_wan_pipeline(c: &mut Criterion) {
|
||||
// TIMED: full push with simulated WAN latency
|
||||
let _ = (dir, path); // keep TempDir alive
|
||||
rt.block_on(async {
|
||||
let server_task =
|
||||
tokio::spawn(serve_with_latency(server, rtt_ms));
|
||||
let server_task = tokio::spawn(serve_with_latency(server, rtt_ms));
|
||||
pipelined_push(addr, &onion, window).await;
|
||||
server_task.await.unwrap();
|
||||
});
|
||||
|
||||
@@ -42,7 +42,11 @@ pub struct PeerCapabilities {
|
||||
|
||||
impl Default for PeerCapabilities {
|
||||
fn default() -> Self {
|
||||
Self { branching: true, dataset_diff: false, protocol_version: 1 }
|
||||
Self {
|
||||
branching: true,
|
||||
dataset_diff: false,
|
||||
protocol_version: 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,10 +58,17 @@ impl Default for PeerCapabilities {
|
||||
/// transport (e.g., QUIC, S3, local copy).
|
||||
pub trait SyncBackend: Send + Sync {
|
||||
/// Push any revisions the remote lacks from `local_path` to `remote`.
|
||||
fn push(&self, local_path: &Path, selector: &SyncSelector) -> impl std::future::Future<Output = Result<SyncStats, AgentSyncError>> + Send;
|
||||
fn push(
|
||||
&self,
|
||||
local_path: &Path,
|
||||
selector: &SyncSelector,
|
||||
) -> impl std::future::Future<Output = Result<SyncStats, AgentSyncError>> + Send;
|
||||
|
||||
/// Pull any revisions the local file lacks from the remote into `local_path`.
|
||||
fn pull(&self, local_path: &Path) -> impl std::future::Future<Output = Result<SyncStats, AgentSyncError>> + Send;
|
||||
fn pull(
|
||||
&self,
|
||||
local_path: &Path,
|
||||
) -> impl std::future::Future<Output = Result<SyncStats, AgentSyncError>> + Send;
|
||||
|
||||
/// Build a manifest for the given path.
|
||||
fn local_manifest(&self, local_path: &Path) -> Result<ClawSyncManifest, AgentSyncError>;
|
||||
@@ -75,14 +86,20 @@ pub struct TcpSyncBackend {
|
||||
|
||||
impl TcpSyncBackend {
|
||||
pub fn new(remote_addr: SocketAddr, agent_id: impl Into<String>) -> Self {
|
||||
Self { remote_addr, agent_id: agent_id.into() }
|
||||
Self {
|
||||
remote_addr,
|
||||
agent_id: agent_id.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SyncBackend for TcpSyncBackend {
|
||||
async fn push(&self, local_path: &Path, selector: &SyncSelector) -> Result<SyncStats, AgentSyncError> {
|
||||
let local_onion = OnionFile::open(local_path)
|
||||
.map_err(AgentSyncError::Onion)?;
|
||||
async fn push(
|
||||
&self,
|
||||
local_path: &Path,
|
||||
selector: &SyncSelector,
|
||||
) -> Result<SyncStats, AgentSyncError> {
|
||||
let local_onion = OnionFile::open(local_path).map_err(AgentSyncError::Onion)?;
|
||||
let h5_base = std::fs::read(local_path).map_err(AgentSyncError::Io)?;
|
||||
let local_manifest = ClawSyncManifest::from_onion(&self.agent_id, &local_onion, &h5_base);
|
||||
|
||||
@@ -107,12 +124,17 @@ impl SyncBackend for TcpSyncBackend {
|
||||
sketch_cells: sketch.cell_count() as u32,
|
||||
sketch: sketch.to_bytes(),
|
||||
};
|
||||
conn.send(&SyncMessage::IbltRequest { sketch: iblt_manifest })
|
||||
conn.send(&SyncMessage::IbltRequest {
|
||||
sketch: iblt_manifest,
|
||||
})
|
||||
.await
|
||||
.map_err(AgentSyncError::Transport)?;
|
||||
|
||||
let missing_from_remote = match conn.recv().await.map_err(AgentSyncError::Transport)? {
|
||||
SyncMessage::IbltResponse { missing_from_remote, .. } => missing_from_remote,
|
||||
SyncMessage::IbltResponse {
|
||||
missing_from_remote,
|
||||
..
|
||||
} => missing_from_remote,
|
||||
SyncMessage::Error { message } => return Err(AgentSyncError::Remote(message)),
|
||||
other => return Err(AgentSyncError::Protocol(format!("unexpected: {other:?}"))),
|
||||
};
|
||||
@@ -127,9 +149,13 @@ impl SyncBackend for TcpSyncBackend {
|
||||
|
||||
for packet in packets {
|
||||
let sz = packet.page_data_size() as u64;
|
||||
conn.send(&SyncMessage::LayerPacket { packet }).await.map_err(AgentSyncError::Transport)?;
|
||||
conn.send(&SyncMessage::LayerPacket { packet })
|
||||
.await
|
||||
.map_err(AgentSyncError::Transport)?;
|
||||
match conn.recv().await.map_err(AgentSyncError::Transport)? {
|
||||
SyncMessage::Ack { .. } => { bytes_sent += sz; }
|
||||
SyncMessage::Ack { .. } => {
|
||||
bytes_sent += sz;
|
||||
}
|
||||
SyncMessage::Error { message } => return Err(AgentSyncError::Remote(message)),
|
||||
other => return Err(AgentSyncError::Protocol(format!("unexpected: {other:?}"))),
|
||||
}
|
||||
@@ -138,10 +164,16 @@ impl SyncBackend for TcpSyncBackend {
|
||||
conn.send(&SyncMessage::SyncComplete {
|
||||
revisions_transferred: total,
|
||||
bytes_transferred: bytes_sent,
|
||||
}).await.map_err(AgentSyncError::Transport)?;
|
||||
})
|
||||
.await
|
||||
.map_err(AgentSyncError::Transport)?;
|
||||
conn.shutdown().await.map_err(AgentSyncError::Transport)?;
|
||||
|
||||
Ok(SyncStats { revisions_transferred: total, bytes_transferred: bytes_sent, revisions_skipped: 0 })
|
||||
Ok(SyncStats {
|
||||
revisions_transferred: total,
|
||||
bytes_transferred: bytes_sent,
|
||||
revisions_skipped: 0,
|
||||
})
|
||||
}
|
||||
|
||||
async fn pull(&self, local_path: &Path) -> Result<SyncStats, AgentSyncError> {
|
||||
@@ -157,7 +189,9 @@ impl SyncBackend for TcpSyncBackend {
|
||||
agent_id: self.agent_id.clone(),
|
||||
head_revision: local_rev_count.saturating_sub(1),
|
||||
revision_count: local_rev_count,
|
||||
}).await.map_err(AgentSyncError::Transport)?;
|
||||
})
|
||||
.await
|
||||
.map_err(AgentSyncError::Transport)?;
|
||||
|
||||
// Get the server's revision count to detect "already in sync".
|
||||
let server_rev_count = match conn.recv().await.map_err(AgentSyncError::Transport)? {
|
||||
@@ -171,12 +205,15 @@ impl SyncBackend for TcpSyncBackend {
|
||||
conn.send(&SyncMessage::SyncComplete {
|
||||
revisions_transferred: 0,
|
||||
bytes_transferred: 0,
|
||||
}).await.map_err(AgentSyncError::Transport)?;
|
||||
})
|
||||
.await
|
||||
.map_err(AgentSyncError::Transport)?;
|
||||
return Ok(SyncStats::default());
|
||||
}
|
||||
|
||||
let mut local_onion = if local_path.exists() {
|
||||
OnionFile::open(local_path).or_else(|_| OnionFile::create_auto(local_path))
|
||||
OnionFile::open(local_path)
|
||||
.or_else(|_| OnionFile::create_auto(local_path))
|
||||
.map_err(AgentSyncError::Onion)?
|
||||
} else {
|
||||
std::fs::write(local_path, b"\x89HDF\r\n\x1a\n").map_err(AgentSyncError::Io)?;
|
||||
@@ -189,10 +226,15 @@ impl SyncBackend for TcpSyncBackend {
|
||||
match conn.recv().await.map_err(AgentSyncError::Transport)? {
|
||||
SyncMessage::LayerPacket { packet } => {
|
||||
let rev = packet.revision;
|
||||
conn.send(&SyncMessage::Ack { revision: rev }).await.map_err(AgentSyncError::Transport)?;
|
||||
conn.send(&SyncMessage::Ack { revision: rev })
|
||||
.await
|
||||
.map_err(AgentSyncError::Transport)?;
|
||||
packets.push(packet);
|
||||
}
|
||||
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => {
|
||||
SyncMessage::SyncComplete {
|
||||
revisions_transferred,
|
||||
bytes_transferred,
|
||||
} => {
|
||||
let stats = merge_packets(&mut local_onion, packets, true)
|
||||
.map_err(AgentSyncError::SyncOnion)?;
|
||||
return Ok(SyncStats {
|
||||
@@ -210,7 +252,11 @@ impl SyncBackend for TcpSyncBackend {
|
||||
fn local_manifest(&self, local_path: &Path) -> Result<ClawSyncManifest, AgentSyncError> {
|
||||
let onion = OnionFile::open(local_path).map_err(AgentSyncError::Onion)?;
|
||||
let h5_base = std::fs::read(local_path).map_err(AgentSyncError::Io)?;
|
||||
Ok(ClawSyncManifest::from_onion(&self.agent_id, &onion, &h5_base))
|
||||
Ok(ClawSyncManifest::from_onion(
|
||||
&self.agent_id,
|
||||
&onion,
|
||||
&h5_base,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -77,8 +77,12 @@ impl PeerCapabilities {
|
||||
let mut out = [0u8; 8];
|
||||
out[..4].copy_from_slice(&self.protocol_version.to_le_bytes());
|
||||
let mut flags: u32 = 0;
|
||||
if self.branching { flags |= 1 << 0; }
|
||||
if self.dataset_diff { flags |= 1 << 1; }
|
||||
if self.branching {
|
||||
flags |= 1 << 0;
|
||||
}
|
||||
if self.dataset_diff {
|
||||
flags |= 1 << 1;
|
||||
}
|
||||
out[4..8].copy_from_slice(&flags.to_le_bytes());
|
||||
out
|
||||
}
|
||||
@@ -105,7 +109,11 @@ mod tests {
|
||||
use crate::backend::PeerCapabilities;
|
||||
|
||||
fn caps(branching: bool, dataset_diff: bool, version: u32) -> PeerCapabilities {
|
||||
PeerCapabilities { branching, dataset_diff, protocol_version: version }
|
||||
PeerCapabilities {
|
||||
branching,
|
||||
dataset_diff,
|
||||
protocol_version: version,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -140,27 +148,43 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn assert_compatible_ok() {
|
||||
let s = SessionCapabilities { branching: true, dataset_diff: false, protocol_version: 1 };
|
||||
let s = SessionCapabilities {
|
||||
branching: true,
|
||||
dataset_diff: false,
|
||||
protocol_version: 1,
|
||||
};
|
||||
assert!(assert_compatible(&s, true, false, 1).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assert_compatible_missing_branching() {
|
||||
let s = SessionCapabilities { branching: false, dataset_diff: false, protocol_version: 1 };
|
||||
let s = SessionCapabilities {
|
||||
branching: false,
|
||||
dataset_diff: false,
|
||||
protocol_version: 1,
|
||||
};
|
||||
let err = assert_compatible(&s, true, false, 1).unwrap_err();
|
||||
assert!(err.contains("branching"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assert_compatible_version_too_low() {
|
||||
let s = SessionCapabilities { branching: true, dataset_diff: false, protocol_version: 1 };
|
||||
let s = SessionCapabilities {
|
||||
branching: true,
|
||||
dataset_diff: false,
|
||||
protocol_version: 1,
|
||||
};
|
||||
let err = assert_compatible(&s, false, false, 2).unwrap_err();
|
||||
assert!(err.contains("protocol_version"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capabilities_wire_roundtrip() {
|
||||
let orig = PeerCapabilities { branching: true, dataset_diff: true, protocol_version: 3 };
|
||||
let orig = PeerCapabilities {
|
||||
branching: true,
|
||||
dataset_diff: true,
|
||||
protocol_version: 3,
|
||||
};
|
||||
let bytes = orig.to_bytes();
|
||||
let decoded = PeerCapabilities::from_bytes(&bytes);
|
||||
assert_eq!(decoded.branching, orig.branching);
|
||||
@@ -170,7 +194,11 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn capabilities_wire_no_features() {
|
||||
let orig = PeerCapabilities { branching: false, dataset_diff: false, protocol_version: 0 };
|
||||
let orig = PeerCapabilities {
|
||||
branching: false,
|
||||
dataset_diff: false,
|
||||
protocol_version: 0,
|
||||
};
|
||||
let bytes = orig.to_bytes();
|
||||
let decoded = PeerCapabilities::from_bytes(&bytes);
|
||||
assert!(!decoded.branching);
|
||||
|
||||
@@ -47,16 +47,28 @@ impl OnionMemory {
|
||||
/// Create a new `OnionMemory` at `h5_path`.
|
||||
pub fn create(config: MemoryConfig) -> Result<Self, AgentSyncError> {
|
||||
let h5_path = config.path.clone();
|
||||
let inner = HDF5Memory::create(config).map_err(|e| AgentSyncError::AgentMemory(e.to_string()))?;
|
||||
let inner =
|
||||
HDF5Memory::create(config).map_err(|e| AgentSyncError::AgentMemory(e.to_string()))?;
|
||||
let onion = OnionFile::create(&h5_path, 4096).map_err(AgentSyncError::Onion)?;
|
||||
Ok(Self { inner, onion, h5_path, auto_checkpoint: true })
|
||||
Ok(Self {
|
||||
inner,
|
||||
onion,
|
||||
h5_path,
|
||||
auto_checkpoint: true,
|
||||
})
|
||||
}
|
||||
|
||||
/// Open an existing `OnionMemory` from `h5_path`.
|
||||
pub fn open(h5_path: &Path) -> Result<Self, AgentSyncError> {
|
||||
let inner = HDF5Memory::open(h5_path).map_err(|e| AgentSyncError::AgentMemory(e.to_string()))?;
|
||||
let inner =
|
||||
HDF5Memory::open(h5_path).map_err(|e| AgentSyncError::AgentMemory(e.to_string()))?;
|
||||
let onion = OnionFile::open(h5_path).map_err(AgentSyncError::Onion)?;
|
||||
Ok(Self { inner, onion, h5_path: h5_path.to_owned(), auto_checkpoint: true })
|
||||
Ok(Self {
|
||||
inner,
|
||||
onion,
|
||||
h5_path: h5_path.to_owned(),
|
||||
auto_checkpoint: true,
|
||||
})
|
||||
}
|
||||
|
||||
/// Commit a new onion revision of the current HDF5 state.
|
||||
@@ -71,14 +83,20 @@ impl OnionMemory {
|
||||
let page_size = self.onion.page_size() as usize;
|
||||
|
||||
// Record all pages from the HDF5 file
|
||||
let mut session = self.onion.begin_session(None).map_err(AgentSyncError::Onion)?;
|
||||
let mut session = self
|
||||
.onion
|
||||
.begin_session(None)
|
||||
.map_err(AgentSyncError::Onion)?;
|
||||
for (chunk_idx, chunk) in h5_bytes.chunks(page_size).enumerate() {
|
||||
let offset = (chunk_idx * page_size) as u64;
|
||||
let mut page = vec![0u8; page_size];
|
||||
page[..chunk.len()].copy_from_slice(chunk);
|
||||
session.record_page(offset, &page);
|
||||
}
|
||||
let rev = self.onion.commit_session(session, annotation).map_err(AgentSyncError::Onion)?;
|
||||
let rev = self
|
||||
.onion
|
||||
.commit_session(session, annotation)
|
||||
.map_err(AgentSyncError::Onion)?;
|
||||
self.onion.flush().map_err(AgentSyncError::Onion)?;
|
||||
Ok(rev)
|
||||
}
|
||||
@@ -89,12 +107,11 @@ impl OnionMemory {
|
||||
/// current file, opens an `HDF5Memory` from it, then removes the temp
|
||||
/// file. Returns the restored memory (detached from the onion sidecar).
|
||||
pub fn restore_revision(&self, revision: u64) -> Result<HDF5Memory, AgentSyncError> {
|
||||
let (h5_bytes, _) = open_revision(&self.h5_path, revision)
|
||||
.map_err(AgentSyncError::Onion)?;
|
||||
let (h5_bytes, _) =
|
||||
open_revision(&self.h5_path, revision).map_err(AgentSyncError::Onion)?;
|
||||
let tmp = self.h5_path.with_extension(format!("h5.rev{revision}.tmp"));
|
||||
std::fs::write(&tmp, &h5_bytes).map_err(AgentSyncError::Io)?;
|
||||
let result = HDF5Memory::open(&tmp)
|
||||
.map_err(|e| AgentSyncError::AgentMemory(e.to_string()));
|
||||
let result = HDF5Memory::open(&tmp).map_err(|e| AgentSyncError::AgentMemory(e.to_string()));
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
result
|
||||
}
|
||||
@@ -353,7 +370,11 @@ mod tests {
|
||||
let head = mem.checkpoint(Some("three")).unwrap();
|
||||
|
||||
let restored = mem.restore_revision(head).unwrap();
|
||||
assert_eq!(restored.count(), mem.count(), "restoring HEAD gives same count");
|
||||
assert_eq!(
|
||||
restored.count(),
|
||||
mem.count(),
|
||||
"restoring HEAD gives same count"
|
||||
);
|
||||
}
|
||||
|
||||
/// Multiple checkpoints can each be individually restored.
|
||||
@@ -373,8 +394,10 @@ mod tests {
|
||||
for i in 0..5u64 {
|
||||
let r = mem.restore_revision(i).unwrap();
|
||||
assert_eq!(
|
||||
r.count() as u64, i + 1,
|
||||
"rev {i} should have {} entries", i + 1
|
||||
r.count() as u64,
|
||||
i + 1,
|
||||
"rev {i} should have {} entries",
|
||||
i + 1
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -387,7 +410,10 @@ mod tests {
|
||||
mem.checkpoint(None).unwrap(); // only rev 0 exists
|
||||
|
||||
let result = mem.restore_revision(99);
|
||||
assert!(result.is_err(), "restoring a non-existent revision must error");
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"restoring a non-existent revision must error"
|
||||
);
|
||||
}
|
||||
|
||||
/// `save_batch` with auto_checkpoint creates exactly one snapshot per call.
|
||||
@@ -417,7 +443,8 @@ mod tests {
|
||||
mem.checkpoint(Some("original")).unwrap();
|
||||
|
||||
// Rev 1: replace with entirely different text
|
||||
mem.save(sample_entry("completely-different-content")).unwrap();
|
||||
mem.save(sample_entry("completely-different-content"))
|
||||
.unwrap();
|
||||
mem.checkpoint(Some("updated")).unwrap();
|
||||
assert_eq!(mem.count(), 2);
|
||||
|
||||
|
||||
@@ -96,7 +96,11 @@ pub fn tcp_scheduler(
|
||||
let addr = remote
|
||||
.parse()
|
||||
.map_err(|_| AgentSyncError::Protocol(format!("invalid address: {remote}")))?;
|
||||
Ok(SyncScheduler::new(TcpSyncBackend::new(addr, agent_id), h5_path, selector))
|
||||
Ok(SyncScheduler::new(
|
||||
TcpSyncBackend::new(addr, agent_id),
|
||||
h5_path,
|
||||
selector,
|
||||
))
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -134,9 +138,19 @@ mod tests {
|
||||
}
|
||||
|
||||
impl SyncBackend for StubBackend {
|
||||
async fn push(&self, path: &Path, _sel: &SyncSelector) -> Result<SyncStats, AgentSyncError> {
|
||||
self.push_calls.lock().unwrap().push(path.to_string_lossy().to_string());
|
||||
Ok(SyncStats { revisions_transferred: 1, ..Default::default() })
|
||||
async fn push(
|
||||
&self,
|
||||
path: &Path,
|
||||
_sel: &SyncSelector,
|
||||
) -> Result<SyncStats, AgentSyncError> {
|
||||
self.push_calls
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(path.to_string_lossy().to_string());
|
||||
Ok(SyncStats {
|
||||
revisions_transferred: 1,
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
async fn pull(&self, _path: &Path) -> Result<SyncStats, AgentSyncError> {
|
||||
Ok(SyncStats::default())
|
||||
@@ -189,7 +203,11 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn push_force_always_pushes() {
|
||||
let backend = StubBackend::new(2);
|
||||
let sched = SyncScheduler::new(backend.clone(), PathBuf::from("/tmp/x.h5"), SyncSelector::All);
|
||||
let sched = SyncScheduler::new(
|
||||
backend.clone(),
|
||||
PathBuf::from("/tmp/x.h5"),
|
||||
SyncSelector::All,
|
||||
);
|
||||
sched.mark_pushed(1); // up-to-date
|
||||
|
||||
// force push anyway
|
||||
|
||||
@@ -48,7 +48,9 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
||||
|
||||
match conn.recv().await.unwrap() {
|
||||
// ── IBLT pre-flight (push direction) ─────────────────────────────
|
||||
SyncMessage::IbltRequest { sketch: client_iblt } => {
|
||||
SyncMessage::IbltRequest {
|
||||
sketch: client_iblt,
|
||||
} => {
|
||||
let server_revs: Vec<u64> = dst.list_revisions().iter().map(|s| s.revision).collect();
|
||||
|
||||
// Decode diff: what client has that server lacks
|
||||
@@ -71,14 +73,18 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
||||
conn.send(&SyncMessage::IbltResponse {
|
||||
sketch: server_iblt,
|
||||
missing_from_remote: diff.only_in_b.clone(), // client has, server lacks
|
||||
}).await.unwrap();
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut packets = Vec::new();
|
||||
loop {
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::LayerPacket { packet } => {
|
||||
let rev = packet.revision;
|
||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
||||
conn.send(&SyncMessage::Ack { revision: rev })
|
||||
.await
|
||||
.unwrap();
|
||||
packets.push(packet);
|
||||
}
|
||||
SyncMessage::SyncComplete { .. } => break,
|
||||
@@ -91,19 +97,30 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
||||
}
|
||||
|
||||
// ── Flat manifest (pull direction) ────────────────────────────────
|
||||
SyncMessage::ManifestRequest { revision_count: remote_rev_count, .. } => {
|
||||
SyncMessage::ManifestRequest {
|
||||
revision_count: remote_rev_count,
|
||||
..
|
||||
} => {
|
||||
let local_rev_count = dst.revision_count();
|
||||
let manifest = ClawSyncManifest::from_onion("server", &dst, H5_MAGIC);
|
||||
conn.send(&SyncMessage::ManifestResponse { manifest }).await.unwrap();
|
||||
conn.send(&SyncMessage::ManifestResponse { manifest })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
if local_rev_count > remote_rev_count {
|
||||
let remote_head = if remote_rev_count == 0 { NO_PARENT } else { remote_rev_count - 1 };
|
||||
let remote_head = if remote_rev_count == 0 {
|
||||
NO_PARENT
|
||||
} else {
|
||||
remote_rev_count - 1
|
||||
};
|
||||
let packets = diff_revisions(&dst, remote_head).unwrap();
|
||||
let total = packets.len() as u64;
|
||||
let mut bytes_sent = 0u64;
|
||||
for packet in packets {
|
||||
bytes_sent += packet.page_data_size() as u64;
|
||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
||||
conn.send(&SyncMessage::LayerPacket { packet })
|
||||
.await
|
||||
.unwrap();
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::Ack { .. } => {}
|
||||
other => panic!("expected Ack, got {other:?}"),
|
||||
@@ -112,14 +129,18 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
||||
conn.send(&SyncMessage::SyncComplete {
|
||||
revisions_transferred: total,
|
||||
bytes_transferred: bytes_sent,
|
||||
}).await.unwrap();
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
} else {
|
||||
let mut packets = Vec::new();
|
||||
loop {
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::LayerPacket { packet } => {
|
||||
let rev = packet.revision;
|
||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
||||
conn.send(&SyncMessage::Ack { revision: rev })
|
||||
.await
|
||||
.unwrap();
|
||||
packets.push(packet);
|
||||
}
|
||||
SyncMessage::SyncComplete { .. } => break,
|
||||
@@ -241,7 +262,10 @@ async fn backend_pull_already_in_sync() {
|
||||
let backend = TcpSyncBackend::new(addr, "agent");
|
||||
let stats = backend.pull(&cli_h5).await.unwrap();
|
||||
|
||||
assert_eq!(stats.revisions_transferred, 0, "nothing to pull when in sync");
|
||||
assert_eq!(
|
||||
stats.revisions_transferred, 0,
|
||||
"nothing to pull when in sync"
|
||||
);
|
||||
let _ = (cli_onion, server_task.await.unwrap());
|
||||
}
|
||||
|
||||
|
||||
@@ -34,7 +34,9 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
||||
|
||||
match conn.recv().await.unwrap() {
|
||||
// ── IBLT pre-flight (push direction) ─────────────────────────────
|
||||
SyncMessage::IbltRequest { sketch: client_iblt } => {
|
||||
SyncMessage::IbltRequest {
|
||||
sketch: client_iblt,
|
||||
} => {
|
||||
let server_revs: Vec<u64> = dst.list_revisions().iter().map(|s| s.revision).collect();
|
||||
|
||||
let diff = client_iblt.diff_against(&server_revs).unwrap();
|
||||
@@ -55,14 +57,18 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
||||
conn.send(&SyncMessage::IbltResponse {
|
||||
sketch: server_iblt,
|
||||
missing_from_remote: diff.only_in_b.clone(),
|
||||
}).await.unwrap();
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut packets = Vec::new();
|
||||
loop {
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::LayerPacket { packet } => {
|
||||
let rev = packet.revision;
|
||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
||||
conn.send(&SyncMessage::Ack { revision: rev })
|
||||
.await
|
||||
.unwrap();
|
||||
packets.push(packet);
|
||||
}
|
||||
SyncMessage::SyncComplete { .. } => break,
|
||||
@@ -75,19 +81,30 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
||||
}
|
||||
|
||||
// ── Flat manifest (pull direction) ────────────────────────────────
|
||||
SyncMessage::ManifestRequest { revision_count: remote_rev_count, .. } => {
|
||||
SyncMessage::ManifestRequest {
|
||||
revision_count: remote_rev_count,
|
||||
..
|
||||
} => {
|
||||
let local_rev_count = dst.revision_count();
|
||||
let manifest = ClawSyncManifest::from_onion("server", &dst, H5_MAGIC);
|
||||
conn.send(&SyncMessage::ManifestResponse { manifest }).await.unwrap();
|
||||
conn.send(&SyncMessage::ManifestResponse { manifest })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
if local_rev_count > remote_rev_count {
|
||||
let remote_head = if remote_rev_count == 0 { NO_PARENT } else { remote_rev_count - 1 };
|
||||
let remote_head = if remote_rev_count == 0 {
|
||||
NO_PARENT
|
||||
} else {
|
||||
remote_rev_count - 1
|
||||
};
|
||||
let packets = diff_revisions(&dst, remote_head).unwrap();
|
||||
let total = packets.len() as u64;
|
||||
let mut bytes_sent = 0u64;
|
||||
for packet in packets {
|
||||
bytes_sent += packet.page_data_size() as u64;
|
||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
||||
conn.send(&SyncMessage::LayerPacket { packet })
|
||||
.await
|
||||
.unwrap();
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::Ack { .. } => {}
|
||||
other => panic!("expected Ack, got {other:?}"),
|
||||
@@ -96,14 +113,18 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
||||
conn.send(&SyncMessage::SyncComplete {
|
||||
revisions_transferred: total,
|
||||
bytes_transferred: bytes_sent,
|
||||
}).await.unwrap();
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
} else {
|
||||
let mut packets = Vec::new();
|
||||
loop {
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::LayerPacket { packet } => {
|
||||
let rev = packet.revision;
|
||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
||||
conn.send(&SyncMessage::Ack { revision: rev })
|
||||
.await
|
||||
.unwrap();
|
||||
packets.push(packet);
|
||||
}
|
||||
SyncMessage::SyncComplete { .. } => break,
|
||||
@@ -124,8 +145,13 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
||||
|
||||
/// Create a temp HDF5 file with `n` committed onion revisions.
|
||||
fn make_onion(n: u8, dir: &TempDir) -> (PathBuf, OnionFile) {
|
||||
let h5 = dir.path().join(format!("data_{n}_{}.h5", std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH).unwrap().subsec_nanos()));
|
||||
let h5 = dir.path().join(format!(
|
||||
"data_{n}_{}.h5",
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.subsec_nanos()
|
||||
));
|
||||
std::fs::write(&h5, H5_MAGIC).unwrap();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
for i in 0..n {
|
||||
@@ -288,7 +314,8 @@ async fn tcp_scheduler_constructor_and_push() {
|
||||
src_h5.clone(),
|
||||
"test-agent",
|
||||
SyncSelector::All,
|
||||
).unwrap();
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let stats = sched.push_force().await.unwrap();
|
||||
assert_eq!(stats.revisions_transferred, 3);
|
||||
|
||||
+223
-112
@@ -26,21 +26,21 @@ use clawhdf5_onion::api::{
|
||||
create_branch, delete_branch, list_branches, list_revisions, rename_branch, rollback,
|
||||
};
|
||||
use clawhdf5_onion::branch::MergeStrategy;
|
||||
use clawhdf5_onion::gc::GcPolicy;
|
||||
use clawhdf5_onion::format::NO_PARENT;
|
||||
use clawhdf5_onion::gc::GcPolicy;
|
||||
use clawhdf5_onion::writer::OnionFile;
|
||||
use clawsync_onion::differ::{diff_revisions, packets_for_revisions};
|
||||
use clawsync_onion::iblt::{IbltSketch, IBLT_SYNC_SEED};
|
||||
use clawsync_onion::iblt::{IBLT_SYNC_SEED, IbltSketch};
|
||||
use clawsync_onion::manifest::{ClawSyncManifest, IbltManifest};
|
||||
use clawsync_onion::merger::merge_packets;
|
||||
use std::sync::Arc;
|
||||
|
||||
use clawsync_fs::{FsSyncClient, FsSyncServer};
|
||||
use clawsync_hdf5::{DatasetManifest, ReceivedDataset, apply_received_payloads};
|
||||
use clawsync_transport::peer::{PipeWriteHalf, SyncPeer};
|
||||
use clawsync_transport::protocol::SyncMessage;
|
||||
use clawsync_transport::quic::{QuicConfig, QuicServer, quic_connect};
|
||||
use clawsync_transport::tcp::{TcpConnection, TcpServer};
|
||||
use clawsync_fs::{FsSyncClient, FsSyncServer};
|
||||
use clawsync_hdf5::{DatasetManifest, ReceivedDataset, apply_received_payloads};
|
||||
|
||||
/// Maximum number of `LayerPacket`s in flight before the sender blocks waiting
|
||||
/// for acks. A window of 16 reduces WAN round-trip overhead by up to 16×
|
||||
@@ -326,7 +326,9 @@ enum BranchAction {
|
||||
|
||||
/// Parse `host:port/path` into `(SocketAddr, PathBuf)`.
|
||||
fn parse_remote(remote: &str) -> Result<(SocketAddr, PathBuf)> {
|
||||
let slash = remote.find('/').context("remote must be 'host:port/path'")?;
|
||||
let slash = remote
|
||||
.find('/')
|
||||
.context("remote must be 'host:port/path'")?;
|
||||
let addr_str = &remote[..slash];
|
||||
let path_str = &remote[slash + 1..];
|
||||
let addr: SocketAddr = addr_str
|
||||
@@ -356,7 +358,9 @@ async fn cmd_list_revisions(h5_path: PathBuf, verbose: bool) -> Result<()> {
|
||||
print!("{:>6} {:>4} {:<30} {}", r.revision, r.branch_id, ann, ts);
|
||||
if verbose {
|
||||
print!(" blake3={}", &r.blake3_hex[..12]);
|
||||
if r.is_snapshot { print!(" [snapshot]"); }
|
||||
if r.is_snapshot {
|
||||
print!(" [snapshot]");
|
||||
}
|
||||
}
|
||||
println!();
|
||||
}
|
||||
@@ -365,7 +369,11 @@ async fn cmd_list_revisions(h5_path: PathBuf, verbose: bool) -> Result<()> {
|
||||
}
|
||||
|
||||
async fn cmd_rollback(h5_path: PathBuf, revision: u64) -> Result<()> {
|
||||
println!("Rolling back {} to revision {}...", h5_path.display(), revision);
|
||||
println!(
|
||||
"Rolling back {} to revision {}...",
|
||||
h5_path.display(),
|
||||
revision
|
||||
);
|
||||
rollback(&h5_path, revision)
|
||||
.with_context(|| format!("rollback failed for {}", h5_path.display()))?;
|
||||
println!("Done.");
|
||||
@@ -389,13 +397,13 @@ async fn cmd_push(
|
||||
);
|
||||
|
||||
// Load local onion — O(sidecar size), does NOT read the large h5 base file.
|
||||
let local_onion = OnionFile::open(&local)
|
||||
.with_context(|| format!("cannot open {}", local.display()))?;
|
||||
let local_onion =
|
||||
OnionFile::open(&local).with_context(|| format!("cannot open {}", local.display()))?;
|
||||
|
||||
// Connect to remote — TCP or QUIC depending on flag.
|
||||
let mut peer: SyncPeer = if quic {
|
||||
let cfg = QuicConfig::self_signed()
|
||||
.with_context(|| "failed to build QUIC self-signed config")?;
|
||||
let cfg =
|
||||
QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
|
||||
let conn = quic_connect(addr, "localhost", cfg)
|
||||
.await
|
||||
.with_context(|| format!("QUIC connect to {addr} failed"))?;
|
||||
@@ -427,11 +435,17 @@ async fn cmd_push(
|
||||
sketch_cells: sketch.cell_count() as u32,
|
||||
sketch: sketch.to_bytes(),
|
||||
};
|
||||
peer.send(&SyncMessage::IbltRequest { sketch: iblt_manifest }).await?;
|
||||
peer.send(&SyncMessage::IbltRequest {
|
||||
sketch: iblt_manifest,
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Receive the server's IBLT response: which revisions we should push.
|
||||
let missing_from_remote = match peer.recv().await? {
|
||||
SyncMessage::IbltResponse { sketch: server_iblt, missing_from_remote } => {
|
||||
SyncMessage::IbltResponse {
|
||||
sketch: server_iblt,
|
||||
missing_from_remote,
|
||||
} => {
|
||||
println!(
|
||||
"Remote has {} revision(s) (IBLT). Pushing {} revision(s).",
|
||||
server_iblt.revision_count,
|
||||
@@ -476,12 +490,15 @@ async fn cmd_push(
|
||||
|
||||
let sem_writer = semaphore.clone();
|
||||
// Return write_half from the task so we can send SyncComplete after the loop.
|
||||
let writer_task: tokio::task::JoinHandle<Result<PipeWriteHalf>> =
|
||||
tokio::spawn(async move {
|
||||
let writer_task: tokio::task::JoinHandle<Result<PipeWriteHalf>> = tokio::spawn(async move {
|
||||
for packet in packets {
|
||||
let rev = packet.revision;
|
||||
let pkt_bytes = packet.page_data_size() as u64;
|
||||
sem_writer.acquire().await.expect("semaphore closed").forget();
|
||||
sem_writer
|
||||
.acquire()
|
||||
.await
|
||||
.expect("semaphore closed")
|
||||
.forget();
|
||||
meta_tx.send((rev, pkt_bytes)).expect("meta channel closed");
|
||||
write_half
|
||||
.send(&SyncMessage::LayerPacket { packet })
|
||||
@@ -548,8 +565,7 @@ async fn cmd_pull(
|
||||
.unwrap_or(0)
|
||||
} else {
|
||||
// Create empty base file
|
||||
std::fs::write(&local, b"\x89HDF\r\n\x1a\n")
|
||||
.context("cannot create local h5")?;
|
||||
std::fs::write(&local, b"\x89HDF\r\n\x1a\n").context("cannot create local h5")?;
|
||||
0
|
||||
};
|
||||
|
||||
@@ -557,8 +573,8 @@ async fn cmd_pull(
|
||||
|
||||
// Connect — TCP or QUIC.
|
||||
let mut peer: SyncPeer = if quic {
|
||||
let cfg = QuicConfig::self_signed()
|
||||
.with_context(|| "failed to build QUIC self-signed config")?;
|
||||
let cfg =
|
||||
QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
|
||||
SyncPeer::Quic(Arc::new(
|
||||
quic_connect(addr, "localhost", cfg)
|
||||
.await
|
||||
@@ -623,7 +639,10 @@ async fn cmd_pull(
|
||||
peer.send(&SyncMessage::Ack { revision: rev }).await?;
|
||||
packets.push(packet);
|
||||
}
|
||||
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => {
|
||||
SyncMessage::SyncComplete {
|
||||
revisions_transferred,
|
||||
bytes_transferred,
|
||||
} => {
|
||||
println!(
|
||||
"Received {} revision(s), {} bytes.",
|
||||
revisions_transferred, bytes_transferred
|
||||
@@ -637,8 +656,7 @@ async fn cmd_pull(
|
||||
|
||||
// Optionally filter to a specific branch before applying.
|
||||
let packets_to_apply = if let Some(ref branch_name) = branch {
|
||||
let selector =
|
||||
clawsync_onion::selector::SyncSelector::Branch(branch_name.clone());
|
||||
let selector = clawsync_onion::selector::SyncSelector::Branch(branch_name.clone());
|
||||
clawsync_onion::selector::filter_packets(packets, &selector, &local_onion)
|
||||
.context("branch filter failed")?
|
||||
} else {
|
||||
@@ -646,8 +664,7 @@ async fn cmd_pull(
|
||||
};
|
||||
|
||||
// Apply packets
|
||||
let stats = merge_packets(&mut local_onion, packets_to_apply, true)
|
||||
.context("merge failed")?;
|
||||
let stats = merge_packets(&mut local_onion, packets_to_apply, true).context("merge failed")?;
|
||||
if let Some(b) = &branch {
|
||||
println!(
|
||||
"Pull complete (branch {b:?}): {} revision(s) merged, {} skipped.",
|
||||
@@ -668,8 +685,8 @@ async fn cmd_pull(
|
||||
/// Serve a local HDF5 file, handling one client connection at a time.
|
||||
async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()> {
|
||||
if quic {
|
||||
let cfg = QuicConfig::self_signed()
|
||||
.with_context(|| "failed to build QUIC self-signed config")?;
|
||||
let cfg =
|
||||
QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
|
||||
let server = QuicServer::bind(bind, cfg)
|
||||
.await
|
||||
.with_context(|| format!("cannot bind QUIC to {bind}"))?;
|
||||
@@ -685,7 +702,11 @@ async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()>
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = handle_client(&mut peer, &h5).await {
|
||||
eprintln!("QUIC client error: {e}");
|
||||
let _ = peer.send(&SyncMessage::Error { message: e.to_string() }).await;
|
||||
let _ = peer
|
||||
.send(&SyncMessage::Error {
|
||||
message: e.to_string(),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -693,7 +714,11 @@ async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()>
|
||||
let server = TcpServer::bind(bind)
|
||||
.await
|
||||
.with_context(|| format!("cannot bind to {bind}"))?;
|
||||
println!("Serving {} on {} (Ctrl-C to stop)", h5_path.display(), server.local_addr);
|
||||
println!(
|
||||
"Serving {} on {} (Ctrl-C to stop)",
|
||||
h5_path.display(),
|
||||
server.local_addr
|
||||
);
|
||||
loop {
|
||||
let (conn, peer_addr) = server.accept().await?;
|
||||
println!("Connection from {peer_addr}");
|
||||
@@ -702,7 +727,11 @@ async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()>
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = handle_client(&mut peer, &h5).await {
|
||||
eprintln!("Client {peer_addr} error: {e}");
|
||||
let _ = peer.send(&SyncMessage::Error { message: e.to_string() }).await;
|
||||
let _ = peer
|
||||
.send(&SyncMessage::Error {
|
||||
message: e.to_string(),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -722,9 +751,11 @@ async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()>
|
||||
async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
|
||||
match conn.recv().await? {
|
||||
// ── IBLT pre-flight (push direction only) ─────────────────────────
|
||||
SyncMessage::IbltRequest { sketch: client_iblt } => {
|
||||
let mut local_onion = OnionFile::open(h5_path)
|
||||
.or_else(|_| OnionFile::create_auto(h5_path))?;
|
||||
SyncMessage::IbltRequest {
|
||||
sketch: client_iblt,
|
||||
} => {
|
||||
let mut local_onion =
|
||||
OnionFile::open(h5_path).or_else(|_| OnionFile::create_auto(h5_path))?;
|
||||
|
||||
let local_revisions: Vec<u64> = local_onion
|
||||
.list_revisions()
|
||||
@@ -762,7 +793,8 @@ async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
|
||||
conn.send(&SyncMessage::IbltResponse {
|
||||
sketch: server_iblt,
|
||||
missing_from_remote: diff.only_in_b.clone(), // client has, server lacks
|
||||
}).await?;
|
||||
})
|
||||
.await?;
|
||||
|
||||
println!(
|
||||
" IBLT: client missing {}, server missing {} revision(s).",
|
||||
@@ -779,7 +811,10 @@ async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
|
||||
conn.send(&SyncMessage::Ack { revision: rev }).await?;
|
||||
packets.push(packet);
|
||||
}
|
||||
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => {
|
||||
SyncMessage::SyncComplete {
|
||||
revisions_transferred,
|
||||
bytes_transferred,
|
||||
} => {
|
||||
println!(
|
||||
" Client sent {revisions_transferred} revision(s), \
|
||||
{bytes_transferred} bytes (IBLT)."
|
||||
@@ -800,14 +835,18 @@ async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
|
||||
}
|
||||
|
||||
// ── Flat manifest (pull direction or legacy push) ──────────────────
|
||||
SyncMessage::ManifestRequest { revision_count: remote_rev_count, .. } => {
|
||||
let mut local_onion = OnionFile::open(h5_path)
|
||||
.or_else(|_| OnionFile::create_auto(h5_path))?;
|
||||
SyncMessage::ManifestRequest {
|
||||
revision_count: remote_rev_count,
|
||||
..
|
||||
} => {
|
||||
let mut local_onion =
|
||||
OnionFile::open(h5_path).or_else(|_| OnionFile::create_auto(h5_path))?;
|
||||
let h5_base = std::fs::read(h5_path)?;
|
||||
let manifest = ClawSyncManifest::from_onion("server", &local_onion, &h5_base);
|
||||
let local_rev_count = manifest.revision_count;
|
||||
|
||||
conn.send(&SyncMessage::ManifestResponse { manifest }).await?;
|
||||
conn.send(&SyncMessage::ManifestResponse { manifest })
|
||||
.await?;
|
||||
|
||||
if local_rev_count > remote_rev_count {
|
||||
// ── Server pushes (client is behind) ──────────────────────
|
||||
@@ -816,8 +855,8 @@ async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
|
||||
} else {
|
||||
remote_rev_count - 1
|
||||
};
|
||||
let packets = diff_revisions(&local_onion, remote_head)
|
||||
.context("diff_revisions failed")?;
|
||||
let packets =
|
||||
diff_revisions(&local_onion, remote_head).context("diff_revisions failed")?;
|
||||
let total = packets.len() as u64;
|
||||
let mut bytes_sent = 0u64;
|
||||
|
||||
@@ -846,7 +885,10 @@ async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
|
||||
conn.send(&SyncMessage::Ack { revision: rev }).await?;
|
||||
packets.push(packet);
|
||||
}
|
||||
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => {
|
||||
SyncMessage::SyncComplete {
|
||||
revisions_transferred,
|
||||
bytes_transferred,
|
||||
} => {
|
||||
println!(
|
||||
" Client sent {revisions_transferred} revision(s), \
|
||||
{bytes_transferred} bytes."
|
||||
@@ -880,11 +922,7 @@ async fn cmd_export_revision(h5_path: PathBuf, revision: u64, dest: PathBuf) ->
|
||||
.with_context(|| format!("cannot open {}", h5_path.display()))?;
|
||||
vf.export_revision(revision, &dest)
|
||||
.with_context(|| format!("export of revision {revision} failed"))?;
|
||||
println!(
|
||||
"Exported revision {} → {}",
|
||||
revision,
|
||||
dest.display()
|
||||
);
|
||||
println!("Exported revision {} → {}", revision, dest.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -921,8 +959,8 @@ async fn cmd_gc(
|
||||
);
|
||||
};
|
||||
|
||||
let mut onion = OnionFile::open(&h5_path)
|
||||
.with_context(|| format!("cannot open {}", h5_path.display()))?;
|
||||
let mut onion =
|
||||
OnionFile::open(&h5_path).with_context(|| format!("cannot open {}", h5_path.display()))?;
|
||||
let stats = onion.gc(policy).context("GC failed")?;
|
||||
onion.flush().context("flush after GC failed")?;
|
||||
|
||||
@@ -935,10 +973,10 @@ async fn cmd_gc(
|
||||
}
|
||||
|
||||
async fn cmd_snapshot(h5_path: PathBuf, message: Option<String>) -> Result<()> {
|
||||
let mut onion = OnionFile::open(&h5_path)
|
||||
.with_context(|| format!("cannot open {}", h5_path.display()))?;
|
||||
let h5_base = std::fs::read(&h5_path)
|
||||
.with_context(|| format!("cannot read {}", h5_path.display()))?;
|
||||
let mut onion =
|
||||
OnionFile::open(&h5_path).with_context(|| format!("cannot open {}", h5_path.display()))?;
|
||||
let h5_base =
|
||||
std::fs::read(&h5_path).with_context(|| format!("cannot read {}", h5_path.display()))?;
|
||||
|
||||
let annotation = message.as_deref().or(Some("manual snapshot"));
|
||||
let snap_rev = onion
|
||||
@@ -980,7 +1018,11 @@ async fn cmd_branch_merge(
|
||||
|
||||
async fn cmd_branch(action: BranchAction) -> Result<()> {
|
||||
match action {
|
||||
BranchAction::Create { h5_path, name, from } => {
|
||||
BranchAction::Create {
|
||||
h5_path,
|
||||
name,
|
||||
from,
|
||||
} => {
|
||||
let id = create_branch(&h5_path, &name, &from)
|
||||
.with_context(|| format!("failed to create branch {name:?}"))?;
|
||||
println!("Branch {name:?} created (id={id}), forked from {from:?}.");
|
||||
@@ -1023,7 +1065,12 @@ async fn cmd_branch(action: BranchAction) -> Result<()> {
|
||||
println!("Branch {from:?} renamed to {to:?}.");
|
||||
}
|
||||
|
||||
BranchAction::Merge { h5_path, source, into, strategy } => {
|
||||
BranchAction::Merge {
|
||||
h5_path,
|
||||
source,
|
||||
into,
|
||||
strategy,
|
||||
} => {
|
||||
cmd_branch_merge(h5_path, source, into, strategy).await?;
|
||||
}
|
||||
}
|
||||
@@ -1037,10 +1084,7 @@ async fn cmd_branch(action: BranchAction) -> Result<()> {
|
||||
fn build_glob_set(patterns: &[String]) -> Result<globset::GlobSet> {
|
||||
let mut builder = globset::GlobSetBuilder::new();
|
||||
for p in patterns {
|
||||
builder.add(
|
||||
globset::Glob::new(p)
|
||||
.with_context(|| format!("invalid glob pattern: {p}"))?,
|
||||
);
|
||||
builder.add(globset::Glob::new(p).with_context(|| format!("invalid glob pattern: {p}"))?);
|
||||
}
|
||||
builder.build().context("failed to build glob set")
|
||||
}
|
||||
@@ -1063,8 +1107,8 @@ async fn cmd_sync(
|
||||
);
|
||||
|
||||
let peer: SyncPeer = if quic {
|
||||
let cfg = QuicConfig::self_signed()
|
||||
.with_context(|| "failed to build QUIC self-signed config")?;
|
||||
let cfg =
|
||||
QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
|
||||
SyncPeer::Quic(Arc::new(
|
||||
quic_connect(addr, "localhost", cfg)
|
||||
.await
|
||||
@@ -1099,8 +1143,8 @@ async fn cmd_serve_fs(
|
||||
let excludes = build_glob_set(&exclude)?;
|
||||
|
||||
if quic {
|
||||
let cfg = QuicConfig::self_signed()
|
||||
.with_context(|| "failed to build QUIC self-signed config")?;
|
||||
let cfg =
|
||||
QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
|
||||
let server = clawsync_transport::quic::QuicServer::bind(bind, cfg)
|
||||
.await
|
||||
.with_context(|| format!("cannot bind QUIC to {bind}"))?;
|
||||
@@ -1148,12 +1192,7 @@ async fn cmd_serve_fs(
|
||||
// HDF5 dataset-granular sync commands
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
async fn cmd_hdf5_sync(
|
||||
local: PathBuf,
|
||||
remote: String,
|
||||
delete: bool,
|
||||
quic: bool,
|
||||
) -> Result<()> {
|
||||
async fn cmd_hdf5_sync(local: PathBuf, remote: String, delete: bool, quic: bool) -> Result<()> {
|
||||
let (addr, remote_path) = parse_remote(&remote)?;
|
||||
let remote_path_str = remote_path.to_string_lossy().into_owned();
|
||||
|
||||
@@ -1195,8 +1234,8 @@ async fn cmd_hdf5_sync(
|
||||
|
||||
// Connect.
|
||||
let mut peer: SyncPeer = if quic {
|
||||
let cfg = QuicConfig::self_signed()
|
||||
.with_context(|| "failed to build QUIC self-signed config")?;
|
||||
let cfg =
|
||||
QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
|
||||
SyncPeer::Quic(Arc::new(
|
||||
quic_connect(addr, "localhost", cfg)
|
||||
.await
|
||||
@@ -1221,7 +1260,10 @@ async fn cmd_hdf5_sync(
|
||||
|
||||
// Receive which datasets the server needs.
|
||||
let (datasets_needed, _to_delete) = match peer.recv().await? {
|
||||
SyncMessage::Hdf5ManifestResponse { datasets_needed, to_delete } => {
|
||||
SyncMessage::Hdf5ManifestResponse {
|
||||
datasets_needed,
|
||||
to_delete,
|
||||
} => {
|
||||
println!(
|
||||
"Server needs {} dataset(s), {} to delete.",
|
||||
datasets_needed.len(),
|
||||
@@ -1237,7 +1279,10 @@ async fn cmd_hdf5_sync(
|
||||
// Nothing to send; wait for completion.
|
||||
match peer.recv().await? {
|
||||
SyncMessage::Hdf5SyncComplete {
|
||||
datasets_added, datasets_modified, datasets_removed, bytes_transferred
|
||||
datasets_added,
|
||||
datasets_modified,
|
||||
datasets_removed,
|
||||
bytes_transferred,
|
||||
} => {
|
||||
println!(
|
||||
"HDF5-sync complete: +{datasets_added} ~{datasets_modified} \
|
||||
@@ -1298,7 +1343,10 @@ async fn cmd_hdf5_sync(
|
||||
// Wait for final completion message.
|
||||
match peer.recv().await? {
|
||||
SyncMessage::Hdf5SyncComplete {
|
||||
datasets_added, datasets_modified, datasets_removed, bytes_transferred
|
||||
datasets_added,
|
||||
datasets_modified,
|
||||
datasets_removed,
|
||||
bytes_transferred,
|
||||
} => {
|
||||
println!(
|
||||
"HDF5-sync complete: +{datasets_added} ~{datasets_modified} \
|
||||
@@ -1321,9 +1369,12 @@ async fn handle_hdf5_client(
|
||||
) -> Result<()> {
|
||||
// Receive the client's manifest.
|
||||
let (remote_path, _client_file_blake3, client_entries, want_delete) = match conn.recv().await? {
|
||||
SyncMessage::Hdf5ManifestRequest { remote_path, file_blake3, entries, want_delete } => {
|
||||
(remote_path, file_blake3, entries, want_delete)
|
||||
}
|
||||
SyncMessage::Hdf5ManifestRequest {
|
||||
remote_path,
|
||||
file_blake3,
|
||||
entries,
|
||||
want_delete,
|
||||
} => (remote_path, file_blake3, entries, want_delete),
|
||||
other => anyhow::bail!("expected Hdf5ManifestRequest, got {other:?}"),
|
||||
};
|
||||
|
||||
@@ -1359,8 +1410,13 @@ async fn handle_hdf5_client(
|
||||
let mut to_delete = Vec::new();
|
||||
|
||||
// Build a quick lookup of client entries by path.
|
||||
let client_map: std::collections::HashMap<&str, &clawsync_transport::protocol::Hdf5DatasetEntry> =
|
||||
client_entries.iter().map(|e| (e.path.as_str(), e)).collect();
|
||||
let client_map: std::collections::HashMap<
|
||||
&str,
|
||||
&clawsync_transport::protocol::Hdf5DatasetEntry,
|
||||
> = client_entries
|
||||
.iter()
|
||||
.map(|e| (e.path.as_str(), e))
|
||||
.collect();
|
||||
|
||||
// Added or Modified: in client manifest but not on server, or hash differs.
|
||||
for entry in &client_entries {
|
||||
@@ -1394,9 +1450,19 @@ async fn handle_hdf5_client(
|
||||
|
||||
for _ in 0..total_needed {
|
||||
match conn.recv().await? {
|
||||
SyncMessage::Hdf5DataPayload { path, data, dtype, shape } => {
|
||||
SyncMessage::Hdf5DataPayload {
|
||||
path,
|
||||
data,
|
||||
dtype,
|
||||
shape,
|
||||
} => {
|
||||
bytes_received += data.len() as u64;
|
||||
payloads.push(ReceivedDataset { path: path.clone(), data, dtype, shape });
|
||||
payloads.push(ReceivedDataset {
|
||||
path: path.clone(),
|
||||
data,
|
||||
dtype,
|
||||
shape,
|
||||
});
|
||||
conn.send(&SyncMessage::Hdf5DataAck { path }).await?;
|
||||
}
|
||||
other => anyhow::bail!("expected Hdf5DataPayload, got {other:?}"),
|
||||
@@ -1440,8 +1506,8 @@ async fn cmd_serve_hdf5(
|
||||
quic: bool,
|
||||
) -> Result<()> {
|
||||
if quic {
|
||||
let cfg = QuicConfig::self_signed()
|
||||
.with_context(|| "failed to build QUIC self-signed config")?;
|
||||
let cfg =
|
||||
QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
|
||||
let server = QuicServer::bind(bind, cfg)
|
||||
.await
|
||||
.with_context(|| format!("cannot bind QUIC to {bind}"))?;
|
||||
@@ -1453,7 +1519,11 @@ async fn cmd_serve_hdf5(
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = handle_hdf5_client(&mut peer, &root, allow_delete).await {
|
||||
eprintln!("HDF5 QUIC client error: {e}");
|
||||
let _ = peer.send(&SyncMessage::Error { message: e.to_string() }).await;
|
||||
let _ = peer
|
||||
.send(&SyncMessage::Error {
|
||||
message: e.to_string(),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -1469,7 +1539,11 @@ async fn cmd_serve_hdf5(
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = handle_hdf5_client(&mut peer, &root, allow_delete).await {
|
||||
eprintln!("HDF5 client {peer_addr} error: {e}");
|
||||
let _ = peer.send(&SyncMessage::Error { message: e.to_string() }).await;
|
||||
let _ = peer
|
||||
.send(&SyncMessage::Error {
|
||||
message: e.to_string(),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -1498,36 +1572,73 @@ fn main() -> Result<()> {
|
||||
Commands::ListRevisions { h5_path, verbose } => {
|
||||
cmd_list_revisions(h5_path, verbose).await
|
||||
}
|
||||
Commands::Rollback { h5_path, revision } => {
|
||||
cmd_rollback(h5_path, revision).await
|
||||
}
|
||||
Commands::Push { local, remote, branch, quic } => {
|
||||
cmd_push(local, remote, branch, quic).await
|
||||
}
|
||||
Commands::Pull { remote, local, branch, quic } => {
|
||||
cmd_pull(remote, local, branch, quic).await
|
||||
}
|
||||
Commands::Serve { h5_path, bind, quic } => cmd_serve(h5_path, bind, quic).await,
|
||||
Commands::Rollback { h5_path, revision } => cmd_rollback(h5_path, revision).await,
|
||||
Commands::Push {
|
||||
local,
|
||||
remote,
|
||||
branch,
|
||||
quic,
|
||||
} => cmd_push(local, remote, branch, quic).await,
|
||||
Commands::Pull {
|
||||
remote,
|
||||
local,
|
||||
branch,
|
||||
quic,
|
||||
} => cmd_pull(remote, local, branch, quic).await,
|
||||
Commands::Serve {
|
||||
h5_path,
|
||||
bind,
|
||||
quic,
|
||||
} => cmd_serve(h5_path, bind, quic).await,
|
||||
Commands::Branch { action } => cmd_branch(action).await,
|
||||
Commands::ExportRevision { h5_path, revision, dest } => {
|
||||
cmd_export_revision(h5_path, revision, dest).await
|
||||
}
|
||||
Commands::Gc { h5_path, keep_last, keep_tagged, keep_since_days, keep_revisions } => {
|
||||
cmd_gc(h5_path, keep_last, keep_tagged, keep_since_days, keep_revisions).await
|
||||
Commands::ExportRevision {
|
||||
h5_path,
|
||||
revision,
|
||||
dest,
|
||||
} => cmd_export_revision(h5_path, revision, dest).await,
|
||||
Commands::Gc {
|
||||
h5_path,
|
||||
keep_last,
|
||||
keep_tagged,
|
||||
keep_since_days,
|
||||
keep_revisions,
|
||||
} => {
|
||||
cmd_gc(
|
||||
h5_path,
|
||||
keep_last,
|
||||
keep_tagged,
|
||||
keep_since_days,
|
||||
keep_revisions,
|
||||
)
|
||||
.await
|
||||
}
|
||||
Commands::Snapshot { h5_path, message } => cmd_snapshot(h5_path, message).await,
|
||||
Commands::Sync { local, remote, delete, exclude, quic } => {
|
||||
cmd_sync(local, remote, delete, exclude, quic).await
|
||||
}
|
||||
Commands::ServeFs { dir, bind, allow_delete, exclude, quic } => {
|
||||
cmd_serve_fs(dir, bind, allow_delete, exclude, quic).await
|
||||
}
|
||||
Commands::Hdf5Sync { local, remote, delete, quic } => {
|
||||
cmd_hdf5_sync(local, remote, delete, quic).await
|
||||
}
|
||||
Commands::ServeHdf5 { dir, bind, allow_delete, quic } => {
|
||||
cmd_serve_hdf5(dir, bind, allow_delete, quic).await
|
||||
}
|
||||
Commands::Sync {
|
||||
local,
|
||||
remote,
|
||||
delete,
|
||||
exclude,
|
||||
quic,
|
||||
} => cmd_sync(local, remote, delete, exclude, quic).await,
|
||||
Commands::ServeFs {
|
||||
dir,
|
||||
bind,
|
||||
allow_delete,
|
||||
exclude,
|
||||
quic,
|
||||
} => cmd_serve_fs(dir, bind, allow_delete, exclude, quic).await,
|
||||
Commands::Hdf5Sync {
|
||||
local,
|
||||
remote,
|
||||
delete,
|
||||
quic,
|
||||
} => cmd_hdf5_sync(local, remote, delete, quic).await,
|
||||
Commands::ServeHdf5 {
|
||||
dir,
|
||||
bind,
|
||||
allow_delete,
|
||||
quic,
|
||||
} => cmd_serve_hdf5(dir, bind, allow_delete, quic).await,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -9,8 +9,8 @@ use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
use clawhdf5_onion::api::{create_branch, delete_branch, list_branches, rename_branch};
|
||||
use clawhdf5_onion::gc::GcPolicy;
|
||||
use clawhdf5_onion::format::NO_PARENT;
|
||||
use clawhdf5_onion::gc::GcPolicy;
|
||||
use clawhdf5_onion::writer::OnionFile;
|
||||
use clawsync_onion::differ::diff_revisions;
|
||||
use clawsync_onion::manifest::ClawSyncManifest;
|
||||
@@ -62,7 +62,9 @@ async fn run_server_one_connection(server: TcpServer, mut dst: OnionFile) -> Oni
|
||||
|
||||
let local_rev_count = dst.revision_count();
|
||||
let manifest = ClawSyncManifest::from_onion("server", &dst, H5_MAGIC);
|
||||
conn.send(&SyncMessage::ManifestResponse { manifest }).await.unwrap();
|
||||
conn.send(&SyncMessage::ManifestResponse { manifest })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
if local_rev_count > remote_rev_count {
|
||||
// Push to client
|
||||
@@ -76,7 +78,9 @@ async fn run_server_one_connection(server: TcpServer, mut dst: OnionFile) -> Oni
|
||||
let mut bytes_sent = 0u64;
|
||||
for packet in packets {
|
||||
bytes_sent += packet.page_data_size() as u64;
|
||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
||||
conn.send(&SyncMessage::LayerPacket { packet })
|
||||
.await
|
||||
.unwrap();
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::Ack { .. } => {}
|
||||
other => panic!("expected Ack, got {other:?}"),
|
||||
@@ -95,7 +99,9 @@ async fn run_server_one_connection(server: TcpServer, mut dst: OnionFile) -> Oni
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::LayerPacket { packet } => {
|
||||
let rev = packet.revision;
|
||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
||||
conn.send(&SyncMessage::Ack { revision: rev })
|
||||
.await
|
||||
.unwrap();
|
||||
packets.push(packet);
|
||||
}
|
||||
SyncMessage::SyncComplete { .. } => break,
|
||||
@@ -139,7 +145,11 @@ async fn push_all_to_empty_server() {
|
||||
|
||||
let remote_head = match conn.recv().await.unwrap() {
|
||||
SyncMessage::ManifestResponse { manifest } => {
|
||||
if manifest.revision_count == 0 { clawhdf5_onion::format::NO_PARENT } else { manifest.head_revision }
|
||||
if manifest.revision_count == 0 {
|
||||
clawhdf5_onion::format::NO_PARENT
|
||||
} else {
|
||||
manifest.head_revision
|
||||
}
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
};
|
||||
@@ -148,14 +158,20 @@ async fn push_all_to_empty_server() {
|
||||
let total = packets.len() as u64;
|
||||
for packet in packets {
|
||||
let rev = packet.revision;
|
||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
||||
conn.send(&SyncMessage::LayerPacket { packet })
|
||||
.await
|
||||
.unwrap();
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::Ack { revision } => assert_eq!(revision, rev),
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
conn.send(&SyncMessage::SyncComplete { revisions_transferred: total, bytes_transferred: 0 })
|
||||
.await.unwrap();
|
||||
conn.send(&SyncMessage::SyncComplete {
|
||||
revisions_transferred: total,
|
||||
bytes_transferred: 0,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let dst_final = server_task.await.unwrap();
|
||||
assert_eq!(dst_final.revision_count(), 5);
|
||||
@@ -180,7 +196,9 @@ async fn push_already_in_sync_is_noop() {
|
||||
agent_id: "c".into(),
|
||||
head_revision: mf.head_revision,
|
||||
revision_count: mf.revision_count,
|
||||
}).await.unwrap();
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::ManifestResponse { .. } => {}
|
||||
@@ -188,8 +206,12 @@ async fn push_already_in_sync_is_noop() {
|
||||
}
|
||||
|
||||
// diff = 0; send SyncComplete immediately
|
||||
conn.send(&SyncMessage::SyncComplete { revisions_transferred: 0, bytes_transferred: 0 })
|
||||
.await.unwrap();
|
||||
conn.send(&SyncMessage::SyncComplete {
|
||||
revisions_transferred: 0,
|
||||
bytes_transferred: 0,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let dst_final = server_task.await.unwrap();
|
||||
assert_eq!(dst_final.revision_count(), 3, "no duplicate revisions");
|
||||
@@ -215,7 +237,9 @@ async fn pull_all_from_server() {
|
||||
agent_id: "pull-client".into(),
|
||||
head_revision: 0,
|
||||
revision_count: 0,
|
||||
}).await.unwrap();
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let server_count = match conn.recv().await.unwrap() {
|
||||
SyncMessage::ManifestResponse { manifest } => manifest.revision_count,
|
||||
@@ -230,7 +254,9 @@ async fn pull_all_from_server() {
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::LayerPacket { packet } => {
|
||||
let rev = packet.revision;
|
||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
||||
conn.send(&SyncMessage::Ack { revision: rev })
|
||||
.await
|
||||
.unwrap();
|
||||
packets.push(packet);
|
||||
}
|
||||
SyncMessage::SyncComplete { .. } => break,
|
||||
@@ -260,7 +286,9 @@ async fn pull_already_in_sync() {
|
||||
agent_id: "pull-client".into(),
|
||||
head_revision: local_count.saturating_sub(1),
|
||||
revision_count: local_count,
|
||||
}).await.unwrap();
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let server_count = match conn.recv().await.unwrap() {
|
||||
SyncMessage::ManifestResponse { manifest } => manifest.revision_count,
|
||||
@@ -269,8 +297,12 @@ async fn pull_already_in_sync() {
|
||||
|
||||
assert!(server_count <= local_count, "already in sync");
|
||||
// Client detects it's up to date — sends SyncComplete immediately.
|
||||
conn.send(&SyncMessage::SyncComplete { revisions_transferred: 0, bytes_transferred: 0 })
|
||||
.await.unwrap();
|
||||
conn.send(&SyncMessage::SyncComplete {
|
||||
revisions_transferred: 0,
|
||||
bytes_transferred: 0,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
server_task.await.unwrap();
|
||||
let _ = cli_h5;
|
||||
@@ -294,7 +326,9 @@ async fn pull_partial_delta() {
|
||||
agent_id: "c".into(),
|
||||
head_revision: K as u64 - 1,
|
||||
revision_count: K as u64,
|
||||
}).await.unwrap();
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let server_count = match conn.recv().await.unwrap() {
|
||||
SyncMessage::ManifestResponse { manifest } => manifest.revision_count,
|
||||
@@ -308,7 +342,9 @@ async fn pull_partial_delta() {
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::LayerPacket { packet } => {
|
||||
let rev = packet.revision;
|
||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
||||
conn.send(&SyncMessage::Ack { revision: rev })
|
||||
.await
|
||||
.unwrap();
|
||||
packets.push(packet);
|
||||
received += 1;
|
||||
}
|
||||
@@ -376,7 +412,9 @@ async fn push_one_revision_within_timeout() {
|
||||
agent_id: "c".into(),
|
||||
head_revision: mf.head_revision,
|
||||
revision_count: mf.revision_count,
|
||||
}).await.unwrap();
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let remote_head = match conn.recv().await.unwrap() {
|
||||
SyncMessage::ManifestResponse { manifest } => {
|
||||
@@ -393,18 +431,27 @@ async fn push_one_revision_within_timeout() {
|
||||
let total = packets.len() as u64;
|
||||
for packet in packets {
|
||||
let rev = packet.revision;
|
||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
||||
conn.send(&SyncMessage::LayerPacket { packet })
|
||||
.await
|
||||
.unwrap();
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::Ack { revision } => assert_eq!(revision, rev),
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
conn.send(&SyncMessage::SyncComplete { revisions_transferred: total, bytes_transferred: 0 })
|
||||
.await.unwrap();
|
||||
conn.send(&SyncMessage::SyncComplete {
|
||||
revisions_transferred: total,
|
||||
bytes_transferred: 0,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(client_result.is_ok(), "push timed out — possible protocol deadlock");
|
||||
assert!(
|
||||
client_result.is_ok(),
|
||||
"push timed out — possible protocol deadlock"
|
||||
);
|
||||
let dst_final = server_handle.await.unwrap();
|
||||
assert_eq!(dst_final.revision_count(), 1);
|
||||
}
|
||||
@@ -495,11 +542,17 @@ async fn push_with_branch_filter_sends_only_branch_revisions() {
|
||||
agent_id: "c".into(),
|
||||
head_revision: mf.head_revision,
|
||||
revision_count: mf.revision_count,
|
||||
}).await.unwrap();
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let remote_head = match conn.recv().await.unwrap() {
|
||||
SyncMessage::ManifestResponse { manifest } => {
|
||||
if manifest.revision_count == 0 { clawhdf5_onion::format::NO_PARENT } else { manifest.head_revision }
|
||||
if manifest.revision_count == 0 {
|
||||
clawhdf5_onion::format::NO_PARENT
|
||||
} else {
|
||||
manifest.head_revision
|
||||
}
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
};
|
||||
@@ -509,22 +562,36 @@ async fn push_with_branch_filter_sends_only_branch_revisions() {
|
||||
let packets = filter_packets(all_packets, &selector, &src).unwrap();
|
||||
|
||||
// Only the 2 feature revisions should be sent (not the 3 main ones).
|
||||
assert_eq!(packets.len(), 2, "only feature branch packets should be sent");
|
||||
assert_eq!(
|
||||
packets.len(),
|
||||
2,
|
||||
"only feature branch packets should be sent"
|
||||
);
|
||||
|
||||
let total = packets.len() as u64;
|
||||
for packet in packets {
|
||||
let rev = packet.revision;
|
||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
||||
conn.send(&SyncMessage::LayerPacket { packet })
|
||||
.await
|
||||
.unwrap();
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::Ack { revision } => assert_eq!(revision, rev),
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
conn.send(&SyncMessage::SyncComplete { revisions_transferred: total, bytes_transferred: 0 })
|
||||
.await.unwrap();
|
||||
conn.send(&SyncMessage::SyncComplete {
|
||||
revisions_transferred: total,
|
||||
bytes_transferred: 0,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let dst_final = server_task.await.unwrap();
|
||||
assert_eq!(dst_final.revision_count(), 2, "destination should have only the 2 feature revisions");
|
||||
assert_eq!(
|
||||
dst_final.revision_count(),
|
||||
2,
|
||||
"destination should have only the 2 feature revisions"
|
||||
);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -567,7 +634,9 @@ async fn push_to_server(src: &OnionFile, src_h5: &PathBuf, addr: SocketAddr) ->
|
||||
let sent = packets.len() as u64;
|
||||
for packet in packets {
|
||||
let rev = packet.revision;
|
||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
||||
conn.send(&SyncMessage::LayerPacket { packet })
|
||||
.await
|
||||
.unwrap();
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::Ack { revision } => assert_eq!(revision, rev),
|
||||
other => panic!("client: expected Ack, got {other:?}"),
|
||||
@@ -597,7 +666,9 @@ async fn run_disk_server(server: TcpServer, h5_path: PathBuf, n: usize) {
|
||||
|
||||
let local_rev_count = onion.revision_count();
|
||||
let manifest = ClawSyncManifest::from_onion("server", &onion, H5_MAGIC);
|
||||
conn.send(&SyncMessage::ManifestResponse { manifest }).await.unwrap();
|
||||
conn.send(&SyncMessage::ManifestResponse { manifest })
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
if local_rev_count > remote_rev_count {
|
||||
// Server pushes (pull direction)
|
||||
@@ -611,7 +682,9 @@ async fn run_disk_server(server: TcpServer, h5_path: PathBuf, n: usize) {
|
||||
let mut bytes_sent = 0u64;
|
||||
for packet in packets {
|
||||
bytes_sent += packet.page_data_size() as u64;
|
||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
||||
conn.send(&SyncMessage::LayerPacket { packet })
|
||||
.await
|
||||
.unwrap();
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::Ack { .. } => {}
|
||||
other => panic!("server: expected Ack, got {other:?}"),
|
||||
@@ -630,7 +703,9 @@ async fn run_disk_server(server: TcpServer, h5_path: PathBuf, n: usize) {
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::LayerPacket { packet } => {
|
||||
let rev = packet.revision;
|
||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
||||
conn.send(&SyncMessage::Ack { revision: rev })
|
||||
.await
|
||||
.unwrap();
|
||||
packets.push(packet);
|
||||
}
|
||||
SyncMessage::SyncComplete { .. } => break,
|
||||
@@ -687,7 +762,11 @@ async fn serve_accumulates_across_two_sequential_pushes() {
|
||||
|
||||
// The on-disk server file must now have all 5 revisions.
|
||||
let final_onion = OnionFile::open(&srv_h5).unwrap();
|
||||
assert_eq!(final_onion.revision_count(), 5, "server should have accumulated all 5 revisions");
|
||||
assert_eq!(
|
||||
final_onion.revision_count(),
|
||||
5,
|
||||
"server should have accumulated all 5 revisions"
|
||||
);
|
||||
let _ = srv_dir;
|
||||
}
|
||||
|
||||
@@ -713,7 +792,10 @@ async fn serve_noop_when_second_client_already_in_sync() {
|
||||
|
||||
// Connection 2: same client — already in sync, should send 0 packets.
|
||||
let sent2 = push_to_server(&src, &src_h5, addr).await;
|
||||
assert_eq!(sent2, 0, "second identical push should transfer 0 revisions");
|
||||
assert_eq!(
|
||||
sent2, 0,
|
||||
"second identical push should transfer 0 revisions"
|
||||
);
|
||||
|
||||
server_task.await.unwrap();
|
||||
|
||||
@@ -738,8 +820,11 @@ fn export_revision_writes_correct_historical_state() {
|
||||
|
||||
let bytes = std::fs::read(&dest).unwrap();
|
||||
// The reconstructed bytes at offset 0 should be the rev-1 page (all 0x01)
|
||||
assert_eq!(bytes[0..4096], vec![0x01u8; 4096][..],
|
||||
"exported rev 1 should contain the page written in rev 1");
|
||||
assert_eq!(
|
||||
bytes[0..4096],
|
||||
vec![0x01u8; 4096][..],
|
||||
"exported rev 1 should contain the page written in rev 1"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -761,7 +846,10 @@ fn export_revision_does_not_modify_source_sidecar() {
|
||||
vf.export_revision(0, &dest).unwrap();
|
||||
|
||||
let after_len = std::fs::metadata(&onion_path).unwrap().len();
|
||||
assert_eq!(before_len, after_len, "sidecar must be unchanged after export");
|
||||
assert_eq!(
|
||||
before_len, after_len,
|
||||
"sidecar must be unchanged after export"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -771,7 +859,10 @@ fn export_revision_nonexistent_errors() {
|
||||
let dest = dest_dir.path().join("out.h5");
|
||||
let vf = clawhdf5_onion::versioned_file::VersionedFile::open(&h5).unwrap();
|
||||
let err = vf.export_revision(999, &dest).unwrap_err();
|
||||
assert!(matches!(err, clawhdf5_onion::OnionError::RevisionNotFound(999)));
|
||||
assert!(matches!(
|
||||
err,
|
||||
clawhdf5_onion::OnionError::RevisionNotFound(999)
|
||||
));
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -802,10 +893,18 @@ fn branch_merge_latest_wins_creates_merge_revision() {
|
||||
.unwrap();
|
||||
|
||||
let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap();
|
||||
assert!(revs.iter().any(|r| r.revision == merge_rev),
|
||||
"merge revision must appear in list_revisions");
|
||||
assert_eq!(revs.iter().find(|r| r.revision == merge_rev).unwrap().branch_id, 0,
|
||||
"merge revision is on main (branch_id 0)");
|
||||
assert!(
|
||||
revs.iter().any(|r| r.revision == merge_rev),
|
||||
"merge revision must appear in list_revisions"
|
||||
);
|
||||
assert_eq!(
|
||||
revs.iter()
|
||||
.find(|r| r.revision == merge_rev)
|
||||
.unwrap()
|
||||
.branch_id,
|
||||
0,
|
||||
"merge revision is on main (branch_id 0)"
|
||||
);
|
||||
|
||||
let _ = h5_base;
|
||||
}
|
||||
@@ -865,8 +964,10 @@ fn gc_keep_tagged_retains_annotated_only() {
|
||||
|
||||
assert_eq!(stats.revisions_removed, 2);
|
||||
let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap();
|
||||
assert!(revs.iter().all(|r| r.annotation.is_some()),
|
||||
"all remaining revisions should be annotated");
|
||||
assert!(
|
||||
revs.iter().all(|r| r.annotation.is_some()),
|
||||
"all remaining revisions should be annotated"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -887,8 +988,10 @@ fn gc_keep_since_days_zero_removes_all_old_revisions() {
|
||||
let stats = onion.gc(GcPolicy::KeepSince(future_cutoff)).unwrap();
|
||||
onion.flush().unwrap();
|
||||
|
||||
assert_eq!(stats.revisions_removed, 4,
|
||||
"all 4 revisions should be pruned when cutoff is in the future");
|
||||
assert_eq!(
|
||||
stats.revisions_removed, 4,
|
||||
"all 4 revisions should be pruned when cutoff is in the future"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -898,8 +1001,10 @@ fn gc_keep_since_days_past_keeps_all() {
|
||||
|
||||
let stats = onion.gc(GcPolicy::KeepSince(0.0)).unwrap();
|
||||
// All revisions are after Unix epoch 0 → 0 removed
|
||||
assert_eq!(stats.revisions_removed, 0,
|
||||
"no revisions should be removed when cutoff is epoch 0");
|
||||
assert_eq!(
|
||||
stats.revisions_removed, 0,
|
||||
"no revisions should be removed when cutoff is epoch 0"
|
||||
);
|
||||
assert_eq!(onion.revision_count(), 4);
|
||||
}
|
||||
|
||||
@@ -926,19 +1031,29 @@ fn gc_keep_revisions_retains_only_listed_and_ancestors() {
|
||||
onion.flush().unwrap();
|
||||
|
||||
// Revisions 3 and 4 are pruned; 0, 1, 2 are kept (2 plus its ancestors).
|
||||
assert_eq!(stats.revisions_removed, 2, "revisions 3 and 4 should be removed");
|
||||
assert_eq!(
|
||||
stats.revisions_removed, 2,
|
||||
"revisions 3 and 4 should be removed"
|
||||
);
|
||||
|
||||
let reloaded = OnionFile::open(&h5).unwrap();
|
||||
assert_eq!(reloaded.revision_count(), 3, "revisions 0, 1, 2 remain");
|
||||
|
||||
let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap();
|
||||
let annotations: Vec<_> = revs.iter()
|
||||
let annotations: Vec<_> = revs
|
||||
.iter()
|
||||
.map(|r| r.annotation.as_deref().unwrap_or(""))
|
||||
.collect();
|
||||
assert!(annotations.contains(&"rev 2"), "rev 2 must survive");
|
||||
// Ancestors are retained too
|
||||
assert!(annotations.contains(&"rev 0"), "ancestor rev 0 must survive");
|
||||
assert!(annotations.contains(&"rev 1"), "ancestor rev 1 must survive");
|
||||
assert!(
|
||||
annotations.contains(&"rev 0"),
|
||||
"ancestor rev 0 must survive"
|
||||
);
|
||||
assert!(
|
||||
annotations.contains(&"rev 1"),
|
||||
"ancestor rev 1 must survive"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -948,7 +1063,10 @@ fn gc_keep_revisions_empty_list_removes_all() {
|
||||
let stats = onion.gc(GcPolicy::KeepRevisions(vec![])).unwrap();
|
||||
onion.flush().unwrap();
|
||||
|
||||
assert_eq!(stats.revisions_removed, 4, "all revisions pruned when keep list is empty");
|
||||
assert_eq!(
|
||||
stats.revisions_removed, 4,
|
||||
"all revisions pruned when keep list is empty"
|
||||
);
|
||||
let reloaded = OnionFile::open(&h5).unwrap();
|
||||
assert_eq!(reloaded.revision_count(), 0);
|
||||
}
|
||||
@@ -967,7 +1085,10 @@ fn snapshot_creates_flagged_revision() {
|
||||
|
||||
let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap();
|
||||
let snap = revs.iter().find(|r| r.revision == snap_rev).unwrap();
|
||||
assert!(snap.is_snapshot, "snapshot revision must have is_snapshot=true");
|
||||
assert!(
|
||||
snap.is_snapshot,
|
||||
"snapshot revision must have is_snapshot=true"
|
||||
);
|
||||
assert_eq!(snap.annotation.as_deref(), Some("snap"));
|
||||
}
|
||||
|
||||
|
||||
@@ -15,8 +15,8 @@ use std::io::{BufRead, BufReader};
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::time::Duration;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use tempfile::TempDir;
|
||||
|
||||
@@ -51,13 +51,13 @@ impl FsServer {
|
||||
|
||||
// Wait for: "ClawSync FS server listening on 127.0.0.1:PORT"
|
||||
let mut line = String::new();
|
||||
reader.read_line(&mut line).expect("failed to read serve-fs startup line");
|
||||
reader
|
||||
.read_line(&mut line)
|
||||
.expect("failed to read serve-fs startup line");
|
||||
let addr = parse_fs_serve_addr(&line);
|
||||
|
||||
// Drain remaining stdout in a background thread so the process doesn't block.
|
||||
thread::spawn(move || {
|
||||
for _ in reader.lines() {}
|
||||
});
|
||||
thread::spawn(move || for _ in reader.lines() {});
|
||||
|
||||
Self { child, addr }
|
||||
}
|
||||
@@ -136,7 +136,13 @@ fn parse_sync_output(out: &str, success: bool) -> SyncResult {
|
||||
}
|
||||
}
|
||||
|
||||
SyncResult { files_added, files_modified, files_removed, bytes_transferred, success }
|
||||
SyncResult {
|
||||
files_added,
|
||||
files_modified,
|
||||
files_removed,
|
||||
bytes_transferred,
|
||||
success,
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -161,7 +167,9 @@ fn assert_files_equal(a: &Path, b: &Path) {
|
||||
fn assert_dir_equal(src: &Path, dst: &Path) {
|
||||
for entry in walkdir::WalkDir::new(src).sort_by_file_name() {
|
||||
let entry: walkdir::DirEntry = entry.unwrap();
|
||||
if !entry.file_type().is_file() { continue; }
|
||||
if !entry.file_type().is_file() {
|
||||
continue;
|
||||
}
|
||||
let rel = entry.path().strip_prefix(src).unwrap();
|
||||
let dst_path = dst.join(rel);
|
||||
assert!(dst_path.exists(), "missing in dst: {}", rel.display());
|
||||
@@ -242,7 +250,11 @@ fn fs_sync_cdc_insertion_minimizes_transfer() {
|
||||
|
||||
// Base: 512 KB of pseudo-random data
|
||||
let base: Vec<u8> = (0..512 * 1024)
|
||||
.map(|i| (i as u64).wrapping_mul(6364136223846793005).wrapping_shr(56) as u8)
|
||||
.map(|i| {
|
||||
(i as u64)
|
||||
.wrapping_mul(6364136223846793005)
|
||||
.wrapping_shr(56) as u8
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Server has the base.
|
||||
@@ -260,7 +272,10 @@ fn fs_sync_cdc_insertion_minimizes_transfer() {
|
||||
assert_eq!(result.files_modified, 1);
|
||||
|
||||
// Verify the content is correct after sync.
|
||||
assert_files_equal(&src.path().join("weights.bin"), &dst.path().join("weights.bin"));
|
||||
assert_files_equal(
|
||||
&src.path().join("weights.bin"),
|
||||
&dst.path().join("weights.bin"),
|
||||
);
|
||||
|
||||
// CDC should transfer much less than the full file (< 40% = 205 KB).
|
||||
// The insertion only invalidates ~2-3 chunk boundaries.
|
||||
@@ -286,7 +301,10 @@ fn fs_sync_delete_removes_server_only_files() {
|
||||
|
||||
assert!(result.success);
|
||||
assert_eq!(result.files_removed, 1);
|
||||
assert!(!dst.path().join("extra.bin").exists(), "extra.bin should have been deleted");
|
||||
assert!(
|
||||
!dst.path().join("extra.bin").exists(),
|
||||
"extra.bin should have been deleted"
|
||||
);
|
||||
assert!(dst.path().join("keep.bin").exists());
|
||||
}
|
||||
|
||||
@@ -304,7 +322,10 @@ fn fs_sync_no_delete_preserves_server_only_files() {
|
||||
|
||||
assert!(result.success);
|
||||
// extra.bin must still be there
|
||||
assert!(dst.path().join("extra.bin").exists(), "extra.bin should be preserved (no --delete)");
|
||||
assert!(
|
||||
dst.path().join("extra.bin").exists(),
|
||||
"extra.bin should be preserved (no --delete)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -319,8 +340,14 @@ fn fs_sync_exclude_pattern_skips_matching_files() {
|
||||
let result = run_sync(src.path(), &server.addr, false, &["*.tmp"]);
|
||||
|
||||
assert!(result.success);
|
||||
assert!(dst.path().join("keep.bin").exists(), "keep.bin should be transferred");
|
||||
assert!(!dst.path().join("skip.tmp").exists(), "skip.tmp should be excluded");
|
||||
assert!(
|
||||
dst.path().join("keep.bin").exists(),
|
||||
"keep.bin should be transferred"
|
||||
);
|
||||
assert!(
|
||||
!dst.path().join("skip.tmp").exists(),
|
||||
"skip.tmp should be excluded"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -356,7 +383,11 @@ fn fs_sync_large_file_roundtrip() {
|
||||
let dst = TempDir::new().unwrap();
|
||||
|
||||
let data: Vec<u8> = (0..8 * 1024 * 1024)
|
||||
.map(|i| (i as u64).wrapping_mul(6364136223846793005).wrapping_shr(40) as u8)
|
||||
.map(|i| {
|
||||
(i as u64)
|
||||
.wrapping_mul(6364136223846793005)
|
||||
.wrapping_shr(40) as u8
|
||||
})
|
||||
.collect();
|
||||
fs::write(src.path().join("large.bin"), &data).unwrap();
|
||||
|
||||
@@ -376,7 +407,11 @@ fn fs_sync_append_to_file_sends_minimal_chunks() {
|
||||
let dst = TempDir::new().unwrap();
|
||||
|
||||
let base: Vec<u8> = (0..256 * 1024)
|
||||
.map(|i| (i as u64).wrapping_mul(6364136223846793005).wrapping_shr(48) as u8)
|
||||
.map(|i| {
|
||||
(i as u64)
|
||||
.wrapping_mul(6364136223846793005)
|
||||
.wrapping_shr(48) as u8
|
||||
})
|
||||
.collect();
|
||||
fs::write(dst.path().join("model.bin"), &base).unwrap();
|
||||
|
||||
|
||||
@@ -99,10 +99,13 @@ fn run_hdf5_sync(local: &Path, addr: SocketAddr, rel_path: &str, delete: bool) {
|
||||
fn hdf5_sync_cold_copy() {
|
||||
let src = TempDir::new().unwrap();
|
||||
let local = src.path().join("model.h5");
|
||||
write_h5(&local, &[
|
||||
write_h5(
|
||||
&local,
|
||||
&[
|
||||
("weights", vec![1.0, 2.0, 3.0, 4.0]),
|
||||
("biases", vec![0.1, 0.2]),
|
||||
]);
|
||||
],
|
||||
);
|
||||
|
||||
let (mut server, addr, serve_dir) = spawn_server(false);
|
||||
run_hdf5_sync(&local, addr, "model.h5", false);
|
||||
@@ -150,19 +153,19 @@ fn hdf5_sync_warm_noop() {
|
||||
fn hdf5_sync_incremental_modified_dataset() {
|
||||
let src = TempDir::new().unwrap();
|
||||
let local = src.path().join("model.h5");
|
||||
write_h5(&local, &[
|
||||
("weights", vec![1.0, 2.0]),
|
||||
("biases", vec![0.5]),
|
||||
]);
|
||||
write_h5(
|
||||
&local,
|
||||
&[("weights", vec![1.0, 2.0]), ("biases", vec![0.5])],
|
||||
);
|
||||
|
||||
let (mut server, addr, serve_dir) = spawn_server(false);
|
||||
run_hdf5_sync(&local, addr, "model.h5", false);
|
||||
|
||||
// Modify weights; biases unchanged.
|
||||
write_h5(&local, &[
|
||||
("weights", vec![9.0, 9.0]),
|
||||
("biases", vec![0.5]),
|
||||
]);
|
||||
write_h5(
|
||||
&local,
|
||||
&[("weights", vec![9.0, 9.0]), ("biases", vec![0.5])],
|
||||
);
|
||||
run_hdf5_sync(&local, addr, "model.h5", false);
|
||||
server.kill().unwrap();
|
||||
server.wait().unwrap();
|
||||
@@ -201,7 +204,10 @@ fn hdf5_sync_added_dataset() {
|
||||
let synced = serve_dir.path().join("model.h5");
|
||||
let m_server = DatasetManifest::from_path(&synced).unwrap();
|
||||
assert!(m_server.get("/alpha").is_some(), "alpha should exist");
|
||||
assert!(m_server.get("/beta").is_some(), "beta should have been added");
|
||||
assert!(
|
||||
m_server.get("/beta").is_some(),
|
||||
"beta should have been added"
|
||||
);
|
||||
}
|
||||
|
||||
/// Delete with `--delete`: dataset absent from client → removed from server.
|
||||
@@ -223,7 +229,10 @@ fn hdf5_sync_delete_removes_server_only_dataset() {
|
||||
let synced = serve_dir.path().join("model.h5");
|
||||
let m_server = DatasetManifest::from_path(&synced).unwrap();
|
||||
assert!(m_server.get("/keep").is_some(), "keep should remain");
|
||||
assert!(m_server.get("/drop").is_none(), "drop should have been removed");
|
||||
assert!(
|
||||
m_server.get("/drop").is_none(),
|
||||
"drop should have been removed"
|
||||
);
|
||||
}
|
||||
|
||||
/// No-delete: dataset absent from client is preserved on server when `--delete` not set.
|
||||
@@ -245,7 +254,10 @@ fn hdf5_sync_no_delete_preserves_server_dataset() {
|
||||
let synced = serve_dir.path().join("model.h5");
|
||||
let m_server = DatasetManifest::from_path(&synced).unwrap();
|
||||
assert!(m_server.get("/keep").is_some(), "keep should remain");
|
||||
assert!(m_server.get("/extra").is_some(), "extra should be preserved (no --delete)");
|
||||
assert!(
|
||||
m_server.get("/extra").is_some(),
|
||||
"extra should be preserved (no --delete)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Nested path: server creates subdirectory as needed.
|
||||
|
||||
@@ -64,7 +64,10 @@ fn make_large_h5(dir: &TempDir, name: &str, n_pages: usize, fill: u8) -> PathBuf
|
||||
let path = dir.path().join(name);
|
||||
let mut data = Vec::with_capacity(n_pages * PAGE_SIZE);
|
||||
data.extend_from_slice(H5_MAGIC);
|
||||
data.extend(std::iter::repeat_n(fill, n_pages * PAGE_SIZE - H5_MAGIC.len()));
|
||||
data.extend(std::iter::repeat_n(
|
||||
fill,
|
||||
n_pages * PAGE_SIZE - H5_MAGIC.len(),
|
||||
));
|
||||
std::fs::write(&path, &data).unwrap();
|
||||
path
|
||||
}
|
||||
@@ -93,7 +96,10 @@ fn make_versioned_large_h5(
|
||||
// Base file: n_pages worth of data
|
||||
let mut base = Vec::with_capacity(n_pages * PAGE_SIZE);
|
||||
base.extend_from_slice(H5_MAGIC);
|
||||
base.extend(std::iter::repeat_n(0u8, n_pages * PAGE_SIZE - H5_MAGIC.len()));
|
||||
base.extend(std::iter::repeat_n(
|
||||
0u8,
|
||||
n_pages * PAGE_SIZE - H5_MAGIC.len(),
|
||||
));
|
||||
std::fs::write(&path, &base).unwrap();
|
||||
|
||||
let mut onion = OnionFile::create(&path, PAGE_SIZE as u32).unwrap();
|
||||
@@ -104,7 +110,9 @@ fn make_versioned_large_h5(
|
||||
let page_idx = (rev * pages_per_rev + p) % n_pages;
|
||||
s.record_page((page_idx * PAGE_SIZE) as u64, &vec![fill; PAGE_SIZE]);
|
||||
}
|
||||
onion.commit_session(s, Some(&format!("rev {rev}"))).unwrap();
|
||||
onion
|
||||
.commit_session(s, Some(&format!("rev {rev}")))
|
||||
.unwrap();
|
||||
}
|
||||
onion.flush().unwrap();
|
||||
path
|
||||
@@ -165,7 +173,10 @@ fn run_rsync(src: &Path, dst: &Path) -> RsyncStats {
|
||||
assert!(output.status.success(), "rsync failed: {}", output.status);
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let mut stats = RsyncStats { elapsed, ..Default::default() };
|
||||
let mut stats = RsyncStats {
|
||||
elapsed,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
for line in stdout.lines() {
|
||||
let line = line.trim();
|
||||
@@ -227,8 +238,8 @@ fn start_clawsync_server(h5_path: &Path) -> (std::process::Child, String) {
|
||||
let mut line = String::new();
|
||||
reader.read_line(&mut line).expect("server startup line");
|
||||
|
||||
let addr = parse_serve_addr(&line)
|
||||
.unwrap_or_else(|| panic!("could not parse addr from: {line:?}"));
|
||||
let addr =
|
||||
parse_serve_addr(&line).unwrap_or_else(|| panic!("could not parse addr from: {line:?}"));
|
||||
|
||||
// Drain remaining stdout so server doesn't get SIGPIPE.
|
||||
std::thread::spawn(move || {
|
||||
@@ -255,11 +266,17 @@ fn run_clawsync_push(src_h5: &Path, dst_h5: &Path) -> ClawStats {
|
||||
server.kill().ok();
|
||||
server.wait().ok();
|
||||
|
||||
assert!(output.status.success(), "clawsync push failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr));
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"clawsync push failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let mut stats = ClawStats { elapsed, ..Default::default() };
|
||||
let mut stats = ClawStats {
|
||||
elapsed,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// "Push complete: N revision(s), M bytes"
|
||||
if let Some(line) = stdout.lines().find(|l| l.starts_with("Push complete:")) {
|
||||
@@ -392,7 +409,13 @@ fn clawsync_vs_rsync_comparison() {
|
||||
|
||||
let rs = run_rsync(&src, &dst_rsync);
|
||||
let cs = run_clawsync_push(&src_claw, &dst_claw);
|
||||
print_row("incremental, 1 page changed (0.4%)", &rs, &cs, n_pages, total);
|
||||
print_row(
|
||||
"incremental, 1 page changed (0.4%)",
|
||||
&rs,
|
||||
&cs,
|
||||
n_pages,
|
||||
total,
|
||||
);
|
||||
}
|
||||
|
||||
// ── 4. Incremental: 25 pages changed (10%) ─────────────────────────────
|
||||
@@ -458,7 +481,10 @@ fn clawsync_vs_rsync_comparison() {
|
||||
|
||||
print_row(
|
||||
&format!("history: {n_revs} revisions (1 page/rev)"),
|
||||
&rs, &cs, n_pages, total,
|
||||
&rs,
|
||||
&cs,
|
||||
n_pages,
|
||||
total,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -480,7 +506,10 @@ fn clawsync_vs_rsync_comparison() {
|
||||
|
||||
print_row(
|
||||
&format!("history: {n_revs} revisions (1 page/rev)"),
|
||||
&rs, &cs, n_pages, total,
|
||||
&rs,
|
||||
&cs,
|
||||
n_pages,
|
||||
total,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -528,7 +557,10 @@ fn make_large_file_small_sidecar(
|
||||
let path = dir.path().join(name);
|
||||
let mut data = Vec::with_capacity(n_pages * PAGE_SIZE);
|
||||
data.extend_from_slice(H5_MAGIC);
|
||||
data.extend(std::iter::repeat_n(fill, n_pages * PAGE_SIZE - H5_MAGIC.len()));
|
||||
data.extend(std::iter::repeat_n(
|
||||
fill,
|
||||
n_pages * PAGE_SIZE - H5_MAGIC.len(),
|
||||
));
|
||||
std::fs::write(&path, &data).unwrap();
|
||||
|
||||
let mut onion = OnionFile::create(&path, PAGE_SIZE as u32).unwrap();
|
||||
@@ -536,7 +568,9 @@ fn make_large_file_small_sidecar(
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
// Only record page 0 — keeps the sidecar tiny regardless of file size.
|
||||
s.record_page(0, &vec![(rev as u8).wrapping_add(1); PAGE_SIZE]);
|
||||
onion.commit_session(s, Some(&format!("rev {rev}"))).unwrap();
|
||||
onion
|
||||
.commit_session(s, Some(&format!("rev {rev}")))
|
||||
.unwrap();
|
||||
}
|
||||
onion.flush().unwrap();
|
||||
path
|
||||
@@ -564,7 +598,10 @@ fn clawsync_preflight_scaling() {
|
||||
"| {:<20} | {:>10} | {:>13} | {:>13} | {:>9} |",
|
||||
"File size", "Revisions", "rsync no-op", "claw no-op", "Speedup"
|
||||
);
|
||||
println!("|{:-<22}|{:-<12}|{:-<15}|{:-<15}|{:-<11}|", "", "", "", "", "");
|
||||
println!(
|
||||
"|{:-<22}|{:-<12}|{:-<15}|{:-<15}|{:-<11}|",
|
||||
"", "", "", "", ""
|
||||
);
|
||||
|
||||
// (n_pages, n_revisions): keep n_revisions small (1) so IBLT sketch is constant.
|
||||
// Only n_pages grows — that's what drives rsync cost.
|
||||
@@ -584,7 +621,10 @@ fn clawsync_preflight_scaling() {
|
||||
{
|
||||
let mut data = Vec::with_capacity(n_pages * PAGE_SIZE);
|
||||
data.extend_from_slice(H5_MAGIC);
|
||||
data.extend(std::iter::repeat_n(0xABu8, n_pages * PAGE_SIZE - H5_MAGIC.len()));
|
||||
data.extend(std::iter::repeat_n(
|
||||
0xABu8,
|
||||
n_pages * PAGE_SIZE - H5_MAGIC.len(),
|
||||
));
|
||||
std::fs::write(&src_h5, &data).unwrap();
|
||||
std::fs::write(&dst_h5, &data).unwrap();
|
||||
}
|
||||
@@ -671,7 +711,11 @@ fn clawsync_unique_features() {
|
||||
.lines()
|
||||
.filter(|l| l.contains("experiment rev"))
|
||||
.collect();
|
||||
assert_eq!(rev_lines.len(), 5, "expected 5 revision lines, got: {revs_output}");
|
||||
assert_eq!(
|
||||
rev_lines.len(),
|
||||
5,
|
||||
"expected 5 revision lines, got: {revs_output}"
|
||||
);
|
||||
println!("rsync equivalent: **none** (rsync tracks no history)");
|
||||
println!();
|
||||
|
||||
@@ -697,7 +741,10 @@ fn clawsync_unique_features() {
|
||||
println!("Exported revision 2 → {} bytes", export_bytes);
|
||||
// Exported file starts with HDF5 magic.
|
||||
let exported = std::fs::read(&export_path).unwrap();
|
||||
assert!(exported.starts_with(H5_MAGIC), "exported file missing HDF5 magic");
|
||||
assert!(
|
||||
exported.starts_with(H5_MAGIC),
|
||||
"exported file missing HDF5 magic"
|
||||
);
|
||||
// Page 0 in revision 2 was filled with 2*10+1=21.
|
||||
let page0_byte = exported[PAGE_SIZE]; // skip magic, first full page
|
||||
assert_eq!(page0_byte, 21u8, "exported revision 2 page0 byte mismatch");
|
||||
@@ -714,12 +761,18 @@ fn clawsync_unique_features() {
|
||||
.args(["rollback", h5.to_str().unwrap(), "4"])
|
||||
.output()
|
||||
.expect("rollback to head failed");
|
||||
assert!(out.status.success(), "rollback to head failed: {}",
|
||||
String::from_utf8_lossy(&out.stderr));
|
||||
assert!(
|
||||
out.status.success(),
|
||||
"rollback to head failed: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
let head_bytes = std::fs::read(&h5).unwrap();
|
||||
let head_byte = head_bytes[PAGE_SIZE];
|
||||
// Rev 4: page filled with 4*10+1=41
|
||||
assert_eq!(head_byte, 41u8, "HEAD page byte should be 41 at rev 4, got {head_byte}");
|
||||
assert_eq!(
|
||||
head_byte, 41u8,
|
||||
"HEAD page byte should be 41 at rev 4, got {head_byte}"
|
||||
);
|
||||
println!("HEAD (rev 4) data-page byte = {} ✓", head_byte);
|
||||
|
||||
let out = Command::new(BIN)
|
||||
@@ -735,8 +788,14 @@ fn clawsync_unique_features() {
|
||||
let rolled_bytes = std::fs::read(&h5).unwrap();
|
||||
let rolled_byte = rolled_bytes[PAGE_SIZE];
|
||||
// After rollback to rev 1: page filled with 1*10+1=11
|
||||
assert_eq!(rolled_byte, 11u8, "after rollback to rev 1, data-page byte should be 11, got {rolled_byte}");
|
||||
println!("After rollback to rev 1: data-page byte = {} ✓ (expected 11)", rolled_byte);
|
||||
assert_eq!(
|
||||
rolled_byte, 11u8,
|
||||
"after rollback to rev 1, data-page byte should be 11, got {rolled_byte}"
|
||||
);
|
||||
println!(
|
||||
"After rollback to rev 1: data-page byte = {} ✓ (expected 11)",
|
||||
rolled_byte
|
||||
);
|
||||
println!("rsync equivalent: **none** (rsync cannot revert to a past state)");
|
||||
println!();
|
||||
|
||||
|
||||
@@ -93,12 +93,7 @@ fn parse_serve_addr(line: &str) -> Option<String> {
|
||||
/// pipe error when it tries to print subsequent log lines.
|
||||
fn start_server(h5_path: &PathBuf) -> (std::process::Child, String) {
|
||||
let mut server = Command::new(BIN)
|
||||
.args([
|
||||
"serve",
|
||||
h5_path.to_str().unwrap(),
|
||||
"--bind",
|
||||
"127.0.0.1:0",
|
||||
])
|
||||
.args(["serve", h5_path.to_str().unwrap(), "--bind", "127.0.0.1:0"])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
@@ -213,7 +208,8 @@ fn subprocess_pull_fetches_all_revisions_from_server() {
|
||||
|
||||
let reloaded = OnionFile::open(&client_h5).expect("cannot open client onion after pull");
|
||||
assert_eq!(
|
||||
reloaded.revision_count(), 4,
|
||||
reloaded.revision_count(),
|
||||
4,
|
||||
"client should have all 4 pulled revisions"
|
||||
);
|
||||
|
||||
@@ -231,7 +227,11 @@ fn subprocess_list_revisions_output_is_correct() {
|
||||
.output()
|
||||
.expect("failed to run clawsync list-revisions");
|
||||
|
||||
assert!(output.status.success(), "list-revisions exited with {}", output.status);
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"list-revisions exited with {}",
|
||||
output.status
|
||||
);
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
assert!(
|
||||
@@ -255,7 +255,11 @@ fn subprocess_list_revisions_timestamp_is_iso_date() {
|
||||
.output()
|
||||
.expect("failed to run clawsync list-revisions");
|
||||
|
||||
assert!(output.status.success(), "list-revisions exited with {}", output.status);
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"list-revisions exited with {}",
|
||||
output.status
|
||||
);
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
assert!(
|
||||
@@ -311,7 +315,11 @@ fn subprocess_rollback_restores_h5_content_and_preserves_history() {
|
||||
|
||||
// Sidecar history must be fully preserved — all 4 revisions still readable.
|
||||
let onion = OnionFile::open(&h5).expect("cannot open onion after rollback");
|
||||
assert_eq!(onion.revision_count(), 4, "rollback must not delete sidecar history");
|
||||
assert_eq!(
|
||||
onion.revision_count(),
|
||||
4,
|
||||
"rollback must not delete sidecar history"
|
||||
);
|
||||
|
||||
// The h5 file must reflect the page content at revision 1.
|
||||
// make_versioned_h5 writes page 0 = vec![i; 4096] for each revision i,
|
||||
@@ -340,7 +348,11 @@ fn subprocess_gc_keep_last_prunes_old_revisions() {
|
||||
assert!(status.success(), "gc exited with {status}");
|
||||
|
||||
let onion = OnionFile::open(&h5).expect("cannot open onion after gc");
|
||||
assert_eq!(onion.revision_count(), 2, "gc --keep-last 2 should leave exactly 2 revisions");
|
||||
assert_eq!(
|
||||
onion.revision_count(),
|
||||
2,
|
||||
"gc --keep-last 2 should leave exactly 2 revisions"
|
||||
);
|
||||
}
|
||||
|
||||
/// `clawsync export-revision`: exports a specific revision to a new file
|
||||
@@ -369,7 +381,11 @@ fn subprocess_export_revision_creates_standalone_file() {
|
||||
|
||||
// Source sidecar must be unchanged — still 3 revisions.
|
||||
let onion = OnionFile::open(&h5).expect("cannot open source onion after export");
|
||||
assert_eq!(onion.revision_count(), 3, "source sidecar must not be modified by export");
|
||||
assert_eq!(
|
||||
onion.revision_count(),
|
||||
3,
|
||||
"source sidecar must not be modified by export"
|
||||
);
|
||||
}
|
||||
|
||||
/// `clawsync snapshot`: creates a snapshot checkpoint revision, incrementing
|
||||
@@ -379,13 +395,22 @@ fn subprocess_snapshot_increments_revision_count() {
|
||||
let (_dir, h5, _) = make_versioned_h5(3);
|
||||
|
||||
let status = Command::new(BIN)
|
||||
.args(["snapshot", h5.to_str().unwrap(), "--message", "test-snapshot"])
|
||||
.args([
|
||||
"snapshot",
|
||||
h5.to_str().unwrap(),
|
||||
"--message",
|
||||
"test-snapshot",
|
||||
])
|
||||
.status()
|
||||
.expect("failed to run clawsync snapshot");
|
||||
assert!(status.success(), "snapshot exited with {status}");
|
||||
|
||||
let onion = OnionFile::open(&h5).expect("cannot open onion after snapshot");
|
||||
assert_eq!(onion.revision_count(), 4, "snapshot should add exactly one revision");
|
||||
assert_eq!(
|
||||
onion.revision_count(),
|
||||
4,
|
||||
"snapshot should add exactly one revision"
|
||||
);
|
||||
}
|
||||
|
||||
/// `clawsync branch create / list / rename / delete`: full lifecycle in order.
|
||||
@@ -406,7 +431,11 @@ fn subprocess_branch_lifecycle() {
|
||||
.args(["branch", "list", h5_str])
|
||||
.output()
|
||||
.expect("failed to run branch list");
|
||||
assert!(output.status.success(), "branch list exited with {}", output.status);
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"branch list exited with {}",
|
||||
output.status
|
||||
);
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
assert!(
|
||||
stdout.contains("experiment"),
|
||||
@@ -427,8 +456,14 @@ fn subprocess_branch_lifecycle() {
|
||||
.expect("failed to run branch list after rename");
|
||||
assert!(output.status.success());
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
assert!(stdout.contains("feature-x"), "expected 'feature-x' after rename:\n{stdout}");
|
||||
assert!(!stdout.contains("experiment"), "old name 'experiment' must be gone:\n{stdout}");
|
||||
assert!(
|
||||
stdout.contains("feature-x"),
|
||||
"expected 'feature-x' after rename:\n{stdout}"
|
||||
);
|
||||
assert!(
|
||||
!stdout.contains("experiment"),
|
||||
"old name 'experiment' must be gone:\n{stdout}"
|
||||
);
|
||||
|
||||
// Delete "feature-x".
|
||||
let status = Command::new(BIN)
|
||||
@@ -444,7 +479,10 @@ fn subprocess_branch_lifecycle() {
|
||||
.expect("failed to run branch list after delete");
|
||||
assert!(output.status.success());
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
assert!(!stdout.contains("feature-x"), "deleted branch must not appear in list:\n{stdout}");
|
||||
assert!(
|
||||
!stdout.contains("feature-x"),
|
||||
"deleted branch must not appear in list:\n{stdout}"
|
||||
);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -483,13 +521,20 @@ fn subprocess_push_delta_transfers_exact_count() {
|
||||
.output()
|
||||
.expect("failed to run clawsync push");
|
||||
|
||||
assert!(output.status.success(), "push exited with {}", output.status);
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"push exited with {}",
|
||||
output.status
|
||||
);
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let transferred = parse_push_revision_count(&stdout)
|
||||
.unwrap_or_else(|| panic!("could not parse push count from:\n{stdout}"));
|
||||
|
||||
assert_eq!(transferred, 4, "expected exactly 4 delta revisions, got {transferred}");
|
||||
assert_eq!(
|
||||
transferred, 4,
|
||||
"expected exactly 4 delta revisions, got {transferred}"
|
||||
);
|
||||
|
||||
// Server must eventually reach 7 revisions total.
|
||||
let count = await_revisions(&server_h5, 7);
|
||||
@@ -515,7 +560,9 @@ fn subprocess_pull_delta_transfers_exact_count() {
|
||||
for i in 0u8..2 {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![i; 4096]);
|
||||
onion.commit_session(s, Some(&format!("local {i}"))).unwrap();
|
||||
onion
|
||||
.commit_session(s, Some(&format!("local {i}")))
|
||||
.unwrap();
|
||||
}
|
||||
onion.flush().unwrap();
|
||||
}
|
||||
@@ -527,20 +574,31 @@ fn subprocess_pull_delta_transfers_exact_count() {
|
||||
.output()
|
||||
.expect("failed to run clawsync pull");
|
||||
|
||||
assert!(output.status.success(), "pull exited with {}", output.status);
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"pull exited with {}",
|
||||
output.status
|
||||
);
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let received = parse_pull_revision_count(&stdout)
|
||||
.unwrap_or_else(|| panic!("could not parse pull count from:\n{stdout}"));
|
||||
|
||||
assert_eq!(received, 4, "expected exactly 4 delta revisions, got {received}");
|
||||
assert_eq!(
|
||||
received, 4,
|
||||
"expected exactly 4 delta revisions, got {received}"
|
||||
);
|
||||
|
||||
server.kill().ok();
|
||||
server.wait().ok();
|
||||
|
||||
// Client must end up with 6 revisions (2 original + 4 pulled).
|
||||
let onion = OnionFile::open(&client_h5).expect("cannot open client onion after pull");
|
||||
assert_eq!(onion.revision_count(), 6, "client should have 6 revisions after delta pull");
|
||||
assert_eq!(
|
||||
onion.revision_count(),
|
||||
6,
|
||||
"client should have 6 revisions after delta pull"
|
||||
);
|
||||
|
||||
let _ = (server_dir, client_dir);
|
||||
}
|
||||
@@ -571,18 +629,28 @@ fn subprocess_push_large_delta_correct_count() {
|
||||
.output()
|
||||
.expect("failed to run clawsync push");
|
||||
|
||||
assert!(output.status.success(), "push exited with {}", output.status);
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"push exited with {}",
|
||||
output.status
|
||||
);
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let transferred = parse_push_revision_count(&stdout)
|
||||
.unwrap_or_else(|| panic!("could not parse push count from:\n{stdout}"));
|
||||
|
||||
assert_eq!(transferred, 40, "expected exactly 40 delta revisions, got {transferred}");
|
||||
assert_eq!(
|
||||
transferred, 40,
|
||||
"expected exactly 40 delta revisions, got {transferred}"
|
||||
);
|
||||
|
||||
let count = await_revisions(&server_h5, 50);
|
||||
|
||||
server.kill().ok();
|
||||
server.wait().ok();
|
||||
|
||||
assert_eq!(count, 50, "server should have 50 revisions after large delta push");
|
||||
assert_eq!(
|
||||
count, 50,
|
||||
"server should have 50 revisions after large delta push"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -35,12 +35,7 @@ pub fn chunk_data(data: &[u8]) -> Vec<Chunk> {
|
||||
}
|
||||
|
||||
/// Split `data` into content-defined chunks with explicit size parameters.
|
||||
pub fn chunk_data_sized(
|
||||
data: &[u8],
|
||||
min_size: u32,
|
||||
avg_size: u32,
|
||||
max_size: u32,
|
||||
) -> Vec<Chunk> {
|
||||
pub fn chunk_data_sized(data: &[u8], min_size: u32, avg_size: u32, max_size: u32) -> Vec<Chunk> {
|
||||
if data.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
@@ -152,8 +147,14 @@ mod tests {
|
||||
for (i, c) in chunks.iter().enumerate() {
|
||||
let is_last = i == chunks.len() - 1;
|
||||
if !is_last {
|
||||
assert!(c.length >= MIN_CHUNK_SIZE as usize, "chunk below min at {i}");
|
||||
assert!(c.length <= MAX_CHUNK_SIZE as usize, "chunk above max at {i}");
|
||||
assert!(
|
||||
c.length >= MIN_CHUNK_SIZE as usize,
|
||||
"chunk below min at {i}"
|
||||
);
|
||||
assert!(
|
||||
c.length <= MAX_CHUNK_SIZE as usize,
|
||||
"chunk above max at {i}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -189,7 +190,10 @@ mod tests {
|
||||
let chunks = chunk_data(&data);
|
||||
let source_set = chunk_hash_set(&chunks);
|
||||
let diff = changed_chunks(&source_set, &chunks);
|
||||
assert!(diff.is_empty(), "identical data should have no changed chunks");
|
||||
assert!(
|
||||
diff.is_empty(),
|
||||
"identical data should have no changed chunks"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -63,10 +63,7 @@ pub fn blake3_hash_large(data: &[u8]) -> [u8; 32] {
|
||||
/// order and page offsets must be incorporated (use `hash_pages` instead).
|
||||
pub fn blake3_hash_batch(chunks: &[&[u8]]) -> Vec<[u8; 32]> {
|
||||
use rayon::prelude::*;
|
||||
chunks
|
||||
.par_iter()
|
||||
.map(|c| blake3_hash_large(c))
|
||||
.collect()
|
||||
chunks.par_iter().map(|c| blake3_hash_large(c)).collect()
|
||||
}
|
||||
|
||||
/// Verify that `data` matches an expected BLAKE3 digest.
|
||||
|
||||
@@ -34,8 +34,7 @@ pub fn lz4_compress(data: &[u8]) -> Result<Vec<u8>, CoreError> {
|
||||
|
||||
/// Decompress lz4-compressed `data` (expects prepended original size).
|
||||
pub fn lz4_decompress(data: &[u8]) -> Result<Vec<u8>, CoreError> {
|
||||
lz4_flex::decompress_size_prepended(data)
|
||||
.map_err(|e| CoreError::Decompress(e.to_string()))
|
||||
lz4_flex::decompress_size_prepended(data).map_err(|e| CoreError::Decompress(e.to_string()))
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -64,10 +63,7 @@ mod tests {
|
||||
fn zstd_compress_reduces_size() {
|
||||
let data = vec![0u8; 65536]; // highly compressible
|
||||
let compressed = zstd_compress(&data, ZSTD_DEFAULT_LEVEL).unwrap();
|
||||
assert!(
|
||||
compressed.len() < data.len(),
|
||||
"zstd should compress zeroes"
|
||||
);
|
||||
assert!(compressed.len() < data.len(), "zstd should compress zeroes");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -58,7 +58,11 @@ pub enum DeltaOp {
|
||||
/// This produces a simple but correct delta — it does not do rolling-hash
|
||||
/// sliding-window matching (that is CDC's job). For block-aligned page data
|
||||
/// (the primary use case) this is optimal.
|
||||
pub fn compute_delta(source_hashes: &[BlockHash], target: &[u8], block_size: usize) -> Vec<DeltaOp> {
|
||||
pub fn compute_delta(
|
||||
source_hashes: &[BlockHash],
|
||||
target: &[u8],
|
||||
block_size: usize,
|
||||
) -> Vec<DeltaOp> {
|
||||
// Build a hash → offset map for O(1) lookups
|
||||
let lookup: HashMap<u64, usize> = source_hashes
|
||||
.iter()
|
||||
@@ -74,21 +78,28 @@ pub fn compute_delta(source_hashes: &[BlockHash], target: &[u8], block_size: usi
|
||||
if let Some(&src_offset) = lookup.get(&hash) {
|
||||
// Coalesce adjacent copies
|
||||
if let Some(DeltaOp::Copy { offset, length }) = ops.last_mut() {
|
||||
if *offset + *length == src_offset && target_offset == {
|
||||
if *offset + *length == src_offset
|
||||
&& target_offset == {
|
||||
let prev_target_block = i.saturating_sub(1) * block_size;
|
||||
prev_target_block + block_size
|
||||
} {
|
||||
}
|
||||
{
|
||||
*length += chunk.len();
|
||||
continue;
|
||||
}
|
||||
}
|
||||
ops.push(DeltaOp::Copy { offset: src_offset, length: chunk.len() });
|
||||
ops.push(DeltaOp::Copy {
|
||||
offset: src_offset,
|
||||
length: chunk.len(),
|
||||
});
|
||||
} else {
|
||||
// Coalesce adjacent inserts
|
||||
if let Some(DeltaOp::Insert { data }) = ops.last_mut() {
|
||||
data.extend_from_slice(chunk);
|
||||
} else {
|
||||
ops.push(DeltaOp::Insert { data: chunk.to_vec() });
|
||||
ops.push(DeltaOp::Insert {
|
||||
data: chunk.to_vec(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -149,7 +160,10 @@ mod tests {
|
||||
"identical data should produce only Copy ops"
|
||||
);
|
||||
let transfer = delta_transfer_size(&ops);
|
||||
assert_eq!(transfer, 0, "no bytes should be transferred for identical data");
|
||||
assert_eq!(
|
||||
transfer, 0,
|
||||
"no bytes should be transferred for identical data"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -10,11 +10,11 @@
|
||||
//! - [`compress`]: zstd and lz4 streaming wrappers
|
||||
//! - [`error`]: [`CoreError`] type
|
||||
|
||||
#[forbid(unsafe_code)]
|
||||
pub mod checksum;
|
||||
#[forbid(unsafe_code)]
|
||||
pub mod cdc;
|
||||
#[forbid(unsafe_code)]
|
||||
pub mod checksum;
|
||||
#[forbid(unsafe_code)]
|
||||
pub mod compress;
|
||||
#[forbid(unsafe_code)]
|
||||
pub mod delta;
|
||||
|
||||
@@ -49,7 +49,11 @@ const fn make_gear() -> [u64; 256] {
|
||||
state ^= state >> 7;
|
||||
state ^= state << 17;
|
||||
// ensure non-zero so hot bytes always change the hash
|
||||
t[i] = if state == 0 { 0x0BAD_CAFE_DEAD_BEEFu64 } else { state };
|
||||
t[i] = if state == 0 {
|
||||
0x0BAD_CAFE_DEAD_BEEFu64
|
||||
} else {
|
||||
state
|
||||
};
|
||||
i += 1;
|
||||
}
|
||||
t
|
||||
@@ -82,7 +86,12 @@ pub fn chunk_data_simd(data: &[u8]) -> Vec<Chunk> {
|
||||
}
|
||||
|
||||
/// Split `data` with explicit size parameters.
|
||||
pub fn chunk_data_sized_simd(data: &[u8], min_size: u32, avg_size: u32, max_size: u32) -> Vec<Chunk> {
|
||||
pub fn chunk_data_sized_simd(
|
||||
data: &[u8],
|
||||
min_size: u32,
|
||||
avg_size: u32,
|
||||
max_size: u32,
|
||||
) -> Vec<Chunk> {
|
||||
let min = min_size as usize;
|
||||
let avg = avg_size as usize;
|
||||
let max = max_size as usize;
|
||||
@@ -222,7 +231,17 @@ mod neon {
|
||||
// Hot byte present, at/past min_size, or approaching max:
|
||||
// scalar fallback handles boundaries correctly.
|
||||
let window = &data[pos..pos + 16];
|
||||
scalar_window(data, window, pos, &mut chunk_start, &mut h, min, max, mask, &mut chunks);
|
||||
scalar_window(
|
||||
data,
|
||||
window,
|
||||
pos,
|
||||
&mut chunk_start,
|
||||
&mut h,
|
||||
min,
|
||||
max,
|
||||
mask,
|
||||
&mut chunks,
|
||||
);
|
||||
pos += 16;
|
||||
}
|
||||
}
|
||||
@@ -230,7 +249,17 @@ mod neon {
|
||||
// Tail bytes (< 16 remaining)
|
||||
if pos < data.len() {
|
||||
let window = &data[pos..];
|
||||
scalar_window(data, window, pos, &mut chunk_start, &mut h, min, max, mask, &mut chunks);
|
||||
scalar_window(
|
||||
data,
|
||||
window,
|
||||
pos,
|
||||
&mut chunk_start,
|
||||
&mut h,
|
||||
min,
|
||||
max,
|
||||
mask,
|
||||
&mut chunks,
|
||||
);
|
||||
}
|
||||
|
||||
if chunk_start < data.len() {
|
||||
@@ -292,14 +321,34 @@ mod sse2 {
|
||||
pos += 16;
|
||||
} else {
|
||||
let window = &data[pos..pos + 16];
|
||||
scalar_window(data, window, pos, &mut chunk_start, &mut h, min, max, mask, &mut chunks);
|
||||
scalar_window(
|
||||
data,
|
||||
window,
|
||||
pos,
|
||||
&mut chunk_start,
|
||||
&mut h,
|
||||
min,
|
||||
max,
|
||||
mask,
|
||||
&mut chunks,
|
||||
);
|
||||
pos += 16;
|
||||
}
|
||||
}
|
||||
|
||||
if pos < data.len() {
|
||||
let window = &data[pos..];
|
||||
scalar_window(data, window, pos, &mut chunk_start, &mut h, min, max, mask, &mut chunks);
|
||||
scalar_window(
|
||||
data,
|
||||
window,
|
||||
pos,
|
||||
&mut chunk_start,
|
||||
&mut h,
|
||||
min,
|
||||
max,
|
||||
mask,
|
||||
&mut chunks,
|
||||
);
|
||||
}
|
||||
|
||||
if chunk_start < data.len() {
|
||||
@@ -410,12 +459,7 @@ mod tests {
|
||||
#[test]
|
||||
fn chunk_sizes_respect_min_max_except_last() {
|
||||
let data = lcg_data(500_000, 7);
|
||||
let chunks = chunk_data_sized_simd(
|
||||
&data,
|
||||
MIN_CHUNK_SIZE,
|
||||
AVG_CHUNK_SIZE,
|
||||
MAX_CHUNK_SIZE,
|
||||
);
|
||||
let chunks = chunk_data_sized_simd(&data, MIN_CHUNK_SIZE, AVG_CHUNK_SIZE, MAX_CHUNK_SIZE);
|
||||
for (i, c) in chunks.iter().enumerate() {
|
||||
let is_last = i == chunks.len() - 1;
|
||||
if !is_last {
|
||||
@@ -511,7 +555,11 @@ mod tests {
|
||||
let data = lcg_data(200_000, 77);
|
||||
for c in chunk_data_simd(&data) {
|
||||
let expected = crate::checksum::xxh3_block(&data[c.offset..c.offset + c.length]);
|
||||
assert_eq!(c.hash, expected, "chunk hash mismatch at offset {}", c.offset);
|
||||
assert_eq!(
|
||||
c.hash, expected,
|
||||
"chunk hash mismatch at offset {}",
|
||||
c.offset
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,9 +9,9 @@
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use clawsync_core::cdc::{chunk_data, Chunk};
|
||||
use clawsync_core::cdc::{Chunk, chunk_data};
|
||||
use clawsync_core::checksum::blake3_hash_large;
|
||||
use clawsync_core::compress::{zstd_compress, zstd_decompress, ZSTD_DEFAULT_LEVEL};
|
||||
use clawsync_core::compress::{ZSTD_DEFAULT_LEVEL, zstd_compress, zstd_decompress};
|
||||
use clawsync_transport::protocol::{FsChunk, FsChunkHash};
|
||||
|
||||
use crate::error::FsSyncError;
|
||||
@@ -40,7 +40,10 @@ pub fn chunk_file_for_request(data: &[u8]) -> (Vec<Chunk>, Vec<FsChunkHash>) {
|
||||
let chunks = chunk_data(data);
|
||||
let hashes = chunks
|
||||
.iter()
|
||||
.map(|c| FsChunkHash { hash: c.hash, length: c.length as u32 })
|
||||
.map(|c| FsChunkHash {
|
||||
hash: c.hash,
|
||||
length: c.length as u32,
|
||||
})
|
||||
.collect();
|
||||
(chunks, hashes)
|
||||
}
|
||||
@@ -72,11 +75,7 @@ pub fn compute_needed_indices(
|
||||
///
|
||||
/// Each chunk is zstd-compressed; raw bytes are used instead when the
|
||||
/// compressed output is not smaller.
|
||||
pub fn build_chunk_data(
|
||||
data: &[u8],
|
||||
all_chunks: &[Chunk],
|
||||
needed_indices: &[u32],
|
||||
) -> Vec<FsChunk> {
|
||||
pub fn build_chunk_data(data: &[u8], all_chunks: &[Chunk], needed_indices: &[u32]) -> Vec<FsChunk> {
|
||||
needed_indices
|
||||
.iter()
|
||||
.map(|&idx| {
|
||||
@@ -90,7 +89,11 @@ pub fn build_chunk_data(
|
||||
_ => (raw.to_vec(), false),
|
||||
};
|
||||
|
||||
FsChunk { hash: c.hash, data: payload, compressed }
|
||||
FsChunk {
|
||||
hash: c.hash,
|
||||
data: payload,
|
||||
compressed,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
@@ -127,9 +130,14 @@ pub fn reconstruct_file(
|
||||
|
||||
for desc in client_chunk_order {
|
||||
if let Some(&src_offset) = server_chunk_map.get(&desc.hash) {
|
||||
plan.push(ReconstructOp::LocalCopy { src_offset, length: desc.length });
|
||||
plan.push(ReconstructOp::LocalCopy {
|
||||
src_offset,
|
||||
length: desc.length,
|
||||
});
|
||||
} else {
|
||||
plan.push(ReconstructOp::Literal { chunk_index: literal_idx });
|
||||
plan.push(ReconstructOp::Literal {
|
||||
chunk_index: literal_idx,
|
||||
});
|
||||
literal_idx += 1;
|
||||
}
|
||||
}
|
||||
@@ -206,8 +214,8 @@ mod tests {
|
||||
|
||||
fn pseudo_random(size: usize, seed: u8) -> Vec<u8> {
|
||||
// Xorshift64 — high entropy, genuinely incompressible output.
|
||||
let mut s: u64 = 0xDEAD_CAFE_0000_0001u64
|
||||
^ (seed as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15);
|
||||
let mut s: u64 =
|
||||
0xDEAD_CAFE_0000_0001u64 ^ (seed as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15);
|
||||
(0..size)
|
||||
.map(|_| {
|
||||
s ^= s << 13;
|
||||
@@ -220,10 +228,7 @@ mod tests {
|
||||
|
||||
/// Run the full CDC delta pipeline: chunk client file, compute needs,
|
||||
/// build literal data, reconstruct, verify BLAKE3.
|
||||
fn full_roundtrip(
|
||||
server_data: &[u8],
|
||||
client_data: &[u8],
|
||||
) -> (Vec<u8>, usize) {
|
||||
fn full_roundtrip(server_data: &[u8], client_data: &[u8]) -> (Vec<u8>, usize) {
|
||||
let expected = blake3_hash_large(client_data);
|
||||
|
||||
let (client_chunks, client_hashes) = chunk_file_for_request(client_data);
|
||||
@@ -252,7 +257,10 @@ mod tests {
|
||||
fn roundtrip_identical_file_sends_zero_chunk_bytes() {
|
||||
let data = pseudo_random(200_000, 1);
|
||||
let (_, literal_bytes) = full_roundtrip(&data, &data);
|
||||
assert_eq!(literal_bytes, 0, "identical file should send 0 literal bytes");
|
||||
assert_eq!(
|
||||
literal_bytes, 0,
|
||||
"identical file should send 0 literal bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -316,7 +324,10 @@ mod tests {
|
||||
let (chunks, _) = chunk_file_for_request(&data);
|
||||
let chunk_data_built = build_chunk_data(&data, &chunks, &[0]);
|
||||
// At least one chunk should be uncompressed (raw) since data is incompressible.
|
||||
assert!(!chunk_data_built[0].compressed, "incompressible chunk should be sent raw");
|
||||
assert!(
|
||||
!chunk_data_built[0].compressed,
|
||||
"incompressible chunk should be sent raw"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -39,8 +39,10 @@ impl FileDiff {
|
||||
///
|
||||
/// Result is sorted by path for deterministic ordering.
|
||||
pub fn diff_manifests(local: &[LocalEntry], remote: &[FsManifestEntry]) -> Vec<FileDiff> {
|
||||
let local_map: HashMap<&str, [u8; 32]> =
|
||||
local.iter().map(|e| (e.rel_path.as_str(), e.blake3)).collect();
|
||||
let local_map: HashMap<&str, [u8; 32]> = local
|
||||
.iter()
|
||||
.map(|e| (e.rel_path.as_str(), e.blake3))
|
||||
.collect();
|
||||
let remote_map: HashMap<&str, [u8; 32]> =
|
||||
remote.iter().map(|e| (e.path.as_str(), e.blake3)).collect();
|
||||
|
||||
@@ -147,14 +149,34 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn mixed_diff_counts_correct() {
|
||||
let l = vec![local("keep.bin", 1), local("new.bin", 3), local("mod.bin", 4)];
|
||||
let r = vec![remote("keep.bin", 1), remote("old.bin", 9), remote("mod.bin", 5)];
|
||||
let l = vec![
|
||||
local("keep.bin", 1),
|
||||
local("new.bin", 3),
|
||||
local("mod.bin", 4),
|
||||
];
|
||||
let r = vec![
|
||||
remote("keep.bin", 1),
|
||||
remote("old.bin", 9),
|
||||
remote("mod.bin", 5),
|
||||
];
|
||||
let diffs = diff_manifests(&l, &r);
|
||||
|
||||
let added: Vec<_> = diffs.iter().filter(|d| matches!(d, FileDiff::Added(_))).collect();
|
||||
let modified: Vec<_> = diffs.iter().filter(|d| matches!(d, FileDiff::Modified(_))).collect();
|
||||
let removed: Vec<_> = diffs.iter().filter(|d| matches!(d, FileDiff::Removed(_))).collect();
|
||||
let unchanged: Vec<_> = diffs.iter().filter(|d| matches!(d, FileDiff::Unchanged(_))).collect();
|
||||
let added: Vec<_> = diffs
|
||||
.iter()
|
||||
.filter(|d| matches!(d, FileDiff::Added(_)))
|
||||
.collect();
|
||||
let modified: Vec<_> = diffs
|
||||
.iter()
|
||||
.filter(|d| matches!(d, FileDiff::Modified(_)))
|
||||
.collect();
|
||||
let removed: Vec<_> = diffs
|
||||
.iter()
|
||||
.filter(|d| matches!(d, FileDiff::Removed(_)))
|
||||
.collect();
|
||||
let unchanged: Vec<_> = diffs
|
||||
.iter()
|
||||
.filter(|d| matches!(d, FileDiff::Unchanged(_)))
|
||||
.collect();
|
||||
|
||||
assert_eq!(added.len(), 1); // new.bin
|
||||
assert_eq!(modified.len(), 1); // mod.bin
|
||||
|
||||
@@ -51,7 +51,9 @@ impl FsManifest {
|
||||
let abs = entry.path().to_path_buf();
|
||||
let rel = abs
|
||||
.strip_prefix(root)
|
||||
.map_err(|_| FsSyncError::Path(format!("cannot strip prefix from {}", abs.display())))?
|
||||
.map_err(|_| {
|
||||
FsSyncError::Path(format!("cannot strip prefix from {}", abs.display()))
|
||||
})?
|
||||
.to_string_lossy()
|
||||
.replace('\\', "/"); // normalise on Windows
|
||||
|
||||
@@ -75,7 +77,13 @@ impl FsManifest {
|
||||
.and_then(|t| t.duration_since(UNIX_EPOCH).ok())
|
||||
.map(|d| d.as_secs() as i64)
|
||||
.unwrap_or(0);
|
||||
Ok(LocalEntry { rel_path: rel, abs_path: abs, blake3, size, mtime })
|
||||
Ok(LocalEntry {
|
||||
rel_path: rel,
|
||||
abs_path: abs,
|
||||
blake3,
|
||||
size,
|
||||
mtime,
|
||||
})
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
@@ -98,9 +106,7 @@ impl FsManifest {
|
||||
|
||||
/// Look up an entry by its relative path.
|
||||
pub fn get(&self, rel_path: &str) -> Option<&LocalEntry> {
|
||||
self.entries
|
||||
.iter()
|
||||
.find(|e| e.rel_path == rel_path)
|
||||
self.entries.iter().find(|e| e.rel_path == rel_path)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,7 +168,11 @@ mod tests {
|
||||
fs::write(dir.path().join("sub").join("nested.bin"), b"nested").unwrap();
|
||||
|
||||
let manifest = FsManifest::build(dir.path(), &empty_excludes()).unwrap();
|
||||
let paths: Vec<&str> = manifest.entries.iter().map(|e| e.rel_path.as_str()).collect();
|
||||
let paths: Vec<&str> = manifest
|
||||
.entries
|
||||
.iter()
|
||||
.map(|e| e.rel_path.as_str())
|
||||
.collect();
|
||||
assert!(paths.contains(&"root.bin"));
|
||||
assert!(paths.contains(&"sub/nested.bin"));
|
||||
}
|
||||
@@ -175,7 +185,11 @@ mod tests {
|
||||
fs::write(dir.path().join("m.bin"), b"m").unwrap();
|
||||
|
||||
let manifest = FsManifest::build(dir.path(), &empty_excludes()).unwrap();
|
||||
let paths: Vec<&str> = manifest.entries.iter().map(|e| e.rel_path.as_str()).collect();
|
||||
let paths: Vec<&str> = manifest
|
||||
.entries
|
||||
.iter()
|
||||
.map(|e| e.rel_path.as_str())
|
||||
.collect();
|
||||
assert_eq!(paths, vec!["a.bin", "m.bin", "z.bin"]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,7 +52,12 @@ pub struct FsSyncClient {
|
||||
|
||||
impl FsSyncClient {
|
||||
pub fn new(conn: SyncPeer, local_root: PathBuf, excludes: GlobSet, delete: bool) -> Self {
|
||||
Self { conn, local_root, excludes, _delete: delete }
|
||||
Self {
|
||||
conn,
|
||||
local_root,
|
||||
excludes,
|
||||
_delete: delete,
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute the full directory sync protocol.
|
||||
@@ -60,7 +65,8 @@ impl FsSyncClient {
|
||||
// ── RTT 1: build local manifest and send it ──────────────────────────
|
||||
let local_root = self.local_root.clone();
|
||||
let excludes = self.excludes.clone();
|
||||
let manifest = tokio::task::spawn_blocking(move || FsManifest::build(&local_root, &excludes))
|
||||
let manifest =
|
||||
tokio::task::spawn_blocking(move || FsManifest::build(&local_root, &excludes))
|
||||
.await
|
||||
.map_err(|e| FsSyncError::Join(e.to_string()))??;
|
||||
|
||||
@@ -72,17 +78,37 @@ impl FsSyncClient {
|
||||
.await?;
|
||||
|
||||
let (needed_files, to_delete) = match self.conn.recv().await? {
|
||||
SyncMessage::FsDirNeed { needed_files, to_delete } => (needed_files, to_delete),
|
||||
SyncMessage::FsDirNeed {
|
||||
needed_files,
|
||||
to_delete,
|
||||
} => (needed_files, to_delete),
|
||||
SyncMessage::Error { message } => return Err(FsSyncError::Protocol(message)),
|
||||
other => return Err(FsSyncError::Protocol(format!("expected FsDirNeed, got {other:?}"))),
|
||||
other => {
|
||||
return Err(FsSyncError::Protocol(format!(
|
||||
"expected FsDirNeed, got {other:?}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
|
||||
// Warm no-op: nothing to do on either side.
|
||||
if needed_files.is_empty() && to_delete.is_empty() {
|
||||
let final_stats = match self.conn.recv().await? {
|
||||
SyncMessage::FsDirComplete { files_added, files_modified, files_removed, bytes_transferred } =>
|
||||
SyncStats { files_added, files_modified, files_removed, bytes_transferred },
|
||||
other => return Err(FsSyncError::Protocol(format!("expected FsDirComplete, got {other:?}"))),
|
||||
SyncMessage::FsDirComplete {
|
||||
files_added,
|
||||
files_modified,
|
||||
files_removed,
|
||||
bytes_transferred,
|
||||
} => SyncStats {
|
||||
files_added,
|
||||
files_modified,
|
||||
files_removed,
|
||||
bytes_transferred,
|
||||
},
|
||||
other => {
|
||||
return Err(FsSyncError::Protocol(format!(
|
||||
"expected FsDirComplete, got {other:?}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
self.conn.shutdown().await?;
|
||||
return Ok(final_stats);
|
||||
@@ -112,16 +138,18 @@ impl FsSyncClient {
|
||||
|
||||
let sem_writer = semaphore.clone();
|
||||
|
||||
let writer_task: tokio::task::JoinHandle<
|
||||
Result<PipeWriteHalf, FsSyncError>,
|
||||
> = tokio::spawn(async move {
|
||||
let writer_task: tokio::task::JoinHandle<Result<PipeWriteHalf, FsSyncError>> =
|
||||
tokio::spawn(async move {
|
||||
for file_need in needed_files {
|
||||
let abs_path = path_map
|
||||
.get(&file_need.path)
|
||||
.cloned()
|
||||
.ok_or_else(|| FsSyncError::Path(format!("local path not found: {}", file_need.path)))?;
|
||||
let abs_path = path_map.get(&file_need.path).cloned().ok_or_else(|| {
|
||||
FsSyncError::Path(format!("local path not found: {}", file_need.path))
|
||||
})?;
|
||||
|
||||
sem_writer.acquire().await.expect("semaphore closed").forget();
|
||||
sem_writer
|
||||
.acquire()
|
||||
.await
|
||||
.expect("semaphore closed")
|
||||
.forget();
|
||||
|
||||
let server_chunks_wire = file_need.server_chunks.clone();
|
||||
|
||||
@@ -161,7 +189,11 @@ impl FsSyncClient {
|
||||
bytes_transferred += meta_rx.recv().await.expect("meta closed early");
|
||||
}
|
||||
SyncMessage::Error { message } => return Err(FsSyncError::Protocol(message)),
|
||||
other => return Err(FsSyncError::Protocol(format!("expected FsFileAck, got {other:?}"))),
|
||||
other => {
|
||||
return Err(FsSyncError::Protocol(format!(
|
||||
"expected FsFileAck, got {other:?}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,9 +203,22 @@ impl FsSyncClient {
|
||||
|
||||
// Receive final FsDirComplete from server.
|
||||
let stats = match read_half.recv().await? {
|
||||
SyncMessage::FsDirComplete { files_added, files_modified, files_removed, .. } =>
|
||||
SyncStats { files_added, files_modified, files_removed, bytes_transferred },
|
||||
other => return Err(FsSyncError::Protocol(format!("expected FsDirComplete, got {other:?}"))),
|
||||
SyncMessage::FsDirComplete {
|
||||
files_added,
|
||||
files_modified,
|
||||
files_removed,
|
||||
..
|
||||
} => SyncStats {
|
||||
files_added,
|
||||
files_modified,
|
||||
files_removed,
|
||||
bytes_transferred,
|
||||
},
|
||||
other => {
|
||||
return Err(FsSyncError::Protocol(format!(
|
||||
"expected FsDirComplete, got {other:?}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
|
||||
write_half.shutdown().await?;
|
||||
@@ -194,13 +239,13 @@ pub struct FsSyncServer {
|
||||
}
|
||||
|
||||
impl FsSyncServer {
|
||||
pub fn new(
|
||||
conn: SyncPeer,
|
||||
serve_root: PathBuf,
|
||||
excludes: GlobSet,
|
||||
allow_delete: bool,
|
||||
) -> Self {
|
||||
Self { conn, serve_root, excludes, allow_delete }
|
||||
pub fn new(conn: SyncPeer, serve_root: PathBuf, excludes: GlobSet, allow_delete: bool) -> Self {
|
||||
Self {
|
||||
conn,
|
||||
serve_root,
|
||||
excludes,
|
||||
allow_delete,
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle one complete client session.
|
||||
@@ -208,7 +253,11 @@ impl FsSyncServer {
|
||||
// Receive client's directory manifest.
|
||||
let client_entries = match self.conn.recv().await? {
|
||||
SyncMessage::FsDirManifest { entries, .. } => entries,
|
||||
other => return Err(FsSyncError::Protocol(format!("expected FsDirManifest, got {other:?}"))),
|
||||
other => {
|
||||
return Err(FsSyncError::Protocol(format!(
|
||||
"expected FsDirManifest, got {other:?}"
|
||||
)));
|
||||
}
|
||||
};
|
||||
|
||||
// Build server's own manifest.
|
||||
@@ -277,9 +326,9 @@ impl FsSyncServer {
|
||||
|
||||
for rel_path in &needed_paths {
|
||||
let is_add = is_add_set.contains(*rel_path);
|
||||
let abs_path = self.serve_root.join(
|
||||
rel_path.replace('/', std::path::MAIN_SEPARATOR_STR)
|
||||
);
|
||||
let abs_path = self
|
||||
.serve_root
|
||||
.join(rel_path.replace('/', std::path::MAIN_SEPARATOR_STR));
|
||||
|
||||
let server_bytes: Vec<u8> = if is_add {
|
||||
vec![]
|
||||
@@ -293,7 +342,10 @@ impl FsSyncServer {
|
||||
let server_chunk_map = build_server_chunk_map(&server_bytes);
|
||||
let server_chunk_hashes: Vec<FsChunkHash> = chunk_data(&server_bytes)
|
||||
.into_iter()
|
||||
.map(|c| FsChunkHash { hash: c.hash, length: c.length as u32 })
|
||||
.map(|c| FsChunkHash {
|
||||
hash: c.hash,
|
||||
length: c.length as u32,
|
||||
})
|
||||
.collect();
|
||||
|
||||
let expected_blake3 = client_entries
|
||||
@@ -308,24 +360,33 @@ impl FsSyncServer {
|
||||
expected_blake3,
|
||||
});
|
||||
|
||||
file_ctx.insert(rel_path.to_string(), FileCtx {
|
||||
file_ctx.insert(
|
||||
rel_path.to_string(),
|
||||
FileCtx {
|
||||
server_bytes,
|
||||
server_chunk_map,
|
||||
expected_blake3,
|
||||
is_add,
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// Send FsDirNeed.
|
||||
self.conn
|
||||
.send(&SyncMessage::FsDirNeed { needed_files, to_delete: to_delete.clone() })
|
||||
.send(&SyncMessage::FsDirNeed {
|
||||
needed_files,
|
||||
to_delete: to_delete.clone(),
|
||||
})
|
||||
.await?;
|
||||
|
||||
// No-op path (nothing to transfer or delete).
|
||||
if file_ctx.is_empty() && to_delete.is_empty() {
|
||||
self.conn
|
||||
.send(&SyncMessage::FsDirComplete {
|
||||
files_added: 0, files_modified: 0, files_removed: 0, bytes_transferred: 0,
|
||||
files_added: 0,
|
||||
files_modified: 0,
|
||||
files_removed: 0,
|
||||
bytes_transferred: 0,
|
||||
})
|
||||
.await?;
|
||||
return Ok(SyncStats::default());
|
||||
@@ -338,7 +399,11 @@ impl FsSyncServer {
|
||||
|
||||
for _ in 0..file_ctx.len() {
|
||||
match self.conn.recv().await? {
|
||||
SyncMessage::FsCdcData { path, chunk_order, chunks } => {
|
||||
SyncMessage::FsCdcData {
|
||||
path,
|
||||
chunk_order,
|
||||
chunks,
|
||||
} => {
|
||||
let ctx = file_ctx.remove(&path).ok_or_else(|| {
|
||||
FsSyncError::Protocol(format!("FsCdcData for unknown path: {path}"))
|
||||
})?;
|
||||
@@ -366,9 +431,9 @@ impl FsSyncServer {
|
||||
bytes_transferred += output.len() as u64;
|
||||
|
||||
// Atomic write: temp file + rename.
|
||||
let abs_path = self.serve_root.join(
|
||||
path.replace('/', std::path::MAIN_SEPARATOR_STR)
|
||||
);
|
||||
let abs_path = self
|
||||
.serve_root
|
||||
.join(path.replace('/', std::path::MAIN_SEPARATOR_STR));
|
||||
// Ensure parent directory exists.
|
||||
if let Some(parent) = abs_path.parent() {
|
||||
tokio::task::spawn_blocking({
|
||||
@@ -391,18 +456,28 @@ impl FsSyncServer {
|
||||
.await
|
||||
.map_err(|e| FsSyncError::Join(e.to_string()))??;
|
||||
|
||||
if is_add { files_added += 1; } else { files_modified += 1; }
|
||||
if is_add {
|
||||
files_added += 1;
|
||||
} else {
|
||||
files_modified += 1;
|
||||
}
|
||||
|
||||
self.conn.send(&SyncMessage::FsFileAck { path }).await?;
|
||||
}
|
||||
SyncMessage::Error { message } => return Err(FsSyncError::Protocol(message)),
|
||||
other => return Err(FsSyncError::Protocol(format!("expected FsCdcData, got {other:?}"))),
|
||||
other => {
|
||||
return Err(FsSyncError::Protocol(format!(
|
||||
"expected FsCdcData, got {other:?}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply deletions after all files reconstructed.
|
||||
for rel in &to_delete {
|
||||
let abs = self.serve_root.join(rel.replace('/', std::path::MAIN_SEPARATOR_STR));
|
||||
let abs = self
|
||||
.serve_root
|
||||
.join(rel.replace('/', std::path::MAIN_SEPARATOR_STR));
|
||||
if abs.exists() {
|
||||
let abs_clone = abs.clone();
|
||||
tokio::task::spawn_blocking(move || std::fs::remove_file(&abs_clone))
|
||||
@@ -421,7 +496,12 @@ impl FsSyncServer {
|
||||
})
|
||||
.await?;
|
||||
|
||||
Ok(SyncStats { files_added, files_modified, files_removed, bytes_transferred })
|
||||
Ok(SyncStats {
|
||||
files_added,
|
||||
files_modified,
|
||||
files_removed,
|
||||
bytes_transferred,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -432,11 +512,11 @@ impl FsSyncServer {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use clawsync_transport::tcp::{TcpConnection, TcpServer};
|
||||
use globset::GlobSetBuilder;
|
||||
use std::fs;
|
||||
use tempfile::TempDir;
|
||||
use clawsync_transport::tcp::{TcpConnection, TcpServer};
|
||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn empty_excludes() -> GlobSet {
|
||||
GlobSetBuilder::new().build().unwrap()
|
||||
@@ -463,7 +543,12 @@ mod tests {
|
||||
});
|
||||
|
||||
let conn = TcpConnection::connect(addr).await.unwrap();
|
||||
let stats = FsSyncClient::new(SyncPeer::Tcp(conn), src_dir.to_path_buf(), empty_excludes(), delete)
|
||||
let stats = FsSyncClient::new(
|
||||
SyncPeer::Tcp(conn),
|
||||
src_dir.to_path_buf(),
|
||||
empty_excludes(),
|
||||
delete,
|
||||
)
|
||||
.run()
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -486,7 +571,10 @@ mod tests {
|
||||
assert_eq!(stats.files_modified, 0);
|
||||
assert!(dst.path().join("a.bin").exists());
|
||||
assert!(dst.path().join("b.bin").exists());
|
||||
assert_eq!(fs::read(dst.path().join("a.bin")).unwrap(), vec![0x11u8; 4096]);
|
||||
assert_eq!(
|
||||
fs::read(dst.path().join("a.bin")).unwrap(),
|
||||
vec![0x11u8; 4096]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -102,7 +102,10 @@ pub fn diff_manifests(old: &DatasetManifest, new: &DatasetManifest) -> DiffResul
|
||||
ord.then_with(|| a.path.cmp(&b.path))
|
||||
});
|
||||
|
||||
DiffResult { patches, unchanged_count }
|
||||
DiffResult {
|
||||
patches,
|
||||
unchanged_count,
|
||||
}
|
||||
}
|
||||
|
||||
/// Given a set of patches keyed by dataset path, look up data bytes in `src`
|
||||
@@ -269,7 +272,10 @@ mod tests {
|
||||
|
||||
let resolved = resolve_patch_data(&diff.patches, &src_file);
|
||||
assert_eq!(resolved.len(), 1);
|
||||
assert!(resolved[0].is_some(), "Added patch should resolve to Some(bytes)");
|
||||
assert!(
|
||||
resolved[0].is_some(),
|
||||
"Added patch should resolve to Some(bytes)"
|
||||
);
|
||||
// 3 f64 values = 24 bytes
|
||||
assert_eq!(resolved[0].as_ref().unwrap().len(), 3 * 8);
|
||||
}
|
||||
@@ -284,7 +290,10 @@ mod tests {
|
||||
// The Removed patch has kind=Removed, so resolve should return None.
|
||||
let resolved = resolve_patch_data(&diff.patches, &src_file);
|
||||
assert_eq!(resolved.len(), 1);
|
||||
assert!(resolved[0].is_none(), "Removed patch should resolve to None");
|
||||
assert!(
|
||||
resolved[0].is_none(),
|
||||
"Removed patch should resolve to None"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -14,7 +14,11 @@ pub enum Hdf5SyncError {
|
||||
DatasetNotFound(String),
|
||||
|
||||
#[error("patch type mismatch for dataset '{path}': expected {expected}, got {actual}")]
|
||||
TypeMismatch { path: String, expected: String, actual: String },
|
||||
TypeMismatch {
|
||||
path: String,
|
||||
expected: String,
|
||||
actual: String,
|
||||
},
|
||||
|
||||
#[error("manifest serialization error: {0}")]
|
||||
Serialize(String),
|
||||
|
||||
@@ -24,5 +24,6 @@ pub mod patcher;
|
||||
pub use differ::{DatasetPatch, DiffResult, PatchKind, diff_manifests};
|
||||
pub use error::Hdf5SyncError;
|
||||
pub use manifest::{DatasetEntry, DatasetManifest};
|
||||
pub use patcher::{PatchStats, ReceivedDataset, apply_patches, apply_patches_to_bytes,
|
||||
apply_received_payloads};
|
||||
pub use patcher::{
|
||||
PatchStats, ReceivedDataset, apply_patches, apply_patches_to_bytes, apply_received_payloads,
|
||||
};
|
||||
|
||||
@@ -48,7 +48,10 @@ impl DatasetManifest {
|
||||
let file = File::open(h5_path)?;
|
||||
let mut datasets = HashMap::new();
|
||||
collect_datasets(file.root(), "", &mut datasets)?;
|
||||
Ok(Self { file_blake3, datasets })
|
||||
Ok(Self {
|
||||
file_blake3,
|
||||
datasets,
|
||||
})
|
||||
}
|
||||
|
||||
/// Build a manifest from already-read bytes (avoids double I/O when the
|
||||
@@ -58,7 +61,10 @@ impl DatasetManifest {
|
||||
let file = File::from_bytes(raw.to_vec())?;
|
||||
let mut datasets = HashMap::new();
|
||||
collect_datasets(file.root(), "", &mut datasets)?;
|
||||
Ok(Self { file_blake3, datasets })
|
||||
Ok(Self {
|
||||
file_blake3,
|
||||
datasets,
|
||||
})
|
||||
}
|
||||
|
||||
/// Return paths of all datasets in this manifest.
|
||||
@@ -113,7 +119,16 @@ fn collect_datasets(
|
||||
let blake3 = blake3_bytes(&raw);
|
||||
let byte_len = raw.len() as u64;
|
||||
|
||||
out.insert(path.clone(), DatasetEntry { path, blake3, shape, dtype, byte_len });
|
||||
out.insert(
|
||||
path.clone(),
|
||||
DatasetEntry {
|
||||
path,
|
||||
blake3,
|
||||
shape,
|
||||
dtype,
|
||||
byte_len,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// Recurse into sub-groups
|
||||
@@ -167,10 +182,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn manifest_counts_datasets() {
|
||||
let bytes = write_test_h5(&[
|
||||
("alpha", vec![1.0, 2.0]),
|
||||
("beta", vec![3.0, 4.0, 5.0]),
|
||||
]);
|
||||
let bytes = write_test_h5(&[("alpha", vec![1.0, 2.0]), ("beta", vec![3.0, 4.0, 5.0])]);
|
||||
let m = DatasetManifest::from_bytes(&bytes).unwrap();
|
||||
assert_eq!(m.len(), 2);
|
||||
}
|
||||
|
||||
@@ -178,7 +178,9 @@ pub fn apply_patches_to_bytes(
|
||||
|
||||
for patch in patches {
|
||||
match patch.kind {
|
||||
PatchKind::Removed => { stats.datasets_removed += 1; }
|
||||
PatchKind::Removed => {
|
||||
stats.datasets_removed += 1;
|
||||
}
|
||||
PatchKind::Added => {
|
||||
if let Some(data) = patched.get(&patch.path) {
|
||||
stats.bytes_written += data.raw.len() as u64;
|
||||
@@ -261,7 +263,11 @@ pub fn apply_received_payloads(
|
||||
// Write received (Added/Modified) datasets.
|
||||
for p in payloads {
|
||||
let dtype = dtype_from_str(&p.dtype);
|
||||
let raw = DatasetRaw { raw: p.data.clone(), dtype, shape: p.shape.clone() };
|
||||
let raw = DatasetRaw {
|
||||
raw: p.data.clone(),
|
||||
dtype,
|
||||
shape: p.shape.clone(),
|
||||
};
|
||||
write_dataset(&mut builder, p.path.trim_start_matches('/'), &raw);
|
||||
// Classify as added or modified based on whether server had it.
|
||||
if existing.contains_key(&p.path) {
|
||||
@@ -383,31 +389,41 @@ fn write_dataset(builder: &mut FileBuilder, name: &str, data: &DatasetRaw) {
|
||||
let ds = builder.create_dataset(name);
|
||||
match &data.dtype {
|
||||
DType::F64 => {
|
||||
let vals: Vec<f64> = data.raw.chunks_exact(8)
|
||||
let vals: Vec<f64> = data
|
||||
.raw
|
||||
.chunks_exact(8)
|
||||
.map(|b| f64::from_le_bytes(b.try_into().unwrap_or([0u8; 8])))
|
||||
.collect();
|
||||
ds.with_f64_data(&vals);
|
||||
}
|
||||
DType::F32 => {
|
||||
let vals: Vec<f32> = data.raw.chunks_exact(4)
|
||||
let vals: Vec<f32> = data
|
||||
.raw
|
||||
.chunks_exact(4)
|
||||
.map(|b| f32::from_le_bytes(b.try_into().unwrap_or([0u8; 4])))
|
||||
.collect();
|
||||
ds.with_f32_data(&vals);
|
||||
}
|
||||
DType::I32 => {
|
||||
let vals: Vec<i32> = data.raw.chunks_exact(4)
|
||||
let vals: Vec<i32> = data
|
||||
.raw
|
||||
.chunks_exact(4)
|
||||
.map(|b| i32::from_le_bytes(b.try_into().unwrap_or([0u8; 4])))
|
||||
.collect();
|
||||
ds.with_i32_data(&vals);
|
||||
}
|
||||
DType::I64 => {
|
||||
let vals: Vec<i64> = data.raw.chunks_exact(8)
|
||||
let vals: Vec<i64> = data
|
||||
.raw
|
||||
.chunks_exact(8)
|
||||
.map(|b| i64::from_le_bytes(b.try_into().unwrap_or([0u8; 8])))
|
||||
.collect();
|
||||
ds.with_i64_data(&vals);
|
||||
}
|
||||
DType::U64 => {
|
||||
let vals: Vec<u64> = data.raw.chunks_exact(8)
|
||||
let vals: Vec<u64> = data
|
||||
.raw
|
||||
.chunks_exact(8)
|
||||
.map(|b| u64::from_le_bytes(b.try_into().unwrap_or([0u8; 8])))
|
||||
.collect();
|
||||
// Use i64 storage for u64 (bit-preserving)
|
||||
@@ -462,7 +478,10 @@ mod tests {
|
||||
|
||||
let result = DatasetManifest::from_bytes(&patched).unwrap();
|
||||
let src_m = DatasetManifest::from_bytes(&src).unwrap();
|
||||
assert_eq!(result.get("/x").unwrap().blake3, src_m.get("/x").unwrap().blake3);
|
||||
assert_eq!(
|
||||
result.get("/x").unwrap().blake3,
|
||||
src_m.get("/x").unwrap().blake3
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -495,8 +514,14 @@ mod tests {
|
||||
|
||||
let orig = DatasetManifest::from_bytes(&bytes).unwrap();
|
||||
let result = DatasetManifest::from_bytes(&patched).unwrap();
|
||||
assert_eq!(orig.get("/a").unwrap().blake3, result.get("/a").unwrap().blake3);
|
||||
assert_eq!(orig.get("/b").unwrap().blake3, result.get("/b").unwrap().blake3);
|
||||
assert_eq!(
|
||||
orig.get("/a").unwrap().blake3,
|
||||
result.get("/a").unwrap().blake3
|
||||
);
|
||||
assert_eq!(
|
||||
orig.get("/b").unwrap().blake3,
|
||||
result.get("/b").unwrap().blake3
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -5,7 +5,9 @@
|
||||
//! creating real HDF5 files in memory and on disk as needed.
|
||||
|
||||
use clawhdf5::{File, FileBuilder};
|
||||
use clawsync_hdf5::{DatasetManifest, PatchKind, apply_patches, apply_patches_to_bytes, diff_manifests};
|
||||
use clawsync_hdf5::{
|
||||
DatasetManifest, PatchKind, apply_patches, apply_patches_to_bytes, diff_manifests,
|
||||
};
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -37,7 +39,10 @@ fn write_temp_h5(bytes: &[u8]) -> (NamedTempFile, std::path::PathBuf) {
|
||||
/// A dataset present in source but absent from target is added after patching.
|
||||
#[test]
|
||||
fn patch_adds_new_dataset() {
|
||||
let src = h5_bytes(&[("temperature", &[20.0, 21.5, 22.0]), ("humidity", &[55.0, 57.0])]);
|
||||
let src = h5_bytes(&[
|
||||
("temperature", &[20.0, 21.5, 22.0]),
|
||||
("humidity", &[55.0, 57.0]),
|
||||
]);
|
||||
let tgt = h5_bytes(&[("temperature", &[20.0, 21.5, 22.0])]);
|
||||
|
||||
let old_m = DatasetManifest::from_bytes(&tgt).unwrap();
|
||||
@@ -51,7 +56,8 @@ fn patch_adds_new_dataset() {
|
||||
let result_m = DatasetManifest::from_bytes(&patched).unwrap();
|
||||
|
||||
// The new dataset must be present with the correct hash.
|
||||
let result_entry = result_m.get("/humidity")
|
||||
let result_entry = result_m
|
||||
.get("/humidity")
|
||||
.expect("humidity should exist after patching");
|
||||
let src_entry = new_m.get("/humidity").unwrap();
|
||||
assert_eq!(result_entry.blake3, src_entry.blake3);
|
||||
@@ -107,13 +113,22 @@ fn patch_removes_dataset() {
|
||||
let diff = diff_manifests(&old_m, &new_m);
|
||||
|
||||
assert_eq!(diff.of_kind(&PatchKind::Removed).count(), 1);
|
||||
assert_eq!(diff.of_kind(&PatchKind::Removed).next().unwrap().path, "/current");
|
||||
assert_eq!(
|
||||
diff.of_kind(&PatchKind::Removed).next().unwrap().path,
|
||||
"/current"
|
||||
);
|
||||
|
||||
let (patched, _) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap();
|
||||
let result_m = DatasetManifest::from_bytes(&patched).unwrap();
|
||||
|
||||
assert!(result_m.get("/current").is_none(), "current should have been removed");
|
||||
assert!(result_m.get("/voltage").is_some(), "voltage should be retained");
|
||||
assert!(
|
||||
result_m.get("/current").is_none(),
|
||||
"current should have been removed"
|
||||
);
|
||||
assert!(
|
||||
result_m.get("/voltage").is_some(),
|
||||
"voltage should be retained"
|
||||
);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -194,17 +209,9 @@ fn patch_multiple_changes() {
|
||||
#[test]
|
||||
fn stats_counts_correct() {
|
||||
// source: "a" (modified), "c" (unchanged), "d" (new)
|
||||
let src = h5_bytes(&[
|
||||
("a", &[9.0, 9.0]),
|
||||
("c", &[3.0]),
|
||||
("d", &[4.0, 5.0]),
|
||||
]);
|
||||
let src = h5_bytes(&[("a", &[9.0, 9.0]), ("c", &[3.0]), ("d", &[4.0, 5.0])]);
|
||||
// target: "a" (old values), "b" (to be removed), "c" (same as source)
|
||||
let tgt = h5_bytes(&[
|
||||
("a", &[1.0, 2.0]),
|
||||
("b", &[0.0]),
|
||||
("c", &[3.0]),
|
||||
]);
|
||||
let tgt = h5_bytes(&[("a", &[1.0, 2.0]), ("b", &[0.0]), ("c", &[3.0])]);
|
||||
|
||||
let old_m = DatasetManifest::from_bytes(&tgt).unwrap();
|
||||
let new_m = DatasetManifest::from_bytes(&src).unwrap();
|
||||
@@ -227,13 +234,8 @@ fn stats_counts_correct() {
|
||||
/// manifest read back from disk matches the source manifest.
|
||||
#[test]
|
||||
fn apply_patches_file_path_api() {
|
||||
let src_bytes = h5_bytes(&[
|
||||
("chan_x", &[1.1, 2.2, 3.3]),
|
||||
("chan_y", &[4.4, 5.5]),
|
||||
]);
|
||||
let tgt_bytes = h5_bytes(&[
|
||||
("chan_x", &[0.0, 0.0, 0.0]),
|
||||
]);
|
||||
let src_bytes = h5_bytes(&[("chan_x", &[1.1, 2.2, 3.3]), ("chan_y", &[4.4, 5.5])]);
|
||||
let tgt_bytes = h5_bytes(&[("chan_x", &[0.0, 0.0, 0.0])]);
|
||||
|
||||
// Write both files to disk using NamedTempFile.
|
||||
let (_src_guard, src_path) = write_temp_h5(&src_bytes);
|
||||
@@ -249,7 +251,10 @@ fn apply_patches_file_path_api() {
|
||||
let stats = apply_patches(&diff.patches, &src_path, &tgt_path).unwrap();
|
||||
|
||||
assert!(stats.datasets_added >= 1, "chan_y should have been added");
|
||||
assert!(stats.datasets_modified >= 1, "chan_x should have been modified");
|
||||
assert!(
|
||||
stats.datasets_modified >= 1,
|
||||
"chan_x should have been modified"
|
||||
);
|
||||
|
||||
// Read the patched file back from disk and verify it matches source.
|
||||
let result_m = DatasetManifest::from_path(&tgt_path).unwrap();
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
//! Run:
|
||||
//! cargo bench -p clawsync-onion -- iblt_bench
|
||||
|
||||
use clawsync_onion::iblt::{IbltSketch, DEFAULT_HASH_COUNT, MIN_CELLS};
|
||||
use clawsync_onion::iblt::{DEFAULT_HASH_COUNT, IbltSketch, MIN_CELLS};
|
||||
use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -28,8 +28,12 @@ fn diff_sketches(n: usize, d: usize) -> (IbltSketch, Vec<u8>) {
|
||||
let m = ((d * 4).max(MIN_CELLS)).max(IbltSketch::recommended_cells(n));
|
||||
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||
for &k in &a_keys { a.insert(k); }
|
||||
for &k in &b_keys { b.insert(k); }
|
||||
for &k in &a_keys {
|
||||
a.insert(k);
|
||||
}
|
||||
for &k in &b_keys {
|
||||
b.insert(k);
|
||||
}
|
||||
let b_bytes = b.to_bytes();
|
||||
a.subtract(&b);
|
||||
(a, b_bytes)
|
||||
@@ -79,8 +83,12 @@ fn bench_iblt(c: &mut Criterion) {
|
||||
let m = (d * 4).max(MIN_CELLS).max(IbltSketch::recommended_cells(n));
|
||||
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||
let mut b_sk = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||
for &k in black_box(ak) { a.insert(k); }
|
||||
for &k in black_box(bk) { b_sk.insert(k); }
|
||||
for &k in black_box(ak) {
|
||||
a.insert(k);
|
||||
}
|
||||
for &k in black_box(bk) {
|
||||
b_sk.insert(k);
|
||||
}
|
||||
a.subtract(&b_sk);
|
||||
black_box(a.decode())
|
||||
});
|
||||
@@ -92,14 +100,20 @@ fn bench_iblt(c: &mut Criterion) {
|
||||
|
||||
// ── Wire size table (not timed) ───────────────────────────────────────────
|
||||
println!("\n=== IBLT sketch size vs flat manifest (N × 60 B) ===");
|
||||
println!("{:>8} {:>10} {:>12} {:>10}", "N", "IBLT (B)", "Flat (B)", "ratio");
|
||||
println!(
|
||||
"{:>8} {:>10} {:>12} {:>10}",
|
||||
"N", "IBLT (B)", "Flat (B)", "ratio"
|
||||
);
|
||||
for &n in &[10usize, 100, 1_000, 10_000, 100_000] {
|
||||
let ks = keys(n);
|
||||
let sz = IbltSketch::from_keys(&ks, SEED).to_bytes().len();
|
||||
let flat = n * 60;
|
||||
println!(
|
||||
"{:>8} {:>10} {:>12} {:>9.1}×",
|
||||
n, sz, flat, flat as f64 / sz as f64
|
||||
n,
|
||||
sz,
|
||||
flat,
|
||||
flat as f64 / sz as f64
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,15 +49,20 @@ pub fn diff_revisions(
|
||||
|
||||
let pages: Vec<OnionPage> = raw_pages
|
||||
.into_iter()
|
||||
.map(|(h5_offset, data, codec, orig_size)| OnionPage { h5_offset, data, codec, orig_size })
|
||||
.map(|(h5_offset, data, codec, orig_size)| OnionPage {
|
||||
h5_offset,
|
||||
data,
|
||||
codec,
|
||||
orig_size,
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Parse the blake3 hex back to bytes
|
||||
let blake3_root = hex_to_bytes32(&summary.blake3_hex)
|
||||
.ok_or_else(|| {
|
||||
SyncOnionError::Onion(clawhdf5_onion::OnionError::Malformed(
|
||||
format!("invalid blake3_hex for rev {}", summary.revision),
|
||||
))
|
||||
let blake3_root = hex_to_bytes32(&summary.blake3_hex).ok_or_else(|| {
|
||||
SyncOnionError::Onion(clawhdf5_onion::OnionError::Malformed(format!(
|
||||
"invalid blake3_hex for rev {}",
|
||||
summary.revision
|
||||
)))
|
||||
})?;
|
||||
|
||||
packets.push(OnionLayerPacket {
|
||||
@@ -103,13 +108,18 @@ pub fn packets_for_revisions(
|
||||
let raw_pages = local.revision_pages_raw(rev)?;
|
||||
let pages: Vec<OnionPage> = raw_pages
|
||||
.into_iter()
|
||||
.map(|(h5_offset, data, codec, orig_size)| OnionPage { h5_offset, data, codec, orig_size })
|
||||
.map(|(h5_offset, data, codec, orig_size)| OnionPage {
|
||||
h5_offset,
|
||||
data,
|
||||
codec,
|
||||
orig_size,
|
||||
})
|
||||
.collect();
|
||||
|
||||
let blake3_root = hex_to_bytes32(&summary.blake3_hex).ok_or_else(|| {
|
||||
SyncOnionError::Onion(clawhdf5_onion::OnionError::Malformed(
|
||||
format!("invalid blake3_hex for rev {rev}"),
|
||||
))
|
||||
SyncOnionError::Onion(clawhdf5_onion::OnionError::Malformed(format!(
|
||||
"invalid blake3_hex for rev {rev}"
|
||||
)))
|
||||
})?;
|
||||
|
||||
packets.push(OnionLayerPacket {
|
||||
@@ -198,7 +208,12 @@ pub fn diff_revisions_merkle(
|
||||
let raw_pages = local.revision_pages_raw(rev)?;
|
||||
let pages: Vec<OnionPage> = raw_pages
|
||||
.into_iter()
|
||||
.map(|(h5_offset, data, codec, orig_size)| OnionPage { h5_offset, data, codec, orig_size })
|
||||
.map(|(h5_offset, data, codec, orig_size)| OnionPage {
|
||||
h5_offset,
|
||||
data,
|
||||
codec,
|
||||
orig_size,
|
||||
})
|
||||
.collect();
|
||||
|
||||
let blake3_root = hex_to_bytes32(&summary.blake3_hex).ok_or_else(|| {
|
||||
|
||||
@@ -139,7 +139,9 @@ impl IbltSketch {
|
||||
pub fn from_keys(keys: &[u64], seed: u64) -> Self {
|
||||
let m = Self::recommended_cells(keys.len());
|
||||
let mut sketch = Self::new(m, DEFAULT_HASH_COUNT, seed);
|
||||
for &k in keys { sketch.insert(k); }
|
||||
for &k in keys {
|
||||
sketch.insert(k);
|
||||
}
|
||||
sketch
|
||||
}
|
||||
|
||||
@@ -182,7 +184,8 @@ impl IbltSketch {
|
||||
/// Panics if `self` and `other` have different cell counts.
|
||||
pub fn subtract(&mut self, other: &IbltSketch) {
|
||||
assert_eq!(
|
||||
self.cells.len(), other.cells.len(),
|
||||
self.cells.len(),
|
||||
other.cells.len(),
|
||||
"cannot subtract sketches of different sizes"
|
||||
);
|
||||
for (a, b) in self.cells.iter_mut().zip(&other.cells) {
|
||||
@@ -213,9 +216,14 @@ impl IbltSketch {
|
||||
|
||||
let Some(pos) = pure_pos else {
|
||||
// No pure cell found.
|
||||
let done = cells.iter().all(|c| c.count == 0 && c.id_sum == 0 && c.hash_sum == 0);
|
||||
let done = cells
|
||||
.iter()
|
||||
.all(|c| c.count == 0 && c.id_sum == 0 && c.hash_sum == 0);
|
||||
return if done {
|
||||
let mut diff = IbltDiff { only_in_a, only_in_b };
|
||||
let mut diff = IbltDiff {
|
||||
only_in_a,
|
||||
only_in_b,
|
||||
};
|
||||
diff.only_in_a.sort_unstable();
|
||||
diff.only_in_b.sort_unstable();
|
||||
IbltDecodeResult::Complete(diff)
|
||||
@@ -227,7 +235,11 @@ impl IbltSketch {
|
||||
let key = cells[pos].id_sum;
|
||||
let positive = cells[pos].count == 1;
|
||||
|
||||
if positive { only_in_a.push(key); } else { only_in_b.push(key); }
|
||||
if positive {
|
||||
only_in_a.push(key);
|
||||
} else {
|
||||
only_in_b.push(key);
|
||||
}
|
||||
|
||||
// Peel this key from all its mapped cells.
|
||||
let sign: i32 = if positive { 1 } else { -1 };
|
||||
@@ -290,10 +302,18 @@ impl IbltSketch {
|
||||
let count = i32::from_le_bytes(data[off..off + 4].try_into().unwrap());
|
||||
let id_sum = u64::from_le_bytes(data[off + 4..off + 12].try_into().unwrap());
|
||||
let hash_sum = u64::from_le_bytes(data[off + 12..off + 20].try_into().unwrap());
|
||||
cells.push(IbltCell { count, id_sum, hash_sum });
|
||||
cells.push(IbltCell {
|
||||
count,
|
||||
id_sum,
|
||||
hash_sum,
|
||||
});
|
||||
off += 20;
|
||||
}
|
||||
Ok(Self { cells, hash_count: k, seed })
|
||||
Ok(Self {
|
||||
cells,
|
||||
hash_count: k,
|
||||
seed,
|
||||
})
|
||||
}
|
||||
|
||||
// ── Private helpers ───────────────────────────────────────────────────────
|
||||
@@ -349,12 +369,16 @@ pub fn reconcile(
|
||||
seed: u64,
|
||||
) -> Result<(IbltDiff, IbltSketch), IbltError> {
|
||||
let b_sketch = IbltSketch::from_bytes(b_sketch_bytes)?;
|
||||
let m_start = b_sketch.cell_count().max(IbltSketch::recommended_cells(a_keys.len()));
|
||||
let m_start = b_sketch
|
||||
.cell_count()
|
||||
.max(IbltSketch::recommended_cells(a_keys.len()));
|
||||
|
||||
let mut m = m_start;
|
||||
loop {
|
||||
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, seed);
|
||||
for &k in a_keys { a.insert(k); }
|
||||
for &k in a_keys {
|
||||
a.insert(k);
|
||||
}
|
||||
|
||||
let mut diff_sketch = a.clone();
|
||||
let b_padded = pad_or_trim_sketch(&b_sketch, m, seed);
|
||||
@@ -372,7 +396,11 @@ pub fn reconcile(
|
||||
fn pad_or_trim_sketch(src: &IbltSketch, target_m: usize, seed: u64) -> IbltSketch {
|
||||
let mut cells = src.cells.clone();
|
||||
cells.resize(target_m, IbltCell::default());
|
||||
IbltSketch { cells, hash_count: src.hash_count, seed }
|
||||
IbltSketch {
|
||||
cells,
|
||||
hash_count: src.hash_count,
|
||||
seed,
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -397,7 +425,11 @@ mod tests {
|
||||
let mut s = IbltSketch::new(64, 3, SEED);
|
||||
s.insert(42);
|
||||
s.remove(42);
|
||||
assert!(s.cells.iter().all(|c| c.count == 0 && c.id_sum == 0 && c.hash_sum == 0));
|
||||
assert!(
|
||||
s.cells
|
||||
.iter()
|
||||
.all(|c| c.count == 0 && c.id_sum == 0 && c.hash_sum == 0)
|
||||
);
|
||||
}
|
||||
|
||||
// ── decode: empty diff ────────────────────────────────────────────────────
|
||||
@@ -409,9 +441,13 @@ mod tests {
|
||||
let b = sketch_of(&keys);
|
||||
a.subtract(&b);
|
||||
let result = a.decode();
|
||||
assert_eq!(result, IbltDecodeResult::Complete(IbltDiff {
|
||||
only_in_a: vec![], only_in_b: vec![],
|
||||
}));
|
||||
assert_eq!(
|
||||
result,
|
||||
IbltDecodeResult::Complete(IbltDiff {
|
||||
only_in_a: vec![],
|
||||
only_in_b: vec![],
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
// ── decode: one-sided diff ────────────────────────────────────────────────
|
||||
@@ -420,14 +456,22 @@ mod tests {
|
||||
fn decode_one_sided_diff_a_has_extra() {
|
||||
let base: Vec<u64> = (0..20).collect();
|
||||
let extra = 9999u64;
|
||||
let a_keys: Vec<u64> = base.iter().chain(std::iter::once(&extra)).copied().collect();
|
||||
let a_keys: Vec<u64> = base
|
||||
.iter()
|
||||
.chain(std::iter::once(&extra))
|
||||
.copied()
|
||||
.collect();
|
||||
let b_keys = base.clone();
|
||||
|
||||
let m = IbltSketch::recommended_cells(a_keys.len()) * 4; // generous m
|
||||
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||
for &k in &a_keys { a.insert(k); }
|
||||
for &k in &b_keys { b.insert(k); }
|
||||
for &k in &a_keys {
|
||||
a.insert(k);
|
||||
}
|
||||
for &k in &b_keys {
|
||||
b.insert(k);
|
||||
}
|
||||
a.subtract(&b);
|
||||
|
||||
match a.decode() {
|
||||
@@ -444,13 +488,21 @@ mod tests {
|
||||
let base: Vec<u64> = (0..20).collect();
|
||||
let extra = 8888u64;
|
||||
let a_keys = base.clone();
|
||||
let b_keys: Vec<u64> = base.iter().chain(std::iter::once(&extra)).copied().collect();
|
||||
let b_keys: Vec<u64> = base
|
||||
.iter()
|
||||
.chain(std::iter::once(&extra))
|
||||
.copied()
|
||||
.collect();
|
||||
|
||||
let m = IbltSketch::recommended_cells(b_keys.len()) * 4;
|
||||
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||
for &k in &a_keys { a.insert(k); }
|
||||
for &k in &b_keys { b.insert(k); }
|
||||
for &k in &a_keys {
|
||||
a.insert(k);
|
||||
}
|
||||
for &k in &b_keys {
|
||||
b.insert(k);
|
||||
}
|
||||
a.subtract(&b);
|
||||
|
||||
match a.decode() {
|
||||
@@ -476,8 +528,12 @@ mod tests {
|
||||
let m = IbltSketch::recommended_cells(a_keys.len().max(b_keys.len())) * 8;
|
||||
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||
for &k in &a_keys { a.insert(k); }
|
||||
for &k in &b_keys { b.insert(k); }
|
||||
for &k in &a_keys {
|
||||
a.insert(k);
|
||||
}
|
||||
for &k in &b_keys {
|
||||
b.insert(k);
|
||||
}
|
||||
a.subtract(&b);
|
||||
|
||||
match a.decode() {
|
||||
@@ -500,8 +556,12 @@ mod tests {
|
||||
let m = 1; // deliberately too small
|
||||
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||
for &k in &a_keys { a.insert(k); }
|
||||
for &k in &b_keys { b.insert(k); }
|
||||
for &k in &a_keys {
|
||||
a.insert(k);
|
||||
}
|
||||
for &k in &b_keys {
|
||||
b.insert(k);
|
||||
}
|
||||
a.subtract(&b);
|
||||
|
||||
// Must not panic — returns NeedMoreCells
|
||||
|
||||
@@ -19,9 +19,11 @@ pub mod merger;
|
||||
pub mod packet;
|
||||
pub mod selector;
|
||||
|
||||
pub use error::SyncOnionError;
|
||||
pub use iblt::{IbltDecodeResult, IbltDiff, IbltSketch, DEFAULT_HASH_COUNT, IBLT_SYNC_SEED, MIN_CELLS};
|
||||
pub use manifest::{ClawSyncManifest, IbltManifest};
|
||||
pub use differ::packets_for_revisions;
|
||||
pub use error::SyncOnionError;
|
||||
pub use iblt::{
|
||||
DEFAULT_HASH_COUNT, IBLT_SYNC_SEED, IbltDecodeResult, IbltDiff, IbltSketch, MIN_CELLS,
|
||||
};
|
||||
pub use manifest::{ClawSyncManifest, IbltManifest};
|
||||
pub use merger::MergeStats;
|
||||
pub use packet::{OnionLayerPacket, OnionPage};
|
||||
|
||||
@@ -10,7 +10,7 @@ use rkyv::{Archive, Deserialize, Serialize};
|
||||
|
||||
use clawhdf5_onion::writer::OnionFile;
|
||||
|
||||
use crate::iblt::{IbltDiff, IbltDecodeResult, IbltSketch, DEFAULT_HASH_COUNT};
|
||||
use crate::iblt::{DEFAULT_HASH_COUNT, IbltDecodeResult, IbltDiff, IbltSketch};
|
||||
|
||||
/// A compact summary of one revision — used in the manifest.
|
||||
#[derive(Archive, Deserialize, Serialize, Debug, Clone, PartialEq)]
|
||||
@@ -60,10 +60,7 @@ impl ClawSyncManifest {
|
||||
})
|
||||
.unwrap_or((0, [0u8; 32]));
|
||||
|
||||
let last_write = summaries
|
||||
.last()
|
||||
.map(|s| s.timestamp)
|
||||
.unwrap_or(0.0);
|
||||
let last_write = summaries.last().map(|s| s.timestamp).unwrap_or(0.0);
|
||||
|
||||
let revisions = summaries
|
||||
.into_iter()
|
||||
@@ -144,7 +141,12 @@ impl IbltManifest {
|
||||
|
||||
let (head_revision, head_blake3) = summaries
|
||||
.last()
|
||||
.map(|s| (s.revision, hex_to_bytes32(&s.blake3_hex).unwrap_or([0u8; 32])))
|
||||
.map(|s| {
|
||||
(
|
||||
s.revision,
|
||||
hex_to_bytes32(&s.blake3_hex).unwrap_or([0u8; 32]),
|
||||
)
|
||||
})
|
||||
.unwrap_or((0, [0u8; 32]));
|
||||
|
||||
let last_write = summaries.last().map(|s| s.timestamp).unwrap_or(0.0);
|
||||
@@ -174,10 +176,7 @@ impl IbltManifest {
|
||||
///
|
||||
/// Returns `Ok(IbltDiff)` or `Err` if the sketch cannot be decoded (too
|
||||
/// small or malformed).
|
||||
pub fn diff_against(
|
||||
&self,
|
||||
local_keys: &[u64],
|
||||
) -> Result<IbltDiff, &'static str> {
|
||||
pub fn diff_against(&self, local_keys: &[u64]) -> Result<IbltDiff, &'static str> {
|
||||
let remote_sketch = IbltSketch::from_bytes(&self.sketch)
|
||||
.map_err(|_| "cannot deserialise remote IBLT sketch")?;
|
||||
|
||||
@@ -189,7 +188,9 @@ impl IbltManifest {
|
||||
.max(IbltSketch::recommended_cells(local_keys.len()));
|
||||
|
||||
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, seed);
|
||||
for &k in local_keys { a.insert(k); }
|
||||
for &k in local_keys {
|
||||
a.insert(k);
|
||||
}
|
||||
|
||||
// Pad remote to same m if needed.
|
||||
let b = if remote_sketch.cell_count() == m {
|
||||
@@ -217,8 +218,7 @@ impl IbltManifest {
|
||||
pub fn from_bytes(bytes: &[u8]) -> Result<Self, String> {
|
||||
let mut aligned = rkyv::util::AlignedVec::<16>::with_capacity(bytes.len());
|
||||
aligned.extend_from_slice(bytes);
|
||||
rkyv::from_bytes::<IbltManifest, rkyv::rancor::Error>(&aligned)
|
||||
.map_err(|e| e.to_string())
|
||||
rkyv::from_bytes::<IbltManifest, rkyv::rancor::Error>(&aligned).map_err(|e| e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -63,8 +63,7 @@ pub fn merge_packets(
|
||||
session.record_page(page.h5_offset, &page.data);
|
||||
total_bytes += page.data.len() as u64;
|
||||
} else {
|
||||
let codec = Codec::from_u8(page.codec)
|
||||
.map_err(SyncOnionError::Onion)?;
|
||||
let codec = Codec::from_u8(page.codec).map_err(SyncOnionError::Onion)?;
|
||||
let raw = decompress_page(&page.data, codec, page.orig_size)
|
||||
.map_err(SyncOnionError::Onion)?;
|
||||
total_bytes += raw.len() as u64;
|
||||
@@ -101,8 +100,7 @@ fn verify_packet_hash(packet: &OnionLayerPacket) -> Result<(), SyncOnionError> {
|
||||
if page.codec == 0 {
|
||||
hasher.update(&page.data);
|
||||
} else {
|
||||
let codec = Codec::from_u8(page.codec)
|
||||
.map_err(SyncOnionError::Onion)?;
|
||||
let codec = Codec::from_u8(page.codec).map_err(SyncOnionError::Onion)?;
|
||||
let raw = decompress_page(&page.data, codec, page.orig_size)
|
||||
.map_err(SyncOnionError::Onion)?;
|
||||
hasher.update(&raw);
|
||||
@@ -204,7 +202,12 @@ mod tests {
|
||||
timestamp: 0.0,
|
||||
annotation: None,
|
||||
blake3_root: [0xDEu8; 32], // wrong hash
|
||||
pages: vec![OnionPage { h5_offset: 0, data: page_data.clone(), codec: 0, orig_size: page_data.len() as u32 }],
|
||||
pages: vec![OnionPage {
|
||||
h5_offset: 0,
|
||||
data: page_data.clone(),
|
||||
codec: 0,
|
||||
orig_size: page_data.len() as u32,
|
||||
}],
|
||||
};
|
||||
bad_packet.blake3_root = [0xDE; 32]; // deliberately wrong
|
||||
|
||||
@@ -229,7 +232,12 @@ mod tests {
|
||||
timestamp: 0.0,
|
||||
annotation: None,
|
||||
blake3_root: hash,
|
||||
pages: vec![OnionPage { h5_offset: 0, data: page_data.clone(), codec: 0, orig_size: page_data.len() as u32 }],
|
||||
pages: vec![OnionPage {
|
||||
h5_offset: 0,
|
||||
data: page_data.clone(),
|
||||
codec: 0,
|
||||
orig_size: page_data.len() as u32,
|
||||
}],
|
||||
};
|
||||
|
||||
let (_h5, mut dst) = make_empty_onion();
|
||||
|
||||
@@ -113,8 +113,18 @@ mod tests {
|
||||
annotation: Some(format!("rev {revision}")),
|
||||
blake3_root: [0xABu8; 32],
|
||||
pages: vec![
|
||||
OnionPage { h5_offset: 0, data: vec![0xAAu8; 4096], codec: 0, orig_size: 4096 },
|
||||
OnionPage { h5_offset: 4096, data: vec![0xBBu8; 4096], codec: 0, orig_size: 4096 },
|
||||
OnionPage {
|
||||
h5_offset: 0,
|
||||
data: vec![0xAAu8; 4096],
|
||||
codec: 0,
|
||||
orig_size: 4096,
|
||||
},
|
||||
OnionPage {
|
||||
h5_offset: 4096,
|
||||
data: vec![0xBBu8; 4096],
|
||||
codec: 0,
|
||||
orig_size: 4096,
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
@@ -193,7 +203,12 @@ mod tests {
|
||||
blake3_root: [0u8; 32],
|
||||
pages: vec![
|
||||
// "compressed" to 1600 bytes, original 4096
|
||||
OnionPage { h5_offset: 0, data: vec![0xCCu8; 1600], codec: 1, orig_size: 4096 },
|
||||
OnionPage {
|
||||
h5_offset: 0,
|
||||
data: vec![0xCCu8; 1600],
|
||||
codec: 1,
|
||||
orig_size: 4096,
|
||||
},
|
||||
],
|
||||
};
|
||||
assert_eq!(pkt.page_data_size(), 4096); // logical (uncompressed)
|
||||
|
||||
@@ -63,10 +63,7 @@ pub fn filter_packets(
|
||||
/// Compute the remote HEAD for a given selector on the local `OnionFile`.
|
||||
///
|
||||
/// Returns [`NO_PARENT`] if no revisions match (full sync needed).
|
||||
pub fn remote_head_for_selector(
|
||||
_selector: &SyncSelector,
|
||||
remote_revision_count: u64,
|
||||
) -> u64 {
|
||||
pub fn remote_head_for_selector(_selector: &SyncSelector, remote_revision_count: u64) -> u64 {
|
||||
// For a simple linear sync, remote HEAD = remote_revision_count - 1
|
||||
// (or NO_PARENT if empty).
|
||||
if remote_revision_count == 0 {
|
||||
@@ -131,8 +128,11 @@ mod tests {
|
||||
fn selector_branch_nonexistent_errors() {
|
||||
let onion = make_onion(2);
|
||||
let packets = diff_revisions(&onion, NO_PARENT).unwrap();
|
||||
let result =
|
||||
filter_packets(packets, &SyncSelector::Branch("no-such".to_string()), &onion);
|
||||
let result = filter_packets(
|
||||
packets,
|
||||
&SyncSelector::Branch("no-such".to_string()),
|
||||
&onion,
|
||||
);
|
||||
assert!(matches!(result, Err(SyncOnionError::BranchNotFound(_))));
|
||||
}
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ use clawhdf5_onion::format::NO_PARENT;
|
||||
use clawhdf5_onion::writer::OnionFile;
|
||||
use clawsync_onion::differ::diff_revisions;
|
||||
use clawsync_onion::merger::merge_packets;
|
||||
use clawsync_onion::selector::{filter_packets, remote_head_for_selector, SyncSelector};
|
||||
use clawsync_onion::selector::{SyncSelector, filter_packets, remote_head_for_selector};
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -309,7 +309,11 @@ fn merge_auto_registers_unknown_branch() {
|
||||
assert_eq!(stats.revisions_merged, 4);
|
||||
// The remote should have both branches registered
|
||||
let branches = dst.list_branches();
|
||||
assert!(branches.len() >= 2, "expected at least 2 branches, got {:?}", branches.len());
|
||||
assert!(
|
||||
branches.len() >= 2,
|
||||
"expected at least 2 branches, got {:?}",
|
||||
branches.len()
|
||||
);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -327,7 +331,10 @@ fn can_branch_on_remote_after_branch_selective_sync() {
|
||||
|
||||
// The remote should now be able to fork its own branch
|
||||
let result = dst.create_branch("experiment", "main");
|
||||
assert!(result.is_ok(), "should be able to create branch after sync: {result:?}");
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"should be able to create branch after sync: {result:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -340,14 +347,12 @@ fn branch_filtered_merge_is_idempotent() {
|
||||
let mut dst = empty_onion();
|
||||
|
||||
let packets1 = diff_revisions(&src, NO_PARENT).unwrap();
|
||||
let filtered1 =
|
||||
filter_packets(packets1, &SyncSelector::Branch("main".into()), &src).unwrap();
|
||||
let filtered1 = filter_packets(packets1, &SyncSelector::Branch("main".into()), &src).unwrap();
|
||||
merge_packets(&mut dst, filtered1, false).unwrap();
|
||||
|
||||
// Apply again — should skip all
|
||||
let packets2 = diff_revisions(&src, NO_PARENT).unwrap();
|
||||
let filtered2 =
|
||||
filter_packets(packets2, &SyncSelector::Branch("main".into()), &src).unwrap();
|
||||
let filtered2 = filter_packets(packets2, &SyncSelector::Branch("main".into()), &src).unwrap();
|
||||
let stats2 = merge_packets(&mut dst, filtered2, false).unwrap();
|
||||
|
||||
assert_eq!(stats2.revisions_merged, 0);
|
||||
|
||||
@@ -16,8 +16,8 @@ pub mod quic;
|
||||
pub mod tcp;
|
||||
|
||||
pub use error::TransportError;
|
||||
pub use mmap::{MmapChannel, DEFAULT_CAPACITY};
|
||||
pub use mmap::{DEFAULT_CAPACITY, MmapChannel};
|
||||
pub use peer::{PipeReadHalf, PipeWriteHalf, SyncPeer};
|
||||
pub use protocol::SyncMessage;
|
||||
pub use tcp::{TcpConnection, TcpServer};
|
||||
pub use quic::{QuicConfig, QuicConnection, QuicServer, quic_connect};
|
||||
pub use tcp::{TcpConnection, TcpServer};
|
||||
|
||||
@@ -26,12 +26,12 @@
|
||||
//! - Capacity is fixed at creation time.
|
||||
//! - No cross-machine support (file must be on a shared filesystem).
|
||||
|
||||
use memmap2::MmapMut;
|
||||
use std::fs::OpenOptions;
|
||||
use std::path::{Path, PathBuf};
|
||||
use memmap2::MmapMut;
|
||||
|
||||
use crate::error::TransportError;
|
||||
use crate::protocol::{SyncMessage, MAX_FRAME_SIZE};
|
||||
use crate::protocol::{MAX_FRAME_SIZE, SyncMessage};
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Constants
|
||||
@@ -92,7 +92,11 @@ impl MmapChannel {
|
||||
write_u64(&mut mmap, 24, capacity as u64); // capacity
|
||||
mmap.flush().map_err(TransportError::Io)?;
|
||||
|
||||
Ok(Self { path: path.to_owned(), mmap, capacity })
|
||||
Ok(Self {
|
||||
path: path.to_owned(),
|
||||
mmap,
|
||||
capacity,
|
||||
})
|
||||
}
|
||||
|
||||
/// Open an existing channel file.
|
||||
@@ -105,10 +109,16 @@ impl MmapChannel {
|
||||
let mmap = unsafe { MmapMut::map_mut(&file).map_err(TransportError::Io)? };
|
||||
|
||||
if &mmap[..8] != MAGIC {
|
||||
return Err(TransportError::Protocol("invalid mmap channel magic".into()));
|
||||
return Err(TransportError::Protocol(
|
||||
"invalid mmap channel magic".into(),
|
||||
));
|
||||
}
|
||||
let capacity = read_u64(&mmap, 24) as usize;
|
||||
Ok(Self { path: path.to_owned(), mmap, capacity })
|
||||
Ok(Self {
|
||||
path: path.to_owned(),
|
||||
mmap,
|
||||
capacity,
|
||||
})
|
||||
}
|
||||
|
||||
/// Send a message into the ring buffer.
|
||||
@@ -194,8 +204,7 @@ impl MmapChannel {
|
||||
write_u64(&mut self.mmap, 16, new_rh);
|
||||
self.mmap.flush_range(0, 32).map_err(TransportError::Io)?;
|
||||
|
||||
SyncMessage::from_bytes(&body)
|
||||
.map_err(TransportError::Protocol)
|
||||
SyncMessage::from_bytes(&body).map_err(TransportError::Protocol)
|
||||
}
|
||||
|
||||
/// Path to the backing file.
|
||||
@@ -231,14 +240,20 @@ impl MmapChannel {
|
||||
impl MmapSender {
|
||||
pub fn open(path: &Path) -> Result<Self, TransportError> {
|
||||
let ch = MmapChannel::open(path)?;
|
||||
Ok(Self { _path: path.to_owned(), _capacity: ch.capacity })
|
||||
Ok(Self {
|
||||
_path: path.to_owned(),
|
||||
_capacity: ch.capacity,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl MmapReceiver {
|
||||
pub fn open(path: &Path) -> Result<Self, TransportError> {
|
||||
let ch = MmapChannel::open(path)?;
|
||||
Ok(Self { _path: path.to_owned(), _capacity: ch.capacity })
|
||||
Ok(Self {
|
||||
_path: path.to_owned(),
|
||||
_capacity: ch.capacity,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -310,7 +325,9 @@ mod tests {
|
||||
let (_f, path) = tmp_path();
|
||||
let mut ch = MmapChannel::create(&path, DEFAULT_CAPACITY).unwrap();
|
||||
|
||||
let msg = SyncMessage::Error { message: "oops".into() };
|
||||
let msg = SyncMessage::Error {
|
||||
message: "oops".into(),
|
||||
};
|
||||
ch.send(&msg).unwrap();
|
||||
let recv = ch.recv().unwrap();
|
||||
assert!(matches!(recv, SyncMessage::Error { .. }));
|
||||
@@ -324,9 +341,13 @@ mod tests {
|
||||
ch.send(&SyncMessage::SyncComplete {
|
||||
revisions_transferred: 100,
|
||||
bytes_transferred: 4096 * 100,
|
||||
}).unwrap();
|
||||
})
|
||||
.unwrap();
|
||||
match ch.recv().unwrap() {
|
||||
SyncMessage::SyncComplete { revisions_transferred, .. } => {
|
||||
SyncMessage::SyncComplete {
|
||||
revisions_transferred,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(revisions_transferred, 100);
|
||||
}
|
||||
other => panic!("unexpected: {other:?}"),
|
||||
|
||||
@@ -63,9 +63,7 @@ impl SyncPeer {
|
||||
let (r, w) = conn.into_split();
|
||||
(PipeReadHalf::Tcp(r), PipeWriteHalf::Tcp(w))
|
||||
}
|
||||
SyncPeer::Quic(arc) => {
|
||||
(PipeReadHalf::Quic(arc.clone()), PipeWriteHalf::Quic(arc))
|
||||
}
|
||||
SyncPeer::Quic(arc) => (PipeReadHalf::Quic(arc.clone()), PipeWriteHalf::Quic(arc)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -167,7 +167,6 @@ pub enum SyncMessage {
|
||||
//
|
||||
// IMPORTANT: rkyv assigns discriminants positionally. These variants MUST
|
||||
// remain appended after `Error` (index 8). Never insert or reorder.
|
||||
|
||||
/// Client announces it wants to sync a single file using CDC delta.
|
||||
///
|
||||
/// Carries the ordered CDC chunk descriptors of the client's version of
|
||||
@@ -205,7 +204,6 @@ pub enum SyncMessage {
|
||||
FsFileAck { path: String },
|
||||
|
||||
// ── FS directory tree sync (variants 13–16) ───────────────────────────────
|
||||
|
||||
/// Client sends a per-file manifest for an entire directory tree.
|
||||
FsDirManifest {
|
||||
/// Root path on the client (informational).
|
||||
@@ -244,7 +242,6 @@ pub enum SyncMessage {
|
||||
//
|
||||
// IMPORTANT: rkyv assigns discriminants positionally. These variants MUST
|
||||
// remain appended after `FsDirDryRun` (index 16). Never insert or reorder.
|
||||
|
||||
/// Client sends its per-dataset manifest and the remote h5 path to sync.
|
||||
///
|
||||
/// The server compares this manifest against its own to find which datasets
|
||||
@@ -310,8 +307,7 @@ impl SyncMessage {
|
||||
pub fn from_bytes(bytes: &[u8]) -> Result<Self, String> {
|
||||
let mut aligned = rkyv::util::AlignedVec::<16>::with_capacity(bytes.len());
|
||||
aligned.extend_from_slice(bytes);
|
||||
rkyv::from_bytes::<SyncMessage, rkyv::rancor::Error>(&aligned)
|
||||
.map_err(|e| e.to_string())
|
||||
rkyv::from_bytes::<SyncMessage, rkyv::rancor::Error>(&aligned).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// Encode to a length-prefixed frame: `[u32 LE length][message bytes]`.
|
||||
@@ -333,10 +329,16 @@ impl SyncMessage {
|
||||
}
|
||||
let len = u32::from_le_bytes(data[..4].try_into().unwrap()) as usize;
|
||||
if len > MAX_FRAME_SIZE {
|
||||
return Err(format!("frame too large: {len} bytes (max {MAX_FRAME_SIZE})"));
|
||||
return Err(format!(
|
||||
"frame too large: {len} bytes (max {MAX_FRAME_SIZE})"
|
||||
));
|
||||
}
|
||||
if data.len() < 4 + len {
|
||||
return Err(format!("incomplete frame: have {} bytes, need {}", data.len(), 4 + len));
|
||||
return Err(format!(
|
||||
"incomplete frame: have {} bytes, need {}",
|
||||
data.len(),
|
||||
4 + len
|
||||
));
|
||||
}
|
||||
let msg = Self::from_bytes(&data[4..4 + len])?;
|
||||
Ok((msg, 4 + len))
|
||||
@@ -377,7 +379,11 @@ mod tests {
|
||||
let bytes = msg.to_bytes().unwrap();
|
||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||
match recovered {
|
||||
SyncMessage::ManifestRequest { agent_id, head_revision, revision_count } => {
|
||||
SyncMessage::ManifestRequest {
|
||||
agent_id,
|
||||
head_revision,
|
||||
revision_count,
|
||||
} => {
|
||||
assert_eq!(agent_id, "test-agent");
|
||||
assert_eq!(head_revision, 5);
|
||||
assert_eq!(revision_count, 6);
|
||||
@@ -419,7 +425,10 @@ mod tests {
|
||||
let bytes = msg.to_bytes().unwrap();
|
||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||
match recovered {
|
||||
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => {
|
||||
SyncMessage::SyncComplete {
|
||||
revisions_transferred,
|
||||
bytes_transferred,
|
||||
} => {
|
||||
assert_eq!(revisions_transferred, 10);
|
||||
assert_eq!(bytes_transferred, 40960);
|
||||
}
|
||||
@@ -429,7 +438,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn error_message_roundtrip() {
|
||||
let msg = SyncMessage::Error { message: "something went wrong".to_string() };
|
||||
let msg = SyncMessage::Error {
|
||||
message: "something went wrong".to_string(),
|
||||
};
|
||||
let bytes = msg.to_bytes().unwrap();
|
||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||
match recovered {
|
||||
@@ -440,8 +451,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn iblt_request_roundtrip() {
|
||||
use clawsync_onion::manifest::IbltManifest;
|
||||
use clawsync_onion::iblt::IbltSketch;
|
||||
use clawsync_onion::manifest::IbltManifest;
|
||||
// Build a minimal IbltManifest
|
||||
let sketch = IbltSketch::from_keys(&[0u64, 1, 2], 0xABCD);
|
||||
let iblt_manifest = IbltManifest {
|
||||
@@ -454,7 +465,9 @@ mod tests {
|
||||
sketch: sketch.to_bytes(),
|
||||
sketch_cells: sketch.cell_count() as u32,
|
||||
};
|
||||
let msg = SyncMessage::IbltRequest { sketch: iblt_manifest };
|
||||
let msg = SyncMessage::IbltRequest {
|
||||
sketch: iblt_manifest,
|
||||
};
|
||||
let bytes = msg.to_bytes().unwrap();
|
||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||
assert!(matches!(recovered, SyncMessage::IbltRequest { .. }));
|
||||
@@ -462,8 +475,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn iblt_response_roundtrip() {
|
||||
use clawsync_onion::manifest::IbltManifest;
|
||||
use clawsync_onion::iblt::IbltSketch;
|
||||
use clawsync_onion::manifest::IbltManifest;
|
||||
let sketch = IbltSketch::from_keys(&[5u64, 6], 0xDEAD);
|
||||
let iblt_manifest = IbltManifest {
|
||||
agent_id: "srv".to_string(),
|
||||
@@ -482,7 +495,10 @@ mod tests {
|
||||
let bytes = msg.to_bytes().unwrap();
|
||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||
match recovered {
|
||||
SyncMessage::IbltResponse { missing_from_remote, .. } => {
|
||||
SyncMessage::IbltResponse {
|
||||
missing_from_remote,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(missing_from_remote, vec![7, 8, 9]);
|
||||
}
|
||||
_ => panic!("wrong variant"),
|
||||
@@ -496,9 +512,18 @@ mod tests {
|
||||
let msg = SyncMessage::FsCdcRequest {
|
||||
path: "data/weights.bin".to_string(),
|
||||
chunk_hashes: vec![
|
||||
FsChunkHash { hash: 0xDEAD, length: 65536 },
|
||||
FsChunkHash { hash: 0xBEEF, length: 32768 },
|
||||
FsChunkHash { hash: 0xCAFE, length: 65536 },
|
||||
FsChunkHash {
|
||||
hash: 0xDEAD,
|
||||
length: 65536,
|
||||
},
|
||||
FsChunkHash {
|
||||
hash: 0xBEEF,
|
||||
length: 32768,
|
||||
},
|
||||
FsChunkHash {
|
||||
hash: 0xCAFE,
|
||||
length: 65536,
|
||||
},
|
||||
],
|
||||
};
|
||||
let bytes = msg.to_bytes().unwrap();
|
||||
@@ -522,7 +547,10 @@ mod tests {
|
||||
let bytes = msg.to_bytes().unwrap();
|
||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||
match recovered {
|
||||
SyncMessage::FsCdcNeed { path, needed_indices } => {
|
||||
SyncMessage::FsCdcNeed {
|
||||
path,
|
||||
needed_indices,
|
||||
} => {
|
||||
assert_eq!(path, "model.bin");
|
||||
assert_eq!(needed_indices, vec![0, 2]);
|
||||
}
|
||||
@@ -535,18 +563,36 @@ mod tests {
|
||||
let msg = SyncMessage::FsCdcData {
|
||||
path: "cfg.bin".to_string(),
|
||||
chunk_order: vec![
|
||||
FsChunkHash { hash: 0x11, length: 65536 },
|
||||
FsChunkHash { hash: 0x22, length: 65536 },
|
||||
FsChunkHash {
|
||||
hash: 0x11,
|
||||
length: 65536,
|
||||
},
|
||||
FsChunkHash {
|
||||
hash: 0x22,
|
||||
length: 65536,
|
||||
},
|
||||
],
|
||||
chunks: vec![
|
||||
FsChunk { hash: 0x11, data: vec![0xAAu8; 100], compressed: true },
|
||||
FsChunk { hash: 0x22, data: vec![0xBBu8; 200], compressed: false },
|
||||
FsChunk {
|
||||
hash: 0x11,
|
||||
data: vec![0xAAu8; 100],
|
||||
compressed: true,
|
||||
},
|
||||
FsChunk {
|
||||
hash: 0x22,
|
||||
data: vec![0xBBu8; 200],
|
||||
compressed: false,
|
||||
},
|
||||
],
|
||||
};
|
||||
let bytes = msg.to_bytes().unwrap();
|
||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||
match recovered {
|
||||
SyncMessage::FsCdcData { path, chunk_order, chunks } => {
|
||||
SyncMessage::FsCdcData {
|
||||
path,
|
||||
chunk_order,
|
||||
chunks,
|
||||
} => {
|
||||
assert_eq!(path, "cfg.bin");
|
||||
assert_eq!(chunk_order.len(), 2);
|
||||
assert_eq!(chunks.len(), 2);
|
||||
@@ -560,10 +606,14 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn fs_file_ack_roundtrip() {
|
||||
let msg = SyncMessage::FsFileAck { path: "embeddings.bin".to_string() };
|
||||
let msg = SyncMessage::FsFileAck {
|
||||
path: "embeddings.bin".to_string(),
|
||||
};
|
||||
let bytes = msg.to_bytes().unwrap();
|
||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||
assert!(matches!(recovered, SyncMessage::FsFileAck { ref path } if path == "embeddings.bin"));
|
||||
assert!(
|
||||
matches!(recovered, SyncMessage::FsFileAck { ref path } if path == "embeddings.bin")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -571,8 +621,18 @@ mod tests {
|
||||
let msg = SyncMessage::FsDirManifest {
|
||||
local_root: "/home/user/data".to_string(),
|
||||
entries: vec![
|
||||
FsManifestEntry { path: "a.bin".to_string(), blake3: [1u8; 32], size: 4096, mtime: 1000 },
|
||||
FsManifestEntry { path: "b.bin".to_string(), blake3: [2u8; 32], size: 8192, mtime: 2000 },
|
||||
FsManifestEntry {
|
||||
path: "a.bin".to_string(),
|
||||
blake3: [1u8; 32],
|
||||
size: 4096,
|
||||
mtime: 1000,
|
||||
},
|
||||
FsManifestEntry {
|
||||
path: "b.bin".to_string(),
|
||||
blake3: [2u8; 32],
|
||||
size: 8192,
|
||||
mtime: 2000,
|
||||
},
|
||||
],
|
||||
};
|
||||
let bytes = msg.to_bytes().unwrap();
|
||||
@@ -590,19 +650,23 @@ mod tests {
|
||||
#[test]
|
||||
fn fs_dir_need_roundtrip() {
|
||||
let msg = SyncMessage::FsDirNeed {
|
||||
needed_files: vec![
|
||||
FsFileNeed {
|
||||
needed_files: vec![FsFileNeed {
|
||||
path: "weights.bin".to_string(),
|
||||
server_chunks: vec![FsChunkHash { hash: 0xAA, length: 65536 }],
|
||||
server_chunks: vec![FsChunkHash {
|
||||
hash: 0xAA,
|
||||
length: 65536,
|
||||
}],
|
||||
expected_blake3: [0xABu8; 32],
|
||||
},
|
||||
],
|
||||
}],
|
||||
to_delete: vec!["old_model.bin".to_string()],
|
||||
};
|
||||
let bytes = msg.to_bytes().unwrap();
|
||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||
match recovered {
|
||||
SyncMessage::FsDirNeed { needed_files, to_delete } => {
|
||||
SyncMessage::FsDirNeed {
|
||||
needed_files,
|
||||
to_delete,
|
||||
} => {
|
||||
assert_eq!(needed_files.len(), 1);
|
||||
assert_eq!(needed_files[0].path, "weights.bin");
|
||||
assert_eq!(to_delete, vec!["old_model.bin"]);
|
||||
@@ -614,12 +678,19 @@ mod tests {
|
||||
#[test]
|
||||
fn fs_dir_complete_roundtrip() {
|
||||
let msg = SyncMessage::FsDirComplete {
|
||||
files_added: 2, files_modified: 3, files_removed: 1, bytes_transferred: 1_048_576,
|
||||
files_added: 2,
|
||||
files_modified: 3,
|
||||
files_removed: 1,
|
||||
bytes_transferred: 1_048_576,
|
||||
};
|
||||
let bytes = msg.to_bytes().unwrap();
|
||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||
match recovered {
|
||||
SyncMessage::FsDirComplete { files_added, bytes_transferred, .. } => {
|
||||
SyncMessage::FsDirComplete {
|
||||
files_added,
|
||||
bytes_transferred,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(files_added, 2);
|
||||
assert_eq!(bytes_transferred, 1_048_576);
|
||||
}
|
||||
@@ -637,8 +708,10 @@ mod tests {
|
||||
// We just verify the roundtrip still produces the exact same ManifestRequest —
|
||||
// any reordering would cause a deserialization error here.
|
||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||
assert!(matches!(recovered, SyncMessage::ManifestRequest { .. }),
|
||||
"ManifestRequest discriminant changed — rkyv variant ordering was modified");
|
||||
assert!(
|
||||
matches!(recovered, SyncMessage::ManifestRequest { .. }),
|
||||
"ManifestRequest discriminant changed — rkyv variant ordering was modified"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -49,10 +49,7 @@ impl QuicConfig {
|
||||
.map_err(|e| TransportError::Tls(e.to_string()))?;
|
||||
|
||||
// Server config
|
||||
let server_config = ServerConfig::with_single_cert(
|
||||
vec![cert_der.clone()],
|
||||
key_der,
|
||||
)
|
||||
let server_config = ServerConfig::with_single_cert(vec![cert_der.clone()], key_der)
|
||||
.map_err(|e| TransportError::Tls(e.to_string()))?;
|
||||
|
||||
// Client config: trust only our self-signed cert
|
||||
@@ -63,7 +60,10 @@ impl QuicConfig {
|
||||
let client_config = ClientConfig::with_root_certificates(Arc::new(roots))
|
||||
.map_err(|e| TransportError::Tls(e.to_string()))?;
|
||||
|
||||
Ok(Self { server_config, client_config })
|
||||
Ok(Self {
|
||||
server_config,
|
||||
client_config,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,9 +83,7 @@ impl QuicConnection {
|
||||
|
||||
/// Send a `SyncMessage` on a new unidirectional QUIC stream.
|
||||
pub async fn send(&self, msg: &SyncMessage) -> Result<(), TransportError> {
|
||||
let body = msg
|
||||
.to_bytes()
|
||||
.map_err(TransportError::Serialization)?;
|
||||
let body = msg.to_bytes().map_err(TransportError::Serialization)?;
|
||||
|
||||
let mut send = self
|
||||
.conn
|
||||
@@ -101,7 +99,8 @@ impl QuicConnection {
|
||||
send.write_all(&body)
|
||||
.await
|
||||
.map_err(|e| TransportError::Io(e.into()))?;
|
||||
send.finish().map_err(|e| TransportError::Quic(e.to_string()))?;
|
||||
send.finish()
|
||||
.map_err(|e| TransportError::Quic(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -147,10 +146,12 @@ pub struct QuicServer {
|
||||
impl QuicServer {
|
||||
/// Bind a QUIC server endpoint.
|
||||
pub async fn bind(addr: SocketAddr, config: QuicConfig) -> Result<Self, TransportError> {
|
||||
let endpoint = Endpoint::server(config.server_config, addr)
|
||||
.map_err(TransportError::Io)?;
|
||||
let endpoint = Endpoint::server(config.server_config, addr).map_err(TransportError::Io)?;
|
||||
let local_addr = endpoint.local_addr()?;
|
||||
Ok(Self { endpoint, local_addr })
|
||||
Ok(Self {
|
||||
endpoint,
|
||||
local_addr,
|
||||
})
|
||||
}
|
||||
|
||||
/// Accept the next incoming QUIC connection.
|
||||
@@ -177,8 +178,8 @@ pub async fn quic_connect(
|
||||
server_name: &str,
|
||||
config: QuicConfig,
|
||||
) -> Result<QuicConnection, TransportError> {
|
||||
let mut endpoint = Endpoint::client("0.0.0.0:0".parse().unwrap())
|
||||
.map_err(TransportError::Io)?;
|
||||
let mut endpoint =
|
||||
Endpoint::client("0.0.0.0:0".parse().unwrap()).map_err(TransportError::Io)?;
|
||||
endpoint.set_default_client_config(config.client_config);
|
||||
|
||||
let conn = endpoint
|
||||
@@ -217,10 +218,15 @@ mod tests {
|
||||
roots.add(cert_der).unwrap();
|
||||
let cli_cfg = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap();
|
||||
|
||||
let server = QuicServer::bind(any_addr(), QuicConfig {
|
||||
let server = QuicServer::bind(
|
||||
any_addr(),
|
||||
QuicConfig {
|
||||
server_config: srv_cfg,
|
||||
client_config: cli_cfg.clone(),
|
||||
}).await.unwrap();
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let addr = server.local_addr;
|
||||
|
||||
let server_task = tokio::spawn(async move {
|
||||
|
||||
@@ -7,11 +7,11 @@
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf};
|
||||
use tokio::net::{TcpListener, TcpStream};
|
||||
|
||||
use crate::error::TransportError;
|
||||
use crate::protocol::{SyncMessage, MAX_FRAME_SIZE};
|
||||
use crate::protocol::{MAX_FRAME_SIZE, SyncMessage};
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// TcpConnection — async read/write of SyncMessages
|
||||
@@ -37,9 +37,7 @@ impl TcpConnection {
|
||||
|
||||
/// Send a `SyncMessage` over the connection.
|
||||
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
||||
let body = msg
|
||||
.to_bytes()
|
||||
.map_err(TransportError::Serialization)?;
|
||||
let body = msg.to_bytes().map_err(TransportError::Serialization)?;
|
||||
let len = body.len() as u32;
|
||||
self.stream.write_all(&len.to_le_bytes()).await?;
|
||||
self.stream.write_all(&body).await?;
|
||||
@@ -65,8 +63,7 @@ impl TcpConnection {
|
||||
let mut body = vec![0u8; len];
|
||||
self.stream.read_exact(&mut body).await?;
|
||||
|
||||
SyncMessage::from_bytes(&body)
|
||||
.map_err(TransportError::Deserialization)
|
||||
SyncMessage::from_bytes(&body).map_err(TransportError::Deserialization)
|
||||
}
|
||||
|
||||
/// Gracefully shut down the write half of the connection.
|
||||
@@ -81,7 +78,10 @@ impl TcpConnection {
|
||||
/// pipelined send/receive (sliding window) without holding a single lock.
|
||||
pub fn into_split(self) -> (TcpReadHalf, TcpWriteHalf) {
|
||||
let (read_half, write_half) = self.stream.into_split();
|
||||
(TcpReadHalf { half: read_half }, TcpWriteHalf { half: write_half })
|
||||
(
|
||||
TcpReadHalf { half: read_half },
|
||||
TcpWriteHalf { half: write_half },
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,7 +102,10 @@ impl TcpReadHalf {
|
||||
let len = u32::from_le_bytes(len_buf) as usize;
|
||||
|
||||
if len > MAX_FRAME_SIZE {
|
||||
return Err(TransportError::FrameTooLarge { size: len, max: MAX_FRAME_SIZE });
|
||||
return Err(TransportError::FrameTooLarge {
|
||||
size: len,
|
||||
max: MAX_FRAME_SIZE,
|
||||
});
|
||||
}
|
||||
|
||||
let mut body = vec![0u8; len];
|
||||
@@ -149,7 +152,10 @@ impl TcpServer {
|
||||
pub async fn bind(addr: SocketAddr) -> Result<Self, TransportError> {
|
||||
let listener = TcpListener::bind(addr).await?;
|
||||
let local_addr = listener.local_addr()?;
|
||||
Ok(Self { listener, local_addr })
|
||||
Ok(Self {
|
||||
listener,
|
||||
local_addr,
|
||||
})
|
||||
}
|
||||
|
||||
/// Accept the next incoming connection.
|
||||
@@ -186,7 +192,10 @@ mod tests {
|
||||
});
|
||||
|
||||
let mut client = TcpConnection::connect(addr).await.unwrap();
|
||||
client.send(&SyncMessage::Ack { revision: 42 }).await.unwrap();
|
||||
client
|
||||
.send(&SyncMessage::Ack { revision: 42 })
|
||||
.await
|
||||
.unwrap();
|
||||
let reply = client.recv().await.unwrap();
|
||||
assert!(matches!(reply, SyncMessage::Ack { revision: 42 }));
|
||||
|
||||
@@ -208,7 +217,10 @@ mod tests {
|
||||
|
||||
let mut client = TcpConnection::connect(addr).await.unwrap();
|
||||
for rev in 0u64..5 {
|
||||
client.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
||||
client
|
||||
.send(&SyncMessage::Ack { revision: rev })
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
server_task.await.unwrap();
|
||||
}
|
||||
@@ -233,7 +245,10 @@ mod tests {
|
||||
client.send(&msg).await.unwrap();
|
||||
let reply = client.recv().await.unwrap();
|
||||
match reply {
|
||||
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => {
|
||||
SyncMessage::SyncComplete {
|
||||
revisions_transferred,
|
||||
bytes_transferred,
|
||||
} => {
|
||||
assert_eq!(revisions_transferred, 7);
|
||||
assert_eq!(bytes_transferred, 28672);
|
||||
}
|
||||
@@ -263,7 +278,11 @@ mod tests {
|
||||
client.send(&msg).await.unwrap();
|
||||
let reply = client.recv().await.unwrap();
|
||||
match reply {
|
||||
SyncMessage::ManifestRequest { agent_id, head_revision, .. } => {
|
||||
SyncMessage::ManifestRequest {
|
||||
agent_id,
|
||||
head_revision,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(agent_id, "my-agent");
|
||||
assert_eq!(head_revision, 3);
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ use clawhdf5_onion::writer::OnionFile;
|
||||
use clawsync_onion::{differ::diff_revisions, manifest::ClawSyncManifest, merger::merge_packets};
|
||||
use clawsync_transport::{
|
||||
protocol::SyncMessage,
|
||||
quic::{quic_connect, QuicConfig, QuicServer},
|
||||
quic::{QuicConfig, QuicServer, quic_connect},
|
||||
};
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
@@ -62,8 +62,7 @@ fn make_quic_configs() -> (QuicConfig, QuicConfig) {
|
||||
|
||||
let mut roots = rustls::RootCertStore::empty();
|
||||
roots.add(srv_cert_der).unwrap();
|
||||
let client_config =
|
||||
quinn::ClientConfig::with_root_certificates(Arc::new(roots)).unwrap();
|
||||
let client_config = quinn::ClientConfig::with_root_certificates(Arc::new(roots)).unwrap();
|
||||
|
||||
// Client-side QuicConfig requires a server_config field; generate a fresh
|
||||
// cert+key pair so there is no key/cert mismatch.
|
||||
@@ -71,8 +70,14 @@ fn make_quic_configs() -> (QuicConfig, QuicConfig) {
|
||||
let cli_server_config =
|
||||
quinn::ServerConfig::with_single_cert(vec![cli_cert_der], cli_key_der).unwrap();
|
||||
|
||||
let srv_cfg = QuicConfig { server_config: srv_server_config, client_config: client_config.clone() };
|
||||
let cli_cfg = QuicConfig { server_config: cli_server_config, client_config };
|
||||
let srv_cfg = QuicConfig {
|
||||
server_config: srv_server_config,
|
||||
client_config: client_config.clone(),
|
||||
};
|
||||
let cli_cfg = QuicConfig {
|
||||
server_config: cli_server_config,
|
||||
client_config,
|
||||
};
|
||||
(srv_cfg, cli_cfg)
|
||||
}
|
||||
|
||||
@@ -99,7 +104,9 @@ async fn run_quic_push(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
||||
match msg {
|
||||
SyncMessage::ManifestRequest { .. } => {
|
||||
let dst_manifest = ClawSyncManifest::from_onion("dst", &dst, H5_BASE);
|
||||
conn.send(&SyncMessage::ManifestResponse { manifest: dst_manifest })
|
||||
conn.send(&SyncMessage::ManifestResponse {
|
||||
manifest: dst_manifest,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
@@ -108,7 +115,9 @@ async fn run_quic_push(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
||||
let rev = packet.revision;
|
||||
merge_packets(&mut dst, vec![packet], false).unwrap();
|
||||
packets_received += 1;
|
||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
||||
conn.send(&SyncMessage::Ack { revision: rev })
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
SyncMessage::SyncComplete { .. } => {
|
||||
@@ -125,7 +134,9 @@ async fn run_quic_push(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
||||
});
|
||||
|
||||
// ── Client logic ─────────────────────────────────────────────────────────
|
||||
let client = quic_connect(server_addr, "localhost", cli_cfg).await.unwrap();
|
||||
let client = quic_connect(server_addr, "localhost", cli_cfg)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
client
|
||||
.send(&SyncMessage::ManifestRequest {
|
||||
@@ -152,7 +163,10 @@ async fn run_quic_push(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
||||
|
||||
for packet in packets {
|
||||
let rev = packet.revision;
|
||||
client.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
||||
client
|
||||
.send(&SyncMessage::LayerPacket { packet })
|
||||
.await
|
||||
.unwrap();
|
||||
match client.recv().await.unwrap() {
|
||||
SyncMessage::Ack { revision } => assert_eq!(revision, rev),
|
||||
other => panic!("client: expected Ack, got {other:?}"),
|
||||
@@ -228,7 +242,11 @@ async fn quic_revision_hashes_match_after_sync() {
|
||||
|
||||
assert_eq!(src_revisions.len(), dst_revisions.len());
|
||||
for (s, d) in src_revisions.iter().zip(dst_revisions.iter()) {
|
||||
assert_eq!(s.blake3_hex, d.blake3_hex, "rev {} hash mismatch", s.revision);
|
||||
assert_eq!(
|
||||
s.blake3_hex, d.blake3_hex,
|
||||
"rev {} hash mismatch",
|
||||
s.revision
|
||||
);
|
||||
assert_eq!(s.annotation, d.annotation);
|
||||
}
|
||||
}
|
||||
@@ -269,7 +287,9 @@ async fn run_quic_pull(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
||||
};
|
||||
|
||||
let srv_manifest = ClawSyncManifest::from_onion("srv", &src, H5_BASE);
|
||||
conn.send(&SyncMessage::ManifestResponse { manifest: srv_manifest })
|
||||
conn.send(&SyncMessage::ManifestResponse {
|
||||
manifest: srv_manifest,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -284,7 +304,9 @@ async fn run_quic_pull(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
||||
let mut bytes_sent = 0u64;
|
||||
for packet in packets {
|
||||
bytes_sent += packet.page_data_size() as u64;
|
||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
||||
conn.send(&SyncMessage::LayerPacket { packet })
|
||||
.await
|
||||
.unwrap();
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::Ack { .. } => {}
|
||||
other => panic!("server: expected Ack, got {other:?}"),
|
||||
@@ -310,7 +332,9 @@ async fn run_quic_pull(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
||||
});
|
||||
|
||||
// ── Client logic ─────────────────────────────────────────────────────────
|
||||
let client = quic_connect(server_addr, "localhost", cli_cfg).await.unwrap();
|
||||
let client = quic_connect(server_addr, "localhost", cli_cfg)
|
||||
.await
|
||||
.unwrap();
|
||||
let local_rev_count = dst.revision_count();
|
||||
|
||||
client
|
||||
@@ -344,10 +368,16 @@ async fn run_quic_pull(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
||||
match client.recv().await.unwrap() {
|
||||
SyncMessage::LayerPacket { packet } => {
|
||||
let rev = packet.revision;
|
||||
client.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
||||
client
|
||||
.send(&SyncMessage::Ack { revision: rev })
|
||||
.await
|
||||
.unwrap();
|
||||
packets.push(packet);
|
||||
}
|
||||
SyncMessage::SyncComplete { revisions_transferred, .. } => {
|
||||
SyncMessage::SyncComplete {
|
||||
revisions_transferred,
|
||||
..
|
||||
} => {
|
||||
merge_packets(&mut dst, packets, true).unwrap();
|
||||
server_handle.await.unwrap();
|
||||
return (revisions_transferred, dst);
|
||||
@@ -412,7 +442,11 @@ async fn quic_pull_revision_hashes_match() {
|
||||
|
||||
assert_eq!(srv_revisions.len(), cli_revisions.len());
|
||||
for (s, c) in srv_revisions.iter().zip(cli_revisions.iter()) {
|
||||
assert_eq!(s.blake3_hex, c.blake3_hex, "rev {} hash mismatch", s.revision);
|
||||
assert_eq!(
|
||||
s.blake3_hex, c.blake3_hex,
|
||||
"rev {} hash mismatch",
|
||||
s.revision
|
||||
);
|
||||
assert_eq!(s.annotation, c.annotation);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,11 +18,7 @@
|
||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||
|
||||
use clawhdf5_onion::writer::OnionFile;
|
||||
use clawsync_onion::{
|
||||
differ::diff_revisions,
|
||||
manifest::ClawSyncManifest,
|
||||
merger::merge_packets,
|
||||
};
|
||||
use clawsync_onion::{differ::diff_revisions, manifest::ClawSyncManifest, merger::merge_packets};
|
||||
use clawsync_transport::{
|
||||
protocol::SyncMessage,
|
||||
tcp::{TcpConnection, TcpServer},
|
||||
@@ -45,9 +41,7 @@ fn make_onion(n: u8) -> (NamedTempFile, OnionFile) {
|
||||
for i in 0..n {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![i; 4096]);
|
||||
onion
|
||||
.commit_session(s, Some(&format!("rev {i}")))
|
||||
.unwrap();
|
||||
onion.commit_session(s, Some(&format!("rev {i}"))).unwrap();
|
||||
}
|
||||
(tmp, onion)
|
||||
}
|
||||
@@ -57,10 +51,7 @@ fn make_onion(n: u8) -> (NamedTempFile, OnionFile) {
|
||||
// number of LayerPackets actually transferred.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
async fn run_push(
|
||||
src: OnionFile,
|
||||
mut dst: OnionFile,
|
||||
) -> (u64, OnionFile) {
|
||||
async fn run_push(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
||||
// Bind the server; the OS picks a free port.
|
||||
let server = TcpServer::bind(localhost_any()).await.unwrap();
|
||||
let server_addr = server.local_addr;
|
||||
@@ -79,8 +70,7 @@ async fn run_push(
|
||||
match msg {
|
||||
SyncMessage::ManifestRequest { head_revision, .. } => {
|
||||
// Build our local manifest and send it back.
|
||||
let dst_manifest =
|
||||
ClawSyncManifest::from_onion("dst", &dst, H5_BASE);
|
||||
let dst_manifest = ClawSyncManifest::from_onion("dst", &dst, H5_BASE);
|
||||
conn.send(&SyncMessage::ManifestResponse {
|
||||
manifest: dst_manifest,
|
||||
})
|
||||
@@ -163,7 +153,10 @@ async fn run_push(
|
||||
.unwrap();
|
||||
|
||||
let (server_received, dst_final) = server_handle.await.unwrap();
|
||||
assert_eq!(server_received, total_sent, "server count must match client send count");
|
||||
assert_eq!(
|
||||
server_received, total_sent,
|
||||
"server count must match client send count"
|
||||
);
|
||||
|
||||
(total_sent, dst_final)
|
||||
}
|
||||
@@ -305,7 +298,9 @@ async fn run_pull(
|
||||
|
||||
// 2. Send back the server manifest.
|
||||
let srv_manifest = ClawSyncManifest::from_onion("srv", &src, H5_BASE);
|
||||
conn.send(&SyncMessage::ManifestResponse { manifest: srv_manifest })
|
||||
conn.send(&SyncMessage::ManifestResponse {
|
||||
manifest: srv_manifest,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -321,7 +316,9 @@ async fn run_pull(
|
||||
let mut bytes_sent = 0u64;
|
||||
for packet in packets {
|
||||
bytes_sent += packet.page_data_size() as u64;
|
||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
||||
conn.send(&SyncMessage::LayerPacket { packet })
|
||||
.await
|
||||
.unwrap();
|
||||
match conn.recv().await.unwrap() {
|
||||
SyncMessage::Ack { .. } => {}
|
||||
other => panic!("server: expected Ack, got {other:?}"),
|
||||
@@ -381,10 +378,16 @@ async fn run_pull(
|
||||
match client.recv().await.unwrap() {
|
||||
SyncMessage::LayerPacket { packet } => {
|
||||
let rev = packet.revision;
|
||||
client.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
||||
client
|
||||
.send(&SyncMessage::Ack { revision: rev })
|
||||
.await
|
||||
.unwrap();
|
||||
packets.push(packet);
|
||||
}
|
||||
SyncMessage::SyncComplete { revisions_transferred, .. } => {
|
||||
SyncMessage::SyncComplete {
|
||||
revisions_transferred,
|
||||
..
|
||||
} => {
|
||||
merge_packets(&mut dst, packets, true).unwrap();
|
||||
server_handle.await.unwrap();
|
||||
return (revisions_transferred, dst);
|
||||
@@ -449,7 +452,11 @@ async fn pull_revision_hashes_match() {
|
||||
|
||||
assert_eq!(srv_revisions.len(), cli_revisions.len());
|
||||
for (s, c) in srv_revisions.iter().zip(cli_revisions.iter()) {
|
||||
assert_eq!(s.blake3_hex, c.blake3_hex, "rev {} hash mismatch", s.revision);
|
||||
assert_eq!(
|
||||
s.blake3_hex, c.blake3_hex,
|
||||
"rev {} hash mismatch",
|
||||
s.revision
|
||||
);
|
||||
assert_eq!(s.annotation, c.annotation);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user