Apply rustfmt to entire workspace
Runs cargo fmt --all; all 573 tests still passing, clippy still clean. No logic changes — formatting only. Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
3d524e2d63
commit
1c107fe58a
@@ -46,7 +46,9 @@ fn bench_epoch_gc(c: &mut Criterion) {
|
|||||||
b.iter_with_setup(
|
b.iter_with_setup(
|
||||||
|| make_onion(n),
|
|| make_onion(n),
|
||||||
|(_tmp, _h5, mut onion)| {
|
|(_tmp, _h5, mut onion)| {
|
||||||
let _ = onion.gc(black_box(GcPolicy::KeepLastN((n / 2) as u64))).unwrap();
|
let _ = onion
|
||||||
|
.gc(black_box(GcPolicy::KeepLastN((n / 2) as u64)))
|
||||||
|
.unwrap();
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@@ -60,9 +62,11 @@ fn bench_epoch_gc(c: &mut Criterion) {
|
|||||||
b.iter_with_setup(
|
b.iter_with_setup(
|
||||||
|| make_onion(n),
|
|| make_onion(n),
|
||||||
|(_tmp, _h5, mut onion)| {
|
|(_tmp, _h5, mut onion)| {
|
||||||
let _ = onion.gc(black_box(GcPolicy::EpochFlip(Box::new(
|
let _ = onion
|
||||||
GcPolicy::KeepLastN((n / 2) as u64),
|
.gc(black_box(GcPolicy::EpochFlip(Box::new(
|
||||||
)))).unwrap();
|
GcPolicy::KeepLastN((n / 2) as u64),
|
||||||
|
))))
|
||||||
|
.unwrap();
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@@ -76,9 +80,11 @@ fn bench_epoch_gc(c: &mut Criterion) {
|
|||||||
b.iter_with_setup(
|
b.iter_with_setup(
|
||||||
|| {
|
|| {
|
||||||
let (tmp, h5, mut onion) = make_onion(n);
|
let (tmp, h5, mut onion) = make_onion(n);
|
||||||
onion.gc(GcPolicy::EpochFlip(Box::new(
|
onion
|
||||||
GcPolicy::KeepLastN((n / 2) as u64),
|
.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(
|
||||||
))).unwrap();
|
(n / 2) as u64,
|
||||||
|
))))
|
||||||
|
.unwrap();
|
||||||
(tmp, h5, onion)
|
(tmp, h5, onion)
|
||||||
},
|
},
|
||||||
|(_tmp, _h5, mut onion)| {
|
|(_tmp, _h5, mut onion)| {
|
||||||
|
|||||||
@@ -51,21 +51,31 @@ fn bench_merkle(c: &mut Criterion) {
|
|||||||
for &n in &ns {
|
for &n in &ns {
|
||||||
let full = RevisionMerkleTree::build(&make_entries(n));
|
let full = RevisionMerkleTree::build(&make_entries(n));
|
||||||
let minus1 = RevisionMerkleTree::build(&make_entries(n - 1));
|
let minus1 = RevisionMerkleTree::build(&make_entries(n - 1));
|
||||||
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, minus1), |b, (f, m)| {
|
group.bench_with_input(
|
||||||
b.iter(|| f.diff_missing_revisions(black_box(m)));
|
BenchmarkId::from_parameter(n),
|
||||||
});
|
&(full, minus1),
|
||||||
|
|b, (f, m)| {
|
||||||
|
b.iter(|| f.diff_missing_revisions(black_box(m)));
|
||||||
|
},
|
||||||
|
);
|
||||||
}
|
}
|
||||||
group.finish();
|
group.finish();
|
||||||
|
|
||||||
// ── Diff walk: D=10 ──────────────────────────────────────────────────────
|
// ── Diff walk: D=10 ──────────────────────────────────────────────────────
|
||||||
let mut group = c.benchmark_group("merkle_index/diff_walk_d10");
|
let mut group = c.benchmark_group("merkle_index/diff_walk_d10");
|
||||||
for &n in &ns {
|
for &n in &ns {
|
||||||
if n < 20 { continue; }
|
if n < 20 {
|
||||||
let full = RevisionMerkleTree::build(&make_entries(n));
|
continue;
|
||||||
let base = RevisionMerkleTree::build(&make_entries(n - 10));
|
}
|
||||||
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, base), |b, (f, base)| {
|
let full = RevisionMerkleTree::build(&make_entries(n));
|
||||||
b.iter(|| f.diff_missing_revisions(black_box(base)));
|
let base = RevisionMerkleTree::build(&make_entries(n - 10));
|
||||||
});
|
group.bench_with_input(
|
||||||
|
BenchmarkId::from_parameter(n),
|
||||||
|
&(full, base),
|
||||||
|
|b, (f, base)| {
|
||||||
|
b.iter(|| f.diff_missing_revisions(black_box(base)));
|
||||||
|
},
|
||||||
|
);
|
||||||
}
|
}
|
||||||
group.finish();
|
group.finish();
|
||||||
|
|
||||||
@@ -74,9 +84,13 @@ fn bench_merkle(c: &mut Criterion) {
|
|||||||
for &n in &[1_000usize, 10_000] {
|
for &n in &[1_000usize, 10_000] {
|
||||||
let full = RevisionMerkleTree::build(&make_entries(n));
|
let full = RevisionMerkleTree::build(&make_entries(n));
|
||||||
let base = RevisionMerkleTree::build(&make_entries(n - 100));
|
let base = RevisionMerkleTree::build(&make_entries(n - 100));
|
||||||
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, base), |b, (f, base)| {
|
group.bench_with_input(
|
||||||
b.iter(|| f.diff_missing_revisions(black_box(base)));
|
BenchmarkId::from_parameter(n),
|
||||||
});
|
&(full, base),
|
||||||
|
|b, (f, base)| {
|
||||||
|
b.iter(|| f.diff_missing_revisions(black_box(base)));
|
||||||
|
},
|
||||||
|
);
|
||||||
}
|
}
|
||||||
group.finish();
|
group.finish();
|
||||||
|
|
||||||
@@ -102,14 +116,19 @@ fn bench_merkle(c: &mut Criterion) {
|
|||||||
|
|
||||||
// ── Serialised size report (not timed) ───────────────────────────────────
|
// ── Serialised size report (not timed) ───────────────────────────────────
|
||||||
println!("\n=== Serialised size vs flat manifest (N × 60 B) ===");
|
println!("\n=== Serialised size vs flat manifest (N × 60 B) ===");
|
||||||
println!("{:>8} {:>12} {:>12} {:>10}", "N", "Merkle (B)", "Flat (B)", "ratio");
|
println!(
|
||||||
|
"{:>8} {:>12} {:>12} {:>10}",
|
||||||
|
"N", "Merkle (B)", "Flat (B)", "ratio"
|
||||||
|
);
|
||||||
for &n in &[10usize, 100, 1_000, 10_000] {
|
for &n in &[10usize, 100, 1_000, 10_000] {
|
||||||
let tree = RevisionMerkleTree::build(&make_entries(n));
|
let tree = RevisionMerkleTree::build(&make_entries(n));
|
||||||
let merkle_sz = tree.serialise().len();
|
let merkle_sz = tree.serialise().len();
|
||||||
let flat_sz = n * 60;
|
let flat_sz = n * 60;
|
||||||
println!(
|
println!(
|
||||||
"{:>8} {:>12} {:>12} {:>10.2}x",
|
"{:>8} {:>12} {:>12} {:>10.2}x",
|
||||||
n, merkle_sz, flat_sz,
|
n,
|
||||||
|
merkle_sz,
|
||||||
|
flat_sz,
|
||||||
flat_sz as f64 / merkle_sz as f64
|
flat_sz as f64 / merkle_sz as f64
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -66,7 +66,10 @@ fn bench_write_one_revision(c: &mut Criterion) {
|
|||||||
b.iter(|| {
|
b.iter(|| {
|
||||||
let mut s = onion.begin_session(None).unwrap();
|
let mut s = onion.begin_session(None).unwrap();
|
||||||
for i in 0u64..4 {
|
for i in 0u64..4 {
|
||||||
s.record_page(i * PAGE_SIZE as u64, black_box(&vec![0xCDu8; PAGE_SIZE as usize]));
|
s.record_page(
|
||||||
|
i * PAGE_SIZE as u64,
|
||||||
|
black_box(&vec![0xCDu8; PAGE_SIZE as usize]),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
black_box(onion.commit_session(s, None).unwrap());
|
black_box(onion.commit_session(s, None).unwrap());
|
||||||
});
|
});
|
||||||
@@ -83,15 +86,15 @@ fn bench_reconstruct_revision(c: &mut Criterion) {
|
|||||||
let (_f, _h5, onion, base) = tmp_onion_with_n(depth);
|
let (_f, _h5, onion, base) = tmp_onion_with_n(depth);
|
||||||
let target_rev = (depth - 1) as u64;
|
let target_rev = (depth - 1) as u64;
|
||||||
|
|
||||||
group.bench_with_input(
|
group.bench_with_input(BenchmarkId::new("no_snapshot", depth), &depth, |b, _| {
|
||||||
BenchmarkId::new("no_snapshot", depth),
|
b.iter(|| {
|
||||||
&depth,
|
black_box(
|
||||||
|b, _| {
|
onion
|
||||||
b.iter(|| {
|
.reconstruct_revision(target_rev, black_box(&base))
|
||||||
black_box(onion.reconstruct_revision(target_rev, black_box(&base)).unwrap());
|
.unwrap(),
|
||||||
});
|
);
|
||||||
},
|
});
|
||||||
);
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
// With snapshot: create a real snapshot at midpoint via create_snapshot()
|
// With snapshot: create a real snapshot at midpoint via create_snapshot()
|
||||||
@@ -107,15 +110,15 @@ fn bench_reconstruct_revision(c: &mut Criterion) {
|
|||||||
}
|
}
|
||||||
let target_rev = onion.revision_count() - 1;
|
let target_rev = onion.revision_count() - 1;
|
||||||
|
|
||||||
group.bench_with_input(
|
group.bench_with_input(BenchmarkId::new("with_snapshot", depth), &depth, |b, _| {
|
||||||
BenchmarkId::new("with_snapshot", depth),
|
b.iter(|| {
|
||||||
&depth,
|
black_box(
|
||||||
|b, _| {
|
onion
|
||||||
b.iter(|| {
|
.reconstruct_revision(target_rev, black_box(&base))
|
||||||
black_box(onion.reconstruct_revision(target_rev, black_box(&base)).unwrap());
|
.unwrap(),
|
||||||
});
|
);
|
||||||
},
|
});
|
||||||
);
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
group.finish();
|
group.finish();
|
||||||
|
|||||||
@@ -35,7 +35,9 @@ fn make_f32_random(n_bytes: usize) -> Vec<u8> {
|
|||||||
let mut state = 0x_dead_beef_u64;
|
let mut state = 0x_dead_beef_u64;
|
||||||
let mut out = Vec::with_capacity(n_bytes);
|
let mut out = Vec::with_capacity(n_bytes);
|
||||||
while out.len() < n_bytes {
|
while out.len() < n_bytes {
|
||||||
state = state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
|
state = state
|
||||||
|
.wrapping_mul(6364136223846793005)
|
||||||
|
.wrapping_add(1442695040888963407);
|
||||||
out.extend_from_slice(&(state as u32).to_le_bytes());
|
out.extend_from_slice(&(state as u32).to_le_bytes());
|
||||||
}
|
}
|
||||||
out.truncate(n_bytes);
|
out.truncate(n_bytes);
|
||||||
@@ -72,16 +74,13 @@ fn bench_tdt_compress(c: &mut Criterion) {
|
|||||||
let sizes = [4096usize, 65536];
|
let sizes = [4096usize, 65536];
|
||||||
|
|
||||||
let datasets: &[(&str, fn(usize) -> Vec<u8>, usize)] = &[
|
let datasets: &[(&str, fn(usize) -> Vec<u8>, usize)] = &[
|
||||||
("f32_smooth", make_f32_smooth, 4),
|
("f32_smooth", make_f32_smooth, 4),
|
||||||
("f32_random", make_f32_random, 4),
|
("f32_random", make_f32_random, 4),
|
||||||
("int32_random", make_int32_random, 4),
|
("int32_random", make_int32_random, 4),
|
||||||
("int8_seq", make_int8_seq, 1),
|
("int8_seq", make_int8_seq, 1),
|
||||||
];
|
];
|
||||||
|
|
||||||
let codecs = [
|
let codecs = [("zstd", Codec::Zstd), ("zstd_tdt", Codec::ZstdTdt)];
|
||||||
("zstd", Codec::Zstd),
|
|
||||||
("zstd_tdt", Codec::ZstdTdt),
|
|
||||||
];
|
|
||||||
|
|
||||||
// ── Throughput benchmark ──────────────────────────────────────────────────
|
// ── Throughput benchmark ──────────────────────────────────────────────────
|
||||||
let mut group = c.benchmark_group("tdt_compress/throughput");
|
let mut group = c.benchmark_group("tdt_compress/throughput");
|
||||||
@@ -90,10 +89,7 @@ fn bench_tdt_compress(c: &mut Criterion) {
|
|||||||
let data = make(size);
|
let data = make(size);
|
||||||
group.throughput(Throughput::Bytes(size as u64));
|
group.throughput(Throughput::Bytes(size as u64));
|
||||||
for &(codec_name, codec) in &codecs {
|
for &(codec_name, codec) in &codecs {
|
||||||
let id = BenchmarkId::new(
|
let id = BenchmarkId::new(format!("{codec_name}/{dtype}"), format!("{size}B"));
|
||||||
format!("{codec_name}/{dtype}"),
|
|
||||||
format!("{size}B"),
|
|
||||||
);
|
|
||||||
group.bench_with_input(id, &data, |b, d| {
|
group.bench_with_input(id, &data, |b, d| {
|
||||||
b.iter(|| bench_compress(d, codec));
|
b.iter(|| bench_compress(d, codec));
|
||||||
});
|
});
|
||||||
@@ -126,16 +122,24 @@ fn bench_tdt_compress(c: &mut Criterion) {
|
|||||||
// ── Compression ratio summary (printed, not timed) ───────────────────────
|
// ── Compression ratio summary (printed, not timed) ───────────────────────
|
||||||
// Run once outside Criterion to print ratio comparison.
|
// Run once outside Criterion to print ratio comparison.
|
||||||
println!("\n─── TDT compression ratio summary ───");
|
println!("\n─── TDT compression ratio summary ───");
|
||||||
println!("{:<20} {:>8} {:>10} {:>10} {:>8}",
|
println!(
|
||||||
"dataset/size", "orig", "zstd", "zstd_tdt", "savings");
|
"{:<20} {:>8} {:>10} {:>10} {:>8}",
|
||||||
|
"dataset/size", "orig", "zstd", "zstd_tdt", "savings"
|
||||||
|
);
|
||||||
for &size in &sizes {
|
for &size in &sizes {
|
||||||
for &(dtype, make, _w) in datasets {
|
for &(dtype, make, _w) in datasets {
|
||||||
let data = make(size);
|
let data = make(size);
|
||||||
let zstd_size = compress_page(&data, Codec::Zstd).unwrap().len();
|
let zstd_size = compress_page(&data, Codec::Zstd).unwrap().len();
|
||||||
let tdt_size = compress_page(&data, Codec::ZstdTdt).unwrap().len();
|
let tdt_size = compress_page(&data, Codec::ZstdTdt).unwrap().len();
|
||||||
let savings_pct = 100.0 * (1.0 - tdt_size as f64 / zstd_size as f64);
|
let savings_pct = 100.0 * (1.0 - tdt_size as f64 / zstd_size as f64);
|
||||||
println!("{:<20} {:>8} {:>10} {:>10} {:>7.1}%",
|
println!(
|
||||||
format!("{dtype}/{size}B"), size, zstd_size, tdt_size, savings_pct);
|
"{:<20} {:>8} {:>10} {:>10} {:>7.1}%",
|
||||||
|
format!("{dtype}/{size}B"),
|
||||||
|
size,
|
||||||
|
zstd_size,
|
||||||
|
tdt_size,
|
||||||
|
savings_pct
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
println!();
|
println!();
|
||||||
|
|||||||
@@ -37,10 +37,7 @@ use crate::writer::OnionFile;
|
|||||||
/// let (bytes, onion) = open_revision("agent.h5", 7)?;
|
/// let (bytes, onion) = open_revision("agent.h5", 7)?;
|
||||||
/// let file = clawhdf5::File::from_bytes(bytes)?;
|
/// let file = clawhdf5::File::from_bytes(bytes)?;
|
||||||
/// ```
|
/// ```
|
||||||
pub fn open_revision(
|
pub fn open_revision(h5_path: &Path, revision: u64) -> Result<(Vec<u8>, OnionFile), OnionError> {
|
||||||
h5_path: &Path,
|
|
||||||
revision: u64,
|
|
||||||
) -> Result<(Vec<u8>, OnionFile), OnionError> {
|
|
||||||
let h5_base = std::fs::read(h5_path)?;
|
let h5_base = std::fs::read(h5_path)?;
|
||||||
let onion = OnionFile::open(h5_path)?;
|
let onion = OnionFile::open(h5_path)?;
|
||||||
let reconstructed = onion.reconstruct_revision(revision, &h5_base)?;
|
let reconstructed = onion.reconstruct_revision(revision, &h5_base)?;
|
||||||
@@ -48,10 +45,7 @@ pub fn open_revision(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Open the HEAD of a named branch.
|
/// Open the HEAD of a named branch.
|
||||||
pub fn open_branch(
|
pub fn open_branch(h5_path: &Path, branch: &str) -> Result<(Vec<u8>, OnionFile), OnionError> {
|
||||||
h5_path: &Path,
|
|
||||||
branch: &str,
|
|
||||||
) -> Result<(Vec<u8>, OnionFile), OnionError> {
|
|
||||||
let h5_base = std::fs::read(h5_path)?;
|
let h5_base = std::fs::read(h5_path)?;
|
||||||
let onion = OnionFile::open(h5_path)?;
|
let onion = OnionFile::open(h5_path)?;
|
||||||
let reconstructed = onion.open_rev(OpenRevision::Branch(branch.to_owned()), &h5_base)?;
|
let reconstructed = onion.open_rev(OpenRevision::Branch(branch.to_owned()), &h5_base)?;
|
||||||
@@ -66,8 +60,10 @@ pub fn open_branch_at(
|
|||||||
) -> Result<(Vec<u8>, OnionFile), OnionError> {
|
) -> Result<(Vec<u8>, OnionFile), OnionError> {
|
||||||
let h5_base = std::fs::read(h5_path)?;
|
let h5_base = std::fs::read(h5_path)?;
|
||||||
let onion = OnionFile::open(h5_path)?;
|
let onion = OnionFile::open(h5_path)?;
|
||||||
let reconstructed =
|
let reconstructed = onion.open_rev(
|
||||||
onion.open_rev(OpenRevision::BranchAt(branch.to_owned(), revision), &h5_base)?;
|
OpenRevision::BranchAt(branch.to_owned(), revision),
|
||||||
|
&h5_base,
|
||||||
|
)?;
|
||||||
Ok((reconstructed, onion))
|
Ok((reconstructed, onion))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -41,11 +41,7 @@ impl OnionFile {
|
|||||||
/// Create a new named branch forked from the current HEAD of `source_branch`.
|
/// Create a new named branch forked from the current HEAD of `source_branch`.
|
||||||
///
|
///
|
||||||
/// Returns the new branch ID.
|
/// Returns the new branch ID.
|
||||||
pub fn create_branch(
|
pub fn create_branch(&mut self, name: &str, source_branch: &str) -> Result<u32, OnionError> {
|
||||||
&mut self,
|
|
||||||
name: &str,
|
|
||||||
source_branch: &str,
|
|
||||||
) -> Result<u32, OnionError> {
|
|
||||||
if self.branch_by_name(name).is_some() {
|
if self.branch_by_name(name).is_some() {
|
||||||
return Err(OnionError::BranchExists(name.to_string()));
|
return Err(OnionError::BranchExists(name.to_string()));
|
||||||
}
|
}
|
||||||
@@ -133,11 +129,7 @@ impl OnionFile {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn merge_latest_wins(
|
fn merge_latest_wins(&mut self, source_id: u32, target_id: u32) -> Result<u64, OnionError> {
|
||||||
&mut self,
|
|
||||||
source_id: u32,
|
|
||||||
target_id: u32,
|
|
||||||
) -> Result<u64, OnionError> {
|
|
||||||
let fork_rev = self
|
let fork_rev = self
|
||||||
.branches
|
.branches
|
||||||
.iter()
|
.iter()
|
||||||
@@ -231,21 +223,21 @@ impl OnionFile {
|
|||||||
self.commit_session(session, Some(&annotation))
|
self.commit_session(session, Some(&annotation))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn merge_three_way(
|
fn merge_three_way(&mut self, source_id: u32, target_id: u32) -> Result<u64, OnionError> {
|
||||||
&mut self,
|
|
||||||
source_id: u32,
|
|
||||||
target_id: u32,
|
|
||||||
) -> Result<u64, OnionError> {
|
|
||||||
let source_head = self
|
let source_head = self
|
||||||
.index
|
.index
|
||||||
.branch_head(source_id)
|
.branch_head(source_id)
|
||||||
.ok_or_else(|| OnionError::Malformed(format!("source branch {source_id} has no revisions")))?
|
.ok_or_else(|| {
|
||||||
|
OnionError::Malformed(format!("source branch {source_id} has no revisions"))
|
||||||
|
})?
|
||||||
.revision;
|
.revision;
|
||||||
|
|
||||||
let target_head = self
|
let target_head = self
|
||||||
.index
|
.index
|
||||||
.branch_head(target_id)
|
.branch_head(target_id)
|
||||||
.ok_or_else(|| OnionError::Malformed(format!("target branch {target_id} has no revisions")))?
|
.ok_or_else(|| {
|
||||||
|
OnionError::Malformed(format!("target branch {target_id} has no revisions"))
|
||||||
|
})?
|
||||||
.revision;
|
.revision;
|
||||||
|
|
||||||
// Find common ancestor of the two branch HEADs
|
// Find common ancestor of the two branch HEADs
|
||||||
@@ -253,13 +245,24 @@ impl OnionFile {
|
|||||||
.index
|
.index
|
||||||
.common_ancestor(source_head, target_head)
|
.common_ancestor(source_head, target_head)
|
||||||
.ok_or_else(|| {
|
.ok_or_else(|| {
|
||||||
let src_name = self.branches.iter().find(|b| b.id == source_id)
|
let src_name = self
|
||||||
|
.branches
|
||||||
|
.iter()
|
||||||
|
.find(|b| b.id == source_id)
|
||||||
.and_then(|b| self.annotations.get(b.name_off))
|
.and_then(|b| self.annotations.get(b.name_off))
|
||||||
.unwrap_or("?").to_owned();
|
.unwrap_or("?")
|
||||||
let tgt_name = self.branches.iter().find(|b| b.id == target_id)
|
.to_owned();
|
||||||
|
let tgt_name = self
|
||||||
|
.branches
|
||||||
|
.iter()
|
||||||
|
.find(|b| b.id == target_id)
|
||||||
.and_then(|b| self.annotations.get(b.name_off))
|
.and_then(|b| self.annotations.get(b.name_off))
|
||||||
.unwrap_or("?").to_owned();
|
.unwrap_or("?")
|
||||||
OnionError::NoCommonAncestor { a: src_name, b: tgt_name }
|
.to_owned();
|
||||||
|
OnionError::NoCommonAncestor {
|
||||||
|
a: src_name,
|
||||||
|
b: tgt_name,
|
||||||
|
}
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
// Collect source delta since ancestor (latest write per offset)
|
// Collect source delta since ancestor (latest write per offset)
|
||||||
@@ -321,7 +324,8 @@ impl OnionFile {
|
|||||||
|
|
||||||
if merged.is_empty() {
|
if merged.is_empty() {
|
||||||
return Err(OnionError::Malformed(
|
return Err(OnionError::Malformed(
|
||||||
"three-way merge: no source changes to apply (target already has all changes)".to_string(),
|
"three-way merge: no source changes to apply (target already has all changes)"
|
||||||
|
.to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -329,9 +333,8 @@ impl OnionFile {
|
|||||||
for (h5_off, bytes) in &merged {
|
for (h5_off, bytes) in &merged {
|
||||||
session.record_page(*h5_off, bytes);
|
session.record_page(*h5_off, bytes);
|
||||||
}
|
}
|
||||||
let annotation = format!(
|
let annotation =
|
||||||
"3-way merge {source_id} → {target_id} [ancestor rev {ancestor_rev}]"
|
format!("3-way merge {source_id} → {target_id} [ancestor rev {ancestor_rev}]");
|
||||||
);
|
|
||||||
self.commit_session(session, Some(&annotation))
|
self.commit_session(session, Some(&annotation))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -371,11 +374,7 @@ impl OnionFile {
|
|||||||
.iter()
|
.iter()
|
||||||
.map(|b| BranchInfo {
|
.map(|b| BranchInfo {
|
||||||
id: b.id,
|
id: b.id,
|
||||||
name: self
|
name: self.annotations.get(b.name_off).unwrap_or("?").to_owned(),
|
||||||
.annotations
|
|
||||||
.get(b.name_off)
|
|
||||||
.unwrap_or("?")
|
|
||||||
.to_owned(),
|
|
||||||
head_rev: b.head_rev,
|
head_rev: b.head_rev,
|
||||||
fork_rev: b.fork_rev,
|
fork_rev: b.fork_rev,
|
||||||
})
|
})
|
||||||
@@ -383,7 +382,10 @@ impl OnionFile {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Return revision entries for a named branch in chronological order.
|
/// Return revision entries for a named branch in chronological order.
|
||||||
pub fn branch_history(&self, name: &str) -> Result<Vec<&crate::format::RevisionEntry>, OnionError> {
|
pub fn branch_history(
|
||||||
|
&self,
|
||||||
|
name: &str,
|
||||||
|
) -> Result<Vec<&crate::format::RevisionEntry>, OnionError> {
|
||||||
let branch = self
|
let branch = self
|
||||||
.branch_by_name(name)
|
.branch_by_name(name)
|
||||||
.ok_or_else(|| OnionError::BranchNotFound(name.to_string()))?;
|
.ok_or_else(|| OnionError::BranchNotFound(name.to_string()))?;
|
||||||
@@ -414,7 +416,9 @@ impl OnionFile {
|
|||||||
.branch_by_name(name)
|
.branch_by_name(name)
|
||||||
.ok_or_else(|| OnionError::BranchNotFound(name.to_string()))?;
|
.ok_or_else(|| OnionError::BranchNotFound(name.to_string()))?;
|
||||||
if branch.id == crate::format::BRANCH_MAIN {
|
if branch.id == crate::format::BRANCH_MAIN {
|
||||||
return Err(OnionError::Malformed("cannot delete the main branch".into()));
|
return Err(OnionError::Malformed(
|
||||||
|
"cannot delete the main branch".into(),
|
||||||
|
));
|
||||||
}
|
}
|
||||||
let id = branch.id;
|
let id = branch.id;
|
||||||
self.branches.retain(|b| b.id != id);
|
self.branches.retain(|b| b.id != id);
|
||||||
@@ -605,14 +609,18 @@ mod tests {
|
|||||||
fn merge_latest_wins_produces_new_revision() {
|
fn merge_latest_wins_produces_new_revision() {
|
||||||
let (_h5, mut onion, _feat_id) = make_fork_scenario();
|
let (_h5, mut onion, _feat_id) = make_fork_scenario();
|
||||||
let pre_count = onion.revision_count();
|
let pre_count = onion.revision_count();
|
||||||
onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap();
|
onion
|
||||||
|
.merge_into("feat", "main", MergeStrategy::LatestWins)
|
||||||
|
.unwrap();
|
||||||
assert_eq!(onion.revision_count(), pre_count + 1);
|
assert_eq!(onion.revision_count(), pre_count + 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn merge_latest_wins_page_content_correct() {
|
fn merge_latest_wins_page_content_correct() {
|
||||||
let (h5, mut onion, _feat_id) = make_fork_scenario();
|
let (h5, mut onion, _feat_id) = make_fork_scenario();
|
||||||
onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap();
|
onion
|
||||||
|
.merge_into("feat", "main", MergeStrategy::LatestWins)
|
||||||
|
.unwrap();
|
||||||
onion.flush().unwrap();
|
onion.flush().unwrap();
|
||||||
|
|
||||||
// Reload from disk and reconstruct main HEAD
|
// Reload from disk and reconstruct main HEAD
|
||||||
@@ -621,8 +629,10 @@ mod tests {
|
|||||||
let base = std::fs::read(&h5).unwrap();
|
let base = std::fs::read(&h5).unwrap();
|
||||||
let state = onion2.reconstruct_revision(main_head, &base).unwrap();
|
let state = onion2.reconstruct_revision(main_head, &base).unwrap();
|
||||||
// Page 0 should now be BB (from feat)
|
// Page 0 should now be BB (from feat)
|
||||||
assert!(state[0..4096].iter().all(|&b| b == 0xBB),
|
assert!(
|
||||||
"page 0 should be BB after latest-wins merge");
|
state[0..4096].iter().all(|&b| b == 0xBB),
|
||||||
|
"page 0 should be BB after latest-wins merge"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -646,15 +656,19 @@ mod tests {
|
|||||||
s2.record_page(0, &vec![0x22u8; 4096]);
|
s2.record_page(0, &vec![0x22u8; 4096]);
|
||||||
onion.commit_session(s2, None).unwrap();
|
onion.commit_session(s2, None).unwrap();
|
||||||
|
|
||||||
onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap();
|
onion
|
||||||
|
.merge_into("feat", "main", MergeStrategy::LatestWins)
|
||||||
|
.unwrap();
|
||||||
onion.flush().unwrap();
|
onion.flush().unwrap();
|
||||||
|
|
||||||
let onion2 = OnionFile::open(&h5).unwrap();
|
let onion2 = OnionFile::open(&h5).unwrap();
|
||||||
let head = onion2.revision_count() - 1;
|
let head = onion2.revision_count() - 1;
|
||||||
let base = std::fs::read(&h5).unwrap();
|
let base = std::fs::read(&h5).unwrap();
|
||||||
let state = onion2.reconstruct_revision(head, &base).unwrap();
|
let state = onion2.reconstruct_revision(head, &base).unwrap();
|
||||||
assert!(state[0..4096].iter().all(|&b| b == 0x22),
|
assert!(
|
||||||
"latest write (0x22) must win in LatestWins merge");
|
state[0..4096].iter().all(|&b| b == 0x22),
|
||||||
|
"latest write (0x22) must win in LatestWins merge"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -666,7 +680,9 @@ mod tests {
|
|||||||
onion.commit_session(s, None).unwrap();
|
onion.commit_session(s, None).unwrap();
|
||||||
// fork but make no commits on feat
|
// fork but make no commits on feat
|
||||||
onion.create_branch("feat", "main").unwrap();
|
onion.create_branch("feat", "main").unwrap();
|
||||||
let err = onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap_err();
|
let err = onion
|
||||||
|
.merge_into("feat", "main", MergeStrategy::LatestWins)
|
||||||
|
.unwrap_err();
|
||||||
assert!(matches!(err, OnionError::Malformed(_)));
|
assert!(matches!(err, OnionError::Malformed(_)));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -682,20 +698,30 @@ mod tests {
|
|||||||
source.to_vec() // just return source
|
source.to_vec() // just return source
|
||||||
};
|
};
|
||||||
onion
|
onion
|
||||||
.merge_into("feat", "main", MergeStrategy::DatasetLevel(Box::new(resolver)))
|
.merge_into(
|
||||||
|
"feat",
|
||||||
|
"main",
|
||||||
|
MergeStrategy::DatasetLevel(Box::new(resolver)),
|
||||||
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(called.load(std::sync::atomic::Ordering::SeqCst), "resolver must be called");
|
assert!(
|
||||||
|
called.load(std::sync::atomic::Ordering::SeqCst),
|
||||||
|
"resolver must be called"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn merge_dataset_level_resolver_controls_output() {
|
fn merge_dataset_level_resolver_controls_output() {
|
||||||
let (h5, mut onion, _feat_id) = make_fork_scenario();
|
let (h5, mut onion, _feat_id) = make_fork_scenario();
|
||||||
// Resolver always returns 0xCC regardless of inputs
|
// Resolver always returns 0xCC regardless of inputs
|
||||||
let resolver = |_path: &str, _target: &[u8], _source: &[u8]| -> Vec<u8> {
|
let resolver =
|
||||||
vec![0xCCu8; 4096]
|
|_path: &str, _target: &[u8], _source: &[u8]| -> Vec<u8> { vec![0xCCu8; 4096] };
|
||||||
};
|
|
||||||
onion
|
onion
|
||||||
.merge_into("feat", "main", MergeStrategy::DatasetLevel(Box::new(resolver)))
|
.merge_into(
|
||||||
|
"feat",
|
||||||
|
"main",
|
||||||
|
MergeStrategy::DatasetLevel(Box::new(resolver)),
|
||||||
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
onion.flush().unwrap();
|
onion.flush().unwrap();
|
||||||
|
|
||||||
@@ -703,8 +729,10 @@ mod tests {
|
|||||||
let head = onion2.revision_count() - 1;
|
let head = onion2.revision_count() - 1;
|
||||||
let base = std::fs::read(&h5).unwrap();
|
let base = std::fs::read(&h5).unwrap();
|
||||||
let state = onion2.reconstruct_revision(head, &base).unwrap();
|
let state = onion2.reconstruct_revision(head, &base).unwrap();
|
||||||
assert!(state[0..4096].iter().all(|&b| b == 0xCC),
|
assert!(
|
||||||
"resolver output 0xCC should be in merged state");
|
state[0..4096].iter().all(|&b| b == 0xCC),
|
||||||
|
"resolver output 0xCC should be in merged state"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Merge: ThreeWay ──────────────────────────────────────────────────────
|
// ── Merge: ThreeWay ──────────────────────────────────────────────────────
|
||||||
@@ -736,7 +764,9 @@ mod tests {
|
|||||||
sf.record_page(0, &vec![0xBBu8; 4096]);
|
sf.record_page(0, &vec![0xBBu8; 4096]);
|
||||||
onion.commit_session(sf, None).unwrap();
|
onion.commit_session(sf, None).unwrap();
|
||||||
|
|
||||||
onion.merge_into("feat", "main", MergeStrategy::ThreeWay).unwrap();
|
onion
|
||||||
|
.merge_into("feat", "main", MergeStrategy::ThreeWay)
|
||||||
|
.unwrap();
|
||||||
onion.flush().unwrap();
|
onion.flush().unwrap();
|
||||||
|
|
||||||
let onion2 = OnionFile::open(&h5).unwrap();
|
let onion2 = OnionFile::open(&h5).unwrap();
|
||||||
@@ -744,10 +774,14 @@ mod tests {
|
|||||||
let base = std::fs::read(&h5).unwrap();
|
let base = std::fs::read(&h5).unwrap();
|
||||||
let state = onion2.reconstruct_revision(head, &base).unwrap();
|
let state = onion2.reconstruct_revision(head, &base).unwrap();
|
||||||
|
|
||||||
assert!(state[0..4096].iter().all(|&b| b == 0xBB),
|
assert!(
|
||||||
"page 0 should be BB (from feat)");
|
state[0..4096].iter().all(|&b| b == 0xBB),
|
||||||
assert!(state[4096..8192].iter().all(|&b| b == 0xCC),
|
"page 0 should be BB (from feat)"
|
||||||
"page 4096 should stay CC (from main — not overwritten by 3-way)");
|
);
|
||||||
|
assert!(
|
||||||
|
state[4096..8192].iter().all(|&b| b == 0xCC),
|
||||||
|
"page 4096 should stay CC (from main — not overwritten by 3-way)"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -772,22 +806,28 @@ mod tests {
|
|||||||
sf.record_page(0, &vec![0xCCu8; 4096]);
|
sf.record_page(0, &vec![0xCCu8; 4096]);
|
||||||
onion.commit_session(sf, None).unwrap();
|
onion.commit_session(sf, None).unwrap();
|
||||||
|
|
||||||
onion.merge_into("feat", "main", MergeStrategy::ThreeWay).unwrap();
|
onion
|
||||||
|
.merge_into("feat", "main", MergeStrategy::ThreeWay)
|
||||||
|
.unwrap();
|
||||||
onion.flush().unwrap();
|
onion.flush().unwrap();
|
||||||
|
|
||||||
let onion2 = OnionFile::open(&h5).unwrap();
|
let onion2 = OnionFile::open(&h5).unwrap();
|
||||||
let head = onion2.revision_count() - 1;
|
let head = onion2.revision_count() - 1;
|
||||||
let base = std::fs::read(&h5).unwrap();
|
let base = std::fs::read(&h5).unwrap();
|
||||||
let state = onion2.reconstruct_revision(head, &base).unwrap();
|
let state = onion2.reconstruct_revision(head, &base).unwrap();
|
||||||
assert!(state[0..4096].iter().all(|&b| b == 0xCC),
|
assert!(
|
||||||
"on conflict, source (CC) should win");
|
state[0..4096].iter().all(|&b| b == 0xCC),
|
||||||
|
"on conflict, source (CC) should win"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn merge_nonexistent_source_errors() {
|
fn merge_nonexistent_source_errors() {
|
||||||
let h5 = tmp_h5();
|
let h5 = tmp_h5();
|
||||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||||
let err = onion.merge_into("no-such", "main", MergeStrategy::LatestWins).unwrap_err();
|
let err = onion
|
||||||
|
.merge_into("no-such", "main", MergeStrategy::LatestWins)
|
||||||
|
.unwrap_err();
|
||||||
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -799,7 +839,9 @@ mod tests {
|
|||||||
s.record_page(0, &vec![0u8; 4096]);
|
s.record_page(0, &vec![0u8; 4096]);
|
||||||
onion.commit_session(s, None).unwrap();
|
onion.commit_session(s, None).unwrap();
|
||||||
onion.create_branch("feat", "main").unwrap();
|
onion.create_branch("feat", "main").unwrap();
|
||||||
let err = onion.merge_into("feat", "no-target", MergeStrategy::LatestWins).unwrap_err();
|
let err = onion
|
||||||
|
.merge_into("feat", "no-target", MergeStrategy::LatestWins)
|
||||||
|
.unwrap_err();
|
||||||
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,26 +13,22 @@ use crate::tdt;
|
|||||||
pub fn compress_page(data: &[u8], codec: Codec) -> Result<Vec<u8>, OnionError> {
|
pub fn compress_page(data: &[u8], codec: Codec) -> Result<Vec<u8>, OnionError> {
|
||||||
match codec {
|
match codec {
|
||||||
Codec::None => Ok(data.to_vec()),
|
Codec::None => Ok(data.to_vec()),
|
||||||
Codec::Zstd => zstd::bulk::compress(data, 3)
|
Codec::Zstd => {
|
||||||
.map_err(|e| OnionError::Compress(e.to_string())),
|
zstd::bulk::compress(data, 3).map_err(|e| OnionError::Compress(e.to_string()))
|
||||||
|
}
|
||||||
Codec::Lz4 => Ok(lz4_flex::compress_prepend_size(data)),
|
Codec::Lz4 => Ok(lz4_flex::compress_prepend_size(data)),
|
||||||
Codec::Brotli => Err(OnionError::Compress(
|
Codec::Brotli => Err(OnionError::Compress(
|
||||||
"brotli not yet implemented".to_string(),
|
"brotli not yet implemented".to_string(),
|
||||||
)),
|
)),
|
||||||
Codec::ZstdTdt => {
|
Codec::ZstdTdt => {
|
||||||
let interleaved = tdt::encode(data, 4);
|
let interleaved = tdt::encode(data, 4);
|
||||||
zstd::bulk::compress(&interleaved, 3)
|
zstd::bulk::compress(&interleaved, 3).map_err(|e| OnionError::Compress(e.to_string()))
|
||||||
.map_err(|e| OnionError::Compress(e.to_string()))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Decompress `data` using the given codec back to `orig_size` bytes.
|
/// Decompress `data` using the given codec back to `orig_size` bytes.
|
||||||
pub fn decompress_page(
|
pub fn decompress_page(data: &[u8], codec: Codec, orig_size: u32) -> Result<Vec<u8>, OnionError> {
|
||||||
data: &[u8],
|
|
||||||
codec: Codec,
|
|
||||||
orig_size: u32,
|
|
||||||
) -> Result<Vec<u8>, OnionError> {
|
|
||||||
match codec {
|
match codec {
|
||||||
Codec::None => {
|
Codec::None => {
|
||||||
if data.len() != orig_size as usize {
|
if data.len() != orig_size as usize {
|
||||||
@@ -161,7 +157,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
let orig = data.len() as u32;
|
let orig = data.len() as u32;
|
||||||
let zstd_size = compress_page(&data, Codec::Zstd).unwrap().len();
|
let zstd_size = compress_page(&data, Codec::Zstd).unwrap().len();
|
||||||
let tdt_size = compress_page(&data, Codec::ZstdTdt).unwrap().len();
|
let tdt_size = compress_page(&data, Codec::ZstdTdt).unwrap().len();
|
||||||
assert!(
|
assert!(
|
||||||
tdt_size < zstd_size,
|
tdt_size < zstd_size,
|
||||||
"ZstdTdt ({tdt_size} B) should beat plain Zstd ({zstd_size} B) on smooth f32 data ({orig} B)"
|
"ZstdTdt ({tdt_size} B) should beat plain Zstd ({zstd_size} B) on smooth f32 data ({orig} B)"
|
||||||
|
|||||||
@@ -94,7 +94,10 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let f = vf.current().unwrap();
|
let f = vf.current().unwrap();
|
||||||
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![1.0, 2.0, 3.0]);
|
assert_eq!(
|
||||||
|
f.dataset("data").unwrap().read_f64().unwrap(),
|
||||||
|
vec![1.0, 2.0, 3.0]
|
||||||
|
);
|
||||||
assert_eq!(vf.onion().revision_count(), 0);
|
assert_eq!(vf.onion().revision_count(), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -106,7 +109,10 @@ mod tests {
|
|||||||
let _ = make_builder(&[1.0]).with_onion(&h5_path, 4096).unwrap();
|
let _ = make_builder(&[1.0]).with_onion(&h5_path, 4096).unwrap();
|
||||||
|
|
||||||
let onion_path = PathBuf::from(format!("{}.onion", h5_path.display()));
|
let onion_path = PathBuf::from(format!("{}.onion", h5_path.display()));
|
||||||
assert!(onion_path.exists(), "sidecar should be flushed by with_onion");
|
assert!(
|
||||||
|
onion_path.exists(),
|
||||||
|
"sidecar should be flushed by with_onion"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -140,6 +146,9 @@ mod tests {
|
|||||||
vf.page_size().is_power_of_two(),
|
vf.page_size().is_power_of_two(),
|
||||||
"auto page size must be a power of two"
|
"auto page size must be a power of two"
|
||||||
);
|
);
|
||||||
assert!(vf.page_size() >= 4096, "auto page size must be at least 4 KiB");
|
assert!(
|
||||||
|
vf.page_size() >= 4096,
|
||||||
|
"auto page size must be at least 4 KiB"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,9 +49,9 @@ pub const HEADER_SIZE: usize = 128;
|
|||||||
|
|
||||||
pub mod feature_flags {
|
pub mod feature_flags {
|
||||||
pub const COMPRESSION: u64 = 1 << 0;
|
pub const COMPRESSION: u64 = 1 << 0;
|
||||||
pub const BRANCHING: u64 = 1 << 1;
|
pub const BRANCHING: u64 = 1 << 1;
|
||||||
pub const PROVENANCE: u64 = 1 << 2;
|
pub const PROVENANCE: u64 = 1 << 2;
|
||||||
pub const SNAPSHOTS: u64 = 1 << 3;
|
pub const SNAPSHOTS: u64 = 1 << 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub const DEFAULT_FEATURE_FLAGS: u64 =
|
pub const DEFAULT_FEATURE_FLAGS: u64 =
|
||||||
@@ -64,13 +64,13 @@ pub const DEFAULT_FEATURE_FLAGS: u64 =
|
|||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
#[repr(u8)]
|
#[repr(u8)]
|
||||||
pub enum Codec {
|
pub enum Codec {
|
||||||
None = 0,
|
None = 0,
|
||||||
Zstd = 1,
|
Zstd = 1,
|
||||||
Lz4 = 2,
|
Lz4 = 2,
|
||||||
Brotli = 3,
|
Brotli = 3,
|
||||||
/// zstd applied after TDT byte-interleaving transform (arXiv:2506.18062).
|
/// zstd applied after TDT byte-interleaving transform (arXiv:2506.18062).
|
||||||
/// Improves compression ratio ~16% for `f32`/`f16` numeric pages.
|
/// Improves compression ratio ~16% for `f32`/`f16` numeric pages.
|
||||||
ZstdTdt = 4,
|
ZstdTdt = 4,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Codec {
|
impl Codec {
|
||||||
@@ -108,19 +108,19 @@ impl Codec {
|
|||||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
||||||
#[repr(C)]
|
#[repr(C)]
|
||||||
pub struct OnionHeader {
|
pub struct OnionHeader {
|
||||||
pub magic: [u8; 9],
|
pub magic: [u8; 9],
|
||||||
pub format_version: u8,
|
pub format_version: u8,
|
||||||
pub _pad_align: [u8; 6],
|
pub _pad_align: [u8; 6],
|
||||||
pub feature_flags: u64,
|
pub feature_flags: u64,
|
||||||
pub page_size: u32,
|
pub page_size: u32,
|
||||||
pub _pad_ps: [u8; 4],
|
pub _pad_ps: [u8; 4],
|
||||||
pub revision_count: u64,
|
pub revision_count: u64,
|
||||||
pub branch_count: u32,
|
pub branch_count: u32,
|
||||||
pub _pad_bc: [u8; 4],
|
pub _pad_bc: [u8; 4],
|
||||||
pub index_offset: u64,
|
pub index_offset: u64,
|
||||||
pub branch_offset: u64,
|
pub branch_offset: u64,
|
||||||
pub created_at: f64,
|
pub created_at: f64,
|
||||||
pub reserved: [u8; 56],
|
pub reserved: [u8; 56],
|
||||||
}
|
}
|
||||||
|
|
||||||
const _: () = assert!(size_of::<OnionHeader>() == HEADER_SIZE);
|
const _: () = assert!(size_of::<OnionHeader>() == HEADER_SIZE);
|
||||||
@@ -180,19 +180,19 @@ impl OnionHeader {
|
|||||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
||||||
#[repr(C)]
|
#[repr(C)]
|
||||||
pub struct RevisionEntry {
|
pub struct RevisionEntry {
|
||||||
pub revision: u64,
|
pub revision: u64,
|
||||||
pub branch_id: u32,
|
pub branch_id: u32,
|
||||||
pub _pad_bi: [u8; 4],
|
pub _pad_bi: [u8; 4],
|
||||||
pub parent_rev: u64,
|
pub parent_rev: u64,
|
||||||
pub page_count: u32,
|
pub page_count: u32,
|
||||||
pub _pad_pc: [u8; 4],
|
pub _pad_pc: [u8; 4],
|
||||||
pub page_table_off: u64,
|
pub page_table_off: u64,
|
||||||
pub timestamp: f64,
|
pub timestamp: f64,
|
||||||
pub blake3: [u8; 32],
|
pub blake3: [u8; 32],
|
||||||
pub session_uuid: [u8; 16],
|
pub session_uuid: [u8; 16],
|
||||||
pub annotation_off: u64,
|
pub annotation_off: u64,
|
||||||
pub flags: u8,
|
pub flags: u8,
|
||||||
pub _pad_flags: [u8; 7],
|
pub _pad_flags: [u8; 7],
|
||||||
}
|
}
|
||||||
|
|
||||||
pub const REV_FLAG_SNAPSHOT: u8 = 1 << 0;
|
pub const REV_FLAG_SNAPSHOT: u8 = 1 << 0;
|
||||||
@@ -233,11 +233,11 @@ impl RevisionEntry {
|
|||||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
||||||
#[repr(C)]
|
#[repr(C)]
|
||||||
pub struct BranchEntry {
|
pub struct BranchEntry {
|
||||||
pub id: u32,
|
pub id: u32,
|
||||||
pub _pad_id: [u8; 4],
|
pub _pad_id: [u8; 4],
|
||||||
pub name_off: u64,
|
pub name_off: u64,
|
||||||
pub head_rev: u64,
|
pub head_rev: u64,
|
||||||
pub fork_rev: u64,
|
pub fork_rev: u64,
|
||||||
pub created_at: f64,
|
pub created_at: f64,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -255,25 +255,25 @@ pub struct BranchEntry {
|
|||||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
||||||
#[repr(C)]
|
#[repr(C)]
|
||||||
pub struct PageTableEntry {
|
pub struct PageTableEntry {
|
||||||
pub h5_offset: u64,
|
pub h5_offset: u64,
|
||||||
pub data_offset: u64,
|
pub data_offset: u64,
|
||||||
pub orig_size: u32,
|
pub orig_size: u32,
|
||||||
pub data_size: u32,
|
pub data_size: u32,
|
||||||
pub codec: u8,
|
pub codec: u8,
|
||||||
pub _pad: [u8; 7],
|
pub _pad: [u8; 7],
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// Compile-time size/alignment assertions
|
// Compile-time size/alignment assertions
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
const _: () = assert!(size_of::<OnionHeader>() == 128);
|
const _: () = assert!(size_of::<OnionHeader>() == 128);
|
||||||
const _: () = assert!(size_of::<RevisionEntry>() == 112);
|
const _: () = assert!(size_of::<RevisionEntry>() == 112);
|
||||||
const _: () = assert!(size_of::<BranchEntry>() == 40);
|
const _: () = assert!(size_of::<BranchEntry>() == 40);
|
||||||
const _: () = assert!(size_of::<PageTableEntry>() == 32);
|
const _: () = assert!(size_of::<PageTableEntry>() == 32);
|
||||||
|
|
||||||
const _: () = assert!(size_of::<RevisionEntry>() % 8 == 0);
|
const _: () = assert!(size_of::<RevisionEntry>() % 8 == 0);
|
||||||
const _: () = assert!(size_of::<BranchEntry>() % 8 == 0);
|
const _: () = assert!(size_of::<BranchEntry>() % 8 == 0);
|
||||||
const _: () = assert!(size_of::<PageTableEntry>() % 8 == 0);
|
const _: () = assert!(size_of::<PageTableEntry>() % 8 == 0);
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -321,7 +321,10 @@ mod tests {
|
|||||||
fn header_unknown_version_rejected() {
|
fn header_unknown_version_rejected() {
|
||||||
let mut hdr = OnionHeader::new(4096, DEFAULT_FEATURE_FLAGS, 0.0);
|
let mut hdr = OnionHeader::new(4096, DEFAULT_FEATURE_FLAGS, 0.0);
|
||||||
hdr.format_version = 42;
|
hdr.format_version = 42;
|
||||||
assert!(matches!(hdr.validate(), Err(OnionError::UnknownVersion(42))));
|
assert!(matches!(
|
||||||
|
hdr.validate(),
|
||||||
|
Err(OnionError::UnknownVersion(42))
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -488,7 +491,10 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn codec_unknown_returns_error() {
|
fn codec_unknown_returns_error() {
|
||||||
assert!(matches!(Codec::from_u8(99), Err(OnionError::UnknownCodec(99))));
|
assert!(matches!(
|
||||||
|
Codec::from_u8(99),
|
||||||
|
Err(OnionError::UnknownCodec(99))
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -69,7 +69,10 @@ impl OnionFile {
|
|||||||
entry.set_epoch(EPOCH_DEAD);
|
entry.set_epoch(EPOCH_DEAD);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(GcStats { revisions_removed: count, bytes_reclaimed: 0 })
|
Ok(GcStats {
|
||||||
|
revisions_removed: count,
|
||||||
|
bytes_reclaimed: 0,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
// ── Immediate path: compact now ───────────────────────────
|
// ── Immediate path: compact now ───────────────────────────
|
||||||
@@ -100,11 +103,7 @@ impl OnionFile {
|
|||||||
GcPolicy::KeepSince(cutoff) => all_revs
|
GcPolicy::KeepSince(cutoff) => all_revs
|
||||||
.iter()
|
.iter()
|
||||||
.copied()
|
.copied()
|
||||||
.filter(|&rev| {
|
.filter(|&rev| self.index.get(rev).is_some_and(|e| e.timestamp >= *cutoff))
|
||||||
self.index
|
|
||||||
.get(rev)
|
|
||||||
.is_some_and(|e| e.timestamp >= *cutoff)
|
|
||||||
})
|
|
||||||
.collect(),
|
.collect(),
|
||||||
GcPolicy::KeepRevisions(explicit) => {
|
GcPolicy::KeepRevisions(explicit) => {
|
||||||
let mut set: HashSet<u64> = explicit.iter().copied().collect();
|
let mut set: HashSet<u64> = explicit.iter().copied().collect();
|
||||||
@@ -118,7 +117,11 @@ impl OnionFile {
|
|||||||
GcPolicy::EpochFlip(inner) => return self.compute_to_remove(inner),
|
GcPolicy::EpochFlip(inner) => return self.compute_to_remove(inner),
|
||||||
};
|
};
|
||||||
|
|
||||||
all_revs.iter().copied().filter(|rev| !keep.contains(rev)).collect()
|
all_revs
|
||||||
|
.iter()
|
||||||
|
.copied()
|
||||||
|
.filter(|rev| !keep.contains(rev))
|
||||||
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Consolidate + compact the given revision set immediately.
|
/// Consolidate + compact the given revision set immediately.
|
||||||
@@ -204,8 +207,7 @@ impl OnionFile {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Compact page_data blob.
|
// Compact page_data blob.
|
||||||
let surviving_revs: Vec<u64> =
|
let surviving_revs: Vec<u64> = self.index.entries().iter().map(|e| e.revision).collect();
|
||||||
self.index.entries().iter().map(|e| e.revision).collect();
|
|
||||||
let mut new_page_data: Vec<u8> = Vec::new();
|
let mut new_page_data: Vec<u8> = Vec::new();
|
||||||
for &rev in &surviving_revs {
|
for &rev in &surviving_revs {
|
||||||
if let Some(table) = self.page_tables.get_mut(rev as usize) {
|
if let Some(table) = self.page_tables.get_mut(rev as usize) {
|
||||||
@@ -222,7 +224,10 @@ impl OnionFile {
|
|||||||
}
|
}
|
||||||
self.page_data = new_page_data;
|
self.page_data = new_page_data;
|
||||||
|
|
||||||
Ok(GcStats { revisions_removed, bytes_reclaimed })
|
Ok(GcStats {
|
||||||
|
revisions_removed,
|
||||||
|
bytes_reclaimed,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Find all entries marked [`EPOCH_DEAD`] and compact them.
|
/// Find all entries marked [`EPOCH_DEAD`] and compact them.
|
||||||
@@ -270,9 +275,7 @@ mod tests {
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
onion
|
onion.commit_session(s, annotation.as_deref()).unwrap();
|
||||||
.commit_session(s, annotation.as_deref())
|
|
||||||
.unwrap();
|
|
||||||
}
|
}
|
||||||
onion
|
onion
|
||||||
}
|
}
|
||||||
@@ -369,7 +372,10 @@ mod tests {
|
|||||||
assert_eq!(stats.revisions_removed, 4);
|
assert_eq!(stats.revisions_removed, 4);
|
||||||
let after = onion.page_data.len();
|
let after = onion.page_data.len();
|
||||||
// page_data must be smaller (or at most equal for all-same pages that compress to 0)
|
// page_data must be smaller (or at most equal for all-same pages that compress to 0)
|
||||||
assert!(after <= before, "page_data must shrink after GC: {before} -> {after}");
|
assert!(
|
||||||
|
after <= before,
|
||||||
|
"page_data must shrink after GC: {before} -> {after}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// After GC + flush, the on-disk file is smaller than before.
|
/// After GC + flush, the on-disk file is smaller than before.
|
||||||
@@ -421,7 +427,10 @@ mod tests {
|
|||||||
// The three surviving revisions must still reconstruct without error.
|
// The three surviving revisions must still reconstruct without error.
|
||||||
for rev in 3..6u64 {
|
for rev in 3..6u64 {
|
||||||
let bytes = onion.reconstruct_revision(rev, &h5_base).unwrap();
|
let bytes = onion.reconstruct_revision(rev, &h5_base).unwrap();
|
||||||
assert!(!bytes.is_empty(), "rev {rev} should produce non-empty bytes");
|
assert!(
|
||||||
|
!bytes.is_empty(),
|
||||||
|
"rev {rev} should produce non-empty bytes"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -456,7 +465,11 @@ mod tests {
|
|||||||
let before_gc = onion.reconstruct_revision(2, &h5_base).unwrap();
|
let before_gc = onion.reconstruct_revision(2, &h5_base).unwrap();
|
||||||
assert_eq!(&before_gc[0..4096], &vec![0xAA_u8; 4096], "pre-GC page0");
|
assert_eq!(&before_gc[0..4096], &vec![0xAA_u8; 4096], "pre-GC page0");
|
||||||
assert_eq!(&before_gc[4096..8192], &vec![0xBB_u8; 4096], "pre-GC page1");
|
assert_eq!(&before_gc[4096..8192], &vec![0xBB_u8; 4096], "pre-GC page1");
|
||||||
assert_eq!(&before_gc[8192..12288], &vec![0xCC_u8; 4096], "pre-GC page2");
|
assert_eq!(
|
||||||
|
&before_gc[8192..12288],
|
||||||
|
&vec![0xCC_u8; 4096],
|
||||||
|
"pre-GC page2"
|
||||||
|
);
|
||||||
|
|
||||||
// GC: keep only rev 2 — revs 0 and 1 are ancestors that will be pruned.
|
// GC: keep only rev 2 — revs 0 and 1 are ancestors that will be pruned.
|
||||||
onion.gc(GcPolicy::KeepLastN(1)).unwrap();
|
onion.gc(GcPolicy::KeepLastN(1)).unwrap();
|
||||||
@@ -490,15 +503,20 @@ mod tests {
|
|||||||
let mut onion = make_onion_with_n_revisions(10);
|
let mut onion = make_onion_with_n_revisions(10);
|
||||||
let page_data_len_before = onion.page_data.len();
|
let page_data_len_before = onion.page_data.len();
|
||||||
|
|
||||||
let stats = onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(3)))).unwrap();
|
let stats = onion
|
||||||
|
.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(3))))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
// Reports the count that will be removed.
|
// Reports the count that will be removed.
|
||||||
assert_eq!(stats.revisions_removed, 7);
|
assert_eq!(stats.revisions_removed, 7);
|
||||||
// bytes_reclaimed is 0 until flush compacts.
|
// bytes_reclaimed is 0 until flush compacts.
|
||||||
assert_eq!(stats.bytes_reclaimed, 0);
|
assert_eq!(stats.bytes_reclaimed, 0);
|
||||||
// page_data must NOT have changed yet.
|
// page_data must NOT have changed yet.
|
||||||
assert_eq!(onion.page_data.len(), page_data_len_before,
|
assert_eq!(
|
||||||
"epoch flip must not compact page_data immediately");
|
onion.page_data.len(),
|
||||||
|
page_data_len_before,
|
||||||
|
"epoch flip must not compact page_data immediately"
|
||||||
|
);
|
||||||
// Index still has all 10 entries (removal deferred).
|
// Index still has all 10 entries (removal deferred).
|
||||||
assert_eq!(onion.revision_count(), 10);
|
assert_eq!(onion.revision_count(), 10);
|
||||||
// Entries 0..6 should be marked EPOCH_DEAD.
|
// Entries 0..6 should be marked EPOCH_DEAD.
|
||||||
@@ -525,7 +543,9 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Epoch flip: defer compaction.
|
// Epoch flip: defer compaction.
|
||||||
onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(2)))).unwrap();
|
onion
|
||||||
|
.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(2))))
|
||||||
|
.unwrap();
|
||||||
assert_eq!(onion.revision_count(), 5, "not compacted yet");
|
assert_eq!(onion.revision_count(), 5, "not compacted yet");
|
||||||
|
|
||||||
// After flush, deferred compaction runs.
|
// After flush, deferred compaction runs.
|
||||||
@@ -533,7 +553,11 @@ mod tests {
|
|||||||
|
|
||||||
// Reload and verify.
|
// Reload and verify.
|
||||||
let reloaded = OnionFile::open(&h5).unwrap();
|
let reloaded = OnionFile::open(&h5).unwrap();
|
||||||
assert_eq!(reloaded.revision_count(), 2, "2 revisions survive after flush");
|
assert_eq!(
|
||||||
|
reloaded.revision_count(),
|
||||||
|
2,
|
||||||
|
"2 revisions survive after flush"
|
||||||
|
);
|
||||||
|
|
||||||
// Surviving revisions are still reconstructable.
|
// Surviving revisions are still reconstructable.
|
||||||
for rev in 3u64..5 {
|
for rev in 3u64..5 {
|
||||||
@@ -559,16 +583,25 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn gc_epoch_flip_nested_policy_keep_tagged() {
|
fn gc_epoch_flip_nested_policy_keep_tagged() {
|
||||||
let mut onion = make_onion_with_n_revisions(9); // tagged at 0,3,6
|
let mut onion = make_onion_with_n_revisions(9); // tagged at 0,3,6
|
||||||
let stats = onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepTagged))).unwrap();
|
let stats = onion
|
||||||
|
.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepTagged)))
|
||||||
|
.unwrap();
|
||||||
assert_eq!(stats.revisions_removed, 6); // keeps 0,3,6
|
assert_eq!(stats.revisions_removed, 6); // keeps 0,3,6
|
||||||
assert_eq!(onion.revision_count(), 9, "deferred — index intact");
|
assert_eq!(onion.revision_count(), 9, "deferred — index intact");
|
||||||
// 0,3,6 are live; rest are dead.
|
// 0,3,6 are live; rest are dead.
|
||||||
for rev in [0u64, 3, 6] {
|
for rev in [0u64, 3, 6] {
|
||||||
assert_eq!(onion.index.get(rev).unwrap().epoch(), 0, "tagged rev {rev} must be live");
|
assert_eq!(
|
||||||
|
onion.index.get(rev).unwrap().epoch(),
|
||||||
|
0,
|
||||||
|
"tagged rev {rev} must be live"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
for rev in [1u64, 2, 4, 5, 7, 8] {
|
for rev in [1u64, 2, 4, 5, 7, 8] {
|
||||||
assert_eq!(onion.index.get(rev).unwrap().epoch(), EPOCH_DEAD,
|
assert_eq!(
|
||||||
"untagged rev {rev} must be EPOCH_DEAD");
|
onion.index.get(rev).unwrap().epoch(),
|
||||||
|
EPOCH_DEAD,
|
||||||
|
"untagged rev {rev} must be EPOCH_DEAD"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
//! RevisionIndex: O(1) lookup + branch-filtered iteration.
|
//! RevisionIndex: O(1) lookup + branch-filtered iteration.
|
||||||
|
|
||||||
use crate::format::{RevisionEntry, NO_PARENT};
|
use crate::format::{NO_PARENT, RevisionEntry};
|
||||||
|
|
||||||
/// In-memory index of all revision entries.
|
/// In-memory index of all revision entries.
|
||||||
///
|
///
|
||||||
@@ -53,7 +53,9 @@ impl RevisionIndex {
|
|||||||
/// monotonically greater than the current maximum.
|
/// monotonically greater than the current maximum.
|
||||||
pub fn append(&mut self, entry: RevisionEntry) {
|
pub fn append(&mut self, entry: RevisionEntry) {
|
||||||
debug_assert!(
|
debug_assert!(
|
||||||
self.entries.last().is_none_or(|e| e.revision < entry.revision),
|
self.entries
|
||||||
|
.last()
|
||||||
|
.is_none_or(|e| e.revision < entry.revision),
|
||||||
"revision numbers must be monotonically increasing"
|
"revision numbers must be monotonically increasing"
|
||||||
);
|
);
|
||||||
self.entries.push(entry);
|
self.entries.push(entry);
|
||||||
@@ -61,15 +63,14 @@ impl RevisionIndex {
|
|||||||
|
|
||||||
/// Iterate over all revisions on a specific branch (by `branch_id`).
|
/// Iterate over all revisions on a specific branch (by `branch_id`).
|
||||||
pub fn branch_revisions(&self, branch_id: u32) -> impl Iterator<Item = &RevisionEntry> {
|
pub fn branch_revisions(&self, branch_id: u32) -> impl Iterator<Item = &RevisionEntry> {
|
||||||
self.entries.iter().filter(move |e| e.branch_id == branch_id)
|
self.entries
|
||||||
|
.iter()
|
||||||
|
.filter(move |e| e.branch_id == branch_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Return the HEAD revision entry for a branch (highest revision number).
|
/// Return the HEAD revision entry for a branch (highest revision number).
|
||||||
pub fn branch_head(&self, branch_id: u32) -> Option<&RevisionEntry> {
|
pub fn branch_head(&self, branch_id: u32) -> Option<&RevisionEntry> {
|
||||||
self.entries
|
self.entries.iter().rev().find(|e| e.branch_id == branch_id)
|
||||||
.iter()
|
|
||||||
.rev()
|
|
||||||
.find(|e| e.branch_id == branch_id)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Walk the DAG from `start_rev` to the root, following `parent_rev`.
|
/// Walk the DAG from `start_rev` to the root, following `parent_rev`.
|
||||||
@@ -191,8 +192,7 @@ mod tests {
|
|||||||
.collect();
|
.collect();
|
||||||
assert_eq!(main_revs, vec![0, 1, 4]);
|
assert_eq!(main_revs, vec![0, 1, 4]);
|
||||||
|
|
||||||
let branch1_revs: Vec<u64> =
|
let branch1_revs: Vec<u64> = idx.branch_revisions(1).map(|e| e.revision).collect();
|
||||||
idx.branch_revisions(1).map(|e| e.revision).collect();
|
|
||||||
assert_eq!(branch1_revs, vec![2, 3]);
|
assert_eq!(branch1_revs, vec![2, 3]);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -252,7 +252,11 @@ mod tests {
|
|||||||
fn remove_revisions() {
|
fn remove_revisions() {
|
||||||
let mut idx = RevisionIndex::new();
|
let mut idx = RevisionIndex::new();
|
||||||
for i in 0u64..5 {
|
for i in 0u64..5 {
|
||||||
idx.append(make_entry(i, BRANCH_MAIN, if i == 0 { NO_PARENT } else { i - 1 }));
|
idx.append(make_entry(
|
||||||
|
i,
|
||||||
|
BRANCH_MAIN,
|
||||||
|
if i == 0 { NO_PARENT } else { i - 1 },
|
||||||
|
));
|
||||||
}
|
}
|
||||||
let to_remove = [1u64, 2].into_iter().collect();
|
let to_remove = [1u64, 2].into_iter().collect();
|
||||||
idx.remove_revisions(&to_remove);
|
idx.remove_revisions(&to_remove);
|
||||||
|
|||||||
@@ -36,19 +36,18 @@ pub mod tdt;
|
|||||||
pub mod versioned_file;
|
pub mod versioned_file;
|
||||||
pub mod writer;
|
pub mod writer;
|
||||||
|
|
||||||
pub use error::OnionError;
|
|
||||||
pub use format::{
|
|
||||||
BranchEntry, Codec, OnionHeader, PageTableEntry, RevisionEntry,
|
|
||||||
BRANCH_MAIN, DEFAULT_FEATURE_FLAGS, EPOCH_DEAD, FORMAT_VERSION, HEADER_SIZE, MAGIC, NO_PARENT,
|
|
||||||
REV_FLAG_SNAPSHOT, feature_flags,
|
|
||||||
};
|
|
||||||
pub use api::{
|
pub use api::{
|
||||||
open_revision, open_branch, open_branch_at,
|
create_branch, delete_branch, list_branches, list_revisions, open_branch, open_branch_at,
|
||||||
list_revisions, rollback,
|
open_revision, rename_branch, rollback,
|
||||||
list_branches, create_branch, delete_branch, rename_branch,
|
|
||||||
};
|
};
|
||||||
pub use branch::{BranchInfo, DatasetResolver, MergeStrategy};
|
pub use branch::{BranchInfo, DatasetResolver, MergeStrategy};
|
||||||
pub use merkle::{MerkleError, MerkleNode, RevisionMerkleTree, WalkStep};
|
pub use error::OnionError;
|
||||||
pub use ext::FileBuilderExt;
|
pub use ext::FileBuilderExt;
|
||||||
pub use versioned_file::{VersionedFile, open_at_revision, open_at_branch};
|
pub use format::{
|
||||||
|
BRANCH_MAIN, BranchEntry, Codec, DEFAULT_FEATURE_FLAGS, EPOCH_DEAD, FORMAT_VERSION,
|
||||||
|
HEADER_SIZE, MAGIC, NO_PARENT, OnionHeader, PageTableEntry, REV_FLAG_SNAPSHOT, RevisionEntry,
|
||||||
|
feature_flags,
|
||||||
|
};
|
||||||
|
pub use merkle::{MerkleError, MerkleNode, RevisionMerkleTree, WalkStep};
|
||||||
|
pub use versioned_file::{VersionedFile, open_at_branch, open_at_revision};
|
||||||
pub use writer::eof_to_page_size;
|
pub use writer::eof_to_page_size;
|
||||||
|
|||||||
@@ -97,7 +97,11 @@ impl RevisionMerkleTree {
|
|||||||
/// The slice **must** be sorted by `revision` in ascending order.
|
/// The slice **must** be sorted by `revision` in ascending order.
|
||||||
pub fn build(entries: &[(u64, [u8; 32])]) -> Self {
|
pub fn build(entries: &[(u64, [u8; 32])]) -> Self {
|
||||||
let leaf_count = entries.len();
|
let leaf_count = entries.len();
|
||||||
let capacity = if leaf_count == 0 { 1 } else { leaf_count.next_power_of_two() };
|
let capacity = if leaf_count == 0 {
|
||||||
|
1
|
||||||
|
} else {
|
||||||
|
leaf_count.next_power_of_two()
|
||||||
|
};
|
||||||
|
|
||||||
// 1-indexed BFS array; index 0 unused.
|
// 1-indexed BFS array; index 0 unused.
|
||||||
let mut hashes = vec![[0u8; 32]; 2 * capacity + 1];
|
let mut hashes = vec![[0u8; 32]; 2 * capacity + 1];
|
||||||
@@ -113,7 +117,12 @@ impl RevisionMerkleTree {
|
|||||||
// Build internal nodes bottom-up
|
// Build internal nodes bottom-up
|
||||||
build_internal(&mut hashes, capacity);
|
build_internal(&mut hashes, capacity);
|
||||||
|
|
||||||
RevisionMerkleTree { hashes, leaf_revisions, leaf_count, capacity }
|
RevisionMerkleTree {
|
||||||
|
hashes,
|
||||||
|
leaf_revisions,
|
||||||
|
leaf_count,
|
||||||
|
capacity,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Queries ───────────────────────────────────────────────────────────────
|
// ── Queries ───────────────────────────────────────────────────────────────
|
||||||
@@ -311,7 +320,7 @@ impl RevisionMerkleTree {
|
|||||||
/// `hashes` must have length `2 * capacity + 1`.
|
/// `hashes` must have length `2 * capacity + 1`.
|
||||||
fn build_internal(hashes: &mut [[u8; 32]], capacity: usize) {
|
fn build_internal(hashes: &mut [[u8; 32]], capacity: usize) {
|
||||||
for i in (1..capacity).rev() {
|
for i in (1..capacity).rev() {
|
||||||
let left = hashes[2 * i];
|
let left = hashes[2 * i];
|
||||||
let right = hashes[2 * i + 1];
|
let right = hashes[2 * i + 1];
|
||||||
hashes[i] = node_hash(&left, &right);
|
hashes[i] = node_hash(&left, &right);
|
||||||
}
|
}
|
||||||
@@ -484,7 +493,10 @@ mod tests {
|
|||||||
let header_only = &full[..13]; // 5 + 8 bytes
|
let header_only = &full[..13]; // 5 + 8 bytes
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
RevisionMerkleTree::deserialise(header_only),
|
RevisionMerkleTree::deserialise(header_only),
|
||||||
Err(MerkleError::TruncatedPayload { expected: 5 + 8 + 5 * 40, got: 13 })
|
Err(MerkleError::TruncatedPayload {
|
||||||
|
expected: 5 + 8 + 5 * 40,
|
||||||
|
got: 13
|
||||||
|
})
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -96,7 +96,10 @@ mod tests {
|
|||||||
// Same pages, different insertion order
|
// Same pages, different insertion order
|
||||||
let h1 = hash_pages(&[(0, a.as_ref()), (4096, b.as_ref())]);
|
let h1 = hash_pages(&[(0, a.as_ref()), (4096, b.as_ref())]);
|
||||||
let h2 = hash_pages(&[(4096, b.as_ref()), (0, a.as_ref())]);
|
let h2 = hash_pages(&[(4096, b.as_ref()), (0, a.as_ref())]);
|
||||||
assert_eq!(h1, h2, "hash must be order-independent (sorted by h5_offset)");
|
assert_eq!(
|
||||||
|
h1, h2,
|
||||||
|
"hash must be order-independent (sorted by h5_offset)"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -3,11 +3,11 @@
|
|||||||
//! Opening a historical revision reads only the `.onion` sidecar —
|
//! Opening a historical revision reads only the `.onion` sidecar —
|
||||||
//! the primary `.h5` file is always at the latest committed state.
|
//! the primary `.h5` file is always at the latest committed state.
|
||||||
|
|
||||||
use std::collections::BTreeMap;
|
|
||||||
use crate::compress::decompress_page;
|
use crate::compress::decompress_page;
|
||||||
use crate::error::OnionError;
|
use crate::error::OnionError;
|
||||||
use crate::format::Codec;
|
use crate::format::Codec;
|
||||||
use crate::writer::OnionFile;
|
use crate::writer::OnionFile;
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
/// `(h5_offset, compressed_data, codec_byte, orig_size)` — raw page as stored.
|
/// `(h5_offset, compressed_data, codec_byte, orig_size)` — raw page as stored.
|
||||||
pub type RawPage = (u64, Vec<u8>, u8, u32);
|
pub type RawPage = (u64, Vec<u8>, u8, u32);
|
||||||
@@ -41,7 +41,11 @@ impl OnionFile {
|
|||||||
|
|
||||||
// Collect all ancestor revisions in order from oldest → newest
|
// Collect all ancestor revisions in order from oldest → newest
|
||||||
let ancestors: Vec<u64> = {
|
let ancestors: Vec<u64> = {
|
||||||
let mut chain = self.index.ancestors(rev).map(|e| e.revision).collect::<Vec<_>>();
|
let mut chain = self
|
||||||
|
.index
|
||||||
|
.ancestors(rev)
|
||||||
|
.map(|e| e.revision)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
chain.reverse(); // oldest first
|
chain.reverse(); // oldest first
|
||||||
chain
|
chain
|
||||||
};
|
};
|
||||||
@@ -53,13 +57,11 @@ impl OnionFile {
|
|||||||
// Optimisation: find the newest snapshot in the ancestor chain and
|
// Optimisation: find the newest snapshot in the ancestor chain and
|
||||||
// start from there instead of from `h5_base`. This bounds
|
// start from there instead of from `h5_base`. This bounds
|
||||||
// reconstruction depth to O(N_since_snapshot · P).
|
// reconstruction depth to O(N_since_snapshot · P).
|
||||||
let snapshot_start_idx = ancestors
|
let snapshot_start_idx = ancestors.iter().rposition(|&r| {
|
||||||
.iter()
|
self.index
|
||||||
.rposition(|&r| {
|
.get(r)
|
||||||
self.index
|
.is_some_and(|e| e.flags & REV_FLAG_SNAPSHOT != 0)
|
||||||
.get(r)
|
});
|
||||||
.is_some_and(|e| e.flags & REV_FLAG_SNAPSHOT != 0)
|
|
||||||
});
|
|
||||||
|
|
||||||
let (start_idx, mut file_bytes) = match snapshot_start_idx {
|
let (start_idx, mut file_bytes) = match snapshot_start_idx {
|
||||||
Some(idx) => {
|
Some(idx) => {
|
||||||
@@ -76,7 +78,9 @@ impl OnionFile {
|
|||||||
let table = self
|
let table = self
|
||||||
.page_tables
|
.page_tables
|
||||||
.get(*ancestor_rev as usize)
|
.get(*ancestor_rev as usize)
|
||||||
.ok_or_else(|| OnionError::Malformed(format!("missing page table for rev {ancestor_rev}")))?;
|
.ok_or_else(|| {
|
||||||
|
OnionError::Malformed(format!("missing page table for rev {ancestor_rev}"))
|
||||||
|
})?;
|
||||||
|
|
||||||
for pt_entry in table {
|
for pt_entry in table {
|
||||||
let codec = Codec::from_u8(pt_entry.codec)?;
|
let codec = Codec::from_u8(pt_entry.codec)?;
|
||||||
@@ -111,11 +115,7 @@ impl OnionFile {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Open a specific revision using an [`OpenRevision`] selector.
|
/// Open a specific revision using an [`OpenRevision`] selector.
|
||||||
pub fn open_rev(
|
pub fn open_rev(&self, selector: OpenRevision, h5_base: &[u8]) -> Result<Vec<u8>, OnionError> {
|
||||||
&self,
|
|
||||||
selector: OpenRevision,
|
|
||||||
h5_base: &[u8],
|
|
||||||
) -> Result<Vec<u8>, OnionError> {
|
|
||||||
let rev = self.resolve_selector(selector)?;
|
let rev = self.resolve_selector(selector)?;
|
||||||
self.reconstruct_revision(rev, h5_base)
|
self.reconstruct_revision(rev, h5_base)
|
||||||
}
|
}
|
||||||
@@ -128,12 +128,11 @@ impl OnionFile {
|
|||||||
self.branch_head_rev(crate::format::BRANCH_MAIN)
|
self.branch_head_rev(crate::format::BRANCH_MAIN)
|
||||||
.ok_or(OnionError::RevisionNotFound(0))
|
.ok_or(OnionError::RevisionNotFound(0))
|
||||||
}
|
}
|
||||||
OpenRevision::At(rev) => {
|
OpenRevision::At(rev) => self
|
||||||
self.index
|
.index
|
||||||
.get(rev)
|
.get(rev)
|
||||||
.map(|_| rev)
|
.map(|_| rev)
|
||||||
.ok_or(OnionError::RevisionNotFound(rev))
|
.ok_or(OnionError::RevisionNotFound(rev)),
|
||||||
}
|
|
||||||
OpenRevision::Branch(name) => {
|
OpenRevision::Branch(name) => {
|
||||||
let branch = self
|
let branch = self
|
||||||
.branch_by_name(&name)
|
.branch_by_name(&name)
|
||||||
@@ -201,10 +200,7 @@ impl OnionFile {
|
|||||||
/// compressed bytes directly and let the receiver decompress.
|
/// compressed bytes directly and let the receiver decompress.
|
||||||
///
|
///
|
||||||
/// Returns `(h5_offset, compressed_data, codec_byte, orig_size)`.
|
/// Returns `(h5_offset, compressed_data, codec_byte, orig_size)`.
|
||||||
pub fn revision_pages_raw(
|
pub fn revision_pages_raw(&self, rev: u64) -> Result<Vec<RawPage>, OnionError> {
|
||||||
&self,
|
|
||||||
rev: u64,
|
|
||||||
) -> Result<Vec<RawPage>, OnionError> {
|
|
||||||
let table = self
|
let table = self
|
||||||
.page_tables
|
.page_tables
|
||||||
.get(rev as usize)
|
.get(rev as usize)
|
||||||
|
|||||||
@@ -117,7 +117,10 @@ mod tests {
|
|||||||
// byte_pos=1: elements 1,5,9,13
|
// byte_pos=1: elements 1,5,9,13
|
||||||
// byte_pos=2: elements 2,6,10,14
|
// byte_pos=2: elements 2,6,10,14
|
||||||
// byte_pos=3: elements 3,7,11,15
|
// byte_pos=3: elements 3,7,11,15
|
||||||
assert_eq!(enc, vec![0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15]);
|
assert_eq!(
|
||||||
|
enc,
|
||||||
|
vec![0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15]
|
||||||
|
);
|
||||||
let dec = decode(&enc, 4, 16);
|
let dec = decode(&enc, 4, 16);
|
||||||
assert_eq!(dec, data);
|
assert_eq!(dec, data);
|
||||||
}
|
}
|
||||||
@@ -180,7 +183,8 @@ mod tests {
|
|||||||
assert!(
|
assert!(
|
||||||
tdt_compressed.len() <= raw_compressed.len(),
|
tdt_compressed.len() <= raw_compressed.len(),
|
||||||
"TDT+zstd ({} bytes) should be no worse than raw zstd ({} bytes) on structured float data",
|
"TDT+zstd ({} bytes) should be no worse than raw zstd ({} bytes) on structured float data",
|
||||||
tdt_compressed.len(), raw_compressed.len()
|
tdt_compressed.len(),
|
||||||
|
raw_compressed.len()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -57,21 +57,21 @@ use crate::writer::OnionFile;
|
|||||||
/// [`export_revision`](VersionedFile::export_revision) to materialise a
|
/// [`export_revision`](VersionedFile::export_revision) to materialise a
|
||||||
/// specific revision to disk.
|
/// specific revision to disk.
|
||||||
pub struct VersionedFile {
|
pub struct VersionedFile {
|
||||||
onion: OnionFile,
|
onion: OnionFile,
|
||||||
h5_path: PathBuf,
|
h5_path: PathBuf,
|
||||||
/// Original `.h5` bytes — the reconstruction base. **Never mutated.**
|
/// Original `.h5` bytes — the reconstruction base. **Never mutated.**
|
||||||
h5_base: Vec<u8>,
|
h5_base: Vec<u8>,
|
||||||
/// In-memory state of the most recently committed revision on the **default**
|
/// In-memory state of the most recently committed revision on the **default**
|
||||||
/// branch. Kept in sync with `branch_state[BRANCH_MAIN]`.
|
/// branch. Kept in sync with `branch_state[BRANCH_MAIN]`.
|
||||||
/// Used by [`snapshot`](VersionedFile::snapshot) and
|
/// Used by [`snapshot`](VersionedFile::snapshot) and
|
||||||
/// [`auto_snapshot`](VersionedFile::auto_snapshot).
|
/// [`auto_snapshot`](VersionedFile::auto_snapshot).
|
||||||
current_h5: Vec<u8>,
|
current_h5: Vec<u8>,
|
||||||
/// Per-branch diff baseline: `branch_id → last-committed bytes on that branch`.
|
/// Per-branch diff baseline: `branch_id → last-committed bytes on that branch`.
|
||||||
///
|
///
|
||||||
/// Ensures that interleaved commits on different branches always diff against
|
/// Ensures that interleaved commits on different branches always diff against
|
||||||
/// the correct prior state for each branch, not just the globally last commit.
|
/// the correct prior state for each branch, not just the globally last commit.
|
||||||
branch_state: HashMap<u32, Vec<u8>>,
|
branch_state: HashMap<u32, Vec<u8>>,
|
||||||
page_size: u32,
|
page_size: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl VersionedFile {
|
impl VersionedFile {
|
||||||
@@ -105,7 +105,14 @@ impl VersionedFile {
|
|||||||
.cloned()
|
.cloned()
|
||||||
.unwrap_or_else(|| h5_base.clone());
|
.unwrap_or_else(|| h5_base.clone());
|
||||||
|
|
||||||
Ok(Self { onion, h5_path, h5_base, current_h5, branch_state, page_size })
|
Ok(Self {
|
||||||
|
onion,
|
||||||
|
h5_path,
|
||||||
|
h5_base,
|
||||||
|
current_h5,
|
||||||
|
branch_state,
|
||||||
|
page_size,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Create a new versioned HDF5 file from a [`clawhdf5::FileBuilder`].
|
/// Create a new versioned HDF5 file from a [`clawhdf5::FileBuilder`].
|
||||||
@@ -144,7 +151,14 @@ impl VersionedFile {
|
|||||||
let h5_base = std::fs::read(&h5_path)?;
|
let h5_base = std::fs::read(&h5_path)?;
|
||||||
let current_h5 = h5_base.clone();
|
let current_h5 = h5_base.clone();
|
||||||
let onion = OnionFile::create(&h5_path, page_size)?;
|
let onion = OnionFile::create(&h5_path, page_size)?;
|
||||||
Ok(Self { onion, h5_path, h5_base, current_h5, branch_state: HashMap::new(), page_size })
|
Ok(Self {
|
||||||
|
onion,
|
||||||
|
h5_path,
|
||||||
|
h5_base,
|
||||||
|
current_h5,
|
||||||
|
branch_state: HashMap::new(),
|
||||||
|
page_size,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Create a new versioned HDF5 file, automatically choosing the page size.
|
/// Create a new versioned HDF5 file, automatically choosing the page size.
|
||||||
@@ -163,7 +177,14 @@ impl VersionedFile {
|
|||||||
let current_h5 = h5_base.clone();
|
let current_h5 = h5_base.clone();
|
||||||
let onion = OnionFile::create_auto(&h5_path)?;
|
let onion = OnionFile::create_auto(&h5_path)?;
|
||||||
let page_size = onion.page_size();
|
let page_size = onion.page_size();
|
||||||
Ok(Self { onion, h5_path, h5_base, current_h5, branch_state: HashMap::new(), page_size })
|
Ok(Self {
|
||||||
|
onion,
|
||||||
|
h5_path,
|
||||||
|
h5_base,
|
||||||
|
current_h5,
|
||||||
|
branch_state: HashMap::new(),
|
||||||
|
page_size,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Read API ──────────────────────────────────────────────────────────────
|
// ── Read API ──────────────────────────────────────────────────────────────
|
||||||
@@ -352,7 +373,9 @@ impl VersionedFile {
|
|||||||
/// vf.auto_snapshot(500)?; // snapshot every 500 revisions
|
/// vf.auto_snapshot(500)?; // snapshot every 500 revisions
|
||||||
/// ```
|
/// ```
|
||||||
pub fn auto_snapshot(&mut self, interval: u64) -> Result<Option<u64>, OnionError> {
|
pub fn auto_snapshot(&mut self, interval: u64) -> Result<Option<u64>, OnionError> {
|
||||||
let result = self.onion.auto_snapshot_if_needed(&self.current_h5, interval)?;
|
let result = self
|
||||||
|
.onion
|
||||||
|
.auto_snapshot_if_needed(&self.current_h5, interval)?;
|
||||||
if result.is_some() {
|
if result.is_some() {
|
||||||
self.onion.flush()?;
|
self.onion.flush()?;
|
||||||
}
|
}
|
||||||
@@ -383,7 +406,8 @@ impl VersionedFile {
|
|||||||
// Refresh the target branch's cached state so future commits on it
|
// Refresh the target branch's cached state so future commits on it
|
||||||
// use the merged content as their diff baseline.
|
// use the merged content as their diff baseline.
|
||||||
let (target_id, target_head) = {
|
let (target_id, target_head) = {
|
||||||
let entry = self.onion
|
let entry = self
|
||||||
|
.onion
|
||||||
.branch_by_name(target)
|
.branch_by_name(target)
|
||||||
.ok_or_else(|| OnionError::BranchNotFound(target.to_string()))?;
|
.ok_or_else(|| OnionError::BranchNotFound(target.to_string()))?;
|
||||||
(entry.id, entry.head_rev)
|
(entry.id, entry.head_rev)
|
||||||
@@ -435,8 +459,7 @@ impl VersionedFile {
|
|||||||
// ── Internal ─────────────────────────────────────────────────────────────
|
// ── Internal ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
fn hdf5_from_bytes(&self, bytes: Vec<u8>) -> Result<clawhdf5::File, OnionError> {
|
fn hdf5_from_bytes(&self, bytes: Vec<u8>) -> Result<clawhdf5::File, OnionError> {
|
||||||
clawhdf5::File::from_bytes(bytes)
|
clawhdf5::File::from_bytes(bytes).map_err(|e| OnionError::Hdf5(e.to_string()))
|
||||||
.map_err(|e| OnionError::Hdf5(e.to_string()))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -510,11 +533,17 @@ mod tests {
|
|||||||
|
|
||||||
// Read back rev 0 — should still be [1.0, 2.0]
|
// Read back rev 0 — should still be [1.0, 2.0]
|
||||||
let f0 = vf.revision(0).unwrap();
|
let f0 = vf.revision(0).unwrap();
|
||||||
assert_eq!(f0.dataset("data").unwrap().read_f64().unwrap(), vec![1.0, 2.0]);
|
assert_eq!(
|
||||||
|
f0.dataset("data").unwrap().read_f64().unwrap(),
|
||||||
|
vec![1.0, 2.0]
|
||||||
|
);
|
||||||
|
|
||||||
// Read back rev 1 — should be [3.0, 4.0]
|
// Read back rev 1 — should be [3.0, 4.0]
|
||||||
let f1 = vf.revision(1).unwrap();
|
let f1 = vf.revision(1).unwrap();
|
||||||
assert_eq!(f1.dataset("data").unwrap().read_f64().unwrap(), vec![3.0, 4.0]);
|
assert_eq!(
|
||||||
|
f1.dataset("data").unwrap().read_f64().unwrap(),
|
||||||
|
vec![3.0, 4.0]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -580,9 +609,15 @@ mod tests {
|
|||||||
let vf2 = VersionedFile::open(&h5_path).unwrap();
|
let vf2 = VersionedFile::open(&h5_path).unwrap();
|
||||||
assert_eq!(vf2.onion().revision_count(), 2);
|
assert_eq!(vf2.onion().revision_count(), 2);
|
||||||
let f0 = vf2.revision(0).unwrap();
|
let f0 = vf2.revision(0).unwrap();
|
||||||
assert_eq!(f0.dataset("data").unwrap().read_f64().unwrap(), vec![10.0, 20.0]);
|
assert_eq!(
|
||||||
|
f0.dataset("data").unwrap().read_f64().unwrap(),
|
||||||
|
vec![10.0, 20.0]
|
||||||
|
);
|
||||||
let f1 = vf2.revision(1).unwrap();
|
let f1 = vf2.revision(1).unwrap();
|
||||||
assert_eq!(f1.dataset("data").unwrap().read_f64().unwrap(), vec![30.0, 40.0]);
|
assert_eq!(
|
||||||
|
f1.dataset("data").unwrap().read_f64().unwrap(),
|
||||||
|
vec![30.0, 40.0]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── merge ────────────────────────────────────────────────────────────────
|
// ── merge ────────────────────────────────────────────────────────────────
|
||||||
@@ -598,7 +633,8 @@ mod tests {
|
|||||||
// Fork feat
|
// Fork feat
|
||||||
let feat_id = vf.onion_mut().create_branch("feat", "main").unwrap();
|
let feat_id = vf.onion_mut().create_branch("feat", "main").unwrap();
|
||||||
// Rev 1: feat writes [3.0, 4.0]
|
// Rev 1: feat writes [3.0, 4.0]
|
||||||
vf.commit_on(Some("feat"), make_h5(&[3.0, 4.0]), Some("feat-v1")).unwrap();
|
vf.commit_on(Some("feat"), make_h5(&[3.0, 4.0]), Some("feat-v1"))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
// Merge feat → main
|
// Merge feat → main
|
||||||
let merge_rev = vf.merge("feat", "main", MergeStrategy::LatestWins).unwrap();
|
let merge_rev = vf.merge("feat", "main", MergeStrategy::LatestWins).unwrap();
|
||||||
@@ -610,7 +646,10 @@ mod tests {
|
|||||||
|
|
||||||
// Content at merge revision reflects feat's state (latest wins)
|
// Content at merge revision reflects feat's state (latest wins)
|
||||||
let f = vf.revision(merge_rev).unwrap();
|
let f = vf.revision(merge_rev).unwrap();
|
||||||
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![3.0, 4.0]);
|
assert_eq!(
|
||||||
|
f.dataset("data").unwrap().read_f64().unwrap(),
|
||||||
|
vec![3.0, 4.0]
|
||||||
|
);
|
||||||
|
|
||||||
let _ = feat_id;
|
let _ = feat_id;
|
||||||
}
|
}
|
||||||
@@ -622,7 +661,8 @@ mod tests {
|
|||||||
|
|
||||||
vf.commit(make_h5(&[1.0]), Some("main-v1")).unwrap();
|
vf.commit(make_h5(&[1.0]), Some("main-v1")).unwrap();
|
||||||
vf.onion_mut().create_branch("feat", "main").unwrap();
|
vf.onion_mut().create_branch("feat", "main").unwrap();
|
||||||
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1")).unwrap();
|
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1"))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
vf.merge("feat", "main", MergeStrategy::LatestWins).unwrap();
|
vf.merge("feat", "main", MergeStrategy::LatestWins).unwrap();
|
||||||
|
|
||||||
@@ -640,7 +680,9 @@ mod tests {
|
|||||||
let mut vf = VersionedFile::create(&h5_path, 4096).unwrap();
|
let mut vf = VersionedFile::create(&h5_path, 4096).unwrap();
|
||||||
vf.commit(make_h5(&[1.0]), None).unwrap();
|
vf.commit(make_h5(&[1.0]), None).unwrap();
|
||||||
|
|
||||||
let err = vf.merge("ghost", "main", MergeStrategy::LatestWins).unwrap_err();
|
let err = vf
|
||||||
|
.merge("ghost", "main", MergeStrategy::LatestWins)
|
||||||
|
.unwrap_err();
|
||||||
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -653,11 +695,12 @@ mod tests {
|
|||||||
|
|
||||||
vf.commit(make_h5(&[1.0]), Some("main-v1")).unwrap();
|
vf.commit(make_h5(&[1.0]), Some("main-v1")).unwrap();
|
||||||
let feat_id = vf.onion_mut().create_branch("feat", "main").unwrap();
|
let feat_id = vf.onion_mut().create_branch("feat", "main").unwrap();
|
||||||
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1")).unwrap();
|
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1"))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let revs = vf.onion().list_revisions();
|
let revs = vf.onion().list_revisions();
|
||||||
assert_eq!(revs.len(), 2);
|
assert_eq!(revs.len(), 2);
|
||||||
assert_eq!(revs[0].branch_id, 0, "rev 0 is on main");
|
assert_eq!(revs[0].branch_id, 0, "rev 0 is on main");
|
||||||
assert_eq!(revs[1].branch_id, feat_id, "rev 1 is on feat");
|
assert_eq!(revs[1].branch_id, feat_id, "rev 1 is on feat");
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -679,7 +722,9 @@ mod tests {
|
|||||||
let (_tmp, h5_path) = tmp_h5(&[1.0]);
|
let (_tmp, h5_path) = tmp_h5(&[1.0]);
|
||||||
let mut vf = VersionedFile::create(&h5_path, 4096).unwrap();
|
let mut vf = VersionedFile::create(&h5_path, 4096).unwrap();
|
||||||
|
|
||||||
let err = vf.commit_on(Some("ghost"), make_h5(&[2.0]), None).unwrap_err();
|
let err = vf
|
||||||
|
.commit_on(Some("ghost"), make_h5(&[2.0]), None)
|
||||||
|
.unwrap_err();
|
||||||
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -698,7 +743,8 @@ mod tests {
|
|||||||
vf.onion_mut().create_branch("feat", "main").unwrap();
|
vf.onion_mut().create_branch("feat", "main").unwrap();
|
||||||
|
|
||||||
// Commit v2 on feat (rev 1: [2.0])
|
// Commit v2 on feat (rev 1: [2.0])
|
||||||
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1")).unwrap();
|
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1"))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
// Commit v2 on main (rev 2: [3.0]) — diff must be against main's last
|
// Commit v2 on main (rev 2: [3.0]) — diff must be against main's last
|
||||||
// commit ([1.0]), not feat's last commit ([2.0]).
|
// commit ([1.0]), not feat's last commit ([2.0]).
|
||||||
@@ -706,16 +752,25 @@ mod tests {
|
|||||||
|
|
||||||
// Reconstruct each revision and verify round-trip fidelity
|
// Reconstruct each revision and verify round-trip fidelity
|
||||||
let f0 = vf.revision(0).unwrap();
|
let f0 = vf.revision(0).unwrap();
|
||||||
assert_eq!(f0.dataset("data").unwrap().read_f64().unwrap(), vec![1.0],
|
assert_eq!(
|
||||||
"rev 0 (main-v1)");
|
f0.dataset("data").unwrap().read_f64().unwrap(),
|
||||||
|
vec![1.0],
|
||||||
|
"rev 0 (main-v1)"
|
||||||
|
);
|
||||||
|
|
||||||
let f1 = vf.revision(1).unwrap();
|
let f1 = vf.revision(1).unwrap();
|
||||||
assert_eq!(f1.dataset("data").unwrap().read_f64().unwrap(), vec![2.0],
|
assert_eq!(
|
||||||
"rev 1 (feat-v1)");
|
f1.dataset("data").unwrap().read_f64().unwrap(),
|
||||||
|
vec![2.0],
|
||||||
|
"rev 1 (feat-v1)"
|
||||||
|
);
|
||||||
|
|
||||||
let f2 = vf.revision(2).unwrap();
|
let f2 = vf.revision(2).unwrap();
|
||||||
assert_eq!(f2.dataset("data").unwrap().read_f64().unwrap(), vec![3.0],
|
assert_eq!(
|
||||||
"rev 2 (main-v2) — would be wrong if diffed against feat state");
|
f2.dataset("data").unwrap().read_f64().unwrap(),
|
||||||
|
vec![3.0],
|
||||||
|
"rev 2 (main-v2) — would be wrong if diffed against feat state"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// After a reload, `open()` must restore per-branch state from the sidecar
|
/// After a reload, `open()` must restore per-branch state from the sidecar
|
||||||
@@ -730,13 +785,21 @@ mod tests {
|
|||||||
let mut s = vf.onion_mut().begin_session(Some(feat_id)).unwrap();
|
let mut s = vf.onion_mut().begin_session(Some(feat_id)).unwrap();
|
||||||
let feat_bytes = make_h5(&[9.0]);
|
let feat_bytes = make_h5(&[9.0]);
|
||||||
let ps = 4096usize;
|
let ps = 4096usize;
|
||||||
let base = vf.onion().reconstruct_revision(0,
|
let base = vf
|
||||||
&std::fs::read(&h5_path).unwrap()).unwrap();
|
.onion()
|
||||||
|
.reconstruct_revision(0, &std::fs::read(&h5_path).unwrap())
|
||||||
|
.unwrap();
|
||||||
for i in 0..(feat_bytes.len().max(base.len())).div_ceil(ps) {
|
for i in 0..(feat_bytes.len().max(base.len())).div_ceil(ps) {
|
||||||
let start = i * ps;
|
let start = i * ps;
|
||||||
if start >= feat_bytes.len() { break; }
|
if start >= feat_bytes.len() {
|
||||||
let ns = &feat_bytes[start..feat_bytes.len().min(start+ps)];
|
break;
|
||||||
let os: &[u8] = if start < base.len() { &base[start..base.len().min(start+ps)] } else { &[] };
|
}
|
||||||
|
let ns = &feat_bytes[start..feat_bytes.len().min(start + ps)];
|
||||||
|
let os: &[u8] = if start < base.len() {
|
||||||
|
&base[start..base.len().min(start + ps)]
|
||||||
|
} else {
|
||||||
|
&[]
|
||||||
|
};
|
||||||
if ns != os {
|
if ns != os {
|
||||||
let mut pad = vec![0u8; ps];
|
let mut pad = vec![0u8; ps];
|
||||||
pad[..ns.len()].copy_from_slice(ns);
|
pad[..ns.len()].copy_from_slice(ns);
|
||||||
@@ -752,8 +815,11 @@ mod tests {
|
|||||||
vf2.commit(make_h5(&[2.0]), Some("main-v2")).unwrap();
|
vf2.commit(make_h5(&[2.0]), Some("main-v2")).unwrap();
|
||||||
|
|
||||||
let f_main = vf2.revision(2).unwrap();
|
let f_main = vf2.revision(2).unwrap();
|
||||||
assert_eq!(f_main.dataset("data").unwrap().read_f64().unwrap(), vec![2.0],
|
assert_eq!(
|
||||||
"main-v2 content must be [2.0] after reload");
|
f_main.dataset("data").unwrap().read_f64().unwrap(),
|
||||||
|
vec![2.0],
|
||||||
|
"main-v2 content must be [2.0] after reload"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── create_auto ──────────────────────────────────────────────────────────
|
// ── create_auto ──────────────────────────────────────────────────────────
|
||||||
@@ -767,7 +833,10 @@ mod tests {
|
|||||||
"auto page size must be a power of two, got {}",
|
"auto page size must be a power of two, got {}",
|
||||||
vf.page_size()
|
vf.page_size()
|
||||||
);
|
);
|
||||||
assert!(vf.page_size() >= 4096, "auto page size must be at least 4 KiB");
|
assert!(
|
||||||
|
vf.page_size() >= 4096,
|
||||||
|
"auto page size must be at least 4 KiB"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -778,7 +847,10 @@ mod tests {
|
|||||||
vf.commit(make_h5(&[7.0, 8.0]), Some("auto-v1")).unwrap();
|
vf.commit(make_h5(&[7.0, 8.0]), Some("auto-v1")).unwrap();
|
||||||
|
|
||||||
let f = vf.revision(0).unwrap();
|
let f = vf.revision(0).unwrap();
|
||||||
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![7.0, 8.0]);
|
assert_eq!(
|
||||||
|
f.dataset("data").unwrap().read_f64().unwrap(),
|
||||||
|
vec![7.0, 8.0]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── snapshot / auto_snapshot ─────────────────────────────────────────────
|
// ── snapshot / auto_snapshot ─────────────────────────────────────────────
|
||||||
@@ -797,7 +869,10 @@ mod tests {
|
|||||||
// The snapshot revision must be flagged as a snapshot.
|
// The snapshot revision must be flagged as a snapshot.
|
||||||
let revs = vf.onion().list_revisions();
|
let revs = vf.onion().list_revisions();
|
||||||
let snap = revs.iter().find(|r| r.revision == snap_rev).unwrap();
|
let snap = revs.iter().find(|r| r.revision == snap_rev).unwrap();
|
||||||
assert!(snap.is_snapshot, "snapshot revision must have is_snapshot == true");
|
assert!(
|
||||||
|
snap.is_snapshot,
|
||||||
|
"snapshot revision must have is_snapshot == true"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -843,7 +918,10 @@ mod tests {
|
|||||||
vf.commit(make_h5(&[i as f64]), None).unwrap();
|
vf.commit(make_h5(&[i as f64]), None).unwrap();
|
||||||
}
|
}
|
||||||
let result = vf.auto_snapshot(5).unwrap();
|
let result = vf.auto_snapshot(5).unwrap();
|
||||||
assert!(result.is_some(), "auto_snapshot should fire when count % interval == 0");
|
assert!(
|
||||||
|
result.is_some(),
|
||||||
|
"auto_snapshot should fire when count % interval == 0"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -855,7 +933,10 @@ mod tests {
|
|||||||
vf.commit(make_h5(&[2.0]), None).unwrap();
|
vf.commit(make_h5(&[2.0]), None).unwrap();
|
||||||
// 2 revisions, interval 5 — should not trigger
|
// 2 revisions, interval 5 — should not trigger
|
||||||
let result = vf.auto_snapshot(5).unwrap();
|
let result = vf.auto_snapshot(5).unwrap();
|
||||||
assert!(result.is_none(), "auto_snapshot should be a no-op when count % interval != 0");
|
assert!(
|
||||||
|
result.is_none(),
|
||||||
|
"auto_snapshot should be a no-op when count % interval != 0"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -908,12 +989,21 @@ mod tests {
|
|||||||
// Record a changed page for the feat branch rev
|
// Record a changed page for the feat branch rev
|
||||||
let feat_bytes = make_h5(&[3.0]);
|
let feat_bytes = make_h5(&[3.0]);
|
||||||
let ps = 4096usize;
|
let ps = 4096usize;
|
||||||
let current = vf.onion().reconstruct_revision(1, &std::fs::read(&h5_path).unwrap()).unwrap();
|
let current = vf
|
||||||
|
.onion()
|
||||||
|
.reconstruct_revision(1, &std::fs::read(&h5_path).unwrap())
|
||||||
|
.unwrap();
|
||||||
for i in 0..(feat_bytes.len().max(current.len())).div_ceil(ps) {
|
for i in 0..(feat_bytes.len().max(current.len())).div_ceil(ps) {
|
||||||
let start = i * ps;
|
let start = i * ps;
|
||||||
if start >= feat_bytes.len() { break; }
|
if start >= feat_bytes.len() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
let new_sl = &feat_bytes[start..feat_bytes.len().min(start + ps)];
|
let new_sl = &feat_bytes[start..feat_bytes.len().min(start + ps)];
|
||||||
let old_sl: &[u8] = if start < current.len() { ¤t[start..current.len().min(start + ps)] } else { &[] };
|
let old_sl: &[u8] = if start < current.len() {
|
||||||
|
¤t[start..current.len().min(start + ps)]
|
||||||
|
} else {
|
||||||
|
&[]
|
||||||
|
};
|
||||||
if new_sl != old_sl {
|
if new_sl != old_sl {
|
||||||
let mut pad = vec![0u8; ps];
|
let mut pad = vec![0u8; ps];
|
||||||
pad[..new_sl.len()].copy_from_slice(new_sl);
|
pad[..new_sl.len()].copy_from_slice(new_sl);
|
||||||
@@ -1009,7 +1099,10 @@ mod tests {
|
|||||||
p.set_extension(&new_ext);
|
p.set_extension(&new_ext);
|
||||||
p
|
p
|
||||||
};
|
};
|
||||||
assert!(onion_path.exists(), ".onion sidecar should exist after from_builder: checked {onion_path:?}");
|
assert!(
|
||||||
|
onion_path.exists(),
|
||||||
|
".onion sidecar should exist after from_builder: checked {onion_path:?}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1026,7 +1119,10 @@ mod tests {
|
|||||||
assert_eq!(rev, 0);
|
assert_eq!(rev, 0);
|
||||||
|
|
||||||
let f = vf.revision(0).unwrap();
|
let f = vf.revision(0).unwrap();
|
||||||
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![3.0, 4.0]);
|
assert_eq!(
|
||||||
|
f.dataset("data").unwrap().read_f64().unwrap(),
|
||||||
|
vec![3.0, 4.0]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── export_revision ───────────────────────────────────────────────────────
|
// ── export_revision ───────────────────────────────────────────────────────
|
||||||
@@ -1046,7 +1142,10 @@ mod tests {
|
|||||||
// Re-open the exported file as a raw clawhdf5::File
|
// Re-open the exported file as a raw clawhdf5::File
|
||||||
let bytes = std::fs::read(&export_path).unwrap();
|
let bytes = std::fs::read(&export_path).unwrap();
|
||||||
let f = clawhdf5::File::from_bytes(bytes).unwrap();
|
let f = clawhdf5::File::from_bytes(bytes).unwrap();
|
||||||
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![1.0, 2.0]);
|
assert_eq!(
|
||||||
|
f.dataset("data").unwrap().read_f64().unwrap(),
|
||||||
|
vec![1.0, 2.0]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1075,7 +1174,10 @@ mod tests {
|
|||||||
|
|
||||||
let bytes = std::fs::read(&export_path).unwrap();
|
let bytes = std::fs::read(&export_path).unwrap();
|
||||||
let f = clawhdf5::File::from_bytes(bytes).unwrap();
|
let f = clawhdf5::File::from_bytes(bytes).unwrap();
|
||||||
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![8.0, 9.0]);
|
assert_eq!(
|
||||||
|
f.dataset("data").unwrap().read_f64().unwrap(),
|
||||||
|
vec![8.0, 9.0]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── multiple datasets ─────────────────────────────────────────────────────
|
// ── multiple datasets ─────────────────────────────────────────────────────
|
||||||
@@ -1098,11 +1200,23 @@ mod tests {
|
|||||||
vf.commit(bytes_v2, Some("update x only")).unwrap();
|
vf.commit(bytes_v2, Some("update x only")).unwrap();
|
||||||
|
|
||||||
let f0 = vf.revision(0).unwrap();
|
let f0 = vf.revision(0).unwrap();
|
||||||
assert_eq!(f0.dataset("x").unwrap().read_f64().unwrap(), vec![1.0, 2.0, 3.0]);
|
assert_eq!(
|
||||||
assert_eq!(f0.dataset("y").unwrap().read_f64().unwrap(), vec![4.0, 5.0, 6.0]);
|
f0.dataset("x").unwrap().read_f64().unwrap(),
|
||||||
|
vec![1.0, 2.0, 3.0]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
f0.dataset("y").unwrap().read_f64().unwrap(),
|
||||||
|
vec![4.0, 5.0, 6.0]
|
||||||
|
);
|
||||||
|
|
||||||
let f1 = vf.revision(1).unwrap();
|
let f1 = vf.revision(1).unwrap();
|
||||||
assert_eq!(f1.dataset("x").unwrap().read_f64().unwrap(), vec![10.0, 20.0, 30.0]);
|
assert_eq!(
|
||||||
assert_eq!(f1.dataset("y").unwrap().read_f64().unwrap(), vec![4.0, 5.0, 6.0]);
|
f1.dataset("x").unwrap().read_f64().unwrap(),
|
||||||
|
vec![10.0, 20.0, 30.0]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
f1.dataset("y").unwrap().read_f64().unwrap(),
|
||||||
|
vec![4.0, 5.0, 6.0]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,8 +18,8 @@ use crate::annotation::AnnotationHeap;
|
|||||||
use crate::compress::compress_page;
|
use crate::compress::compress_page;
|
||||||
use crate::error::OnionError;
|
use crate::error::OnionError;
|
||||||
use crate::format::{
|
use crate::format::{
|
||||||
BranchEntry, Codec, OnionHeader, PageTableEntry, RevisionEntry,
|
BRANCH_MAIN, BranchEntry, Codec, DEFAULT_FEATURE_FLAGS, HEADER_SIZE, NO_PARENT, OnionHeader,
|
||||||
BRANCH_MAIN, DEFAULT_FEATURE_FLAGS, HEADER_SIZE, NO_PARENT,
|
PageTableEntry, RevisionEntry,
|
||||||
};
|
};
|
||||||
use crate::index::RevisionIndex;
|
use crate::index::RevisionIndex;
|
||||||
use crate::provenance::{SessionId, hash_pages};
|
use crate::provenance::{SessionId, hash_pages};
|
||||||
@@ -204,9 +204,7 @@ impl OnionFile {
|
|||||||
annotation: Option<&str>,
|
annotation: Option<&str>,
|
||||||
) -> Result<u64, OnionError> {
|
) -> Result<u64, OnionError> {
|
||||||
let revision = self.index.len() as u64;
|
let revision = self.index.len() as u64;
|
||||||
let parent_rev = self
|
let parent_rev = self.branch_head_rev(session.branch_id).unwrap_or(NO_PARENT);
|
||||||
.branch_head_rev(session.branch_id)
|
|
||||||
.unwrap_or(NO_PARENT);
|
|
||||||
|
|
||||||
// Compress pages and build BLAKE3 input
|
// Compress pages and build BLAKE3 input
|
||||||
let mut pages: Vec<(u64, Vec<u8>, u32, Codec)> = Vec::new();
|
let mut pages: Vec<(u64, Vec<u8>, u32, Codec)> = Vec::new();
|
||||||
@@ -226,9 +224,7 @@ impl OnionFile {
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Annotation
|
// Annotation
|
||||||
let annotation_off = annotation
|
let annotation_off = annotation.map(|a| self.annotations.push(a)).unwrap_or(0);
|
||||||
.map(|a| self.annotations.push(a))
|
|
||||||
.unwrap_or(0);
|
|
||||||
|
|
||||||
// Append page table entries
|
// Append page table entries
|
||||||
let mut table: Vec<PageTableEntry> = Vec::new();
|
let mut table: Vec<PageTableEntry> = Vec::new();
|
||||||
@@ -451,9 +447,9 @@ impl OnionFile {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn branch_by_name(&self, name: &str) -> Option<&BranchEntry> {
|
pub fn branch_by_name(&self, name: &str) -> Option<&BranchEntry> {
|
||||||
self.branches.iter().find(|b| {
|
self.branches
|
||||||
self.annotations.get(b.name_off) == Some(name)
|
.iter()
|
||||||
})
|
.find(|b| self.annotations.get(b.name_off) == Some(name))
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Serialise the entire `.onion` file to bytes.
|
/// Serialise the entire `.onion` file to bytes.
|
||||||
@@ -480,8 +476,7 @@ impl OnionFile {
|
|||||||
let branch_offset = index_offset + (n_revisions * rev_entry_size) as u64;
|
let branch_offset = index_offset + (n_revisions * rev_entry_size) as u64;
|
||||||
|
|
||||||
// Calculate PageTable section start
|
// Calculate PageTable section start
|
||||||
let page_table_section_start =
|
let page_table_section_start = branch_offset + (n_branches * branch_entry_size) as u64;
|
||||||
branch_offset + (n_branches * branch_entry_size) as u64;
|
|
||||||
|
|
||||||
// Calculate per-revision page_table_off values and total page table size
|
// Calculate per-revision page_table_off values and total page table size
|
||||||
let mut page_table_offsets: Vec<u64> = Vec::with_capacity(n_revisions);
|
let mut page_table_offsets: Vec<u64> = Vec::with_capacity(n_revisions);
|
||||||
@@ -599,7 +594,11 @@ impl OnionFile {
|
|||||||
// 0-4). We extend the Vec with empty entries so that
|
// 0-4). We extend the Vec with empty entries so that
|
||||||
// page_tables[rev_number] is always valid.
|
// page_tables[rev_number] is always valid.
|
||||||
let max_rev = entries.iter().map(|e| e.revision).max().unwrap_or(0);
|
let max_rev = entries.iter().map(|e| e.revision).max().unwrap_or(0);
|
||||||
let table_len = if entries.is_empty() { 0 } else { max_rev as usize + 1 };
|
let table_len = if entries.is_empty() {
|
||||||
|
0
|
||||||
|
} else {
|
||||||
|
max_rev as usize + 1
|
||||||
|
};
|
||||||
let mut page_tables: Vec<Vec<PageTableEntry>> = vec![Vec::new(); table_len];
|
let mut page_tables: Vec<Vec<PageTableEntry>> = vec![Vec::new(); table_len];
|
||||||
for entry in &entries {
|
for entry in &entries {
|
||||||
let pt_start = entry.page_table_off as usize;
|
let pt_start = entry.page_table_off as usize;
|
||||||
@@ -792,7 +791,10 @@ mod tests {
|
|||||||
fn create_accepts_valid_page_sizes() {
|
fn create_accepts_valid_page_sizes() {
|
||||||
for &size in &[512u32, 1024, 4096, 65536] {
|
for &size in &[512u32, 1024, 4096, 65536] {
|
||||||
let path = Path::new("/tmp/dummy.h5");
|
let path = Path::new("/tmp/dummy.h5");
|
||||||
assert!(OnionFile::create(path, size).is_ok(), "size {size} rejected");
|
assert!(
|
||||||
|
OnionFile::create(path, size).is_ok(),
|
||||||
|
"size {size} rejected"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -973,7 +975,9 @@ mod tests {
|
|||||||
.list_revisions()
|
.list_revisions()
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.filter(|r| {
|
.filter(|r| {
|
||||||
onion.index.get(r.revision)
|
onion
|
||||||
|
.index
|
||||||
|
.get(r.revision)
|
||||||
.map_or(false, |e| e.flags & crate::format::REV_FLAG_SNAPSHOT != 0)
|
.map_or(false, |e| e.flags & crate::format::REV_FLAG_SNAPSHOT != 0)
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
@@ -1041,7 +1045,10 @@ mod tests {
|
|||||||
fn eof_to_page_size_always_power_of_two() {
|
fn eof_to_page_size_always_power_of_two() {
|
||||||
for eof in [1, 100, 1_000, 10_000, 100_000, 1_000_000, 100_000_000u64] {
|
for eof in [1, 100, 1_000, 10_000, 100_000, 1_000_000, 100_000_000u64] {
|
||||||
let ps = eof_to_page_size(eof);
|
let ps = eof_to_page_size(eof);
|
||||||
assert!(ps.is_power_of_two(), "page_size {ps} is not a power of two for eof={eof}");
|
assert!(
|
||||||
|
ps.is_power_of_two(),
|
||||||
|
"page_size {ps} is not a power of two for eof={eof}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -20,23 +20,27 @@ use tempfile::NamedTempFile;
|
|||||||
// Layout constants (must match CLAWONION_SPEC.md §2 + §3)
|
// Layout constants (must match CLAWONION_SPEC.md §2 + §3)
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
const HEADER_SIZE: usize = 128;
|
const HEADER_SIZE: usize = 128;
|
||||||
const REV_ENTRY_SIZE: usize = 112;
|
const REV_ENTRY_SIZE: usize = 112;
|
||||||
const BRANCH_ENTRY_SIZE: usize = 40;
|
const BRANCH_ENTRY_SIZE: usize = 40;
|
||||||
const PAGE_ENTRY_SIZE: usize = 32;
|
const PAGE_ENTRY_SIZE: usize = 32;
|
||||||
const PAGE_SIZE: u32 = 4096;
|
const PAGE_SIZE: u32 = 4096;
|
||||||
|
|
||||||
const INDEX_OFFSET: usize = HEADER_SIZE; // 128
|
const INDEX_OFFSET: usize = HEADER_SIZE; // 128
|
||||||
const BRANCH_OFFSET: usize = INDEX_OFFSET + REV_ENTRY_SIZE; // 240
|
const BRANCH_OFFSET: usize = INDEX_OFFSET + REV_ENTRY_SIZE; // 240
|
||||||
const PT_OFFSET: usize = BRANCH_OFFSET + BRANCH_ENTRY_SIZE; // 280
|
const PT_OFFSET: usize = BRANCH_OFFSET + BRANCH_ENTRY_SIZE; // 280
|
||||||
const PD_OFFSET: usize = PT_OFFSET + PAGE_ENTRY_SIZE; // 312
|
const PD_OFFSET: usize = PT_OFFSET + PAGE_ENTRY_SIZE; // 312
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// Helpers
|
// Helpers
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
fn le64(v: u64) -> [u8; 8] { v.to_le_bytes() }
|
fn le64(v: u64) -> [u8; 8] {
|
||||||
fn le32(v: u32) -> [u8; 4] { v.to_le_bytes() }
|
v.to_le_bytes()
|
||||||
|
}
|
||||||
|
fn le32(v: u32) -> [u8; 4] {
|
||||||
|
v.to_le_bytes()
|
||||||
|
}
|
||||||
|
|
||||||
/// Build the expected BLAKE3 for one 4096-byte page of 0xAB at h5_offset = 0.
|
/// Build the expected BLAKE3 for one 4096-byte page of 0xAB at h5_offset = 0.
|
||||||
///
|
///
|
||||||
@@ -82,7 +86,8 @@ fn golden_minimal_onion_byte_layout() {
|
|||||||
let ann_heap: &[u8] = &[0x00, 0x04, 0x00, 0x00, 0x00, b'm', b'a', b'i', b'n'];
|
let ann_heap: &[u8] = &[0x00, 0x04, 0x00, 0x00, 0x00, b'm', b'a', b'i', b'n'];
|
||||||
let expected_total = PD_OFFSET + PAGE_SIZE as usize + ann_heap.len(); // 4417
|
let expected_total = PD_OFFSET + PAGE_SIZE as usize + ann_heap.len(); // 4417
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
bytes.len(), expected_total,
|
bytes.len(),
|
||||||
|
expected_total,
|
||||||
"total file size mismatch (got {}, expected {expected_total})",
|
"total file size mismatch (got {}, expected {expected_total})",
|
||||||
bytes.len()
|
bytes.len()
|
||||||
);
|
);
|
||||||
@@ -127,10 +132,17 @@ fn golden_minimal_onion_byte_layout() {
|
|||||||
// [64..72] created_at — f64, time-dependent; skip exact value check,
|
// [64..72] created_at — f64, time-dependent; skip exact value check,
|
||||||
// but assert it's non-zero (a real Unix timestamp).
|
// but assert it's non-zero (a real Unix timestamp).
|
||||||
let created_at = f64::from_le_bytes(bytes[64..72].try_into().unwrap());
|
let created_at = f64::from_le_bytes(bytes[64..72].try_into().unwrap());
|
||||||
assert!(created_at > 0.0, "created_at should be a positive Unix timestamp");
|
assert!(
|
||||||
|
created_at > 0.0,
|
||||||
|
"created_at should be a positive Unix timestamp"
|
||||||
|
);
|
||||||
|
|
||||||
// [72..128] reserved — 56 bytes, must all be zero
|
// [72..128] reserved — 56 bytes, must all be zero
|
||||||
assert_eq!(&bytes[72..128], &[0u8; 56], "reserved header bytes must be zero");
|
assert_eq!(
|
||||||
|
&bytes[72..128],
|
||||||
|
&[0u8; 56],
|
||||||
|
"reserved header bytes must be zero"
|
||||||
|
);
|
||||||
|
|
||||||
// ══════════════════════════════════════════════════════════════════════════
|
// ══════════════════════════════════════════════════════════════════════════
|
||||||
// §3.2 RevisionEntry — 112 bytes @ offset 128
|
// §3.2 RevisionEntry — 112 bytes @ offset 128
|
||||||
@@ -139,45 +151,69 @@ fn golden_minimal_onion_byte_layout() {
|
|||||||
let re = INDEX_OFFSET; // 128
|
let re = INDEX_OFFSET; // 128
|
||||||
|
|
||||||
// [re+0..re+8] revision = 0
|
// [re+0..re+8] revision = 0
|
||||||
assert_eq!(&bytes[re..re+8], &le64(0), "revision number");
|
assert_eq!(&bytes[re..re + 8], &le64(0), "revision number");
|
||||||
|
|
||||||
// [re+8..re+12] branch_id = 0 (main)
|
// [re+8..re+12] branch_id = 0 (main)
|
||||||
assert_eq!(&bytes[re+8..re+12], &le32(0), "branch_id");
|
assert_eq!(&bytes[re + 8..re + 12], &le32(0), "branch_id");
|
||||||
|
|
||||||
// [re+12..re+16] _pad_bi — must be zero
|
// [re+12..re+16] _pad_bi — must be zero
|
||||||
assert_eq!(&bytes[re+12..re+16], &[0u8; 4], "_pad_bi must be zero");
|
assert_eq!(&bytes[re + 12..re + 16], &[0u8; 4], "_pad_bi must be zero");
|
||||||
|
|
||||||
// [re+16..re+24] parent_rev = u64::MAX (root has no parent)
|
// [re+16..re+24] parent_rev = u64::MAX (root has no parent)
|
||||||
assert_eq!(&bytes[re+16..re+24], &le64(u64::MAX), "parent_rev (NO_PARENT sentinel)");
|
assert_eq!(
|
||||||
|
&bytes[re + 16..re + 24],
|
||||||
|
&le64(u64::MAX),
|
||||||
|
"parent_rev (NO_PARENT sentinel)"
|
||||||
|
);
|
||||||
|
|
||||||
// [re+24..re+28] page_count = 1
|
// [re+24..re+28] page_count = 1
|
||||||
assert_eq!(&bytes[re+24..re+28], &le32(1), "page_count");
|
assert_eq!(&bytes[re + 24..re + 28], &le32(1), "page_count");
|
||||||
|
|
||||||
// [re+28..re+32] _pad_pc — must be zero
|
// [re+28..re+32] _pad_pc — must be zero
|
||||||
assert_eq!(&bytes[re+28..re+32], &[0u8; 4], "_pad_pc must be zero");
|
assert_eq!(&bytes[re + 28..re + 32], &[0u8; 4], "_pad_pc must be zero");
|
||||||
|
|
||||||
// [re+32..re+40] page_table_off = 280 (PT_OFFSET)
|
// [re+32..re+40] page_table_off = 280 (PT_OFFSET)
|
||||||
assert_eq!(&bytes[re+32..re+40], &le64(PT_OFFSET as u64), "page_table_off");
|
assert_eq!(
|
||||||
|
&bytes[re + 32..re + 40],
|
||||||
|
&le64(PT_OFFSET as u64),
|
||||||
|
"page_table_off"
|
||||||
|
);
|
||||||
|
|
||||||
// [re+40..re+48] timestamp — f64, time-dependent; assert positive
|
// [re+40..re+48] timestamp — f64, time-dependent; assert positive
|
||||||
let ts = f64::from_le_bytes(bytes[re+40..re+48].try_into().unwrap());
|
let ts = f64::from_le_bytes(bytes[re + 40..re + 48].try_into().unwrap());
|
||||||
assert!(ts > 0.0, "revision timestamp should be positive");
|
assert!(ts > 0.0, "revision timestamp should be positive");
|
||||||
|
|
||||||
// [re+48..re+80] blake3 — verify against spec §5 algorithm
|
// [re+48..re+80] blake3 — verify against spec §5 algorithm
|
||||||
let computed_blake3 = expected_blake3(&page_bytes);
|
let computed_blake3 = expected_blake3(&page_bytes);
|
||||||
assert_eq!(&bytes[re+48..re+80], &computed_blake3, "BLAKE3 hash mismatch");
|
assert_eq!(
|
||||||
|
&bytes[re + 48..re + 80],
|
||||||
|
&computed_blake3,
|
||||||
|
"BLAKE3 hash mismatch"
|
||||||
|
);
|
||||||
|
|
||||||
// [re+80..re+96] session_uuid — 16 bytes UUIDv7; must be non-zero
|
// [re+80..re+96] session_uuid — 16 bytes UUIDv7; must be non-zero
|
||||||
assert_ne!(&bytes[re+80..re+96], &[0u8; 16], "session_uuid must not be all zeros");
|
assert_ne!(
|
||||||
|
&bytes[re + 80..re + 96],
|
||||||
|
&[0u8; 16],
|
||||||
|
"session_uuid must not be all zeros"
|
||||||
|
);
|
||||||
|
|
||||||
// [re+96..re+104] annotation_off = 0 (no annotation)
|
// [re+96..re+104] annotation_off = 0 (no annotation)
|
||||||
assert_eq!(&bytes[re+96..re+104], &le64(0), "annotation_off must be 0 (no annotation)");
|
assert_eq!(
|
||||||
|
&bytes[re + 96..re + 104],
|
||||||
|
&le64(0),
|
||||||
|
"annotation_off must be 0 (no annotation)"
|
||||||
|
);
|
||||||
|
|
||||||
// [re+104] flags = 0 (not a snapshot)
|
// [re+104] flags = 0 (not a snapshot)
|
||||||
assert_eq!(bytes[re+104], 0x00, "revision flags");
|
assert_eq!(bytes[re + 104], 0x00, "revision flags");
|
||||||
|
|
||||||
// [re+105..re+112] _pad_flags — must be zero
|
// [re+105..re+112] _pad_flags — must be zero
|
||||||
assert_eq!(&bytes[re+105..re+112], &[0u8; 7], "_pad_flags must be zero");
|
assert_eq!(
|
||||||
|
&bytes[re + 105..re + 112],
|
||||||
|
&[0u8; 7],
|
||||||
|
"_pad_flags must be zero"
|
||||||
|
);
|
||||||
|
|
||||||
// ══════════════════════════════════════════════════════════════════════════
|
// ══════════════════════════════════════════════════════════════════════════
|
||||||
// §3.3 BranchEntry — 40 bytes @ offset 240
|
// §3.3 BranchEntry — 40 bytes @ offset 240
|
||||||
@@ -186,22 +222,34 @@ fn golden_minimal_onion_byte_layout() {
|
|||||||
let be = BRANCH_OFFSET; // 240
|
let be = BRANCH_OFFSET; // 240
|
||||||
|
|
||||||
// [be+0..be+4] id = 0 (main)
|
// [be+0..be+4] id = 0 (main)
|
||||||
assert_eq!(&bytes[be..be+4], &le32(0), "branch id");
|
assert_eq!(&bytes[be..be + 4], &le32(0), "branch id");
|
||||||
|
|
||||||
// [be+4..be+8] _pad_id — must be zero
|
// [be+4..be+8] _pad_id — must be zero
|
||||||
assert_eq!(&bytes[be+4..be+8], &[0u8; 4], "_pad_id must be zero");
|
assert_eq!(&bytes[be + 4..be + 8], &[0u8; 4], "_pad_id must be zero");
|
||||||
|
|
||||||
// [be+8..be+16] name_off = 1 (heap-relative; first string after reserved null)
|
// [be+8..be+16] name_off = 1 (heap-relative; first string after reserved null)
|
||||||
assert_eq!(&bytes[be+8..be+16], &le64(1), "name_off for main branch");
|
assert_eq!(
|
||||||
|
&bytes[be + 8..be + 16],
|
||||||
|
&le64(1),
|
||||||
|
"name_off for main branch"
|
||||||
|
);
|
||||||
|
|
||||||
// [be+16..be+24] head_rev = 0 (updated after commit)
|
// [be+16..be+24] head_rev = 0 (updated after commit)
|
||||||
assert_eq!(&bytes[be+16..be+24], &le64(0), "head_rev after first commit");
|
assert_eq!(
|
||||||
|
&bytes[be + 16..be + 24],
|
||||||
|
&le64(0),
|
||||||
|
"head_rev after first commit"
|
||||||
|
);
|
||||||
|
|
||||||
// [be+24..be+32] fork_rev = u64::MAX (main is not forked)
|
// [be+24..be+32] fork_rev = u64::MAX (main is not forked)
|
||||||
assert_eq!(&bytes[be+24..be+32], &le64(u64::MAX), "fork_rev (NO_PARENT for main)");
|
assert_eq!(
|
||||||
|
&bytes[be + 24..be + 32],
|
||||||
|
&le64(u64::MAX),
|
||||||
|
"fork_rev (NO_PARENT for main)"
|
||||||
|
);
|
||||||
|
|
||||||
// [be+32..be+40] created_at — f64, time-dependent; assert positive
|
// [be+32..be+40] created_at — f64, time-dependent; assert positive
|
||||||
let branch_ts = f64::from_le_bytes(bytes[be+32..be+40].try_into().unwrap());
|
let branch_ts = f64::from_le_bytes(bytes[be + 32..be + 40].try_into().unwrap());
|
||||||
assert!(branch_ts > 0.0, "branch created_at should be positive");
|
assert!(branch_ts > 0.0, "branch created_at should be positive");
|
||||||
|
|
||||||
// ══════════════════════════════════════════════════════════════════════════
|
// ══════════════════════════════════════════════════════════════════════════
|
||||||
@@ -211,36 +259,51 @@ fn golden_minimal_onion_byte_layout() {
|
|||||||
let pt = PT_OFFSET; // 280
|
let pt = PT_OFFSET; // 280
|
||||||
|
|
||||||
// [pt+0..pt+8] h5_offset = 0
|
// [pt+0..pt+8] h5_offset = 0
|
||||||
assert_eq!(&bytes[pt..pt+8], &le64(0), "h5_offset");
|
assert_eq!(&bytes[pt..pt + 8], &le64(0), "h5_offset");
|
||||||
|
|
||||||
// [pt+8..pt+16] data_offset = 312 (PD_OFFSET, absolute file position)
|
// [pt+8..pt+16] data_offset = 312 (PD_OFFSET, absolute file position)
|
||||||
assert_eq!(&bytes[pt+8..pt+16], &le64(PD_OFFSET as u64), "data_offset (absolute)");
|
assert_eq!(
|
||||||
|
&bytes[pt + 8..pt + 16],
|
||||||
|
&le64(PD_OFFSET as u64),
|
||||||
|
"data_offset (absolute)"
|
||||||
|
);
|
||||||
|
|
||||||
// [pt+16..pt+20] orig_size = 4096
|
// [pt+16..pt+20] orig_size = 4096
|
||||||
assert_eq!(&bytes[pt+16..pt+20], &le32(4096), "orig_size");
|
assert_eq!(&bytes[pt + 16..pt + 20], &le32(4096), "orig_size");
|
||||||
|
|
||||||
// [pt+20..pt+24] data_size = 4096 (codec=None → no compression)
|
// [pt+20..pt+24] data_size = 4096 (codec=None → no compression)
|
||||||
assert_eq!(&bytes[pt+20..pt+24], &le32(4096), "data_size (codec=None, uncompressed)");
|
assert_eq!(
|
||||||
|
&bytes[pt + 20..pt + 24],
|
||||||
|
&le32(4096),
|
||||||
|
"data_size (codec=None, uncompressed)"
|
||||||
|
);
|
||||||
|
|
||||||
// [pt+24] codec = 0 (None)
|
// [pt+24] codec = 0 (None)
|
||||||
assert_eq!(bytes[pt+24], 0x00, "codec = None (0)");
|
assert_eq!(bytes[pt + 24], 0x00, "codec = None (0)");
|
||||||
|
|
||||||
// [pt+25..pt+32] _pad — must be zero
|
// [pt+25..pt+32] _pad — must be zero
|
||||||
assert_eq!(&bytes[pt+25..pt+32], &[0u8; 7], "_pad must be zero");
|
assert_eq!(&bytes[pt + 25..pt + 32], &[0u8; 7], "_pad must be zero");
|
||||||
|
|
||||||
// ══════════════════════════════════════════════════════════════════════════
|
// ══════════════════════════════════════════════════════════════════════════
|
||||||
// PageData — 4096 bytes @ offset 312
|
// PageData — 4096 bytes @ offset 312
|
||||||
// ══════════════════════════════════════════════════════════════════════════
|
// ══════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
assert_eq!(&bytes[PD_OFFSET..PD_OFFSET + 4096], page_bytes.as_slice(),
|
assert_eq!(
|
||||||
"page data must equal the uncompressed original bytes");
|
&bytes[PD_OFFSET..PD_OFFSET + 4096],
|
||||||
|
page_bytes.as_slice(),
|
||||||
|
"page data must equal the uncompressed original bytes"
|
||||||
|
);
|
||||||
|
|
||||||
// ══════════════════════════════════════════════════════════════════════════
|
// ══════════════════════════════════════════════════════════════════════════
|
||||||
// §3.5 AnnotationHeap — tail of file
|
// §3.5 AnnotationHeap — tail of file
|
||||||
// ══════════════════════════════════════════════════════════════════════════
|
// ══════════════════════════════════════════════════════════════════════════
|
||||||
|
|
||||||
// Last 9 bytes: [0x00 reserved] [u32 LE length=4] ["main"]
|
// Last 9 bytes: [0x00 reserved] [u32 LE length=4] ["main"]
|
||||||
assert_eq!(&bytes[bytes.len()-9..], ann_heap, "annotation heap content");
|
assert_eq!(
|
||||||
|
&bytes[bytes.len() - 9..],
|
||||||
|
ann_heap,
|
||||||
|
"annotation heap content"
|
||||||
|
);
|
||||||
|
|
||||||
// ══════════════════════════════════════════════════════════════════════════
|
// ══════════════════════════════════════════════════════════════════════════
|
||||||
// §7 Reconstruction — verify round-trip
|
// §7 Reconstruction — verify round-trip
|
||||||
@@ -270,10 +333,10 @@ fn golden_minimal_onion_byte_layout() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn byte_exact_format_freeze() {
|
fn byte_exact_format_freeze() {
|
||||||
let re = INDEX_OFFSET; // 128 — RevisionEntry start
|
let re = INDEX_OFFSET; // 128 — RevisionEntry start
|
||||||
let be = BRANCH_OFFSET; // 240 — BranchEntry start
|
let be = BRANCH_OFFSET; // 240 — BranchEntry start
|
||||||
let pt = PT_OFFSET; // 280 — PageTableEntry start
|
let pt = PT_OFFSET; // 280 — PageTableEntry start
|
||||||
let pd = PD_OFFSET; // 312 — PageData start
|
let pd = PD_OFFSET; // 312 — PageData start
|
||||||
|
|
||||||
// ── Build the actual serialised file ────────────────────────────────────
|
// ── Build the actual serialised file ────────────────────────────────────
|
||||||
|
|
||||||
@@ -298,71 +361,75 @@ fn byte_exact_format_freeze() {
|
|||||||
let mut expected = vec![0u8; total];
|
let mut expected = vec![0u8; total];
|
||||||
|
|
||||||
// §3.1 OnionHeader (128 bytes)
|
// §3.1 OnionHeader (128 bytes)
|
||||||
expected[0..9].copy_from_slice(b"CLAWONION"); // magic
|
expected[0..9].copy_from_slice(b"CLAWONION"); // magic
|
||||||
expected[9] = 0x01; // format_version
|
expected[9] = 0x01; // format_version
|
||||||
// [10..16] = 0 (_pad_align)
|
// [10..16] = 0 (_pad_align)
|
||||||
expected[16..24].copy_from_slice(&le64(0x07)); // feature_flags
|
expected[16..24].copy_from_slice(&le64(0x07)); // feature_flags
|
||||||
expected[24..28].copy_from_slice(&le32(4096)); // page_size
|
expected[24..28].copy_from_slice(&le32(4096)); // page_size
|
||||||
// [28..32] = 0 (_pad_ps)
|
// [28..32] = 0 (_pad_ps)
|
||||||
expected[32..40].copy_from_slice(&le64(1)); // revision_count = 1
|
expected[32..40].copy_from_slice(&le64(1)); // revision_count = 1
|
||||||
expected[40..44].copy_from_slice(&le32(1)); // branch_count = 1
|
expected[40..44].copy_from_slice(&le32(1)); // branch_count = 1
|
||||||
// [44..48] = 0 (_pad_bc)
|
// [44..48] = 0 (_pad_bc)
|
||||||
expected[48..56].copy_from_slice(&le64(128)); // index_offset
|
expected[48..56].copy_from_slice(&le64(128)); // index_offset
|
||||||
expected[56..64].copy_from_slice(&le64(240)); // branch_offset
|
expected[56..64].copy_from_slice(&le64(240)); // branch_offset
|
||||||
// [64..72] = 0 (created_at — zeroed, dynamic)
|
// [64..72] = 0 (created_at — zeroed, dynamic)
|
||||||
// [72..128] = 0 (reserved)
|
// [72..128] = 0 (reserved)
|
||||||
|
|
||||||
// §3.2 RevisionEntry (112 bytes @ 128)
|
// §3.2 RevisionEntry (112 bytes @ 128)
|
||||||
expected[re..re+8] .copy_from_slice(&le64(0)); // revision = 0
|
expected[re..re + 8].copy_from_slice(&le64(0)); // revision = 0
|
||||||
expected[re+8..re+12].copy_from_slice(&le32(0)); // branch_id = 0
|
expected[re + 8..re + 12].copy_from_slice(&le32(0)); // branch_id = 0
|
||||||
// [re+12..re+16] = 0 (_pad_bi)
|
// [re+12..re+16] = 0 (_pad_bi)
|
||||||
expected[re+16..re+24].copy_from_slice(&le64(u64::MAX)); // parent_rev = NO_PARENT
|
expected[re + 16..re + 24].copy_from_slice(&le64(u64::MAX)); // parent_rev = NO_PARENT
|
||||||
expected[re+24..re+28].copy_from_slice(&le32(1)); // page_count = 1
|
expected[re + 24..re + 28].copy_from_slice(&le32(1)); // page_count = 1
|
||||||
// [re+28..re+32] = 0 (_pad_pc)
|
// [re+28..re+32] = 0 (_pad_pc)
|
||||||
expected[re+32..re+40].copy_from_slice(&le64(pt as u64)); // page_table_off
|
expected[re + 32..re + 40].copy_from_slice(&le64(pt as u64)); // page_table_off
|
||||||
// [re+40..re+48] = 0 (timestamp — zeroed, dynamic)
|
// [re+40..re+48] = 0 (timestamp — zeroed, dynamic)
|
||||||
expected[re+48..re+80].copy_from_slice(&expected_blake3(&page_bytes)); // blake3
|
expected[re + 48..re + 80].copy_from_slice(&expected_blake3(&page_bytes)); // blake3
|
||||||
// [re+80..re+96] = 0 (session_uuid — zeroed, dynamic)
|
// [re+80..re+96] = 0 (session_uuid — zeroed, dynamic)
|
||||||
// [re+96..re+104] = 0 (annotation_off = 0, no annotation)
|
// [re+96..re+104] = 0 (annotation_off = 0, no annotation)
|
||||||
// [re+104] = 0 (flags = 0, not a snapshot)
|
// [re+104] = 0 (flags = 0, not a snapshot)
|
||||||
// [re+105..re+112] = 0 (_pad_flags)
|
// [re+105..re+112] = 0 (_pad_flags)
|
||||||
|
|
||||||
// §3.3 BranchEntry (40 bytes @ 240)
|
// §3.3 BranchEntry (40 bytes @ 240)
|
||||||
expected[be..be+4] .copy_from_slice(&le32(0)); // id = 0 (main)
|
expected[be..be + 4].copy_from_slice(&le32(0)); // id = 0 (main)
|
||||||
// [be+4..be+8] = 0 (_pad_id)
|
// [be+4..be+8] = 0 (_pad_id)
|
||||||
expected[be+8..be+16].copy_from_slice(&le64(1)); // name_off = 1 (heap offset of "main")
|
expected[be + 8..be + 16].copy_from_slice(&le64(1)); // name_off = 1 (heap offset of "main")
|
||||||
expected[be+16..be+24].copy_from_slice(&le64(0)); // head_rev = 0
|
expected[be + 16..be + 24].copy_from_slice(&le64(0)); // head_rev = 0
|
||||||
expected[be+24..be+32].copy_from_slice(&le64(u64::MAX)); // fork_rev = NO_PARENT
|
expected[be + 24..be + 32].copy_from_slice(&le64(u64::MAX)); // fork_rev = NO_PARENT
|
||||||
// [be+32..be+40] = 0 (created_at — zeroed, dynamic)
|
// [be+32..be+40] = 0 (created_at — zeroed, dynamic)
|
||||||
|
|
||||||
// §3.4 PageTableEntry (32 bytes @ 280)
|
// §3.4 PageTableEntry (32 bytes @ 280)
|
||||||
expected[pt..pt+8] .copy_from_slice(&le64(0)); // h5_offset = 0
|
expected[pt..pt + 8].copy_from_slice(&le64(0)); // h5_offset = 0
|
||||||
expected[pt+8..pt+16].copy_from_slice(&le64(pd as u64)); // data_offset
|
expected[pt + 8..pt + 16].copy_from_slice(&le64(pd as u64)); // data_offset
|
||||||
expected[pt+16..pt+20].copy_from_slice(&le32(4096)); // orig_size
|
expected[pt + 16..pt + 20].copy_from_slice(&le32(4096)); // orig_size
|
||||||
expected[pt+20..pt+24].copy_from_slice(&le32(4096)); // data_size (no compression)
|
expected[pt + 20..pt + 24].copy_from_slice(&le32(4096)); // data_size (no compression)
|
||||||
// [pt+24] = 0 (codec = None)
|
// [pt+24] = 0 (codec = None)
|
||||||
// [pt+25..pt+32] = 0 (_pad)
|
// [pt+25..pt+32] = 0 (_pad)
|
||||||
|
|
||||||
// PageData (4096 bytes @ 312)
|
// PageData (4096 bytes @ 312)
|
||||||
expected[pd..pd+4096].fill(0xAB);
|
expected[pd..pd + 4096].fill(0xAB);
|
||||||
|
|
||||||
// §3.5 AnnotationHeap (9 bytes at end)
|
// §3.5 AnnotationHeap (9 bytes at end)
|
||||||
expected[pd+4096..].copy_from_slice(ann_heap);
|
expected[pd + 4096..].copy_from_slice(ann_heap);
|
||||||
|
|
||||||
// ── Zero dynamic fields in both buffers ──────────────────────────────────
|
// ── Zero dynamic fields in both buffers ──────────────────────────────────
|
||||||
// OnionHeader.created_at
|
// OnionHeader.created_at
|
||||||
actual[64..72].fill(0);
|
actual[64..72].fill(0);
|
||||||
// RevisionEntry.timestamp
|
// RevisionEntry.timestamp
|
||||||
actual[re+40..re+48].fill(0);
|
actual[re + 40..re + 48].fill(0);
|
||||||
// RevisionEntry.session_uuid
|
// RevisionEntry.session_uuid
|
||||||
actual[re+80..re+96].fill(0);
|
actual[re + 80..re + 96].fill(0);
|
||||||
// BranchEntry.created_at
|
// BranchEntry.created_at
|
||||||
actual[be+32..be+40].fill(0);
|
actual[be + 32..be + 40].fill(0);
|
||||||
|
|
||||||
// ── Compare byte-for-byte ────────────────────────────────────────────────
|
// ── Compare byte-for-byte ────────────────────────────────────────────────
|
||||||
assert_eq!(actual.len(), expected.len(),
|
assert_eq!(
|
||||||
|
actual.len(),
|
||||||
|
expected.len(),
|
||||||
"total file size mismatch: actual={} expected={}",
|
"total file size mismatch: actual={} expected={}",
|
||||||
actual.len(), expected.len());
|
actual.len(),
|
||||||
|
expected.len()
|
||||||
|
);
|
||||||
|
|
||||||
// Find the first differing byte for a useful failure message.
|
// Find the first differing byte for a useful failure message.
|
||||||
if actual != expected {
|
if actual != expected {
|
||||||
@@ -387,13 +454,29 @@ fn byte_exact_format_freeze() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn struct_sizes_match_spec() {
|
fn struct_sizes_match_spec() {
|
||||||
|
use clawhdf5_onion::format::{BranchEntry, OnionHeader, PageTableEntry, RevisionEntry};
|
||||||
use std::mem::size_of;
|
use std::mem::size_of;
|
||||||
use clawhdf5_onion::format::{OnionHeader, RevisionEntry, BranchEntry, PageTableEntry};
|
|
||||||
|
|
||||||
assert_eq!(size_of::<OnionHeader>(), 128, "OnionHeader must be 128 bytes (§3.1)");
|
assert_eq!(
|
||||||
assert_eq!(size_of::<RevisionEntry>(), 112, "RevisionEntry must be 112 bytes (§3.2)");
|
size_of::<OnionHeader>(),
|
||||||
assert_eq!(size_of::<BranchEntry>(), 40, "BranchEntry must be 40 bytes (§3.3)");
|
128,
|
||||||
assert_eq!(size_of::<PageTableEntry>(), 32, "PageTableEntry must be 32 bytes (§3.4)");
|
"OnionHeader must be 128 bytes (§3.1)"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
size_of::<RevisionEntry>(),
|
||||||
|
112,
|
||||||
|
"RevisionEntry must be 112 bytes (§3.2)"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
size_of::<BranchEntry>(),
|
||||||
|
40,
|
||||||
|
"BranchEntry must be 40 bytes (§3.3)"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
size_of::<PageTableEntry>(),
|
||||||
|
32,
|
||||||
|
"PageTableEntry must be 32 bytes (§3.4)"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|||||||
@@ -258,8 +258,13 @@ fn assert_gc_invariants(onion: &OnionFile) {
|
|||||||
|
|
||||||
// 5. blake3_hex has correct length (64 chars)
|
// 5. blake3_hex has correct length (64 chars)
|
||||||
for rev in &revisions {
|
for rev in &revisions {
|
||||||
assert_eq!(rev.blake3_hex.len(), 64,
|
assert_eq!(
|
||||||
"revision {} has blake3_hex of wrong length {}", rev.revision, rev.blake3_hex.len());
|
rev.blake3_hex.len(),
|
||||||
|
64,
|
||||||
|
"revision {} has blake3_hex of wrong length {}",
|
||||||
|
rev.revision,
|
||||||
|
rev.blake3_hex.len()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 6. Pages are accessible for every revision
|
// 6. Pages are accessible for every revision
|
||||||
|
|||||||
@@ -48,7 +48,11 @@ struct Round {
|
|||||||
|
|
||||||
fn arb_round() -> impl Strategy<Value = Round> {
|
fn arb_round() -> impl Strategy<Value = Round> {
|
||||||
(any::<bool>(), 0usize..8usize, 1u8..=255u8).prop_map(|(create_branch, branch_idx, fill)| {
|
(any::<bool>(), 0usize..8usize, 1u8..=255u8).prop_map(|(create_branch, branch_idx, fill)| {
|
||||||
Round { create_branch, branch_idx, fill }
|
Round {
|
||||||
|
create_branch,
|
||||||
|
branch_idx,
|
||||||
|
fill,
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -72,10 +76,7 @@ fn make_vf() -> (TempDir, std::path::PathBuf, VersionedFile) {
|
|||||||
/// Apply `rounds` to `vf`, returning a Vec of `(revision_number, expected_page_bytes)`.
|
/// Apply `rounds` to `vf`, returning a Vec of `(revision_number, expected_page_bytes)`.
|
||||||
///
|
///
|
||||||
/// `branches` starts as `["main"]` and grows as new branches are created.
|
/// `branches` starts as `["main"]` and grows as new branches are created.
|
||||||
fn apply_rounds(
|
fn apply_rounds(vf: &mut VersionedFile, rounds: &[Round]) -> Vec<(u64, Vec<u8>)> {
|
||||||
vf: &mut VersionedFile,
|
|
||||||
rounds: &[Round],
|
|
||||||
) -> Vec<(u64, Vec<u8>)> {
|
|
||||||
let mut branches: Vec<String> = vec!["main".to_string()];
|
let mut branches: Vec<String> = vec!["main".to_string()];
|
||||||
let mut expectations: Vec<(u64, Vec<u8>)> = Vec::new();
|
let mut expectations: Vec<(u64, Vec<u8>)> = Vec::new();
|
||||||
let mut branch_counter: usize = 0;
|
let mut branch_counter: usize = 0;
|
||||||
@@ -124,7 +125,8 @@ fn verify_expectations(
|
|||||||
prop_assert_eq!(
|
prop_assert_eq!(
|
||||||
&actual[..page_end],
|
&actual[..page_end],
|
||||||
expected.as_slice(),
|
expected.as_slice(),
|
||||||
"revision {}: page 0 mismatch", rev
|
"revision {}: page 0 mismatch",
|
||||||
|
rev
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|||||||
@@ -78,17 +78,25 @@ async fn serve_one(server: TcpServer, server_onion_path: PathBuf) {
|
|||||||
let server_onion = OnionFile::open(&server_onion_path).unwrap();
|
let server_onion = OnionFile::open(&server_onion_path).unwrap();
|
||||||
let server_rev_count = server_onion.revision_count();
|
let server_rev_count = server_onion.revision_count();
|
||||||
let manifest = ClawSyncManifest::from_onion("bench-server", &server_onion, H5_MAGIC);
|
let manifest = ClawSyncManifest::from_onion("bench-server", &server_onion, H5_MAGIC);
|
||||||
conn.send(&SyncMessage::ManifestResponse { manifest }).await.unwrap();
|
conn.send(&SyncMessage::ManifestResponse { manifest })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
if server_rev_count > client_rev_count {
|
if server_rev_count > client_rev_count {
|
||||||
// Server is ahead — push its extra revisions to client
|
// Server is ahead — push its extra revisions to client
|
||||||
let client_head = if client_rev_count == 0 { NO_PARENT } else { client_rev_count - 1 };
|
let client_head = if client_rev_count == 0 {
|
||||||
|
NO_PARENT
|
||||||
|
} else {
|
||||||
|
client_rev_count - 1
|
||||||
|
};
|
||||||
let packets = diff_revisions(&server_onion, client_head).unwrap();
|
let packets = diff_revisions(&server_onion, client_head).unwrap();
|
||||||
let total = packets.len() as u64;
|
let total = packets.len() as u64;
|
||||||
let mut bytes_sent = 0u64;
|
let mut bytes_sent = 0u64;
|
||||||
for packet in packets {
|
for packet in packets {
|
||||||
bytes_sent += packet.page_data_size() as u64;
|
bytes_sent += packet.page_data_size() as u64;
|
||||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
conn.send(&SyncMessage::LayerPacket { packet })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::Ack { .. } => {}
|
SyncMessage::Ack { .. } => {}
|
||||||
other => panic!("server: expected Ack, got {other:?}"),
|
other => panic!("server: expected Ack, got {other:?}"),
|
||||||
@@ -96,8 +104,10 @@ async fn serve_one(server: TcpServer, server_onion_path: PathBuf) {
|
|||||||
}
|
}
|
||||||
conn.send(&SyncMessage::SyncComplete {
|
conn.send(&SyncMessage::SyncComplete {
|
||||||
revisions_transferred: total,
|
revisions_transferred: total,
|
||||||
bytes_transferred: bytes_sent,
|
bytes_transferred: bytes_sent,
|
||||||
}).await.unwrap();
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
} else {
|
} else {
|
||||||
// Server is behind or equal — receive packets from client
|
// Server is behind or equal — receive packets from client
|
||||||
let mut server_onion_rw = OnionFile::open(&server_onion_path).unwrap();
|
let mut server_onion_rw = OnionFile::open(&server_onion_path).unwrap();
|
||||||
@@ -106,7 +116,9 @@ async fn serve_one(server: TcpServer, server_onion_path: PathBuf) {
|
|||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::LayerPacket { packet } => {
|
SyncMessage::LayerPacket { packet } => {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
conn.send(&SyncMessage::Ack { revision: rev })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
packets.push(packet);
|
packets.push(packet);
|
||||||
}
|
}
|
||||||
SyncMessage::SyncComplete { .. } => {
|
SyncMessage::SyncComplete { .. } => {
|
||||||
@@ -157,7 +169,10 @@ fn bench_e2e(c: &mut Criterion) {
|
|||||||
rt.block_on(async {
|
rt.block_on(async {
|
||||||
let server_task = tokio::spawn(serve_one(server, dst_h5));
|
let server_task = tokio::spawn(serve_one(server, dst_h5));
|
||||||
let backend = TcpSyncBackend::new(addr, "bench-agent");
|
let backend = TcpSyncBackend::new(addr, "bench-agent");
|
||||||
backend.push(Path::new(&src_h5), &SyncSelector::All).await.unwrap();
|
backend
|
||||||
|
.push(Path::new(&src_h5), &SyncSelector::All)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
server_task.await.unwrap();
|
server_task.await.unwrap();
|
||||||
});
|
});
|
||||||
drop(src_dir);
|
drop(src_dir);
|
||||||
@@ -275,7 +290,10 @@ fn bench_e2e(c: &mut Criterion) {
|
|||||||
let addr = server.local_addr;
|
let addr = server.local_addr;
|
||||||
let server_task = tokio::spawn(serve_one(server, dst_h5));
|
let server_task = tokio::spawn(serve_one(server, dst_h5));
|
||||||
let backend = TcpSyncBackend::new(addr, "bench-agent");
|
let backend = TcpSyncBackend::new(addr, "bench-agent");
|
||||||
let stats = backend.push(Path::new(&src_h5), &SyncSelector::All).await.unwrap();
|
let stats = backend
|
||||||
|
.push(Path::new(&src_h5), &SyncSelector::All)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
server_task.await.unwrap();
|
server_task.await.unwrap();
|
||||||
stats.revisions_transferred
|
stats.revisions_transferred
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -78,13 +78,13 @@ async fn serve_with_latency(server: TcpServer, rtt_ms: f64) {
|
|||||||
|
|
||||||
conn.send(&SyncMessage::ManifestResponse {
|
conn.send(&SyncMessage::ManifestResponse {
|
||||||
manifest: ClawSyncManifest {
|
manifest: ClawSyncManifest {
|
||||||
agent_id: "bench-server".to_string(),
|
agent_id: "bench-server".to_string(),
|
||||||
file_blake3: [0u8; 32],
|
file_blake3: [0u8; 32],
|
||||||
revision_count: 0,
|
revision_count: 0,
|
||||||
head_revision: 0,
|
head_revision: 0,
|
||||||
head_blake3: [0u8; 32],
|
head_blake3: [0u8; 32],
|
||||||
revisions: vec![],
|
revisions: vec![],
|
||||||
last_write: 0.0,
|
last_write: 0.0,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
@@ -98,7 +98,9 @@ async fn serve_with_latency(server: TcpServer, rtt_ms: f64) {
|
|||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
// Inject simulated WAN RTT before sending Ack.
|
// Inject simulated WAN RTT before sending Ack.
|
||||||
tokio::time::sleep(delay).await;
|
tokio::time::sleep(delay).await;
|
||||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
conn.send(&SyncMessage::Ack { revision: rev })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
received += 1;
|
received += 1;
|
||||||
}
|
}
|
||||||
SyncMessage::SyncComplete { .. } => break,
|
SyncMessage::SyncComplete { .. } => break,
|
||||||
@@ -120,8 +122,8 @@ async fn pipelined_push(addr: SocketAddr, onion: &OnionFile, window: usize) {
|
|||||||
|
|
||||||
let rev_count = onion.revision_count();
|
let rev_count = onion.revision_count();
|
||||||
conn.send(&SyncMessage::ManifestRequest {
|
conn.send(&SyncMessage::ManifestRequest {
|
||||||
agent_id: "bench-client".to_string(),
|
agent_id: "bench-client".to_string(),
|
||||||
head_revision: rev_count.saturating_sub(1),
|
head_revision: rev_count.saturating_sub(1),
|
||||||
revision_count: rev_count,
|
revision_count: rev_count,
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
@@ -170,7 +172,7 @@ async fn pipelined_push(addr: SocketAddr, onion: &OnionFile, window: usize) {
|
|||||||
write_half
|
write_half
|
||||||
.send(&SyncMessage::SyncComplete {
|
.send(&SyncMessage::SyncComplete {
|
||||||
revisions_transferred: sent,
|
revisions_transferred: sent,
|
||||||
bytes_transferred: bytes_sent,
|
bytes_transferred: bytes_sent,
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -197,12 +199,12 @@ fn bench_wan_pipeline(c: &mut Criterion) {
|
|||||||
// At 1 ms RTT: W=1 ≈ 100 ms/iter, W=16 ≈ 7 ms/iter
|
// At 1 ms RTT: W=1 ≈ 100 ms/iter, W=16 ≈ 7 ms/iter
|
||||||
// At 5 ms RTT: W=1 ≈ 500 ms/iter, W=16 ≈ 32 ms/iter
|
// At 5 ms RTT: W=1 ≈ 500 ms/iter, W=16 ≈ 32 ms/iter
|
||||||
let params: &[(f64, usize, &str)] = &[
|
let params: &[(f64, usize, &str)] = &[
|
||||||
(1.0, 1, "rtt_1ms_W1"),
|
(1.0, 1, "rtt_1ms_W1"),
|
||||||
(1.0, 4, "rtt_1ms_W4"),
|
(1.0, 4, "rtt_1ms_W4"),
|
||||||
(1.0, 16, "rtt_1ms_W16"),
|
(1.0, 16, "rtt_1ms_W16"),
|
||||||
(5.0, 1, "rtt_5ms_W1"),
|
(5.0, 1, "rtt_5ms_W1"),
|
||||||
(5.0, 4, "rtt_5ms_W4"),
|
(5.0, 4, "rtt_5ms_W4"),
|
||||||
(5.0, 16, "rtt_5ms_W16"),
|
(5.0, 16, "rtt_5ms_W16"),
|
||||||
];
|
];
|
||||||
|
|
||||||
for &(rtt_ms, window, label) in params {
|
for &(rtt_ms, window, label) in params {
|
||||||
@@ -223,8 +225,7 @@ fn bench_wan_pipeline(c: &mut Criterion) {
|
|||||||
// TIMED: full push with simulated WAN latency
|
// TIMED: full push with simulated WAN latency
|
||||||
let _ = (dir, path); // keep TempDir alive
|
let _ = (dir, path); // keep TempDir alive
|
||||||
rt.block_on(async {
|
rt.block_on(async {
|
||||||
let server_task =
|
let server_task = tokio::spawn(serve_with_latency(server, rtt_ms));
|
||||||
tokio::spawn(serve_with_latency(server, rtt_ms));
|
|
||||||
pipelined_push(addr, &onion, window).await;
|
pipelined_push(addr, &onion, window).await;
|
||||||
server_task.await.unwrap();
|
server_task.await.unwrap();
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -42,7 +42,11 @@ pub struct PeerCapabilities {
|
|||||||
|
|
||||||
impl Default for PeerCapabilities {
|
impl Default for PeerCapabilities {
|
||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self { branching: true, dataset_diff: false, protocol_version: 1 }
|
Self {
|
||||||
|
branching: true,
|
||||||
|
dataset_diff: false,
|
||||||
|
protocol_version: 1,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,10 +58,17 @@ impl Default for PeerCapabilities {
|
|||||||
/// transport (e.g., QUIC, S3, local copy).
|
/// transport (e.g., QUIC, S3, local copy).
|
||||||
pub trait SyncBackend: Send + Sync {
|
pub trait SyncBackend: Send + Sync {
|
||||||
/// Push any revisions the remote lacks from `local_path` to `remote`.
|
/// Push any revisions the remote lacks from `local_path` to `remote`.
|
||||||
fn push(&self, local_path: &Path, selector: &SyncSelector) -> impl std::future::Future<Output = Result<SyncStats, AgentSyncError>> + Send;
|
fn push(
|
||||||
|
&self,
|
||||||
|
local_path: &Path,
|
||||||
|
selector: &SyncSelector,
|
||||||
|
) -> impl std::future::Future<Output = Result<SyncStats, AgentSyncError>> + Send;
|
||||||
|
|
||||||
/// Pull any revisions the local file lacks from the remote into `local_path`.
|
/// Pull any revisions the local file lacks from the remote into `local_path`.
|
||||||
fn pull(&self, local_path: &Path) -> impl std::future::Future<Output = Result<SyncStats, AgentSyncError>> + Send;
|
fn pull(
|
||||||
|
&self,
|
||||||
|
local_path: &Path,
|
||||||
|
) -> impl std::future::Future<Output = Result<SyncStats, AgentSyncError>> + Send;
|
||||||
|
|
||||||
/// Build a manifest for the given path.
|
/// Build a manifest for the given path.
|
||||||
fn local_manifest(&self, local_path: &Path) -> Result<ClawSyncManifest, AgentSyncError>;
|
fn local_manifest(&self, local_path: &Path) -> Result<ClawSyncManifest, AgentSyncError>;
|
||||||
@@ -75,14 +86,20 @@ pub struct TcpSyncBackend {
|
|||||||
|
|
||||||
impl TcpSyncBackend {
|
impl TcpSyncBackend {
|
||||||
pub fn new(remote_addr: SocketAddr, agent_id: impl Into<String>) -> Self {
|
pub fn new(remote_addr: SocketAddr, agent_id: impl Into<String>) -> Self {
|
||||||
Self { remote_addr, agent_id: agent_id.into() }
|
Self {
|
||||||
|
remote_addr,
|
||||||
|
agent_id: agent_id.into(),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SyncBackend for TcpSyncBackend {
|
impl SyncBackend for TcpSyncBackend {
|
||||||
async fn push(&self, local_path: &Path, selector: &SyncSelector) -> Result<SyncStats, AgentSyncError> {
|
async fn push(
|
||||||
let local_onion = OnionFile::open(local_path)
|
&self,
|
||||||
.map_err(AgentSyncError::Onion)?;
|
local_path: &Path,
|
||||||
|
selector: &SyncSelector,
|
||||||
|
) -> Result<SyncStats, AgentSyncError> {
|
||||||
|
let local_onion = OnionFile::open(local_path).map_err(AgentSyncError::Onion)?;
|
||||||
let h5_base = std::fs::read(local_path).map_err(AgentSyncError::Io)?;
|
let h5_base = std::fs::read(local_path).map_err(AgentSyncError::Io)?;
|
||||||
let local_manifest = ClawSyncManifest::from_onion(&self.agent_id, &local_onion, &h5_base);
|
let local_manifest = ClawSyncManifest::from_onion(&self.agent_id, &local_onion, &h5_base);
|
||||||
|
|
||||||
@@ -98,21 +115,26 @@ impl SyncBackend for TcpSyncBackend {
|
|||||||
.collect();
|
.collect();
|
||||||
let sketch = IbltSketch::from_keys(&local_rev_numbers, IBLT_SYNC_SEED);
|
let sketch = IbltSketch::from_keys(&local_rev_numbers, IBLT_SYNC_SEED);
|
||||||
let iblt_manifest = IbltManifest {
|
let iblt_manifest = IbltManifest {
|
||||||
agent_id: self.agent_id.clone(),
|
agent_id: self.agent_id.clone(),
|
||||||
file_blake3: local_manifest.file_blake3,
|
file_blake3: local_manifest.file_blake3,
|
||||||
revision_count: local_manifest.revision_count,
|
revision_count: local_manifest.revision_count,
|
||||||
head_revision: local_manifest.head_revision,
|
head_revision: local_manifest.head_revision,
|
||||||
head_blake3: local_manifest.head_blake3,
|
head_blake3: local_manifest.head_blake3,
|
||||||
last_write: local_manifest.last_write,
|
last_write: local_manifest.last_write,
|
||||||
sketch_cells: sketch.cell_count() as u32,
|
sketch_cells: sketch.cell_count() as u32,
|
||||||
sketch: sketch.to_bytes(),
|
sketch: sketch.to_bytes(),
|
||||||
};
|
};
|
||||||
conn.send(&SyncMessage::IbltRequest { sketch: iblt_manifest })
|
conn.send(&SyncMessage::IbltRequest {
|
||||||
.await
|
sketch: iblt_manifest,
|
||||||
.map_err(AgentSyncError::Transport)?;
|
})
|
||||||
|
.await
|
||||||
|
.map_err(AgentSyncError::Transport)?;
|
||||||
|
|
||||||
let missing_from_remote = match conn.recv().await.map_err(AgentSyncError::Transport)? {
|
let missing_from_remote = match conn.recv().await.map_err(AgentSyncError::Transport)? {
|
||||||
SyncMessage::IbltResponse { missing_from_remote, .. } => missing_from_remote,
|
SyncMessage::IbltResponse {
|
||||||
|
missing_from_remote,
|
||||||
|
..
|
||||||
|
} => missing_from_remote,
|
||||||
SyncMessage::Error { message } => return Err(AgentSyncError::Remote(message)),
|
SyncMessage::Error { message } => return Err(AgentSyncError::Remote(message)),
|
||||||
other => return Err(AgentSyncError::Protocol(format!("unexpected: {other:?}"))),
|
other => return Err(AgentSyncError::Protocol(format!("unexpected: {other:?}"))),
|
||||||
};
|
};
|
||||||
@@ -127,9 +149,13 @@ impl SyncBackend for TcpSyncBackend {
|
|||||||
|
|
||||||
for packet in packets {
|
for packet in packets {
|
||||||
let sz = packet.page_data_size() as u64;
|
let sz = packet.page_data_size() as u64;
|
||||||
conn.send(&SyncMessage::LayerPacket { packet }).await.map_err(AgentSyncError::Transport)?;
|
conn.send(&SyncMessage::LayerPacket { packet })
|
||||||
|
.await
|
||||||
|
.map_err(AgentSyncError::Transport)?;
|
||||||
match conn.recv().await.map_err(AgentSyncError::Transport)? {
|
match conn.recv().await.map_err(AgentSyncError::Transport)? {
|
||||||
SyncMessage::Ack { .. } => { bytes_sent += sz; }
|
SyncMessage::Ack { .. } => {
|
||||||
|
bytes_sent += sz;
|
||||||
|
}
|
||||||
SyncMessage::Error { message } => return Err(AgentSyncError::Remote(message)),
|
SyncMessage::Error { message } => return Err(AgentSyncError::Remote(message)),
|
||||||
other => return Err(AgentSyncError::Protocol(format!("unexpected: {other:?}"))),
|
other => return Err(AgentSyncError::Protocol(format!("unexpected: {other:?}"))),
|
||||||
}
|
}
|
||||||
@@ -138,10 +164,16 @@ impl SyncBackend for TcpSyncBackend {
|
|||||||
conn.send(&SyncMessage::SyncComplete {
|
conn.send(&SyncMessage::SyncComplete {
|
||||||
revisions_transferred: total,
|
revisions_transferred: total,
|
||||||
bytes_transferred: bytes_sent,
|
bytes_transferred: bytes_sent,
|
||||||
}).await.map_err(AgentSyncError::Transport)?;
|
})
|
||||||
|
.await
|
||||||
|
.map_err(AgentSyncError::Transport)?;
|
||||||
conn.shutdown().await.map_err(AgentSyncError::Transport)?;
|
conn.shutdown().await.map_err(AgentSyncError::Transport)?;
|
||||||
|
|
||||||
Ok(SyncStats { revisions_transferred: total, bytes_transferred: bytes_sent, revisions_skipped: 0 })
|
Ok(SyncStats {
|
||||||
|
revisions_transferred: total,
|
||||||
|
bytes_transferred: bytes_sent,
|
||||||
|
revisions_skipped: 0,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn pull(&self, local_path: &Path) -> Result<SyncStats, AgentSyncError> {
|
async fn pull(&self, local_path: &Path) -> Result<SyncStats, AgentSyncError> {
|
||||||
@@ -157,7 +189,9 @@ impl SyncBackend for TcpSyncBackend {
|
|||||||
agent_id: self.agent_id.clone(),
|
agent_id: self.agent_id.clone(),
|
||||||
head_revision: local_rev_count.saturating_sub(1),
|
head_revision: local_rev_count.saturating_sub(1),
|
||||||
revision_count: local_rev_count,
|
revision_count: local_rev_count,
|
||||||
}).await.map_err(AgentSyncError::Transport)?;
|
})
|
||||||
|
.await
|
||||||
|
.map_err(AgentSyncError::Transport)?;
|
||||||
|
|
||||||
// Get the server's revision count to detect "already in sync".
|
// Get the server's revision count to detect "already in sync".
|
||||||
let server_rev_count = match conn.recv().await.map_err(AgentSyncError::Transport)? {
|
let server_rev_count = match conn.recv().await.map_err(AgentSyncError::Transport)? {
|
||||||
@@ -171,12 +205,15 @@ impl SyncBackend for TcpSyncBackend {
|
|||||||
conn.send(&SyncMessage::SyncComplete {
|
conn.send(&SyncMessage::SyncComplete {
|
||||||
revisions_transferred: 0,
|
revisions_transferred: 0,
|
||||||
bytes_transferred: 0,
|
bytes_transferred: 0,
|
||||||
}).await.map_err(AgentSyncError::Transport)?;
|
})
|
||||||
|
.await
|
||||||
|
.map_err(AgentSyncError::Transport)?;
|
||||||
return Ok(SyncStats::default());
|
return Ok(SyncStats::default());
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut local_onion = if local_path.exists() {
|
let mut local_onion = if local_path.exists() {
|
||||||
OnionFile::open(local_path).or_else(|_| OnionFile::create_auto(local_path))
|
OnionFile::open(local_path)
|
||||||
|
.or_else(|_| OnionFile::create_auto(local_path))
|
||||||
.map_err(AgentSyncError::Onion)?
|
.map_err(AgentSyncError::Onion)?
|
||||||
} else {
|
} else {
|
||||||
std::fs::write(local_path, b"\x89HDF\r\n\x1a\n").map_err(AgentSyncError::Io)?;
|
std::fs::write(local_path, b"\x89HDF\r\n\x1a\n").map_err(AgentSyncError::Io)?;
|
||||||
@@ -189,10 +226,15 @@ impl SyncBackend for TcpSyncBackend {
|
|||||||
match conn.recv().await.map_err(AgentSyncError::Transport)? {
|
match conn.recv().await.map_err(AgentSyncError::Transport)? {
|
||||||
SyncMessage::LayerPacket { packet } => {
|
SyncMessage::LayerPacket { packet } => {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
conn.send(&SyncMessage::Ack { revision: rev }).await.map_err(AgentSyncError::Transport)?;
|
conn.send(&SyncMessage::Ack { revision: rev })
|
||||||
|
.await
|
||||||
|
.map_err(AgentSyncError::Transport)?;
|
||||||
packets.push(packet);
|
packets.push(packet);
|
||||||
}
|
}
|
||||||
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => {
|
SyncMessage::SyncComplete {
|
||||||
|
revisions_transferred,
|
||||||
|
bytes_transferred,
|
||||||
|
} => {
|
||||||
let stats = merge_packets(&mut local_onion, packets, true)
|
let stats = merge_packets(&mut local_onion, packets, true)
|
||||||
.map_err(AgentSyncError::SyncOnion)?;
|
.map_err(AgentSyncError::SyncOnion)?;
|
||||||
return Ok(SyncStats {
|
return Ok(SyncStats {
|
||||||
@@ -210,7 +252,11 @@ impl SyncBackend for TcpSyncBackend {
|
|||||||
fn local_manifest(&self, local_path: &Path) -> Result<ClawSyncManifest, AgentSyncError> {
|
fn local_manifest(&self, local_path: &Path) -> Result<ClawSyncManifest, AgentSyncError> {
|
||||||
let onion = OnionFile::open(local_path).map_err(AgentSyncError::Onion)?;
|
let onion = OnionFile::open(local_path).map_err(AgentSyncError::Onion)?;
|
||||||
let h5_base = std::fs::read(local_path).map_err(AgentSyncError::Io)?;
|
let h5_base = std::fs::read(local_path).map_err(AgentSyncError::Io)?;
|
||||||
Ok(ClawSyncManifest::from_onion(&self.agent_id, &onion, &h5_base))
|
Ok(ClawSyncManifest::from_onion(
|
||||||
|
&self.agent_id,
|
||||||
|
&onion,
|
||||||
|
&h5_base,
|
||||||
|
))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -77,8 +77,12 @@ impl PeerCapabilities {
|
|||||||
let mut out = [0u8; 8];
|
let mut out = [0u8; 8];
|
||||||
out[..4].copy_from_slice(&self.protocol_version.to_le_bytes());
|
out[..4].copy_from_slice(&self.protocol_version.to_le_bytes());
|
||||||
let mut flags: u32 = 0;
|
let mut flags: u32 = 0;
|
||||||
if self.branching { flags |= 1 << 0; }
|
if self.branching {
|
||||||
if self.dataset_diff { flags |= 1 << 1; }
|
flags |= 1 << 0;
|
||||||
|
}
|
||||||
|
if self.dataset_diff {
|
||||||
|
flags |= 1 << 1;
|
||||||
|
}
|
||||||
out[4..8].copy_from_slice(&flags.to_le_bytes());
|
out[4..8].copy_from_slice(&flags.to_le_bytes());
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
@@ -105,7 +109,11 @@ mod tests {
|
|||||||
use crate::backend::PeerCapabilities;
|
use crate::backend::PeerCapabilities;
|
||||||
|
|
||||||
fn caps(branching: bool, dataset_diff: bool, version: u32) -> PeerCapabilities {
|
fn caps(branching: bool, dataset_diff: bool, version: u32) -> PeerCapabilities {
|
||||||
PeerCapabilities { branching, dataset_diff, protocol_version: version }
|
PeerCapabilities {
|
||||||
|
branching,
|
||||||
|
dataset_diff,
|
||||||
|
protocol_version: version,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -123,7 +131,7 @@ mod tests {
|
|||||||
let a = caps(true, true, 2);
|
let a = caps(true, true, 2);
|
||||||
let b = caps(false, true, 1);
|
let b = caps(false, true, 1);
|
||||||
let s = negotiate(&a, &b);
|
let s = negotiate(&a, &b);
|
||||||
assert!(!s.branching); // b doesn't support it
|
assert!(!s.branching); // b doesn't support it
|
||||||
assert!(s.dataset_diff); // both support
|
assert!(s.dataset_diff); // both support
|
||||||
assert_eq!(s.protocol_version, 1); // min
|
assert_eq!(s.protocol_version, 1); // min
|
||||||
}
|
}
|
||||||
@@ -140,27 +148,43 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn assert_compatible_ok() {
|
fn assert_compatible_ok() {
|
||||||
let s = SessionCapabilities { branching: true, dataset_diff: false, protocol_version: 1 };
|
let s = SessionCapabilities {
|
||||||
|
branching: true,
|
||||||
|
dataset_diff: false,
|
||||||
|
protocol_version: 1,
|
||||||
|
};
|
||||||
assert!(assert_compatible(&s, true, false, 1).is_ok());
|
assert!(assert_compatible(&s, true, false, 1).is_ok());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn assert_compatible_missing_branching() {
|
fn assert_compatible_missing_branching() {
|
||||||
let s = SessionCapabilities { branching: false, dataset_diff: false, protocol_version: 1 };
|
let s = SessionCapabilities {
|
||||||
|
branching: false,
|
||||||
|
dataset_diff: false,
|
||||||
|
protocol_version: 1,
|
||||||
|
};
|
||||||
let err = assert_compatible(&s, true, false, 1).unwrap_err();
|
let err = assert_compatible(&s, true, false, 1).unwrap_err();
|
||||||
assert!(err.contains("branching"));
|
assert!(err.contains("branching"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn assert_compatible_version_too_low() {
|
fn assert_compatible_version_too_low() {
|
||||||
let s = SessionCapabilities { branching: true, dataset_diff: false, protocol_version: 1 };
|
let s = SessionCapabilities {
|
||||||
|
branching: true,
|
||||||
|
dataset_diff: false,
|
||||||
|
protocol_version: 1,
|
||||||
|
};
|
||||||
let err = assert_compatible(&s, false, false, 2).unwrap_err();
|
let err = assert_compatible(&s, false, false, 2).unwrap_err();
|
||||||
assert!(err.contains("protocol_version"));
|
assert!(err.contains("protocol_version"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn capabilities_wire_roundtrip() {
|
fn capabilities_wire_roundtrip() {
|
||||||
let orig = PeerCapabilities { branching: true, dataset_diff: true, protocol_version: 3 };
|
let orig = PeerCapabilities {
|
||||||
|
branching: true,
|
||||||
|
dataset_diff: true,
|
||||||
|
protocol_version: 3,
|
||||||
|
};
|
||||||
let bytes = orig.to_bytes();
|
let bytes = orig.to_bytes();
|
||||||
let decoded = PeerCapabilities::from_bytes(&bytes);
|
let decoded = PeerCapabilities::from_bytes(&bytes);
|
||||||
assert_eq!(decoded.branching, orig.branching);
|
assert_eq!(decoded.branching, orig.branching);
|
||||||
@@ -170,7 +194,11 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn capabilities_wire_no_features() {
|
fn capabilities_wire_no_features() {
|
||||||
let orig = PeerCapabilities { branching: false, dataset_diff: false, protocol_version: 0 };
|
let orig = PeerCapabilities {
|
||||||
|
branching: false,
|
||||||
|
dataset_diff: false,
|
||||||
|
protocol_version: 0,
|
||||||
|
};
|
||||||
let bytes = orig.to_bytes();
|
let bytes = orig.to_bytes();
|
||||||
let decoded = PeerCapabilities::from_bytes(&bytes);
|
let decoded = PeerCapabilities::from_bytes(&bytes);
|
||||||
assert!(!decoded.branching);
|
assert!(!decoded.branching);
|
||||||
|
|||||||
@@ -47,16 +47,28 @@ impl OnionMemory {
|
|||||||
/// Create a new `OnionMemory` at `h5_path`.
|
/// Create a new `OnionMemory` at `h5_path`.
|
||||||
pub fn create(config: MemoryConfig) -> Result<Self, AgentSyncError> {
|
pub fn create(config: MemoryConfig) -> Result<Self, AgentSyncError> {
|
||||||
let h5_path = config.path.clone();
|
let h5_path = config.path.clone();
|
||||||
let inner = HDF5Memory::create(config).map_err(|e| AgentSyncError::AgentMemory(e.to_string()))?;
|
let inner =
|
||||||
|
HDF5Memory::create(config).map_err(|e| AgentSyncError::AgentMemory(e.to_string()))?;
|
||||||
let onion = OnionFile::create(&h5_path, 4096).map_err(AgentSyncError::Onion)?;
|
let onion = OnionFile::create(&h5_path, 4096).map_err(AgentSyncError::Onion)?;
|
||||||
Ok(Self { inner, onion, h5_path, auto_checkpoint: true })
|
Ok(Self {
|
||||||
|
inner,
|
||||||
|
onion,
|
||||||
|
h5_path,
|
||||||
|
auto_checkpoint: true,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Open an existing `OnionMemory` from `h5_path`.
|
/// Open an existing `OnionMemory` from `h5_path`.
|
||||||
pub fn open(h5_path: &Path) -> Result<Self, AgentSyncError> {
|
pub fn open(h5_path: &Path) -> Result<Self, AgentSyncError> {
|
||||||
let inner = HDF5Memory::open(h5_path).map_err(|e| AgentSyncError::AgentMemory(e.to_string()))?;
|
let inner =
|
||||||
|
HDF5Memory::open(h5_path).map_err(|e| AgentSyncError::AgentMemory(e.to_string()))?;
|
||||||
let onion = OnionFile::open(h5_path).map_err(AgentSyncError::Onion)?;
|
let onion = OnionFile::open(h5_path).map_err(AgentSyncError::Onion)?;
|
||||||
Ok(Self { inner, onion, h5_path: h5_path.to_owned(), auto_checkpoint: true })
|
Ok(Self {
|
||||||
|
inner,
|
||||||
|
onion,
|
||||||
|
h5_path: h5_path.to_owned(),
|
||||||
|
auto_checkpoint: true,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Commit a new onion revision of the current HDF5 state.
|
/// Commit a new onion revision of the current HDF5 state.
|
||||||
@@ -71,14 +83,20 @@ impl OnionMemory {
|
|||||||
let page_size = self.onion.page_size() as usize;
|
let page_size = self.onion.page_size() as usize;
|
||||||
|
|
||||||
// Record all pages from the HDF5 file
|
// Record all pages from the HDF5 file
|
||||||
let mut session = self.onion.begin_session(None).map_err(AgentSyncError::Onion)?;
|
let mut session = self
|
||||||
|
.onion
|
||||||
|
.begin_session(None)
|
||||||
|
.map_err(AgentSyncError::Onion)?;
|
||||||
for (chunk_idx, chunk) in h5_bytes.chunks(page_size).enumerate() {
|
for (chunk_idx, chunk) in h5_bytes.chunks(page_size).enumerate() {
|
||||||
let offset = (chunk_idx * page_size) as u64;
|
let offset = (chunk_idx * page_size) as u64;
|
||||||
let mut page = vec![0u8; page_size];
|
let mut page = vec![0u8; page_size];
|
||||||
page[..chunk.len()].copy_from_slice(chunk);
|
page[..chunk.len()].copy_from_slice(chunk);
|
||||||
session.record_page(offset, &page);
|
session.record_page(offset, &page);
|
||||||
}
|
}
|
||||||
let rev = self.onion.commit_session(session, annotation).map_err(AgentSyncError::Onion)?;
|
let rev = self
|
||||||
|
.onion
|
||||||
|
.commit_session(session, annotation)
|
||||||
|
.map_err(AgentSyncError::Onion)?;
|
||||||
self.onion.flush().map_err(AgentSyncError::Onion)?;
|
self.onion.flush().map_err(AgentSyncError::Onion)?;
|
||||||
Ok(rev)
|
Ok(rev)
|
||||||
}
|
}
|
||||||
@@ -89,12 +107,11 @@ impl OnionMemory {
|
|||||||
/// current file, opens an `HDF5Memory` from it, then removes the temp
|
/// current file, opens an `HDF5Memory` from it, then removes the temp
|
||||||
/// file. Returns the restored memory (detached from the onion sidecar).
|
/// file. Returns the restored memory (detached from the onion sidecar).
|
||||||
pub fn restore_revision(&self, revision: u64) -> Result<HDF5Memory, AgentSyncError> {
|
pub fn restore_revision(&self, revision: u64) -> Result<HDF5Memory, AgentSyncError> {
|
||||||
let (h5_bytes, _) = open_revision(&self.h5_path, revision)
|
let (h5_bytes, _) =
|
||||||
.map_err(AgentSyncError::Onion)?;
|
open_revision(&self.h5_path, revision).map_err(AgentSyncError::Onion)?;
|
||||||
let tmp = self.h5_path.with_extension(format!("h5.rev{revision}.tmp"));
|
let tmp = self.h5_path.with_extension(format!("h5.rev{revision}.tmp"));
|
||||||
std::fs::write(&tmp, &h5_bytes).map_err(AgentSyncError::Io)?;
|
std::fs::write(&tmp, &h5_bytes).map_err(AgentSyncError::Io)?;
|
||||||
let result = HDF5Memory::open(&tmp)
|
let result = HDF5Memory::open(&tmp).map_err(|e| AgentSyncError::AgentMemory(e.to_string()));
|
||||||
.map_err(|e| AgentSyncError::AgentMemory(e.to_string()));
|
|
||||||
let _ = std::fs::remove_file(&tmp);
|
let _ = std::fs::remove_file(&tmp);
|
||||||
result
|
result
|
||||||
}
|
}
|
||||||
@@ -353,7 +370,11 @@ mod tests {
|
|||||||
let head = mem.checkpoint(Some("three")).unwrap();
|
let head = mem.checkpoint(Some("three")).unwrap();
|
||||||
|
|
||||||
let restored = mem.restore_revision(head).unwrap();
|
let restored = mem.restore_revision(head).unwrap();
|
||||||
assert_eq!(restored.count(), mem.count(), "restoring HEAD gives same count");
|
assert_eq!(
|
||||||
|
restored.count(),
|
||||||
|
mem.count(),
|
||||||
|
"restoring HEAD gives same count"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Multiple checkpoints can each be individually restored.
|
/// Multiple checkpoints can each be individually restored.
|
||||||
@@ -373,8 +394,10 @@ mod tests {
|
|||||||
for i in 0..5u64 {
|
for i in 0..5u64 {
|
||||||
let r = mem.restore_revision(i).unwrap();
|
let r = mem.restore_revision(i).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
r.count() as u64, i + 1,
|
r.count() as u64,
|
||||||
"rev {i} should have {} entries", i + 1
|
i + 1,
|
||||||
|
"rev {i} should have {} entries",
|
||||||
|
i + 1
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -387,7 +410,10 @@ mod tests {
|
|||||||
mem.checkpoint(None).unwrap(); // only rev 0 exists
|
mem.checkpoint(None).unwrap(); // only rev 0 exists
|
||||||
|
|
||||||
let result = mem.restore_revision(99);
|
let result = mem.restore_revision(99);
|
||||||
assert!(result.is_err(), "restoring a non-existent revision must error");
|
assert!(
|
||||||
|
result.is_err(),
|
||||||
|
"restoring a non-existent revision must error"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `save_batch` with auto_checkpoint creates exactly one snapshot per call.
|
/// `save_batch` with auto_checkpoint creates exactly one snapshot per call.
|
||||||
@@ -417,7 +443,8 @@ mod tests {
|
|||||||
mem.checkpoint(Some("original")).unwrap();
|
mem.checkpoint(Some("original")).unwrap();
|
||||||
|
|
||||||
// Rev 1: replace with entirely different text
|
// Rev 1: replace with entirely different text
|
||||||
mem.save(sample_entry("completely-different-content")).unwrap();
|
mem.save(sample_entry("completely-different-content"))
|
||||||
|
.unwrap();
|
||||||
mem.checkpoint(Some("updated")).unwrap();
|
mem.checkpoint(Some("updated")).unwrap();
|
||||||
assert_eq!(mem.count(), 2);
|
assert_eq!(mem.count(), 2);
|
||||||
|
|
||||||
|
|||||||
@@ -96,7 +96,11 @@ pub fn tcp_scheduler(
|
|||||||
let addr = remote
|
let addr = remote
|
||||||
.parse()
|
.parse()
|
||||||
.map_err(|_| AgentSyncError::Protocol(format!("invalid address: {remote}")))?;
|
.map_err(|_| AgentSyncError::Protocol(format!("invalid address: {remote}")))?;
|
||||||
Ok(SyncScheduler::new(TcpSyncBackend::new(addr, agent_id), h5_path, selector))
|
Ok(SyncScheduler::new(
|
||||||
|
TcpSyncBackend::new(addr, agent_id),
|
||||||
|
h5_path,
|
||||||
|
selector,
|
||||||
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -134,9 +138,19 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl SyncBackend for StubBackend {
|
impl SyncBackend for StubBackend {
|
||||||
async fn push(&self, path: &Path, _sel: &SyncSelector) -> Result<SyncStats, AgentSyncError> {
|
async fn push(
|
||||||
self.push_calls.lock().unwrap().push(path.to_string_lossy().to_string());
|
&self,
|
||||||
Ok(SyncStats { revisions_transferred: 1, ..Default::default() })
|
path: &Path,
|
||||||
|
_sel: &SyncSelector,
|
||||||
|
) -> Result<SyncStats, AgentSyncError> {
|
||||||
|
self.push_calls
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.push(path.to_string_lossy().to_string());
|
||||||
|
Ok(SyncStats {
|
||||||
|
revisions_transferred: 1,
|
||||||
|
..Default::default()
|
||||||
|
})
|
||||||
}
|
}
|
||||||
async fn pull(&self, _path: &Path) -> Result<SyncStats, AgentSyncError> {
|
async fn pull(&self, _path: &Path) -> Result<SyncStats, AgentSyncError> {
|
||||||
Ok(SyncStats::default())
|
Ok(SyncStats::default())
|
||||||
@@ -189,7 +203,11 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn push_force_always_pushes() {
|
async fn push_force_always_pushes() {
|
||||||
let backend = StubBackend::new(2);
|
let backend = StubBackend::new(2);
|
||||||
let sched = SyncScheduler::new(backend.clone(), PathBuf::from("/tmp/x.h5"), SyncSelector::All);
|
let sched = SyncScheduler::new(
|
||||||
|
backend.clone(),
|
||||||
|
PathBuf::from("/tmp/x.h5"),
|
||||||
|
SyncSelector::All,
|
||||||
|
);
|
||||||
sched.mark_pushed(1); // up-to-date
|
sched.mark_pushed(1); // up-to-date
|
||||||
|
|
||||||
// force push anyway
|
// force push anyway
|
||||||
|
|||||||
@@ -48,7 +48,9 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
|||||||
|
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
// ── IBLT pre-flight (push direction) ─────────────────────────────
|
// ── IBLT pre-flight (push direction) ─────────────────────────────
|
||||||
SyncMessage::IbltRequest { sketch: client_iblt } => {
|
SyncMessage::IbltRequest {
|
||||||
|
sketch: client_iblt,
|
||||||
|
} => {
|
||||||
let server_revs: Vec<u64> = dst.list_revisions().iter().map(|s| s.revision).collect();
|
let server_revs: Vec<u64> = dst.list_revisions().iter().map(|s| s.revision).collect();
|
||||||
|
|
||||||
// Decode diff: what client has that server lacks
|
// Decode diff: what client has that server lacks
|
||||||
@@ -59,26 +61,30 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
|||||||
let seed = client_sketch.seed();
|
let seed = client_sketch.seed();
|
||||||
let srv_sketch = IbltSketch::from_keys(&server_revs, seed);
|
let srv_sketch = IbltSketch::from_keys(&server_revs, seed);
|
||||||
let server_iblt = IbltManifest {
|
let server_iblt = IbltManifest {
|
||||||
agent_id: "test-server".to_string(),
|
agent_id: "test-server".to_string(),
|
||||||
file_blake3: [0u8; 32],
|
file_blake3: [0u8; 32],
|
||||||
revision_count: server_revs.len() as u64,
|
revision_count: server_revs.len() as u64,
|
||||||
head_revision: server_revs.last().copied().unwrap_or(0),
|
head_revision: server_revs.last().copied().unwrap_or(0),
|
||||||
head_blake3: [0u8; 32],
|
head_blake3: [0u8; 32],
|
||||||
last_write: 0.0,
|
last_write: 0.0,
|
||||||
sketch_cells: srv_sketch.cell_count() as u32,
|
sketch_cells: srv_sketch.cell_count() as u32,
|
||||||
sketch: srv_sketch.to_bytes(),
|
sketch: srv_sketch.to_bytes(),
|
||||||
};
|
};
|
||||||
conn.send(&SyncMessage::IbltResponse {
|
conn.send(&SyncMessage::IbltResponse {
|
||||||
sketch: server_iblt,
|
sketch: server_iblt,
|
||||||
missing_from_remote: diff.only_in_b.clone(), // client has, server lacks
|
missing_from_remote: diff.only_in_b.clone(), // client has, server lacks
|
||||||
}).await.unwrap();
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let mut packets = Vec::new();
|
let mut packets = Vec::new();
|
||||||
loop {
|
loop {
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::LayerPacket { packet } => {
|
SyncMessage::LayerPacket { packet } => {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
conn.send(&SyncMessage::Ack { revision: rev })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
packets.push(packet);
|
packets.push(packet);
|
||||||
}
|
}
|
||||||
SyncMessage::SyncComplete { .. } => break,
|
SyncMessage::SyncComplete { .. } => break,
|
||||||
@@ -91,19 +97,30 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ── Flat manifest (pull direction) ────────────────────────────────
|
// ── Flat manifest (pull direction) ────────────────────────────────
|
||||||
SyncMessage::ManifestRequest { revision_count: remote_rev_count, .. } => {
|
SyncMessage::ManifestRequest {
|
||||||
|
revision_count: remote_rev_count,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
let local_rev_count = dst.revision_count();
|
let local_rev_count = dst.revision_count();
|
||||||
let manifest = ClawSyncManifest::from_onion("server", &dst, H5_MAGIC);
|
let manifest = ClawSyncManifest::from_onion("server", &dst, H5_MAGIC);
|
||||||
conn.send(&SyncMessage::ManifestResponse { manifest }).await.unwrap();
|
conn.send(&SyncMessage::ManifestResponse { manifest })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
if local_rev_count > remote_rev_count {
|
if local_rev_count > remote_rev_count {
|
||||||
let remote_head = if remote_rev_count == 0 { NO_PARENT } else { remote_rev_count - 1 };
|
let remote_head = if remote_rev_count == 0 {
|
||||||
|
NO_PARENT
|
||||||
|
} else {
|
||||||
|
remote_rev_count - 1
|
||||||
|
};
|
||||||
let packets = diff_revisions(&dst, remote_head).unwrap();
|
let packets = diff_revisions(&dst, remote_head).unwrap();
|
||||||
let total = packets.len() as u64;
|
let total = packets.len() as u64;
|
||||||
let mut bytes_sent = 0u64;
|
let mut bytes_sent = 0u64;
|
||||||
for packet in packets {
|
for packet in packets {
|
||||||
bytes_sent += packet.page_data_size() as u64;
|
bytes_sent += packet.page_data_size() as u64;
|
||||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
conn.send(&SyncMessage::LayerPacket { packet })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::Ack { .. } => {}
|
SyncMessage::Ack { .. } => {}
|
||||||
other => panic!("expected Ack, got {other:?}"),
|
other => panic!("expected Ack, got {other:?}"),
|
||||||
@@ -112,14 +129,18 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
|||||||
conn.send(&SyncMessage::SyncComplete {
|
conn.send(&SyncMessage::SyncComplete {
|
||||||
revisions_transferred: total,
|
revisions_transferred: total,
|
||||||
bytes_transferred: bytes_sent,
|
bytes_transferred: bytes_sent,
|
||||||
}).await.unwrap();
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
} else {
|
} else {
|
||||||
let mut packets = Vec::new();
|
let mut packets = Vec::new();
|
||||||
loop {
|
loop {
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::LayerPacket { packet } => {
|
SyncMessage::LayerPacket { packet } => {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
conn.send(&SyncMessage::Ack { revision: rev })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
packets.push(packet);
|
packets.push(packet);
|
||||||
}
|
}
|
||||||
SyncMessage::SyncComplete { .. } => break,
|
SyncMessage::SyncComplete { .. } => break,
|
||||||
@@ -241,7 +262,10 @@ async fn backend_pull_already_in_sync() {
|
|||||||
let backend = TcpSyncBackend::new(addr, "agent");
|
let backend = TcpSyncBackend::new(addr, "agent");
|
||||||
let stats = backend.pull(&cli_h5).await.unwrap();
|
let stats = backend.pull(&cli_h5).await.unwrap();
|
||||||
|
|
||||||
assert_eq!(stats.revisions_transferred, 0, "nothing to pull when in sync");
|
assert_eq!(
|
||||||
|
stats.revisions_transferred, 0,
|
||||||
|
"nothing to pull when in sync"
|
||||||
|
);
|
||||||
let _ = (cli_onion, server_task.await.unwrap());
|
let _ = (cli_onion, server_task.await.unwrap());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -34,7 +34,9 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
|||||||
|
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
// ── IBLT pre-flight (push direction) ─────────────────────────────
|
// ── IBLT pre-flight (push direction) ─────────────────────────────
|
||||||
SyncMessage::IbltRequest { sketch: client_iblt } => {
|
SyncMessage::IbltRequest {
|
||||||
|
sketch: client_iblt,
|
||||||
|
} => {
|
||||||
let server_revs: Vec<u64> = dst.list_revisions().iter().map(|s| s.revision).collect();
|
let server_revs: Vec<u64> = dst.list_revisions().iter().map(|s| s.revision).collect();
|
||||||
|
|
||||||
let diff = client_iblt.diff_against(&server_revs).unwrap();
|
let diff = client_iblt.diff_against(&server_revs).unwrap();
|
||||||
@@ -43,26 +45,30 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
|||||||
let seed = client_sketch.seed();
|
let seed = client_sketch.seed();
|
||||||
let srv_sketch = IbltSketch::from_keys(&server_revs, seed);
|
let srv_sketch = IbltSketch::from_keys(&server_revs, seed);
|
||||||
let server_iblt = IbltManifest {
|
let server_iblt = IbltManifest {
|
||||||
agent_id: "test-server".to_string(),
|
agent_id: "test-server".to_string(),
|
||||||
file_blake3: [0u8; 32],
|
file_blake3: [0u8; 32],
|
||||||
revision_count: server_revs.len() as u64,
|
revision_count: server_revs.len() as u64,
|
||||||
head_revision: server_revs.last().copied().unwrap_or(0),
|
head_revision: server_revs.last().copied().unwrap_or(0),
|
||||||
head_blake3: [0u8; 32],
|
head_blake3: [0u8; 32],
|
||||||
last_write: 0.0,
|
last_write: 0.0,
|
||||||
sketch_cells: srv_sketch.cell_count() as u32,
|
sketch_cells: srv_sketch.cell_count() as u32,
|
||||||
sketch: srv_sketch.to_bytes(),
|
sketch: srv_sketch.to_bytes(),
|
||||||
};
|
};
|
||||||
conn.send(&SyncMessage::IbltResponse {
|
conn.send(&SyncMessage::IbltResponse {
|
||||||
sketch: server_iblt,
|
sketch: server_iblt,
|
||||||
missing_from_remote: diff.only_in_b.clone(),
|
missing_from_remote: diff.only_in_b.clone(),
|
||||||
}).await.unwrap();
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let mut packets = Vec::new();
|
let mut packets = Vec::new();
|
||||||
loop {
|
loop {
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::LayerPacket { packet } => {
|
SyncMessage::LayerPacket { packet } => {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
conn.send(&SyncMessage::Ack { revision: rev })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
packets.push(packet);
|
packets.push(packet);
|
||||||
}
|
}
|
||||||
SyncMessage::SyncComplete { .. } => break,
|
SyncMessage::SyncComplete { .. } => break,
|
||||||
@@ -75,19 +81,30 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ── Flat manifest (pull direction) ────────────────────────────────
|
// ── Flat manifest (pull direction) ────────────────────────────────
|
||||||
SyncMessage::ManifestRequest { revision_count: remote_rev_count, .. } => {
|
SyncMessage::ManifestRequest {
|
||||||
|
revision_count: remote_rev_count,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
let local_rev_count = dst.revision_count();
|
let local_rev_count = dst.revision_count();
|
||||||
let manifest = ClawSyncManifest::from_onion("server", &dst, H5_MAGIC);
|
let manifest = ClawSyncManifest::from_onion("server", &dst, H5_MAGIC);
|
||||||
conn.send(&SyncMessage::ManifestResponse { manifest }).await.unwrap();
|
conn.send(&SyncMessage::ManifestResponse { manifest })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
if local_rev_count > remote_rev_count {
|
if local_rev_count > remote_rev_count {
|
||||||
let remote_head = if remote_rev_count == 0 { NO_PARENT } else { remote_rev_count - 1 };
|
let remote_head = if remote_rev_count == 0 {
|
||||||
|
NO_PARENT
|
||||||
|
} else {
|
||||||
|
remote_rev_count - 1
|
||||||
|
};
|
||||||
let packets = diff_revisions(&dst, remote_head).unwrap();
|
let packets = diff_revisions(&dst, remote_head).unwrap();
|
||||||
let total = packets.len() as u64;
|
let total = packets.len() as u64;
|
||||||
let mut bytes_sent = 0u64;
|
let mut bytes_sent = 0u64;
|
||||||
for packet in packets {
|
for packet in packets {
|
||||||
bytes_sent += packet.page_data_size() as u64;
|
bytes_sent += packet.page_data_size() as u64;
|
||||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
conn.send(&SyncMessage::LayerPacket { packet })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::Ack { .. } => {}
|
SyncMessage::Ack { .. } => {}
|
||||||
other => panic!("expected Ack, got {other:?}"),
|
other => panic!("expected Ack, got {other:?}"),
|
||||||
@@ -96,14 +113,18 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
|||||||
conn.send(&SyncMessage::SyncComplete {
|
conn.send(&SyncMessage::SyncComplete {
|
||||||
revisions_transferred: total,
|
revisions_transferred: total,
|
||||||
bytes_transferred: bytes_sent,
|
bytes_transferred: bytes_sent,
|
||||||
}).await.unwrap();
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
} else {
|
} else {
|
||||||
let mut packets = Vec::new();
|
let mut packets = Vec::new();
|
||||||
loop {
|
loop {
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::LayerPacket { packet } => {
|
SyncMessage::LayerPacket { packet } => {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
conn.send(&SyncMessage::Ack { revision: rev })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
packets.push(packet);
|
packets.push(packet);
|
||||||
}
|
}
|
||||||
SyncMessage::SyncComplete { .. } => break,
|
SyncMessage::SyncComplete { .. } => break,
|
||||||
@@ -124,8 +145,13 @@ async fn run_server(server: TcpServer, mut dst: OnionFile) -> OnionFile {
|
|||||||
|
|
||||||
/// Create a temp HDF5 file with `n` committed onion revisions.
|
/// Create a temp HDF5 file with `n` committed onion revisions.
|
||||||
fn make_onion(n: u8, dir: &TempDir) -> (PathBuf, OnionFile) {
|
fn make_onion(n: u8, dir: &TempDir) -> (PathBuf, OnionFile) {
|
||||||
let h5 = dir.path().join(format!("data_{n}_{}.h5", std::time::SystemTime::now()
|
let h5 = dir.path().join(format!(
|
||||||
.duration_since(std::time::UNIX_EPOCH).unwrap().subsec_nanos()));
|
"data_{n}_{}.h5",
|
||||||
|
std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.unwrap()
|
||||||
|
.subsec_nanos()
|
||||||
|
));
|
||||||
std::fs::write(&h5, H5_MAGIC).unwrap();
|
std::fs::write(&h5, H5_MAGIC).unwrap();
|
||||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
@@ -288,7 +314,8 @@ async fn tcp_scheduler_constructor_and_push() {
|
|||||||
src_h5.clone(),
|
src_h5.clone(),
|
||||||
"test-agent",
|
"test-agent",
|
||||||
SyncSelector::All,
|
SyncSelector::All,
|
||||||
).unwrap();
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let stats = sched.push_force().await.unwrap();
|
let stats = sched.push_force().await.unwrap();
|
||||||
assert_eq!(stats.revisions_transferred, 3);
|
assert_eq!(stats.revisions_transferred, 3);
|
||||||
|
|||||||
+259
-148
@@ -26,21 +26,21 @@ use clawhdf5_onion::api::{
|
|||||||
create_branch, delete_branch, list_branches, list_revisions, rename_branch, rollback,
|
create_branch, delete_branch, list_branches, list_revisions, rename_branch, rollback,
|
||||||
};
|
};
|
||||||
use clawhdf5_onion::branch::MergeStrategy;
|
use clawhdf5_onion::branch::MergeStrategy;
|
||||||
use clawhdf5_onion::gc::GcPolicy;
|
|
||||||
use clawhdf5_onion::format::NO_PARENT;
|
use clawhdf5_onion::format::NO_PARENT;
|
||||||
|
use clawhdf5_onion::gc::GcPolicy;
|
||||||
use clawhdf5_onion::writer::OnionFile;
|
use clawhdf5_onion::writer::OnionFile;
|
||||||
use clawsync_onion::differ::{diff_revisions, packets_for_revisions};
|
use clawsync_onion::differ::{diff_revisions, packets_for_revisions};
|
||||||
use clawsync_onion::iblt::{IbltSketch, IBLT_SYNC_SEED};
|
use clawsync_onion::iblt::{IBLT_SYNC_SEED, IbltSketch};
|
||||||
use clawsync_onion::manifest::{ClawSyncManifest, IbltManifest};
|
use clawsync_onion::manifest::{ClawSyncManifest, IbltManifest};
|
||||||
use clawsync_onion::merger::merge_packets;
|
use clawsync_onion::merger::merge_packets;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use clawsync_fs::{FsSyncClient, FsSyncServer};
|
||||||
|
use clawsync_hdf5::{DatasetManifest, ReceivedDataset, apply_received_payloads};
|
||||||
use clawsync_transport::peer::{PipeWriteHalf, SyncPeer};
|
use clawsync_transport::peer::{PipeWriteHalf, SyncPeer};
|
||||||
use clawsync_transport::protocol::SyncMessage;
|
use clawsync_transport::protocol::SyncMessage;
|
||||||
use clawsync_transport::quic::{QuicConfig, QuicServer, quic_connect};
|
use clawsync_transport::quic::{QuicConfig, QuicServer, quic_connect};
|
||||||
use clawsync_transport::tcp::{TcpConnection, TcpServer};
|
use clawsync_transport::tcp::{TcpConnection, TcpServer};
|
||||||
use clawsync_fs::{FsSyncClient, FsSyncServer};
|
|
||||||
use clawsync_hdf5::{DatasetManifest, ReceivedDataset, apply_received_payloads};
|
|
||||||
|
|
||||||
/// Maximum number of `LayerPacket`s in flight before the sender blocks waiting
|
/// Maximum number of `LayerPacket`s in flight before the sender blocks waiting
|
||||||
/// for acks. A window of 16 reduces WAN round-trip overhead by up to 16×
|
/// for acks. A window of 16 reduces WAN round-trip overhead by up to 16×
|
||||||
@@ -326,7 +326,9 @@ enum BranchAction {
|
|||||||
|
|
||||||
/// Parse `host:port/path` into `(SocketAddr, PathBuf)`.
|
/// Parse `host:port/path` into `(SocketAddr, PathBuf)`.
|
||||||
fn parse_remote(remote: &str) -> Result<(SocketAddr, PathBuf)> {
|
fn parse_remote(remote: &str) -> Result<(SocketAddr, PathBuf)> {
|
||||||
let slash = remote.find('/').context("remote must be 'host:port/path'")?;
|
let slash = remote
|
||||||
|
.find('/')
|
||||||
|
.context("remote must be 'host:port/path'")?;
|
||||||
let addr_str = &remote[..slash];
|
let addr_str = &remote[..slash];
|
||||||
let path_str = &remote[slash + 1..];
|
let path_str = &remote[slash + 1..];
|
||||||
let addr: SocketAddr = addr_str
|
let addr: SocketAddr = addr_str
|
||||||
@@ -356,7 +358,9 @@ async fn cmd_list_revisions(h5_path: PathBuf, verbose: bool) -> Result<()> {
|
|||||||
print!("{:>6} {:>4} {:<30} {}", r.revision, r.branch_id, ann, ts);
|
print!("{:>6} {:>4} {:<30} {}", r.revision, r.branch_id, ann, ts);
|
||||||
if verbose {
|
if verbose {
|
||||||
print!(" blake3={}", &r.blake3_hex[..12]);
|
print!(" blake3={}", &r.blake3_hex[..12]);
|
||||||
if r.is_snapshot { print!(" [snapshot]"); }
|
if r.is_snapshot {
|
||||||
|
print!(" [snapshot]");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
println!();
|
println!();
|
||||||
}
|
}
|
||||||
@@ -365,7 +369,11 @@ async fn cmd_list_revisions(h5_path: PathBuf, verbose: bool) -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn cmd_rollback(h5_path: PathBuf, revision: u64) -> Result<()> {
|
async fn cmd_rollback(h5_path: PathBuf, revision: u64) -> Result<()> {
|
||||||
println!("Rolling back {} to revision {}...", h5_path.display(), revision);
|
println!(
|
||||||
|
"Rolling back {} to revision {}...",
|
||||||
|
h5_path.display(),
|
||||||
|
revision
|
||||||
|
);
|
||||||
rollback(&h5_path, revision)
|
rollback(&h5_path, revision)
|
||||||
.with_context(|| format!("rollback failed for {}", h5_path.display()))?;
|
.with_context(|| format!("rollback failed for {}", h5_path.display()))?;
|
||||||
println!("Done.");
|
println!("Done.");
|
||||||
@@ -389,13 +397,13 @@ async fn cmd_push(
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Load local onion — O(sidecar size), does NOT read the large h5 base file.
|
// Load local onion — O(sidecar size), does NOT read the large h5 base file.
|
||||||
let local_onion = OnionFile::open(&local)
|
let local_onion =
|
||||||
.with_context(|| format!("cannot open {}", local.display()))?;
|
OnionFile::open(&local).with_context(|| format!("cannot open {}", local.display()))?;
|
||||||
|
|
||||||
// Connect to remote — TCP or QUIC depending on flag.
|
// Connect to remote — TCP or QUIC depending on flag.
|
||||||
let mut peer: SyncPeer = if quic {
|
let mut peer: SyncPeer = if quic {
|
||||||
let cfg = QuicConfig::self_signed()
|
let cfg =
|
||||||
.with_context(|| "failed to build QUIC self-signed config")?;
|
QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
|
||||||
let conn = quic_connect(addr, "localhost", cfg)
|
let conn = quic_connect(addr, "localhost", cfg)
|
||||||
.await
|
.await
|
||||||
.with_context(|| format!("QUIC connect to {addr} failed"))?;
|
.with_context(|| format!("QUIC connect to {addr} failed"))?;
|
||||||
@@ -418,20 +426,26 @@ async fn cmd_push(
|
|||||||
.collect();
|
.collect();
|
||||||
let sketch = IbltSketch::from_keys(&local_rev_numbers, IBLT_SYNC_SEED);
|
let sketch = IbltSketch::from_keys(&local_rev_numbers, IBLT_SYNC_SEED);
|
||||||
let iblt_manifest = IbltManifest {
|
let iblt_manifest = IbltManifest {
|
||||||
agent_id: "push-client".to_string(),
|
agent_id: "push-client".to_string(),
|
||||||
file_blake3: [0u8; 32], // filled lazily; not needed for IBLT pre-flight
|
file_blake3: [0u8; 32], // filled lazily; not needed for IBLT pre-flight
|
||||||
revision_count: local_rev_numbers.len() as u64,
|
revision_count: local_rev_numbers.len() as u64,
|
||||||
head_revision: local_rev_numbers.last().copied().unwrap_or(0),
|
head_revision: local_rev_numbers.last().copied().unwrap_or(0),
|
||||||
head_blake3: [0u8; 32],
|
head_blake3: [0u8; 32],
|
||||||
last_write: 0.0,
|
last_write: 0.0,
|
||||||
sketch_cells: sketch.cell_count() as u32,
|
sketch_cells: sketch.cell_count() as u32,
|
||||||
sketch: sketch.to_bytes(),
|
sketch: sketch.to_bytes(),
|
||||||
};
|
};
|
||||||
peer.send(&SyncMessage::IbltRequest { sketch: iblt_manifest }).await?;
|
peer.send(&SyncMessage::IbltRequest {
|
||||||
|
sketch: iblt_manifest,
|
||||||
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
// Receive the server's IBLT response: which revisions we should push.
|
// Receive the server's IBLT response: which revisions we should push.
|
||||||
let missing_from_remote = match peer.recv().await? {
|
let missing_from_remote = match peer.recv().await? {
|
||||||
SyncMessage::IbltResponse { sketch: server_iblt, missing_from_remote } => {
|
SyncMessage::IbltResponse {
|
||||||
|
sketch: server_iblt,
|
||||||
|
missing_from_remote,
|
||||||
|
} => {
|
||||||
println!(
|
println!(
|
||||||
"Remote has {} revision(s) (IBLT). Pushing {} revision(s).",
|
"Remote has {} revision(s) (IBLT). Pushing {} revision(s).",
|
||||||
server_iblt.revision_count,
|
server_iblt.revision_count,
|
||||||
@@ -476,20 +490,23 @@ async fn cmd_push(
|
|||||||
|
|
||||||
let sem_writer = semaphore.clone();
|
let sem_writer = semaphore.clone();
|
||||||
// Return write_half from the task so we can send SyncComplete after the loop.
|
// Return write_half from the task so we can send SyncComplete after the loop.
|
||||||
let writer_task: tokio::task::JoinHandle<Result<PipeWriteHalf>> =
|
let writer_task: tokio::task::JoinHandle<Result<PipeWriteHalf>> = tokio::spawn(async move {
|
||||||
tokio::spawn(async move {
|
for packet in packets {
|
||||||
for packet in packets {
|
let rev = packet.revision;
|
||||||
let rev = packet.revision;
|
let pkt_bytes = packet.page_data_size() as u64;
|
||||||
let pkt_bytes = packet.page_data_size() as u64;
|
sem_writer
|
||||||
sem_writer.acquire().await.expect("semaphore closed").forget();
|
.acquire()
|
||||||
meta_tx.send((rev, pkt_bytes)).expect("meta channel closed");
|
.await
|
||||||
write_half
|
.expect("semaphore closed")
|
||||||
.send(&SyncMessage::LayerPacket { packet })
|
.forget();
|
||||||
.await
|
meta_tx.send((rev, pkt_bytes)).expect("meta channel closed");
|
||||||
.map_err(|e| anyhow::anyhow!("send error: {e}"))?;
|
write_half
|
||||||
}
|
.send(&SyncMessage::LayerPacket { packet })
|
||||||
Ok(write_half)
|
.await
|
||||||
});
|
.map_err(|e| anyhow::anyhow!("send error: {e}"))?;
|
||||||
|
}
|
||||||
|
Ok(write_half)
|
||||||
|
});
|
||||||
|
|
||||||
let mut sent = 0u64;
|
let mut sent = 0u64;
|
||||||
let mut bytes_sent = 0u64;
|
let mut bytes_sent = 0u64;
|
||||||
@@ -516,7 +533,7 @@ async fn cmd_push(
|
|||||||
write_half
|
write_half
|
||||||
.send(&SyncMessage::SyncComplete {
|
.send(&SyncMessage::SyncComplete {
|
||||||
revisions_transferred: sent,
|
revisions_transferred: sent,
|
||||||
bytes_transferred: bytes_sent,
|
bytes_transferred: bytes_sent,
|
||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
write_half.shutdown().await?;
|
write_half.shutdown().await?;
|
||||||
@@ -548,8 +565,7 @@ async fn cmd_pull(
|
|||||||
.unwrap_or(0)
|
.unwrap_or(0)
|
||||||
} else {
|
} else {
|
||||||
// Create empty base file
|
// Create empty base file
|
||||||
std::fs::write(&local, b"\x89HDF\r\n\x1a\n")
|
std::fs::write(&local, b"\x89HDF\r\n\x1a\n").context("cannot create local h5")?;
|
||||||
.context("cannot create local h5")?;
|
|
||||||
0
|
0
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -557,8 +573,8 @@ async fn cmd_pull(
|
|||||||
|
|
||||||
// Connect — TCP or QUIC.
|
// Connect — TCP or QUIC.
|
||||||
let mut peer: SyncPeer = if quic {
|
let mut peer: SyncPeer = if quic {
|
||||||
let cfg = QuicConfig::self_signed()
|
let cfg =
|
||||||
.with_context(|| "failed to build QUIC self-signed config")?;
|
QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
|
||||||
SyncPeer::Quic(Arc::new(
|
SyncPeer::Quic(Arc::new(
|
||||||
quic_connect(addr, "localhost", cfg)
|
quic_connect(addr, "localhost", cfg)
|
||||||
.await
|
.await
|
||||||
@@ -623,7 +639,10 @@ async fn cmd_pull(
|
|||||||
peer.send(&SyncMessage::Ack { revision: rev }).await?;
|
peer.send(&SyncMessage::Ack { revision: rev }).await?;
|
||||||
packets.push(packet);
|
packets.push(packet);
|
||||||
}
|
}
|
||||||
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => {
|
SyncMessage::SyncComplete {
|
||||||
|
revisions_transferred,
|
||||||
|
bytes_transferred,
|
||||||
|
} => {
|
||||||
println!(
|
println!(
|
||||||
"Received {} revision(s), {} bytes.",
|
"Received {} revision(s), {} bytes.",
|
||||||
revisions_transferred, bytes_transferred
|
revisions_transferred, bytes_transferred
|
||||||
@@ -637,8 +656,7 @@ async fn cmd_pull(
|
|||||||
|
|
||||||
// Optionally filter to a specific branch before applying.
|
// Optionally filter to a specific branch before applying.
|
||||||
let packets_to_apply = if let Some(ref branch_name) = branch {
|
let packets_to_apply = if let Some(ref branch_name) = branch {
|
||||||
let selector =
|
let selector = clawsync_onion::selector::SyncSelector::Branch(branch_name.clone());
|
||||||
clawsync_onion::selector::SyncSelector::Branch(branch_name.clone());
|
|
||||||
clawsync_onion::selector::filter_packets(packets, &selector, &local_onion)
|
clawsync_onion::selector::filter_packets(packets, &selector, &local_onion)
|
||||||
.context("branch filter failed")?
|
.context("branch filter failed")?
|
||||||
} else {
|
} else {
|
||||||
@@ -646,8 +664,7 @@ async fn cmd_pull(
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Apply packets
|
// Apply packets
|
||||||
let stats = merge_packets(&mut local_onion, packets_to_apply, true)
|
let stats = merge_packets(&mut local_onion, packets_to_apply, true).context("merge failed")?;
|
||||||
.context("merge failed")?;
|
|
||||||
if let Some(b) = &branch {
|
if let Some(b) = &branch {
|
||||||
println!(
|
println!(
|
||||||
"Pull complete (branch {b:?}): {} revision(s) merged, {} skipped.",
|
"Pull complete (branch {b:?}): {} revision(s) merged, {} skipped.",
|
||||||
@@ -668,8 +685,8 @@ async fn cmd_pull(
|
|||||||
/// Serve a local HDF5 file, handling one client connection at a time.
|
/// Serve a local HDF5 file, handling one client connection at a time.
|
||||||
async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()> {
|
async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()> {
|
||||||
if quic {
|
if quic {
|
||||||
let cfg = QuicConfig::self_signed()
|
let cfg =
|
||||||
.with_context(|| "failed to build QUIC self-signed config")?;
|
QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
|
||||||
let server = QuicServer::bind(bind, cfg)
|
let server = QuicServer::bind(bind, cfg)
|
||||||
.await
|
.await
|
||||||
.with_context(|| format!("cannot bind QUIC to {bind}"))?;
|
.with_context(|| format!("cannot bind QUIC to {bind}"))?;
|
||||||
@@ -685,7 +702,11 @@ async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()>
|
|||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
if let Err(e) = handle_client(&mut peer, &h5).await {
|
if let Err(e) = handle_client(&mut peer, &h5).await {
|
||||||
eprintln!("QUIC client error: {e}");
|
eprintln!("QUIC client error: {e}");
|
||||||
let _ = peer.send(&SyncMessage::Error { message: e.to_string() }).await;
|
let _ = peer
|
||||||
|
.send(&SyncMessage::Error {
|
||||||
|
message: e.to_string(),
|
||||||
|
})
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -693,7 +714,11 @@ async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()>
|
|||||||
let server = TcpServer::bind(bind)
|
let server = TcpServer::bind(bind)
|
||||||
.await
|
.await
|
||||||
.with_context(|| format!("cannot bind to {bind}"))?;
|
.with_context(|| format!("cannot bind to {bind}"))?;
|
||||||
println!("Serving {} on {} (Ctrl-C to stop)", h5_path.display(), server.local_addr);
|
println!(
|
||||||
|
"Serving {} on {} (Ctrl-C to stop)",
|
||||||
|
h5_path.display(),
|
||||||
|
server.local_addr
|
||||||
|
);
|
||||||
loop {
|
loop {
|
||||||
let (conn, peer_addr) = server.accept().await?;
|
let (conn, peer_addr) = server.accept().await?;
|
||||||
println!("Connection from {peer_addr}");
|
println!("Connection from {peer_addr}");
|
||||||
@@ -702,7 +727,11 @@ async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()>
|
|||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
if let Err(e) = handle_client(&mut peer, &h5).await {
|
if let Err(e) = handle_client(&mut peer, &h5).await {
|
||||||
eprintln!("Client {peer_addr} error: {e}");
|
eprintln!("Client {peer_addr} error: {e}");
|
||||||
let _ = peer.send(&SyncMessage::Error { message: e.to_string() }).await;
|
let _ = peer
|
||||||
|
.send(&SyncMessage::Error {
|
||||||
|
message: e.to_string(),
|
||||||
|
})
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -722,9 +751,11 @@ async fn cmd_serve(h5_path: PathBuf, bind: SocketAddr, quic: bool) -> Result<()>
|
|||||||
async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
|
async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
|
||||||
match conn.recv().await? {
|
match conn.recv().await? {
|
||||||
// ── IBLT pre-flight (push direction only) ─────────────────────────
|
// ── IBLT pre-flight (push direction only) ─────────────────────────
|
||||||
SyncMessage::IbltRequest { sketch: client_iblt } => {
|
SyncMessage::IbltRequest {
|
||||||
let mut local_onion = OnionFile::open(h5_path)
|
sketch: client_iblt,
|
||||||
.or_else(|_| OnionFile::create_auto(h5_path))?;
|
} => {
|
||||||
|
let mut local_onion =
|
||||||
|
OnionFile::open(h5_path).or_else(|_| OnionFile::create_auto(h5_path))?;
|
||||||
|
|
||||||
let local_revisions: Vec<u64> = local_onion
|
let local_revisions: Vec<u64> = local_onion
|
||||||
.list_revisions()
|
.list_revisions()
|
||||||
@@ -749,25 +780,26 @@ async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
|
|||||||
// not validated during IBLT pre-flight, so we use zeros to avoid
|
// not validated during IBLT pre-flight, so we use zeros to avoid
|
||||||
// an O(file_size) read of the h5 base.
|
// an O(file_size) read of the h5 base.
|
||||||
let server_iblt = IbltManifest {
|
let server_iblt = IbltManifest {
|
||||||
agent_id: "server".to_string(),
|
agent_id: "server".to_string(),
|
||||||
file_blake3: [0u8; 32],
|
file_blake3: [0u8; 32],
|
||||||
revision_count: local_revisions.len() as u64,
|
revision_count: local_revisions.len() as u64,
|
||||||
head_revision: local_revisions.last().copied().unwrap_or(0),
|
head_revision: local_revisions.last().copied().unwrap_or(0),
|
||||||
head_blake3: [0u8; 32],
|
head_blake3: [0u8; 32],
|
||||||
last_write: 0.0,
|
last_write: 0.0,
|
||||||
sketch_cells: server_sketch.cell_count() as u32,
|
sketch_cells: server_sketch.cell_count() as u32,
|
||||||
sketch: server_sketch.to_bytes(),
|
sketch: server_sketch.to_bytes(),
|
||||||
};
|
};
|
||||||
|
|
||||||
conn.send(&SyncMessage::IbltResponse {
|
conn.send(&SyncMessage::IbltResponse {
|
||||||
sketch: server_iblt,
|
sketch: server_iblt,
|
||||||
missing_from_remote: diff.only_in_b.clone(), // client has, server lacks
|
missing_from_remote: diff.only_in_b.clone(), // client has, server lacks
|
||||||
}).await?;
|
})
|
||||||
|
.await?;
|
||||||
|
|
||||||
println!(
|
println!(
|
||||||
" IBLT: client missing {}, server missing {} revision(s).",
|
" IBLT: client missing {}, server missing {} revision(s).",
|
||||||
diff.only_in_a.len(), // server has, client lacks
|
diff.only_in_a.len(), // server has, client lacks
|
||||||
diff.only_in_b.len(), // client has, server lacks
|
diff.only_in_b.len(), // client has, server lacks
|
||||||
);
|
);
|
||||||
|
|
||||||
// Receive exactly the packets the client is pushing.
|
// Receive exactly the packets the client is pushing.
|
||||||
@@ -779,7 +811,10 @@ async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
|
|||||||
conn.send(&SyncMessage::Ack { revision: rev }).await?;
|
conn.send(&SyncMessage::Ack { revision: rev }).await?;
|
||||||
packets.push(packet);
|
packets.push(packet);
|
||||||
}
|
}
|
||||||
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => {
|
SyncMessage::SyncComplete {
|
||||||
|
revisions_transferred,
|
||||||
|
bytes_transferred,
|
||||||
|
} => {
|
||||||
println!(
|
println!(
|
||||||
" Client sent {revisions_transferred} revision(s), \
|
" Client sent {revisions_transferred} revision(s), \
|
||||||
{bytes_transferred} bytes (IBLT)."
|
{bytes_transferred} bytes (IBLT)."
|
||||||
@@ -800,14 +835,18 @@ async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ── Flat manifest (pull direction or legacy push) ──────────────────
|
// ── Flat manifest (pull direction or legacy push) ──────────────────
|
||||||
SyncMessage::ManifestRequest { revision_count: remote_rev_count, .. } => {
|
SyncMessage::ManifestRequest {
|
||||||
let mut local_onion = OnionFile::open(h5_path)
|
revision_count: remote_rev_count,
|
||||||
.or_else(|_| OnionFile::create_auto(h5_path))?;
|
..
|
||||||
|
} => {
|
||||||
|
let mut local_onion =
|
||||||
|
OnionFile::open(h5_path).or_else(|_| OnionFile::create_auto(h5_path))?;
|
||||||
let h5_base = std::fs::read(h5_path)?;
|
let h5_base = std::fs::read(h5_path)?;
|
||||||
let manifest = ClawSyncManifest::from_onion("server", &local_onion, &h5_base);
|
let manifest = ClawSyncManifest::from_onion("server", &local_onion, &h5_base);
|
||||||
let local_rev_count = manifest.revision_count;
|
let local_rev_count = manifest.revision_count;
|
||||||
|
|
||||||
conn.send(&SyncMessage::ManifestResponse { manifest }).await?;
|
conn.send(&SyncMessage::ManifestResponse { manifest })
|
||||||
|
.await?;
|
||||||
|
|
||||||
if local_rev_count > remote_rev_count {
|
if local_rev_count > remote_rev_count {
|
||||||
// ── Server pushes (client is behind) ──────────────────────
|
// ── Server pushes (client is behind) ──────────────────────
|
||||||
@@ -816,8 +855,8 @@ async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
|
|||||||
} else {
|
} else {
|
||||||
remote_rev_count - 1
|
remote_rev_count - 1
|
||||||
};
|
};
|
||||||
let packets = diff_revisions(&local_onion, remote_head)
|
let packets =
|
||||||
.context("diff_revisions failed")?;
|
diff_revisions(&local_onion, remote_head).context("diff_revisions failed")?;
|
||||||
let total = packets.len() as u64;
|
let total = packets.len() as u64;
|
||||||
let mut bytes_sent = 0u64;
|
let mut bytes_sent = 0u64;
|
||||||
|
|
||||||
@@ -846,7 +885,10 @@ async fn handle_client(conn: &mut SyncPeer, h5_path: &PathBuf) -> Result<()> {
|
|||||||
conn.send(&SyncMessage::Ack { revision: rev }).await?;
|
conn.send(&SyncMessage::Ack { revision: rev }).await?;
|
||||||
packets.push(packet);
|
packets.push(packet);
|
||||||
}
|
}
|
||||||
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => {
|
SyncMessage::SyncComplete {
|
||||||
|
revisions_transferred,
|
||||||
|
bytes_transferred,
|
||||||
|
} => {
|
||||||
println!(
|
println!(
|
||||||
" Client sent {revisions_transferred} revision(s), \
|
" Client sent {revisions_transferred} revision(s), \
|
||||||
{bytes_transferred} bytes."
|
{bytes_transferred} bytes."
|
||||||
@@ -880,11 +922,7 @@ async fn cmd_export_revision(h5_path: PathBuf, revision: u64, dest: PathBuf) ->
|
|||||||
.with_context(|| format!("cannot open {}", h5_path.display()))?;
|
.with_context(|| format!("cannot open {}", h5_path.display()))?;
|
||||||
vf.export_revision(revision, &dest)
|
vf.export_revision(revision, &dest)
|
||||||
.with_context(|| format!("export of revision {revision} failed"))?;
|
.with_context(|| format!("export of revision {revision} failed"))?;
|
||||||
println!(
|
println!("Exported revision {} → {}", revision, dest.display());
|
||||||
"Exported revision {} → {}",
|
|
||||||
revision,
|
|
||||||
dest.display()
|
|
||||||
);
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -921,8 +959,8 @@ async fn cmd_gc(
|
|||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut onion = OnionFile::open(&h5_path)
|
let mut onion =
|
||||||
.with_context(|| format!("cannot open {}", h5_path.display()))?;
|
OnionFile::open(&h5_path).with_context(|| format!("cannot open {}", h5_path.display()))?;
|
||||||
let stats = onion.gc(policy).context("GC failed")?;
|
let stats = onion.gc(policy).context("GC failed")?;
|
||||||
onion.flush().context("flush after GC failed")?;
|
onion.flush().context("flush after GC failed")?;
|
||||||
|
|
||||||
@@ -935,10 +973,10 @@ async fn cmd_gc(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn cmd_snapshot(h5_path: PathBuf, message: Option<String>) -> Result<()> {
|
async fn cmd_snapshot(h5_path: PathBuf, message: Option<String>) -> Result<()> {
|
||||||
let mut onion = OnionFile::open(&h5_path)
|
let mut onion =
|
||||||
.with_context(|| format!("cannot open {}", h5_path.display()))?;
|
OnionFile::open(&h5_path).with_context(|| format!("cannot open {}", h5_path.display()))?;
|
||||||
let h5_base = std::fs::read(&h5_path)
|
let h5_base =
|
||||||
.with_context(|| format!("cannot read {}", h5_path.display()))?;
|
std::fs::read(&h5_path).with_context(|| format!("cannot read {}", h5_path.display()))?;
|
||||||
|
|
||||||
let annotation = message.as_deref().or(Some("manual snapshot"));
|
let annotation = message.as_deref().or(Some("manual snapshot"));
|
||||||
let snap_rev = onion
|
let snap_rev = onion
|
||||||
@@ -960,7 +998,7 @@ async fn cmd_branch_merge(
|
|||||||
|
|
||||||
let merge_strategy = match strategy.as_str() {
|
let merge_strategy = match strategy.as_str() {
|
||||||
"latest-wins" => MergeStrategy::LatestWins,
|
"latest-wins" => MergeStrategy::LatestWins,
|
||||||
"three-way" => MergeStrategy::ThreeWay,
|
"three-way" => MergeStrategy::ThreeWay,
|
||||||
other => anyhow::bail!("unknown strategy {other:?}; use latest-wins or three-way"),
|
other => anyhow::bail!("unknown strategy {other:?}; use latest-wins or three-way"),
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -980,7 +1018,11 @@ async fn cmd_branch_merge(
|
|||||||
|
|
||||||
async fn cmd_branch(action: BranchAction) -> Result<()> {
|
async fn cmd_branch(action: BranchAction) -> Result<()> {
|
||||||
match action {
|
match action {
|
||||||
BranchAction::Create { h5_path, name, from } => {
|
BranchAction::Create {
|
||||||
|
h5_path,
|
||||||
|
name,
|
||||||
|
from,
|
||||||
|
} => {
|
||||||
let id = create_branch(&h5_path, &name, &from)
|
let id = create_branch(&h5_path, &name, &from)
|
||||||
.with_context(|| format!("failed to create branch {name:?}"))?;
|
.with_context(|| format!("failed to create branch {name:?}"))?;
|
||||||
println!("Branch {name:?} created (id={id}), forked from {from:?}.");
|
println!("Branch {name:?} created (id={id}), forked from {from:?}.");
|
||||||
@@ -1023,7 +1065,12 @@ async fn cmd_branch(action: BranchAction) -> Result<()> {
|
|||||||
println!("Branch {from:?} renamed to {to:?}.");
|
println!("Branch {from:?} renamed to {to:?}.");
|
||||||
}
|
}
|
||||||
|
|
||||||
BranchAction::Merge { h5_path, source, into, strategy } => {
|
BranchAction::Merge {
|
||||||
|
h5_path,
|
||||||
|
source,
|
||||||
|
into,
|
||||||
|
strategy,
|
||||||
|
} => {
|
||||||
cmd_branch_merge(h5_path, source, into, strategy).await?;
|
cmd_branch_merge(h5_path, source, into, strategy).await?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1037,10 +1084,7 @@ async fn cmd_branch(action: BranchAction) -> Result<()> {
|
|||||||
fn build_glob_set(patterns: &[String]) -> Result<globset::GlobSet> {
|
fn build_glob_set(patterns: &[String]) -> Result<globset::GlobSet> {
|
||||||
let mut builder = globset::GlobSetBuilder::new();
|
let mut builder = globset::GlobSetBuilder::new();
|
||||||
for p in patterns {
|
for p in patterns {
|
||||||
builder.add(
|
builder.add(globset::Glob::new(p).with_context(|| format!("invalid glob pattern: {p}"))?);
|
||||||
globset::Glob::new(p)
|
|
||||||
.with_context(|| format!("invalid glob pattern: {p}"))?,
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
builder.build().context("failed to build glob set")
|
builder.build().context("failed to build glob set")
|
||||||
}
|
}
|
||||||
@@ -1063,8 +1107,8 @@ async fn cmd_sync(
|
|||||||
);
|
);
|
||||||
|
|
||||||
let peer: SyncPeer = if quic {
|
let peer: SyncPeer = if quic {
|
||||||
let cfg = QuicConfig::self_signed()
|
let cfg =
|
||||||
.with_context(|| "failed to build QUIC self-signed config")?;
|
QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
|
||||||
SyncPeer::Quic(Arc::new(
|
SyncPeer::Quic(Arc::new(
|
||||||
quic_connect(addr, "localhost", cfg)
|
quic_connect(addr, "localhost", cfg)
|
||||||
.await
|
.await
|
||||||
@@ -1099,8 +1143,8 @@ async fn cmd_serve_fs(
|
|||||||
let excludes = build_glob_set(&exclude)?;
|
let excludes = build_glob_set(&exclude)?;
|
||||||
|
|
||||||
if quic {
|
if quic {
|
||||||
let cfg = QuicConfig::self_signed()
|
let cfg =
|
||||||
.with_context(|| "failed to build QUIC self-signed config")?;
|
QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
|
||||||
let server = clawsync_transport::quic::QuicServer::bind(bind, cfg)
|
let server = clawsync_transport::quic::QuicServer::bind(bind, cfg)
|
||||||
.await
|
.await
|
||||||
.with_context(|| format!("cannot bind QUIC to {bind}"))?;
|
.with_context(|| format!("cannot bind QUIC to {bind}"))?;
|
||||||
@@ -1148,12 +1192,7 @@ async fn cmd_serve_fs(
|
|||||||
// HDF5 dataset-granular sync commands
|
// HDF5 dataset-granular sync commands
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
async fn cmd_hdf5_sync(
|
async fn cmd_hdf5_sync(local: PathBuf, remote: String, delete: bool, quic: bool) -> Result<()> {
|
||||||
local: PathBuf,
|
|
||||||
remote: String,
|
|
||||||
delete: bool,
|
|
||||||
quic: bool,
|
|
||||||
) -> Result<()> {
|
|
||||||
let (addr, remote_path) = parse_remote(&remote)?;
|
let (addr, remote_path) = parse_remote(&remote)?;
|
||||||
let remote_path_str = remote_path.to_string_lossy().into_owned();
|
let remote_path_str = remote_path.to_string_lossy().into_owned();
|
||||||
|
|
||||||
@@ -1185,18 +1224,18 @@ async fn cmd_hdf5_sync(
|
|||||||
.datasets
|
.datasets
|
||||||
.values()
|
.values()
|
||||||
.map(|e| clawsync_transport::protocol::Hdf5DatasetEntry {
|
.map(|e| clawsync_transport::protocol::Hdf5DatasetEntry {
|
||||||
path: e.path.clone(),
|
path: e.path.clone(),
|
||||||
blake3: e.blake3,
|
blake3: e.blake3,
|
||||||
byte_len: e.byte_len,
|
byte_len: e.byte_len,
|
||||||
shape: e.shape.clone(),
|
shape: e.shape.clone(),
|
||||||
dtype: e.dtype.clone(),
|
dtype: e.dtype.clone(),
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
// Connect.
|
// Connect.
|
||||||
let mut peer: SyncPeer = if quic {
|
let mut peer: SyncPeer = if quic {
|
||||||
let cfg = QuicConfig::self_signed()
|
let cfg =
|
||||||
.with_context(|| "failed to build QUIC self-signed config")?;
|
QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
|
||||||
SyncPeer::Quic(Arc::new(
|
SyncPeer::Quic(Arc::new(
|
||||||
quic_connect(addr, "localhost", cfg)
|
quic_connect(addr, "localhost", cfg)
|
||||||
.await
|
.await
|
||||||
@@ -1221,7 +1260,10 @@ async fn cmd_hdf5_sync(
|
|||||||
|
|
||||||
// Receive which datasets the server needs.
|
// Receive which datasets the server needs.
|
||||||
let (datasets_needed, _to_delete) = match peer.recv().await? {
|
let (datasets_needed, _to_delete) = match peer.recv().await? {
|
||||||
SyncMessage::Hdf5ManifestResponse { datasets_needed, to_delete } => {
|
SyncMessage::Hdf5ManifestResponse {
|
||||||
|
datasets_needed,
|
||||||
|
to_delete,
|
||||||
|
} => {
|
||||||
println!(
|
println!(
|
||||||
"Server needs {} dataset(s), {} to delete.",
|
"Server needs {} dataset(s), {} to delete.",
|
||||||
datasets_needed.len(),
|
datasets_needed.len(),
|
||||||
@@ -1237,7 +1279,10 @@ async fn cmd_hdf5_sync(
|
|||||||
// Nothing to send; wait for completion.
|
// Nothing to send; wait for completion.
|
||||||
match peer.recv().await? {
|
match peer.recv().await? {
|
||||||
SyncMessage::Hdf5SyncComplete {
|
SyncMessage::Hdf5SyncComplete {
|
||||||
datasets_added, datasets_modified, datasets_removed, bytes_transferred
|
datasets_added,
|
||||||
|
datasets_modified,
|
||||||
|
datasets_removed,
|
||||||
|
bytes_transferred,
|
||||||
} => {
|
} => {
|
||||||
println!(
|
println!(
|
||||||
"HDF5-sync complete: +{datasets_added} ~{datasets_modified} \
|
"HDF5-sync complete: +{datasets_added} ~{datasets_modified} \
|
||||||
@@ -1298,7 +1343,10 @@ async fn cmd_hdf5_sync(
|
|||||||
// Wait for final completion message.
|
// Wait for final completion message.
|
||||||
match peer.recv().await? {
|
match peer.recv().await? {
|
||||||
SyncMessage::Hdf5SyncComplete {
|
SyncMessage::Hdf5SyncComplete {
|
||||||
datasets_added, datasets_modified, datasets_removed, bytes_transferred
|
datasets_added,
|
||||||
|
datasets_modified,
|
||||||
|
datasets_removed,
|
||||||
|
bytes_transferred,
|
||||||
} => {
|
} => {
|
||||||
println!(
|
println!(
|
||||||
"HDF5-sync complete: +{datasets_added} ~{datasets_modified} \
|
"HDF5-sync complete: +{datasets_added} ~{datasets_modified} \
|
||||||
@@ -1321,9 +1369,12 @@ async fn handle_hdf5_client(
|
|||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
// Receive the client's manifest.
|
// Receive the client's manifest.
|
||||||
let (remote_path, _client_file_blake3, client_entries, want_delete) = match conn.recv().await? {
|
let (remote_path, _client_file_blake3, client_entries, want_delete) = match conn.recv().await? {
|
||||||
SyncMessage::Hdf5ManifestRequest { remote_path, file_blake3, entries, want_delete } => {
|
SyncMessage::Hdf5ManifestRequest {
|
||||||
(remote_path, file_blake3, entries, want_delete)
|
remote_path,
|
||||||
}
|
file_blake3,
|
||||||
|
entries,
|
||||||
|
want_delete,
|
||||||
|
} => (remote_path, file_blake3, entries, want_delete),
|
||||||
other => anyhow::bail!("expected Hdf5ManifestRequest, got {other:?}"),
|
other => anyhow::bail!("expected Hdf5ManifestRequest, got {other:?}"),
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -1359,8 +1410,13 @@ async fn handle_hdf5_client(
|
|||||||
let mut to_delete = Vec::new();
|
let mut to_delete = Vec::new();
|
||||||
|
|
||||||
// Build a quick lookup of client entries by path.
|
// Build a quick lookup of client entries by path.
|
||||||
let client_map: std::collections::HashMap<&str, &clawsync_transport::protocol::Hdf5DatasetEntry> =
|
let client_map: std::collections::HashMap<
|
||||||
client_entries.iter().map(|e| (e.path.as_str(), e)).collect();
|
&str,
|
||||||
|
&clawsync_transport::protocol::Hdf5DatasetEntry,
|
||||||
|
> = client_entries
|
||||||
|
.iter()
|
||||||
|
.map(|e| (e.path.as_str(), e))
|
||||||
|
.collect();
|
||||||
|
|
||||||
// Added or Modified: in client manifest but not on server, or hash differs.
|
// Added or Modified: in client manifest but not on server, or hash differs.
|
||||||
for entry in &client_entries {
|
for entry in &client_entries {
|
||||||
@@ -1394,9 +1450,19 @@ async fn handle_hdf5_client(
|
|||||||
|
|
||||||
for _ in 0..total_needed {
|
for _ in 0..total_needed {
|
||||||
match conn.recv().await? {
|
match conn.recv().await? {
|
||||||
SyncMessage::Hdf5DataPayload { path, data, dtype, shape } => {
|
SyncMessage::Hdf5DataPayload {
|
||||||
|
path,
|
||||||
|
data,
|
||||||
|
dtype,
|
||||||
|
shape,
|
||||||
|
} => {
|
||||||
bytes_received += data.len() as u64;
|
bytes_received += data.len() as u64;
|
||||||
payloads.push(ReceivedDataset { path: path.clone(), data, dtype, shape });
|
payloads.push(ReceivedDataset {
|
||||||
|
path: path.clone(),
|
||||||
|
data,
|
||||||
|
dtype,
|
||||||
|
shape,
|
||||||
|
});
|
||||||
conn.send(&SyncMessage::Hdf5DataAck { path }).await?;
|
conn.send(&SyncMessage::Hdf5DataAck { path }).await?;
|
||||||
}
|
}
|
||||||
other => anyhow::bail!("expected Hdf5DataPayload, got {other:?}"),
|
other => anyhow::bail!("expected Hdf5DataPayload, got {other:?}"),
|
||||||
@@ -1419,9 +1485,9 @@ async fn handle_hdf5_client(
|
|||||||
tokio::fs::rename(&tmp_path, &target_path).await?;
|
tokio::fs::rename(&tmp_path, &target_path).await?;
|
||||||
|
|
||||||
conn.send(&SyncMessage::Hdf5SyncComplete {
|
conn.send(&SyncMessage::Hdf5SyncComplete {
|
||||||
datasets_added: stats.datasets_added,
|
datasets_added: stats.datasets_added,
|
||||||
datasets_modified: stats.datasets_modified,
|
datasets_modified: stats.datasets_modified,
|
||||||
datasets_removed: stats.datasets_removed,
|
datasets_removed: stats.datasets_removed,
|
||||||
bytes_transferred: bytes_received,
|
bytes_transferred: bytes_received,
|
||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
@@ -1440,8 +1506,8 @@ async fn cmd_serve_hdf5(
|
|||||||
quic: bool,
|
quic: bool,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
if quic {
|
if quic {
|
||||||
let cfg = QuicConfig::self_signed()
|
let cfg =
|
||||||
.with_context(|| "failed to build QUIC self-signed config")?;
|
QuicConfig::self_signed().with_context(|| "failed to build QUIC self-signed config")?;
|
||||||
let server = QuicServer::bind(bind, cfg)
|
let server = QuicServer::bind(bind, cfg)
|
||||||
.await
|
.await
|
||||||
.with_context(|| format!("cannot bind QUIC to {bind}"))?;
|
.with_context(|| format!("cannot bind QUIC to {bind}"))?;
|
||||||
@@ -1453,7 +1519,11 @@ async fn cmd_serve_hdf5(
|
|||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
if let Err(e) = handle_hdf5_client(&mut peer, &root, allow_delete).await {
|
if let Err(e) = handle_hdf5_client(&mut peer, &root, allow_delete).await {
|
||||||
eprintln!("HDF5 QUIC client error: {e}");
|
eprintln!("HDF5 QUIC client error: {e}");
|
||||||
let _ = peer.send(&SyncMessage::Error { message: e.to_string() }).await;
|
let _ = peer
|
||||||
|
.send(&SyncMessage::Error {
|
||||||
|
message: e.to_string(),
|
||||||
|
})
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -1469,7 +1539,11 @@ async fn cmd_serve_hdf5(
|
|||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
if let Err(e) = handle_hdf5_client(&mut peer, &root, allow_delete).await {
|
if let Err(e) = handle_hdf5_client(&mut peer, &root, allow_delete).await {
|
||||||
eprintln!("HDF5 client {peer_addr} error: {e}");
|
eprintln!("HDF5 client {peer_addr} error: {e}");
|
||||||
let _ = peer.send(&SyncMessage::Error { message: e.to_string() }).await;
|
let _ = peer
|
||||||
|
.send(&SyncMessage::Error {
|
||||||
|
message: e.to_string(),
|
||||||
|
})
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -1498,36 +1572,73 @@ fn main() -> Result<()> {
|
|||||||
Commands::ListRevisions { h5_path, verbose } => {
|
Commands::ListRevisions { h5_path, verbose } => {
|
||||||
cmd_list_revisions(h5_path, verbose).await
|
cmd_list_revisions(h5_path, verbose).await
|
||||||
}
|
}
|
||||||
Commands::Rollback { h5_path, revision } => {
|
Commands::Rollback { h5_path, revision } => cmd_rollback(h5_path, revision).await,
|
||||||
cmd_rollback(h5_path, revision).await
|
Commands::Push {
|
||||||
}
|
local,
|
||||||
Commands::Push { local, remote, branch, quic } => {
|
remote,
|
||||||
cmd_push(local, remote, branch, quic).await
|
branch,
|
||||||
}
|
quic,
|
||||||
Commands::Pull { remote, local, branch, quic } => {
|
} => cmd_push(local, remote, branch, quic).await,
|
||||||
cmd_pull(remote, local, branch, quic).await
|
Commands::Pull {
|
||||||
}
|
remote,
|
||||||
Commands::Serve { h5_path, bind, quic } => cmd_serve(h5_path, bind, quic).await,
|
local,
|
||||||
|
branch,
|
||||||
|
quic,
|
||||||
|
} => cmd_pull(remote, local, branch, quic).await,
|
||||||
|
Commands::Serve {
|
||||||
|
h5_path,
|
||||||
|
bind,
|
||||||
|
quic,
|
||||||
|
} => cmd_serve(h5_path, bind, quic).await,
|
||||||
Commands::Branch { action } => cmd_branch(action).await,
|
Commands::Branch { action } => cmd_branch(action).await,
|
||||||
Commands::ExportRevision { h5_path, revision, dest } => {
|
Commands::ExportRevision {
|
||||||
cmd_export_revision(h5_path, revision, dest).await
|
h5_path,
|
||||||
}
|
revision,
|
||||||
Commands::Gc { h5_path, keep_last, keep_tagged, keep_since_days, keep_revisions } => {
|
dest,
|
||||||
cmd_gc(h5_path, keep_last, keep_tagged, keep_since_days, keep_revisions).await
|
} => cmd_export_revision(h5_path, revision, dest).await,
|
||||||
|
Commands::Gc {
|
||||||
|
h5_path,
|
||||||
|
keep_last,
|
||||||
|
keep_tagged,
|
||||||
|
keep_since_days,
|
||||||
|
keep_revisions,
|
||||||
|
} => {
|
||||||
|
cmd_gc(
|
||||||
|
h5_path,
|
||||||
|
keep_last,
|
||||||
|
keep_tagged,
|
||||||
|
keep_since_days,
|
||||||
|
keep_revisions,
|
||||||
|
)
|
||||||
|
.await
|
||||||
}
|
}
|
||||||
Commands::Snapshot { h5_path, message } => cmd_snapshot(h5_path, message).await,
|
Commands::Snapshot { h5_path, message } => cmd_snapshot(h5_path, message).await,
|
||||||
Commands::Sync { local, remote, delete, exclude, quic } => {
|
Commands::Sync {
|
||||||
cmd_sync(local, remote, delete, exclude, quic).await
|
local,
|
||||||
}
|
remote,
|
||||||
Commands::ServeFs { dir, bind, allow_delete, exclude, quic } => {
|
delete,
|
||||||
cmd_serve_fs(dir, bind, allow_delete, exclude, quic).await
|
exclude,
|
||||||
}
|
quic,
|
||||||
Commands::Hdf5Sync { local, remote, delete, quic } => {
|
} => cmd_sync(local, remote, delete, exclude, quic).await,
|
||||||
cmd_hdf5_sync(local, remote, delete, quic).await
|
Commands::ServeFs {
|
||||||
}
|
dir,
|
||||||
Commands::ServeHdf5 { dir, bind, allow_delete, quic } => {
|
bind,
|
||||||
cmd_serve_hdf5(dir, bind, allow_delete, quic).await
|
allow_delete,
|
||||||
}
|
exclude,
|
||||||
|
quic,
|
||||||
|
} => cmd_serve_fs(dir, bind, allow_delete, exclude, quic).await,
|
||||||
|
Commands::Hdf5Sync {
|
||||||
|
local,
|
||||||
|
remote,
|
||||||
|
delete,
|
||||||
|
quic,
|
||||||
|
} => cmd_hdf5_sync(local, remote, delete, quic).await,
|
||||||
|
Commands::ServeHdf5 {
|
||||||
|
dir,
|
||||||
|
bind,
|
||||||
|
allow_delete,
|
||||||
|
quic,
|
||||||
|
} => cmd_serve_hdf5(dir, bind, allow_delete, quic).await,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,8 +9,8 @@ use std::path::PathBuf;
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use clawhdf5_onion::api::{create_branch, delete_branch, list_branches, rename_branch};
|
use clawhdf5_onion::api::{create_branch, delete_branch, list_branches, rename_branch};
|
||||||
use clawhdf5_onion::gc::GcPolicy;
|
|
||||||
use clawhdf5_onion::format::NO_PARENT;
|
use clawhdf5_onion::format::NO_PARENT;
|
||||||
|
use clawhdf5_onion::gc::GcPolicy;
|
||||||
use clawhdf5_onion::writer::OnionFile;
|
use clawhdf5_onion::writer::OnionFile;
|
||||||
use clawsync_onion::differ::diff_revisions;
|
use clawsync_onion::differ::diff_revisions;
|
||||||
use clawsync_onion::manifest::ClawSyncManifest;
|
use clawsync_onion::manifest::ClawSyncManifest;
|
||||||
@@ -62,7 +62,9 @@ async fn run_server_one_connection(server: TcpServer, mut dst: OnionFile) -> Oni
|
|||||||
|
|
||||||
let local_rev_count = dst.revision_count();
|
let local_rev_count = dst.revision_count();
|
||||||
let manifest = ClawSyncManifest::from_onion("server", &dst, H5_MAGIC);
|
let manifest = ClawSyncManifest::from_onion("server", &dst, H5_MAGIC);
|
||||||
conn.send(&SyncMessage::ManifestResponse { manifest }).await.unwrap();
|
conn.send(&SyncMessage::ManifestResponse { manifest })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
if local_rev_count > remote_rev_count {
|
if local_rev_count > remote_rev_count {
|
||||||
// Push to client
|
// Push to client
|
||||||
@@ -76,7 +78,9 @@ async fn run_server_one_connection(server: TcpServer, mut dst: OnionFile) -> Oni
|
|||||||
let mut bytes_sent = 0u64;
|
let mut bytes_sent = 0u64;
|
||||||
for packet in packets {
|
for packet in packets {
|
||||||
bytes_sent += packet.page_data_size() as u64;
|
bytes_sent += packet.page_data_size() as u64;
|
||||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
conn.send(&SyncMessage::LayerPacket { packet })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::Ack { .. } => {}
|
SyncMessage::Ack { .. } => {}
|
||||||
other => panic!("expected Ack, got {other:?}"),
|
other => panic!("expected Ack, got {other:?}"),
|
||||||
@@ -95,7 +99,9 @@ async fn run_server_one_connection(server: TcpServer, mut dst: OnionFile) -> Oni
|
|||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::LayerPacket { packet } => {
|
SyncMessage::LayerPacket { packet } => {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
conn.send(&SyncMessage::Ack { revision: rev })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
packets.push(packet);
|
packets.push(packet);
|
||||||
}
|
}
|
||||||
SyncMessage::SyncComplete { .. } => break,
|
SyncMessage::SyncComplete { .. } => break,
|
||||||
@@ -139,7 +145,11 @@ async fn push_all_to_empty_server() {
|
|||||||
|
|
||||||
let remote_head = match conn.recv().await.unwrap() {
|
let remote_head = match conn.recv().await.unwrap() {
|
||||||
SyncMessage::ManifestResponse { manifest } => {
|
SyncMessage::ManifestResponse { manifest } => {
|
||||||
if manifest.revision_count == 0 { clawhdf5_onion::format::NO_PARENT } else { manifest.head_revision }
|
if manifest.revision_count == 0 {
|
||||||
|
clawhdf5_onion::format::NO_PARENT
|
||||||
|
} else {
|
||||||
|
manifest.head_revision
|
||||||
|
}
|
||||||
}
|
}
|
||||||
other => panic!("{other:?}"),
|
other => panic!("{other:?}"),
|
||||||
};
|
};
|
||||||
@@ -148,14 +158,20 @@ async fn push_all_to_empty_server() {
|
|||||||
let total = packets.len() as u64;
|
let total = packets.len() as u64;
|
||||||
for packet in packets {
|
for packet in packets {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
conn.send(&SyncMessage::LayerPacket { packet })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::Ack { revision } => assert_eq!(revision, rev),
|
SyncMessage::Ack { revision } => assert_eq!(revision, rev),
|
||||||
other => panic!("{other:?}"),
|
other => panic!("{other:?}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
conn.send(&SyncMessage::SyncComplete { revisions_transferred: total, bytes_transferred: 0 })
|
conn.send(&SyncMessage::SyncComplete {
|
||||||
.await.unwrap();
|
revisions_transferred: total,
|
||||||
|
bytes_transferred: 0,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let dst_final = server_task.await.unwrap();
|
let dst_final = server_task.await.unwrap();
|
||||||
assert_eq!(dst_final.revision_count(), 5);
|
assert_eq!(dst_final.revision_count(), 5);
|
||||||
@@ -180,7 +196,9 @@ async fn push_already_in_sync_is_noop() {
|
|||||||
agent_id: "c".into(),
|
agent_id: "c".into(),
|
||||||
head_revision: mf.head_revision,
|
head_revision: mf.head_revision,
|
||||||
revision_count: mf.revision_count,
|
revision_count: mf.revision_count,
|
||||||
}).await.unwrap();
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::ManifestResponse { .. } => {}
|
SyncMessage::ManifestResponse { .. } => {}
|
||||||
@@ -188,8 +206,12 @@ async fn push_already_in_sync_is_noop() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// diff = 0; send SyncComplete immediately
|
// diff = 0; send SyncComplete immediately
|
||||||
conn.send(&SyncMessage::SyncComplete { revisions_transferred: 0, bytes_transferred: 0 })
|
conn.send(&SyncMessage::SyncComplete {
|
||||||
.await.unwrap();
|
revisions_transferred: 0,
|
||||||
|
bytes_transferred: 0,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let dst_final = server_task.await.unwrap();
|
let dst_final = server_task.await.unwrap();
|
||||||
assert_eq!(dst_final.revision_count(), 3, "no duplicate revisions");
|
assert_eq!(dst_final.revision_count(), 3, "no duplicate revisions");
|
||||||
@@ -215,7 +237,9 @@ async fn pull_all_from_server() {
|
|||||||
agent_id: "pull-client".into(),
|
agent_id: "pull-client".into(),
|
||||||
head_revision: 0,
|
head_revision: 0,
|
||||||
revision_count: 0,
|
revision_count: 0,
|
||||||
}).await.unwrap();
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let server_count = match conn.recv().await.unwrap() {
|
let server_count = match conn.recv().await.unwrap() {
|
||||||
SyncMessage::ManifestResponse { manifest } => manifest.revision_count,
|
SyncMessage::ManifestResponse { manifest } => manifest.revision_count,
|
||||||
@@ -230,7 +254,9 @@ async fn pull_all_from_server() {
|
|||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::LayerPacket { packet } => {
|
SyncMessage::LayerPacket { packet } => {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
conn.send(&SyncMessage::Ack { revision: rev })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
packets.push(packet);
|
packets.push(packet);
|
||||||
}
|
}
|
||||||
SyncMessage::SyncComplete { .. } => break,
|
SyncMessage::SyncComplete { .. } => break,
|
||||||
@@ -260,7 +286,9 @@ async fn pull_already_in_sync() {
|
|||||||
agent_id: "pull-client".into(),
|
agent_id: "pull-client".into(),
|
||||||
head_revision: local_count.saturating_sub(1),
|
head_revision: local_count.saturating_sub(1),
|
||||||
revision_count: local_count,
|
revision_count: local_count,
|
||||||
}).await.unwrap();
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let server_count = match conn.recv().await.unwrap() {
|
let server_count = match conn.recv().await.unwrap() {
|
||||||
SyncMessage::ManifestResponse { manifest } => manifest.revision_count,
|
SyncMessage::ManifestResponse { manifest } => manifest.revision_count,
|
||||||
@@ -269,8 +297,12 @@ async fn pull_already_in_sync() {
|
|||||||
|
|
||||||
assert!(server_count <= local_count, "already in sync");
|
assert!(server_count <= local_count, "already in sync");
|
||||||
// Client detects it's up to date — sends SyncComplete immediately.
|
// Client detects it's up to date — sends SyncComplete immediately.
|
||||||
conn.send(&SyncMessage::SyncComplete { revisions_transferred: 0, bytes_transferred: 0 })
|
conn.send(&SyncMessage::SyncComplete {
|
||||||
.await.unwrap();
|
revisions_transferred: 0,
|
||||||
|
bytes_transferred: 0,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
server_task.await.unwrap();
|
server_task.await.unwrap();
|
||||||
let _ = cli_h5;
|
let _ = cli_h5;
|
||||||
@@ -294,7 +326,9 @@ async fn pull_partial_delta() {
|
|||||||
agent_id: "c".into(),
|
agent_id: "c".into(),
|
||||||
head_revision: K as u64 - 1,
|
head_revision: K as u64 - 1,
|
||||||
revision_count: K as u64,
|
revision_count: K as u64,
|
||||||
}).await.unwrap();
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let server_count = match conn.recv().await.unwrap() {
|
let server_count = match conn.recv().await.unwrap() {
|
||||||
SyncMessage::ManifestResponse { manifest } => manifest.revision_count,
|
SyncMessage::ManifestResponse { manifest } => manifest.revision_count,
|
||||||
@@ -308,7 +342,9 @@ async fn pull_partial_delta() {
|
|||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::LayerPacket { packet } => {
|
SyncMessage::LayerPacket { packet } => {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
conn.send(&SyncMessage::Ack { revision: rev })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
packets.push(packet);
|
packets.push(packet);
|
||||||
received += 1;
|
received += 1;
|
||||||
}
|
}
|
||||||
@@ -376,7 +412,9 @@ async fn push_one_revision_within_timeout() {
|
|||||||
agent_id: "c".into(),
|
agent_id: "c".into(),
|
||||||
head_revision: mf.head_revision,
|
head_revision: mf.head_revision,
|
||||||
revision_count: mf.revision_count,
|
revision_count: mf.revision_count,
|
||||||
}).await.unwrap();
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let remote_head = match conn.recv().await.unwrap() {
|
let remote_head = match conn.recv().await.unwrap() {
|
||||||
SyncMessage::ManifestResponse { manifest } => {
|
SyncMessage::ManifestResponse { manifest } => {
|
||||||
@@ -393,18 +431,27 @@ async fn push_one_revision_within_timeout() {
|
|||||||
let total = packets.len() as u64;
|
let total = packets.len() as u64;
|
||||||
for packet in packets {
|
for packet in packets {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
conn.send(&SyncMessage::LayerPacket { packet })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::Ack { revision } => assert_eq!(revision, rev),
|
SyncMessage::Ack { revision } => assert_eq!(revision, rev),
|
||||||
other => panic!("{other:?}"),
|
other => panic!("{other:?}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
conn.send(&SyncMessage::SyncComplete { revisions_transferred: total, bytes_transferred: 0 })
|
conn.send(&SyncMessage::SyncComplete {
|
||||||
.await.unwrap();
|
revisions_transferred: total,
|
||||||
|
bytes_transferred: 0,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
})
|
})
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
assert!(client_result.is_ok(), "push timed out — possible protocol deadlock");
|
assert!(
|
||||||
|
client_result.is_ok(),
|
||||||
|
"push timed out — possible protocol deadlock"
|
||||||
|
);
|
||||||
let dst_final = server_handle.await.unwrap();
|
let dst_final = server_handle.await.unwrap();
|
||||||
assert_eq!(dst_final.revision_count(), 1);
|
assert_eq!(dst_final.revision_count(), 1);
|
||||||
}
|
}
|
||||||
@@ -495,11 +542,17 @@ async fn push_with_branch_filter_sends_only_branch_revisions() {
|
|||||||
agent_id: "c".into(),
|
agent_id: "c".into(),
|
||||||
head_revision: mf.head_revision,
|
head_revision: mf.head_revision,
|
||||||
revision_count: mf.revision_count,
|
revision_count: mf.revision_count,
|
||||||
}).await.unwrap();
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let remote_head = match conn.recv().await.unwrap() {
|
let remote_head = match conn.recv().await.unwrap() {
|
||||||
SyncMessage::ManifestResponse { manifest } => {
|
SyncMessage::ManifestResponse { manifest } => {
|
||||||
if manifest.revision_count == 0 { clawhdf5_onion::format::NO_PARENT } else { manifest.head_revision }
|
if manifest.revision_count == 0 {
|
||||||
|
clawhdf5_onion::format::NO_PARENT
|
||||||
|
} else {
|
||||||
|
manifest.head_revision
|
||||||
|
}
|
||||||
}
|
}
|
||||||
other => panic!("{other:?}"),
|
other => panic!("{other:?}"),
|
||||||
};
|
};
|
||||||
@@ -509,22 +562,36 @@ async fn push_with_branch_filter_sends_only_branch_revisions() {
|
|||||||
let packets = filter_packets(all_packets, &selector, &src).unwrap();
|
let packets = filter_packets(all_packets, &selector, &src).unwrap();
|
||||||
|
|
||||||
// Only the 2 feature revisions should be sent (not the 3 main ones).
|
// Only the 2 feature revisions should be sent (not the 3 main ones).
|
||||||
assert_eq!(packets.len(), 2, "only feature branch packets should be sent");
|
assert_eq!(
|
||||||
|
packets.len(),
|
||||||
|
2,
|
||||||
|
"only feature branch packets should be sent"
|
||||||
|
);
|
||||||
|
|
||||||
let total = packets.len() as u64;
|
let total = packets.len() as u64;
|
||||||
for packet in packets {
|
for packet in packets {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
conn.send(&SyncMessage::LayerPacket { packet })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::Ack { revision } => assert_eq!(revision, rev),
|
SyncMessage::Ack { revision } => assert_eq!(revision, rev),
|
||||||
other => panic!("{other:?}"),
|
other => panic!("{other:?}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
conn.send(&SyncMessage::SyncComplete { revisions_transferred: total, bytes_transferred: 0 })
|
conn.send(&SyncMessage::SyncComplete {
|
||||||
.await.unwrap();
|
revisions_transferred: total,
|
||||||
|
bytes_transferred: 0,
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let dst_final = server_task.await.unwrap();
|
let dst_final = server_task.await.unwrap();
|
||||||
assert_eq!(dst_final.revision_count(), 2, "destination should have only the 2 feature revisions");
|
assert_eq!(
|
||||||
|
dst_final.revision_count(),
|
||||||
|
2,
|
||||||
|
"destination should have only the 2 feature revisions"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -567,7 +634,9 @@ async fn push_to_server(src: &OnionFile, src_h5: &PathBuf, addr: SocketAddr) ->
|
|||||||
let sent = packets.len() as u64;
|
let sent = packets.len() as u64;
|
||||||
for packet in packets {
|
for packet in packets {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
conn.send(&SyncMessage::LayerPacket { packet })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::Ack { revision } => assert_eq!(revision, rev),
|
SyncMessage::Ack { revision } => assert_eq!(revision, rev),
|
||||||
other => panic!("client: expected Ack, got {other:?}"),
|
other => panic!("client: expected Ack, got {other:?}"),
|
||||||
@@ -597,7 +666,9 @@ async fn run_disk_server(server: TcpServer, h5_path: PathBuf, n: usize) {
|
|||||||
|
|
||||||
let local_rev_count = onion.revision_count();
|
let local_rev_count = onion.revision_count();
|
||||||
let manifest = ClawSyncManifest::from_onion("server", &onion, H5_MAGIC);
|
let manifest = ClawSyncManifest::from_onion("server", &onion, H5_MAGIC);
|
||||||
conn.send(&SyncMessage::ManifestResponse { manifest }).await.unwrap();
|
conn.send(&SyncMessage::ManifestResponse { manifest })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
if local_rev_count > remote_rev_count {
|
if local_rev_count > remote_rev_count {
|
||||||
// Server pushes (pull direction)
|
// Server pushes (pull direction)
|
||||||
@@ -611,7 +682,9 @@ async fn run_disk_server(server: TcpServer, h5_path: PathBuf, n: usize) {
|
|||||||
let mut bytes_sent = 0u64;
|
let mut bytes_sent = 0u64;
|
||||||
for packet in packets {
|
for packet in packets {
|
||||||
bytes_sent += packet.page_data_size() as u64;
|
bytes_sent += packet.page_data_size() as u64;
|
||||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
conn.send(&SyncMessage::LayerPacket { packet })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::Ack { .. } => {}
|
SyncMessage::Ack { .. } => {}
|
||||||
other => panic!("server: expected Ack, got {other:?}"),
|
other => panic!("server: expected Ack, got {other:?}"),
|
||||||
@@ -630,7 +703,9 @@ async fn run_disk_server(server: TcpServer, h5_path: PathBuf, n: usize) {
|
|||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::LayerPacket { packet } => {
|
SyncMessage::LayerPacket { packet } => {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
conn.send(&SyncMessage::Ack { revision: rev })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
packets.push(packet);
|
packets.push(packet);
|
||||||
}
|
}
|
||||||
SyncMessage::SyncComplete { .. } => break,
|
SyncMessage::SyncComplete { .. } => break,
|
||||||
@@ -687,7 +762,11 @@ async fn serve_accumulates_across_two_sequential_pushes() {
|
|||||||
|
|
||||||
// The on-disk server file must now have all 5 revisions.
|
// The on-disk server file must now have all 5 revisions.
|
||||||
let final_onion = OnionFile::open(&srv_h5).unwrap();
|
let final_onion = OnionFile::open(&srv_h5).unwrap();
|
||||||
assert_eq!(final_onion.revision_count(), 5, "server should have accumulated all 5 revisions");
|
assert_eq!(
|
||||||
|
final_onion.revision_count(),
|
||||||
|
5,
|
||||||
|
"server should have accumulated all 5 revisions"
|
||||||
|
);
|
||||||
let _ = srv_dir;
|
let _ = srv_dir;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -713,7 +792,10 @@ async fn serve_noop_when_second_client_already_in_sync() {
|
|||||||
|
|
||||||
// Connection 2: same client — already in sync, should send 0 packets.
|
// Connection 2: same client — already in sync, should send 0 packets.
|
||||||
let sent2 = push_to_server(&src, &src_h5, addr).await;
|
let sent2 = push_to_server(&src, &src_h5, addr).await;
|
||||||
assert_eq!(sent2, 0, "second identical push should transfer 0 revisions");
|
assert_eq!(
|
||||||
|
sent2, 0,
|
||||||
|
"second identical push should transfer 0 revisions"
|
||||||
|
);
|
||||||
|
|
||||||
server_task.await.unwrap();
|
server_task.await.unwrap();
|
||||||
|
|
||||||
@@ -738,8 +820,11 @@ fn export_revision_writes_correct_historical_state() {
|
|||||||
|
|
||||||
let bytes = std::fs::read(&dest).unwrap();
|
let bytes = std::fs::read(&dest).unwrap();
|
||||||
// The reconstructed bytes at offset 0 should be the rev-1 page (all 0x01)
|
// The reconstructed bytes at offset 0 should be the rev-1 page (all 0x01)
|
||||||
assert_eq!(bytes[0..4096], vec![0x01u8; 4096][..],
|
assert_eq!(
|
||||||
"exported rev 1 should contain the page written in rev 1");
|
bytes[0..4096],
|
||||||
|
vec![0x01u8; 4096][..],
|
||||||
|
"exported rev 1 should contain the page written in rev 1"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -761,7 +846,10 @@ fn export_revision_does_not_modify_source_sidecar() {
|
|||||||
vf.export_revision(0, &dest).unwrap();
|
vf.export_revision(0, &dest).unwrap();
|
||||||
|
|
||||||
let after_len = std::fs::metadata(&onion_path).unwrap().len();
|
let after_len = std::fs::metadata(&onion_path).unwrap().len();
|
||||||
assert_eq!(before_len, after_len, "sidecar must be unchanged after export");
|
assert_eq!(
|
||||||
|
before_len, after_len,
|
||||||
|
"sidecar must be unchanged after export"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -771,7 +859,10 @@ fn export_revision_nonexistent_errors() {
|
|||||||
let dest = dest_dir.path().join("out.h5");
|
let dest = dest_dir.path().join("out.h5");
|
||||||
let vf = clawhdf5_onion::versioned_file::VersionedFile::open(&h5).unwrap();
|
let vf = clawhdf5_onion::versioned_file::VersionedFile::open(&h5).unwrap();
|
||||||
let err = vf.export_revision(999, &dest).unwrap_err();
|
let err = vf.export_revision(999, &dest).unwrap_err();
|
||||||
assert!(matches!(err, clawhdf5_onion::OnionError::RevisionNotFound(999)));
|
assert!(matches!(
|
||||||
|
err,
|
||||||
|
clawhdf5_onion::OnionError::RevisionNotFound(999)
|
||||||
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -802,10 +893,18 @@ fn branch_merge_latest_wins_creates_merge_revision() {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap();
|
let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap();
|
||||||
assert!(revs.iter().any(|r| r.revision == merge_rev),
|
assert!(
|
||||||
"merge revision must appear in list_revisions");
|
revs.iter().any(|r| r.revision == merge_rev),
|
||||||
assert_eq!(revs.iter().find(|r| r.revision == merge_rev).unwrap().branch_id, 0,
|
"merge revision must appear in list_revisions"
|
||||||
"merge revision is on main (branch_id 0)");
|
);
|
||||||
|
assert_eq!(
|
||||||
|
revs.iter()
|
||||||
|
.find(|r| r.revision == merge_rev)
|
||||||
|
.unwrap()
|
||||||
|
.branch_id,
|
||||||
|
0,
|
||||||
|
"merge revision is on main (branch_id 0)"
|
||||||
|
);
|
||||||
|
|
||||||
let _ = h5_base;
|
let _ = h5_base;
|
||||||
}
|
}
|
||||||
@@ -865,8 +964,10 @@ fn gc_keep_tagged_retains_annotated_only() {
|
|||||||
|
|
||||||
assert_eq!(stats.revisions_removed, 2);
|
assert_eq!(stats.revisions_removed, 2);
|
||||||
let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap();
|
let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap();
|
||||||
assert!(revs.iter().all(|r| r.annotation.is_some()),
|
assert!(
|
||||||
"all remaining revisions should be annotated");
|
revs.iter().all(|r| r.annotation.is_some()),
|
||||||
|
"all remaining revisions should be annotated"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -887,8 +988,10 @@ fn gc_keep_since_days_zero_removes_all_old_revisions() {
|
|||||||
let stats = onion.gc(GcPolicy::KeepSince(future_cutoff)).unwrap();
|
let stats = onion.gc(GcPolicy::KeepSince(future_cutoff)).unwrap();
|
||||||
onion.flush().unwrap();
|
onion.flush().unwrap();
|
||||||
|
|
||||||
assert_eq!(stats.revisions_removed, 4,
|
assert_eq!(
|
||||||
"all 4 revisions should be pruned when cutoff is in the future");
|
stats.revisions_removed, 4,
|
||||||
|
"all 4 revisions should be pruned when cutoff is in the future"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -898,8 +1001,10 @@ fn gc_keep_since_days_past_keeps_all() {
|
|||||||
|
|
||||||
let stats = onion.gc(GcPolicy::KeepSince(0.0)).unwrap();
|
let stats = onion.gc(GcPolicy::KeepSince(0.0)).unwrap();
|
||||||
// All revisions are after Unix epoch 0 → 0 removed
|
// All revisions are after Unix epoch 0 → 0 removed
|
||||||
assert_eq!(stats.revisions_removed, 0,
|
assert_eq!(
|
||||||
"no revisions should be removed when cutoff is epoch 0");
|
stats.revisions_removed, 0,
|
||||||
|
"no revisions should be removed when cutoff is epoch 0"
|
||||||
|
);
|
||||||
assert_eq!(onion.revision_count(), 4);
|
assert_eq!(onion.revision_count(), 4);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -926,19 +1031,29 @@ fn gc_keep_revisions_retains_only_listed_and_ancestors() {
|
|||||||
onion.flush().unwrap();
|
onion.flush().unwrap();
|
||||||
|
|
||||||
// Revisions 3 and 4 are pruned; 0, 1, 2 are kept (2 plus its ancestors).
|
// Revisions 3 and 4 are pruned; 0, 1, 2 are kept (2 plus its ancestors).
|
||||||
assert_eq!(stats.revisions_removed, 2, "revisions 3 and 4 should be removed");
|
assert_eq!(
|
||||||
|
stats.revisions_removed, 2,
|
||||||
|
"revisions 3 and 4 should be removed"
|
||||||
|
);
|
||||||
|
|
||||||
let reloaded = OnionFile::open(&h5).unwrap();
|
let reloaded = OnionFile::open(&h5).unwrap();
|
||||||
assert_eq!(reloaded.revision_count(), 3, "revisions 0, 1, 2 remain");
|
assert_eq!(reloaded.revision_count(), 3, "revisions 0, 1, 2 remain");
|
||||||
|
|
||||||
let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap();
|
let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap();
|
||||||
let annotations: Vec<_> = revs.iter()
|
let annotations: Vec<_> = revs
|
||||||
|
.iter()
|
||||||
.map(|r| r.annotation.as_deref().unwrap_or(""))
|
.map(|r| r.annotation.as_deref().unwrap_or(""))
|
||||||
.collect();
|
.collect();
|
||||||
assert!(annotations.contains(&"rev 2"), "rev 2 must survive");
|
assert!(annotations.contains(&"rev 2"), "rev 2 must survive");
|
||||||
// Ancestors are retained too
|
// Ancestors are retained too
|
||||||
assert!(annotations.contains(&"rev 0"), "ancestor rev 0 must survive");
|
assert!(
|
||||||
assert!(annotations.contains(&"rev 1"), "ancestor rev 1 must survive");
|
annotations.contains(&"rev 0"),
|
||||||
|
"ancestor rev 0 must survive"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
annotations.contains(&"rev 1"),
|
||||||
|
"ancestor rev 1 must survive"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -948,7 +1063,10 @@ fn gc_keep_revisions_empty_list_removes_all() {
|
|||||||
let stats = onion.gc(GcPolicy::KeepRevisions(vec![])).unwrap();
|
let stats = onion.gc(GcPolicy::KeepRevisions(vec![])).unwrap();
|
||||||
onion.flush().unwrap();
|
onion.flush().unwrap();
|
||||||
|
|
||||||
assert_eq!(stats.revisions_removed, 4, "all revisions pruned when keep list is empty");
|
assert_eq!(
|
||||||
|
stats.revisions_removed, 4,
|
||||||
|
"all revisions pruned when keep list is empty"
|
||||||
|
);
|
||||||
let reloaded = OnionFile::open(&h5).unwrap();
|
let reloaded = OnionFile::open(&h5).unwrap();
|
||||||
assert_eq!(reloaded.revision_count(), 0);
|
assert_eq!(reloaded.revision_count(), 0);
|
||||||
}
|
}
|
||||||
@@ -967,7 +1085,10 @@ fn snapshot_creates_flagged_revision() {
|
|||||||
|
|
||||||
let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap();
|
let revs = clawhdf5_onion::api::list_revisions(&h5).unwrap();
|
||||||
let snap = revs.iter().find(|r| r.revision == snap_rev).unwrap();
|
let snap = revs.iter().find(|r| r.revision == snap_rev).unwrap();
|
||||||
assert!(snap.is_snapshot, "snapshot revision must have is_snapshot=true");
|
assert!(
|
||||||
|
snap.is_snapshot,
|
||||||
|
"snapshot revision must have is_snapshot=true"
|
||||||
|
);
|
||||||
assert_eq!(snap.annotation.as_deref(), Some("snap"));
|
assert_eq!(snap.annotation.as_deref(), Some("snap"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -15,8 +15,8 @@ use std::io::{BufRead, BufReader};
|
|||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::process::{Child, Command, Stdio};
|
use std::process::{Child, Command, Stdio};
|
||||||
use std::time::Duration;
|
|
||||||
use std::thread;
|
use std::thread;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use tempfile::TempDir;
|
use tempfile::TempDir;
|
||||||
|
|
||||||
@@ -51,13 +51,13 @@ impl FsServer {
|
|||||||
|
|
||||||
// Wait for: "ClawSync FS server listening on 127.0.0.1:PORT"
|
// Wait for: "ClawSync FS server listening on 127.0.0.1:PORT"
|
||||||
let mut line = String::new();
|
let mut line = String::new();
|
||||||
reader.read_line(&mut line).expect("failed to read serve-fs startup line");
|
reader
|
||||||
|
.read_line(&mut line)
|
||||||
|
.expect("failed to read serve-fs startup line");
|
||||||
let addr = parse_fs_serve_addr(&line);
|
let addr = parse_fs_serve_addr(&line);
|
||||||
|
|
||||||
// Drain remaining stdout in a background thread so the process doesn't block.
|
// Drain remaining stdout in a background thread so the process doesn't block.
|
||||||
thread::spawn(move || {
|
thread::spawn(move || for _ in reader.lines() {});
|
||||||
for _ in reader.lines() {}
|
|
||||||
});
|
|
||||||
|
|
||||||
Self { child, addr }
|
Self { child, addr }
|
||||||
}
|
}
|
||||||
@@ -85,11 +85,11 @@ fn parse_fs_serve_addr(line: &str) -> String {
|
|||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
struct SyncResult {
|
struct SyncResult {
|
||||||
files_added: u32,
|
files_added: u32,
|
||||||
files_modified: u32,
|
files_modified: u32,
|
||||||
files_removed: u32,
|
files_removed: u32,
|
||||||
bytes_transferred: u64,
|
bytes_transferred: u64,
|
||||||
success: bool,
|
success: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn run_sync(src: &Path, server_addr: &str, delete: bool, excludes: &[&str]) -> SyncResult {
|
fn run_sync(src: &Path, server_addr: &str, delete: bool, excludes: &[&str]) -> SyncResult {
|
||||||
@@ -136,7 +136,13 @@ fn parse_sync_output(out: &str, success: bool) -> SyncResult {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
SyncResult { files_added, files_modified, files_removed, bytes_transferred, success }
|
SyncResult {
|
||||||
|
files_added,
|
||||||
|
files_modified,
|
||||||
|
files_removed,
|
||||||
|
bytes_transferred,
|
||||||
|
success,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -161,7 +167,9 @@ fn assert_files_equal(a: &Path, b: &Path) {
|
|||||||
fn assert_dir_equal(src: &Path, dst: &Path) {
|
fn assert_dir_equal(src: &Path, dst: &Path) {
|
||||||
for entry in walkdir::WalkDir::new(src).sort_by_file_name() {
|
for entry in walkdir::WalkDir::new(src).sort_by_file_name() {
|
||||||
let entry: walkdir::DirEntry = entry.unwrap();
|
let entry: walkdir::DirEntry = entry.unwrap();
|
||||||
if !entry.file_type().is_file() { continue; }
|
if !entry.file_type().is_file() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
let rel = entry.path().strip_prefix(src).unwrap();
|
let rel = entry.path().strip_prefix(src).unwrap();
|
||||||
let dst_path = dst.join(rel);
|
let dst_path = dst.join(rel);
|
||||||
assert!(dst_path.exists(), "missing in dst: {}", rel.display());
|
assert!(dst_path.exists(), "missing in dst: {}", rel.display());
|
||||||
@@ -242,7 +250,11 @@ fn fs_sync_cdc_insertion_minimizes_transfer() {
|
|||||||
|
|
||||||
// Base: 512 KB of pseudo-random data
|
// Base: 512 KB of pseudo-random data
|
||||||
let base: Vec<u8> = (0..512 * 1024)
|
let base: Vec<u8> = (0..512 * 1024)
|
||||||
.map(|i| (i as u64).wrapping_mul(6364136223846793005).wrapping_shr(56) as u8)
|
.map(|i| {
|
||||||
|
(i as u64)
|
||||||
|
.wrapping_mul(6364136223846793005)
|
||||||
|
.wrapping_shr(56) as u8
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
// Server has the base.
|
// Server has the base.
|
||||||
@@ -260,7 +272,10 @@ fn fs_sync_cdc_insertion_minimizes_transfer() {
|
|||||||
assert_eq!(result.files_modified, 1);
|
assert_eq!(result.files_modified, 1);
|
||||||
|
|
||||||
// Verify the content is correct after sync.
|
// Verify the content is correct after sync.
|
||||||
assert_files_equal(&src.path().join("weights.bin"), &dst.path().join("weights.bin"));
|
assert_files_equal(
|
||||||
|
&src.path().join("weights.bin"),
|
||||||
|
&dst.path().join("weights.bin"),
|
||||||
|
);
|
||||||
|
|
||||||
// CDC should transfer much less than the full file (< 40% = 205 KB).
|
// CDC should transfer much less than the full file (< 40% = 205 KB).
|
||||||
// The insertion only invalidates ~2-3 chunk boundaries.
|
// The insertion only invalidates ~2-3 chunk boundaries.
|
||||||
@@ -286,7 +301,10 @@ fn fs_sync_delete_removes_server_only_files() {
|
|||||||
|
|
||||||
assert!(result.success);
|
assert!(result.success);
|
||||||
assert_eq!(result.files_removed, 1);
|
assert_eq!(result.files_removed, 1);
|
||||||
assert!(!dst.path().join("extra.bin").exists(), "extra.bin should have been deleted");
|
assert!(
|
||||||
|
!dst.path().join("extra.bin").exists(),
|
||||||
|
"extra.bin should have been deleted"
|
||||||
|
);
|
||||||
assert!(dst.path().join("keep.bin").exists());
|
assert!(dst.path().join("keep.bin").exists());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -304,7 +322,10 @@ fn fs_sync_no_delete_preserves_server_only_files() {
|
|||||||
|
|
||||||
assert!(result.success);
|
assert!(result.success);
|
||||||
// extra.bin must still be there
|
// extra.bin must still be there
|
||||||
assert!(dst.path().join("extra.bin").exists(), "extra.bin should be preserved (no --delete)");
|
assert!(
|
||||||
|
dst.path().join("extra.bin").exists(),
|
||||||
|
"extra.bin should be preserved (no --delete)"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -319,8 +340,14 @@ fn fs_sync_exclude_pattern_skips_matching_files() {
|
|||||||
let result = run_sync(src.path(), &server.addr, false, &["*.tmp"]);
|
let result = run_sync(src.path(), &server.addr, false, &["*.tmp"]);
|
||||||
|
|
||||||
assert!(result.success);
|
assert!(result.success);
|
||||||
assert!(dst.path().join("keep.bin").exists(), "keep.bin should be transferred");
|
assert!(
|
||||||
assert!(!dst.path().join("skip.tmp").exists(), "skip.tmp should be excluded");
|
dst.path().join("keep.bin").exists(),
|
||||||
|
"keep.bin should be transferred"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!dst.path().join("skip.tmp").exists(),
|
||||||
|
"skip.tmp should be excluded"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -356,7 +383,11 @@ fn fs_sync_large_file_roundtrip() {
|
|||||||
let dst = TempDir::new().unwrap();
|
let dst = TempDir::new().unwrap();
|
||||||
|
|
||||||
let data: Vec<u8> = (0..8 * 1024 * 1024)
|
let data: Vec<u8> = (0..8 * 1024 * 1024)
|
||||||
.map(|i| (i as u64).wrapping_mul(6364136223846793005).wrapping_shr(40) as u8)
|
.map(|i| {
|
||||||
|
(i as u64)
|
||||||
|
.wrapping_mul(6364136223846793005)
|
||||||
|
.wrapping_shr(40) as u8
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
fs::write(src.path().join("large.bin"), &data).unwrap();
|
fs::write(src.path().join("large.bin"), &data).unwrap();
|
||||||
|
|
||||||
@@ -376,7 +407,11 @@ fn fs_sync_append_to_file_sends_minimal_chunks() {
|
|||||||
let dst = TempDir::new().unwrap();
|
let dst = TempDir::new().unwrap();
|
||||||
|
|
||||||
let base: Vec<u8> = (0..256 * 1024)
|
let base: Vec<u8> = (0..256 * 1024)
|
||||||
.map(|i| (i as u64).wrapping_mul(6364136223846793005).wrapping_shr(48) as u8)
|
.map(|i| {
|
||||||
|
(i as u64)
|
||||||
|
.wrapping_mul(6364136223846793005)
|
||||||
|
.wrapping_shr(48) as u8
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
fs::write(dst.path().join("model.bin"), &base).unwrap();
|
fs::write(dst.path().join("model.bin"), &base).unwrap();
|
||||||
|
|
||||||
|
|||||||
@@ -99,10 +99,13 @@ fn run_hdf5_sync(local: &Path, addr: SocketAddr, rel_path: &str, delete: bool) {
|
|||||||
fn hdf5_sync_cold_copy() {
|
fn hdf5_sync_cold_copy() {
|
||||||
let src = TempDir::new().unwrap();
|
let src = TempDir::new().unwrap();
|
||||||
let local = src.path().join("model.h5");
|
let local = src.path().join("model.h5");
|
||||||
write_h5(&local, &[
|
write_h5(
|
||||||
("weights", vec![1.0, 2.0, 3.0, 4.0]),
|
&local,
|
||||||
("biases", vec![0.1, 0.2]),
|
&[
|
||||||
]);
|
("weights", vec![1.0, 2.0, 3.0, 4.0]),
|
||||||
|
("biases", vec![0.1, 0.2]),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
|
||||||
let (mut server, addr, serve_dir) = spawn_server(false);
|
let (mut server, addr, serve_dir) = spawn_server(false);
|
||||||
run_hdf5_sync(&local, addr, "model.h5", false);
|
run_hdf5_sync(&local, addr, "model.h5", false);
|
||||||
@@ -150,19 +153,19 @@ fn hdf5_sync_warm_noop() {
|
|||||||
fn hdf5_sync_incremental_modified_dataset() {
|
fn hdf5_sync_incremental_modified_dataset() {
|
||||||
let src = TempDir::new().unwrap();
|
let src = TempDir::new().unwrap();
|
||||||
let local = src.path().join("model.h5");
|
let local = src.path().join("model.h5");
|
||||||
write_h5(&local, &[
|
write_h5(
|
||||||
("weights", vec![1.0, 2.0]),
|
&local,
|
||||||
("biases", vec![0.5]),
|
&[("weights", vec![1.0, 2.0]), ("biases", vec![0.5])],
|
||||||
]);
|
);
|
||||||
|
|
||||||
let (mut server, addr, serve_dir) = spawn_server(false);
|
let (mut server, addr, serve_dir) = spawn_server(false);
|
||||||
run_hdf5_sync(&local, addr, "model.h5", false);
|
run_hdf5_sync(&local, addr, "model.h5", false);
|
||||||
|
|
||||||
// Modify weights; biases unchanged.
|
// Modify weights; biases unchanged.
|
||||||
write_h5(&local, &[
|
write_h5(
|
||||||
("weights", vec![9.0, 9.0]),
|
&local,
|
||||||
("biases", vec![0.5]),
|
&[("weights", vec![9.0, 9.0]), ("biases", vec![0.5])],
|
||||||
]);
|
);
|
||||||
run_hdf5_sync(&local, addr, "model.h5", false);
|
run_hdf5_sync(&local, addr, "model.h5", false);
|
||||||
server.kill().unwrap();
|
server.kill().unwrap();
|
||||||
server.wait().unwrap();
|
server.wait().unwrap();
|
||||||
@@ -201,7 +204,10 @@ fn hdf5_sync_added_dataset() {
|
|||||||
let synced = serve_dir.path().join("model.h5");
|
let synced = serve_dir.path().join("model.h5");
|
||||||
let m_server = DatasetManifest::from_path(&synced).unwrap();
|
let m_server = DatasetManifest::from_path(&synced).unwrap();
|
||||||
assert!(m_server.get("/alpha").is_some(), "alpha should exist");
|
assert!(m_server.get("/alpha").is_some(), "alpha should exist");
|
||||||
assert!(m_server.get("/beta").is_some(), "beta should have been added");
|
assert!(
|
||||||
|
m_server.get("/beta").is_some(),
|
||||||
|
"beta should have been added"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Delete with `--delete`: dataset absent from client → removed from server.
|
/// Delete with `--delete`: dataset absent from client → removed from server.
|
||||||
@@ -223,7 +229,10 @@ fn hdf5_sync_delete_removes_server_only_dataset() {
|
|||||||
let synced = serve_dir.path().join("model.h5");
|
let synced = serve_dir.path().join("model.h5");
|
||||||
let m_server = DatasetManifest::from_path(&synced).unwrap();
|
let m_server = DatasetManifest::from_path(&synced).unwrap();
|
||||||
assert!(m_server.get("/keep").is_some(), "keep should remain");
|
assert!(m_server.get("/keep").is_some(), "keep should remain");
|
||||||
assert!(m_server.get("/drop").is_none(), "drop should have been removed");
|
assert!(
|
||||||
|
m_server.get("/drop").is_none(),
|
||||||
|
"drop should have been removed"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// No-delete: dataset absent from client is preserved on server when `--delete` not set.
|
/// No-delete: dataset absent from client is preserved on server when `--delete` not set.
|
||||||
@@ -244,8 +253,11 @@ fn hdf5_sync_no_delete_preserves_server_dataset() {
|
|||||||
|
|
||||||
let synced = serve_dir.path().join("model.h5");
|
let synced = serve_dir.path().join("model.h5");
|
||||||
let m_server = DatasetManifest::from_path(&synced).unwrap();
|
let m_server = DatasetManifest::from_path(&synced).unwrap();
|
||||||
assert!(m_server.get("/keep").is_some(), "keep should remain");
|
assert!(m_server.get("/keep").is_some(), "keep should remain");
|
||||||
assert!(m_server.get("/extra").is_some(), "extra should be preserved (no --delete)");
|
assert!(
|
||||||
|
m_server.get("/extra").is_some(),
|
||||||
|
"extra should be preserved (no --delete)"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Nested path: server creates subdirectory as needed.
|
/// Nested path: server creates subdirectory as needed.
|
||||||
|
|||||||
@@ -64,7 +64,10 @@ fn make_large_h5(dir: &TempDir, name: &str, n_pages: usize, fill: u8) -> PathBuf
|
|||||||
let path = dir.path().join(name);
|
let path = dir.path().join(name);
|
||||||
let mut data = Vec::with_capacity(n_pages * PAGE_SIZE);
|
let mut data = Vec::with_capacity(n_pages * PAGE_SIZE);
|
||||||
data.extend_from_slice(H5_MAGIC);
|
data.extend_from_slice(H5_MAGIC);
|
||||||
data.extend(std::iter::repeat_n(fill, n_pages * PAGE_SIZE - H5_MAGIC.len()));
|
data.extend(std::iter::repeat_n(
|
||||||
|
fill,
|
||||||
|
n_pages * PAGE_SIZE - H5_MAGIC.len(),
|
||||||
|
));
|
||||||
std::fs::write(&path, &data).unwrap();
|
std::fs::write(&path, &data).unwrap();
|
||||||
path
|
path
|
||||||
}
|
}
|
||||||
@@ -93,7 +96,10 @@ fn make_versioned_large_h5(
|
|||||||
// Base file: n_pages worth of data
|
// Base file: n_pages worth of data
|
||||||
let mut base = Vec::with_capacity(n_pages * PAGE_SIZE);
|
let mut base = Vec::with_capacity(n_pages * PAGE_SIZE);
|
||||||
base.extend_from_slice(H5_MAGIC);
|
base.extend_from_slice(H5_MAGIC);
|
||||||
base.extend(std::iter::repeat_n(0u8, n_pages * PAGE_SIZE - H5_MAGIC.len()));
|
base.extend(std::iter::repeat_n(
|
||||||
|
0u8,
|
||||||
|
n_pages * PAGE_SIZE - H5_MAGIC.len(),
|
||||||
|
));
|
||||||
std::fs::write(&path, &base).unwrap();
|
std::fs::write(&path, &base).unwrap();
|
||||||
|
|
||||||
let mut onion = OnionFile::create(&path, PAGE_SIZE as u32).unwrap();
|
let mut onion = OnionFile::create(&path, PAGE_SIZE as u32).unwrap();
|
||||||
@@ -104,7 +110,9 @@ fn make_versioned_large_h5(
|
|||||||
let page_idx = (rev * pages_per_rev + p) % n_pages;
|
let page_idx = (rev * pages_per_rev + p) % n_pages;
|
||||||
s.record_page((page_idx * PAGE_SIZE) as u64, &vec![fill; PAGE_SIZE]);
|
s.record_page((page_idx * PAGE_SIZE) as u64, &vec![fill; PAGE_SIZE]);
|
||||||
}
|
}
|
||||||
onion.commit_session(s, Some(&format!("rev {rev}"))).unwrap();
|
onion
|
||||||
|
.commit_session(s, Some(&format!("rev {rev}")))
|
||||||
|
.unwrap();
|
||||||
}
|
}
|
||||||
onion.flush().unwrap();
|
onion.flush().unwrap();
|
||||||
path
|
path
|
||||||
@@ -165,7 +173,10 @@ fn run_rsync(src: &Path, dst: &Path) -> RsyncStats {
|
|||||||
assert!(output.status.success(), "rsync failed: {}", output.status);
|
assert!(output.status.success(), "rsync failed: {}", output.status);
|
||||||
|
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
let mut stats = RsyncStats { elapsed, ..Default::default() };
|
let mut stats = RsyncStats {
|
||||||
|
elapsed,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
for line in stdout.lines() {
|
for line in stdout.lines() {
|
||||||
let line = line.trim();
|
let line = line.trim();
|
||||||
@@ -227,8 +238,8 @@ fn start_clawsync_server(h5_path: &Path) -> (std::process::Child, String) {
|
|||||||
let mut line = String::new();
|
let mut line = String::new();
|
||||||
reader.read_line(&mut line).expect("server startup line");
|
reader.read_line(&mut line).expect("server startup line");
|
||||||
|
|
||||||
let addr = parse_serve_addr(&line)
|
let addr =
|
||||||
.unwrap_or_else(|| panic!("could not parse addr from: {line:?}"));
|
parse_serve_addr(&line).unwrap_or_else(|| panic!("could not parse addr from: {line:?}"));
|
||||||
|
|
||||||
// Drain remaining stdout so server doesn't get SIGPIPE.
|
// Drain remaining stdout so server doesn't get SIGPIPE.
|
||||||
std::thread::spawn(move || {
|
std::thread::spawn(move || {
|
||||||
@@ -255,11 +266,17 @@ fn run_clawsync_push(src_h5: &Path, dst_h5: &Path) -> ClawStats {
|
|||||||
server.kill().ok();
|
server.kill().ok();
|
||||||
server.wait().ok();
|
server.wait().ok();
|
||||||
|
|
||||||
assert!(output.status.success(), "clawsync push failed: {}",
|
assert!(
|
||||||
String::from_utf8_lossy(&output.stderr));
|
output.status.success(),
|
||||||
|
"clawsync push failed: {}",
|
||||||
|
String::from_utf8_lossy(&output.stderr)
|
||||||
|
);
|
||||||
|
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
let mut stats = ClawStats { elapsed, ..Default::default() };
|
let mut stats = ClawStats {
|
||||||
|
elapsed,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
// "Push complete: N revision(s), M bytes"
|
// "Push complete: N revision(s), M bytes"
|
||||||
if let Some(line) = stdout.lines().find(|l| l.starts_with("Push complete:")) {
|
if let Some(line) = stdout.lines().find(|l| l.starts_with("Push complete:")) {
|
||||||
@@ -341,7 +358,7 @@ fn clawsync_vs_rsync_comparison() {
|
|||||||
|
|
||||||
let src = make_large_h5(&src_dir, "src.h5", n_pages, 0xAB);
|
let src = make_large_h5(&src_dir, "src.h5", n_pages, 0xAB);
|
||||||
let dst_rsync = dst_dir.path().join("dst_rsync.h5");
|
let dst_rsync = dst_dir.path().join("dst_rsync.h5");
|
||||||
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 0, 0);
|
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 0, 0);
|
||||||
// src as clawsync source: 1 revision with all pages
|
// src as clawsync source: 1 revision with all pages
|
||||||
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, 1, n_pages);
|
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, 1, n_pages);
|
||||||
|
|
||||||
@@ -387,12 +404,18 @@ fn clawsync_vs_rsync_comparison() {
|
|||||||
|
|
||||||
// clawsync: server has 1 revision, source adds 1 more (1 page)
|
// clawsync: server has 1 revision, source adds 1 more (1 page)
|
||||||
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, 1, n_pages);
|
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, 1, n_pages);
|
||||||
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 1, n_pages);
|
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 1, n_pages);
|
||||||
add_revision(&src_claw, 1, 0xFF); // 1 new page in new revision
|
add_revision(&src_claw, 1, 0xFF); // 1 new page in new revision
|
||||||
|
|
||||||
let rs = run_rsync(&src, &dst_rsync);
|
let rs = run_rsync(&src, &dst_rsync);
|
||||||
let cs = run_clawsync_push(&src_claw, &dst_claw);
|
let cs = run_clawsync_push(&src_claw, &dst_claw);
|
||||||
print_row("incremental, 1 page changed (0.4%)", &rs, &cs, n_pages, total);
|
print_row(
|
||||||
|
"incremental, 1 page changed (0.4%)",
|
||||||
|
&rs,
|
||||||
|
&cs,
|
||||||
|
n_pages,
|
||||||
|
total,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── 4. Incremental: 25 pages changed (10%) ─────────────────────────────
|
// ── 4. Incremental: 25 pages changed (10%) ─────────────────────────────
|
||||||
@@ -409,7 +432,7 @@ fn clawsync_vs_rsync_comparison() {
|
|||||||
overwrite_pages(&src, 0, changed, 0xFF);
|
overwrite_pages(&src, 0, changed, 0xFF);
|
||||||
|
|
||||||
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, 1, n_pages);
|
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, 1, n_pages);
|
||||||
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 1, n_pages);
|
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 1, n_pages);
|
||||||
add_revision(&src_claw, changed, 0xFF);
|
add_revision(&src_claw, changed, 0xFF);
|
||||||
|
|
||||||
let rs = run_rsync(&src, &dst_rsync);
|
let rs = run_rsync(&src, &dst_rsync);
|
||||||
@@ -430,7 +453,7 @@ fn clawsync_vs_rsync_comparison() {
|
|||||||
overwrite_pages(&src, 512, 1, 0xFF);
|
overwrite_pages(&src, 512, 1, 0xFF);
|
||||||
|
|
||||||
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, 1, n_pages);
|
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, 1, n_pages);
|
||||||
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 1, n_pages);
|
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 1, n_pages);
|
||||||
add_revision(&src_claw, 1, 0xFF);
|
add_revision(&src_claw, 1, 0xFF);
|
||||||
|
|
||||||
let rs = run_rsync(&src, &dst_rsync);
|
let rs = run_rsync(&src, &dst_rsync);
|
||||||
@@ -453,12 +476,15 @@ fn clawsync_vs_rsync_comparison() {
|
|||||||
|
|
||||||
// clawsync: transfers all 10 revisions with full history
|
// clawsync: transfers all 10 revisions with full history
|
||||||
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, n_revs, 1);
|
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, n_revs, 1);
|
||||||
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 0, 0);
|
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 0, 0);
|
||||||
let cs = run_clawsync_push(&src_claw, &dst_claw);
|
let cs = run_clawsync_push(&src_claw, &dst_claw);
|
||||||
|
|
||||||
print_row(
|
print_row(
|
||||||
&format!("history: {n_revs} revisions (1 page/rev)"),
|
&format!("history: {n_revs} revisions (1 page/rev)"),
|
||||||
&rs, &cs, n_pages, total,
|
&rs,
|
||||||
|
&cs,
|
||||||
|
n_pages,
|
||||||
|
total,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -475,12 +501,15 @@ fn clawsync_vs_rsync_comparison() {
|
|||||||
let rs = run_rsync(&src_final, &dst_rsync);
|
let rs = run_rsync(&src_final, &dst_rsync);
|
||||||
|
|
||||||
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, n_revs, 1);
|
let src_claw = make_versioned_large_h5(&src_dir, "src_claw.h5", n_pages, n_revs, 1);
|
||||||
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 0, 0);
|
let dst_claw = make_versioned_large_h5(&dst_dir, "dst_claw.h5", n_pages, 0, 0);
|
||||||
let cs = run_clawsync_push(&src_claw, &dst_claw);
|
let cs = run_clawsync_push(&src_claw, &dst_claw);
|
||||||
|
|
||||||
print_row(
|
print_row(
|
||||||
&format!("history: {n_revs} revisions (1 page/rev)"),
|
&format!("history: {n_revs} revisions (1 page/rev)"),
|
||||||
&rs, &cs, n_pages, total,
|
&rs,
|
||||||
|
&cs,
|
||||||
|
n_pages,
|
||||||
|
total,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -528,7 +557,10 @@ fn make_large_file_small_sidecar(
|
|||||||
let path = dir.path().join(name);
|
let path = dir.path().join(name);
|
||||||
let mut data = Vec::with_capacity(n_pages * PAGE_SIZE);
|
let mut data = Vec::with_capacity(n_pages * PAGE_SIZE);
|
||||||
data.extend_from_slice(H5_MAGIC);
|
data.extend_from_slice(H5_MAGIC);
|
||||||
data.extend(std::iter::repeat_n(fill, n_pages * PAGE_SIZE - H5_MAGIC.len()));
|
data.extend(std::iter::repeat_n(
|
||||||
|
fill,
|
||||||
|
n_pages * PAGE_SIZE - H5_MAGIC.len(),
|
||||||
|
));
|
||||||
std::fs::write(&path, &data).unwrap();
|
std::fs::write(&path, &data).unwrap();
|
||||||
|
|
||||||
let mut onion = OnionFile::create(&path, PAGE_SIZE as u32).unwrap();
|
let mut onion = OnionFile::create(&path, PAGE_SIZE as u32).unwrap();
|
||||||
@@ -536,7 +568,9 @@ fn make_large_file_small_sidecar(
|
|||||||
let mut s = onion.begin_session(None).unwrap();
|
let mut s = onion.begin_session(None).unwrap();
|
||||||
// Only record page 0 — keeps the sidecar tiny regardless of file size.
|
// Only record page 0 — keeps the sidecar tiny regardless of file size.
|
||||||
s.record_page(0, &vec![(rev as u8).wrapping_add(1); PAGE_SIZE]);
|
s.record_page(0, &vec![(rev as u8).wrapping_add(1); PAGE_SIZE]);
|
||||||
onion.commit_session(s, Some(&format!("rev {rev}"))).unwrap();
|
onion
|
||||||
|
.commit_session(s, Some(&format!("rev {rev}")))
|
||||||
|
.unwrap();
|
||||||
}
|
}
|
||||||
onion.flush().unwrap();
|
onion.flush().unwrap();
|
||||||
path
|
path
|
||||||
@@ -564,13 +598,16 @@ fn clawsync_preflight_scaling() {
|
|||||||
"| {:<20} | {:>10} | {:>13} | {:>13} | {:>9} |",
|
"| {:<20} | {:>10} | {:>13} | {:>13} | {:>9} |",
|
||||||
"File size", "Revisions", "rsync no-op", "claw no-op", "Speedup"
|
"File size", "Revisions", "rsync no-op", "claw no-op", "Speedup"
|
||||||
);
|
);
|
||||||
println!("|{:-<22}|{:-<12}|{:-<15}|{:-<15}|{:-<11}|", "", "", "", "", "");
|
println!(
|
||||||
|
"|{:-<22}|{:-<12}|{:-<15}|{:-<15}|{:-<11}|",
|
||||||
|
"", "", "", "", ""
|
||||||
|
);
|
||||||
|
|
||||||
// (n_pages, n_revisions): keep n_revisions small (1) so IBLT sketch is constant.
|
// (n_pages, n_revisions): keep n_revisions small (1) so IBLT sketch is constant.
|
||||||
// Only n_pages grows — that's what drives rsync cost.
|
// Only n_pages grows — that's what drives rsync cost.
|
||||||
let cases: &[(usize, usize, &str)] = &[
|
let cases: &[(usize, usize, &str)] = &[
|
||||||
(64, 1, " 256 KiB"),
|
(64, 1, " 256 KiB"),
|
||||||
(256, 1, " 1 MiB"),
|
(256, 1, " 1 MiB"),
|
||||||
(1024, 1, " 4 MiB"),
|
(1024, 1, " 4 MiB"),
|
||||||
(4096, 1, " 16 MiB"),
|
(4096, 1, " 16 MiB"),
|
||||||
];
|
];
|
||||||
@@ -584,7 +621,10 @@ fn clawsync_preflight_scaling() {
|
|||||||
{
|
{
|
||||||
let mut data = Vec::with_capacity(n_pages * PAGE_SIZE);
|
let mut data = Vec::with_capacity(n_pages * PAGE_SIZE);
|
||||||
data.extend_from_slice(H5_MAGIC);
|
data.extend_from_slice(H5_MAGIC);
|
||||||
data.extend(std::iter::repeat_n(0xABu8, n_pages * PAGE_SIZE - H5_MAGIC.len()));
|
data.extend(std::iter::repeat_n(
|
||||||
|
0xABu8,
|
||||||
|
n_pages * PAGE_SIZE - H5_MAGIC.len(),
|
||||||
|
));
|
||||||
std::fs::write(&src_h5, &data).unwrap();
|
std::fs::write(&src_h5, &data).unwrap();
|
||||||
std::fs::write(&dst_h5, &data).unwrap();
|
std::fs::write(&dst_h5, &data).unwrap();
|
||||||
}
|
}
|
||||||
@@ -671,7 +711,11 @@ fn clawsync_unique_features() {
|
|||||||
.lines()
|
.lines()
|
||||||
.filter(|l| l.contains("experiment rev"))
|
.filter(|l| l.contains("experiment rev"))
|
||||||
.collect();
|
.collect();
|
||||||
assert_eq!(rev_lines.len(), 5, "expected 5 revision lines, got: {revs_output}");
|
assert_eq!(
|
||||||
|
rev_lines.len(),
|
||||||
|
5,
|
||||||
|
"expected 5 revision lines, got: {revs_output}"
|
||||||
|
);
|
||||||
println!("rsync equivalent: **none** (rsync tracks no history)");
|
println!("rsync equivalent: **none** (rsync tracks no history)");
|
||||||
println!();
|
println!();
|
||||||
|
|
||||||
@@ -697,7 +741,10 @@ fn clawsync_unique_features() {
|
|||||||
println!("Exported revision 2 → {} bytes", export_bytes);
|
println!("Exported revision 2 → {} bytes", export_bytes);
|
||||||
// Exported file starts with HDF5 magic.
|
// Exported file starts with HDF5 magic.
|
||||||
let exported = std::fs::read(&export_path).unwrap();
|
let exported = std::fs::read(&export_path).unwrap();
|
||||||
assert!(exported.starts_with(H5_MAGIC), "exported file missing HDF5 magic");
|
assert!(
|
||||||
|
exported.starts_with(H5_MAGIC),
|
||||||
|
"exported file missing HDF5 magic"
|
||||||
|
);
|
||||||
// Page 0 in revision 2 was filled with 2*10+1=21.
|
// Page 0 in revision 2 was filled with 2*10+1=21.
|
||||||
let page0_byte = exported[PAGE_SIZE]; // skip magic, first full page
|
let page0_byte = exported[PAGE_SIZE]; // skip magic, first full page
|
||||||
assert_eq!(page0_byte, 21u8, "exported revision 2 page0 byte mismatch");
|
assert_eq!(page0_byte, 21u8, "exported revision 2 page0 byte mismatch");
|
||||||
@@ -714,12 +761,18 @@ fn clawsync_unique_features() {
|
|||||||
.args(["rollback", h5.to_str().unwrap(), "4"])
|
.args(["rollback", h5.to_str().unwrap(), "4"])
|
||||||
.output()
|
.output()
|
||||||
.expect("rollback to head failed");
|
.expect("rollback to head failed");
|
||||||
assert!(out.status.success(), "rollback to head failed: {}",
|
assert!(
|
||||||
String::from_utf8_lossy(&out.stderr));
|
out.status.success(),
|
||||||
|
"rollback to head failed: {}",
|
||||||
|
String::from_utf8_lossy(&out.stderr)
|
||||||
|
);
|
||||||
let head_bytes = std::fs::read(&h5).unwrap();
|
let head_bytes = std::fs::read(&h5).unwrap();
|
||||||
let head_byte = head_bytes[PAGE_SIZE];
|
let head_byte = head_bytes[PAGE_SIZE];
|
||||||
// Rev 4: page filled with 4*10+1=41
|
// Rev 4: page filled with 4*10+1=41
|
||||||
assert_eq!(head_byte, 41u8, "HEAD page byte should be 41 at rev 4, got {head_byte}");
|
assert_eq!(
|
||||||
|
head_byte, 41u8,
|
||||||
|
"HEAD page byte should be 41 at rev 4, got {head_byte}"
|
||||||
|
);
|
||||||
println!("HEAD (rev 4) data-page byte = {} ✓", head_byte);
|
println!("HEAD (rev 4) data-page byte = {} ✓", head_byte);
|
||||||
|
|
||||||
let out = Command::new(BIN)
|
let out = Command::new(BIN)
|
||||||
@@ -735,8 +788,14 @@ fn clawsync_unique_features() {
|
|||||||
let rolled_bytes = std::fs::read(&h5).unwrap();
|
let rolled_bytes = std::fs::read(&h5).unwrap();
|
||||||
let rolled_byte = rolled_bytes[PAGE_SIZE];
|
let rolled_byte = rolled_bytes[PAGE_SIZE];
|
||||||
// After rollback to rev 1: page filled with 1*10+1=11
|
// After rollback to rev 1: page filled with 1*10+1=11
|
||||||
assert_eq!(rolled_byte, 11u8, "after rollback to rev 1, data-page byte should be 11, got {rolled_byte}");
|
assert_eq!(
|
||||||
println!("After rollback to rev 1: data-page byte = {} ✓ (expected 11)", rolled_byte);
|
rolled_byte, 11u8,
|
||||||
|
"after rollback to rev 1, data-page byte should be 11, got {rolled_byte}"
|
||||||
|
);
|
||||||
|
println!(
|
||||||
|
"After rollback to rev 1: data-page byte = {} ✓ (expected 11)",
|
||||||
|
rolled_byte
|
||||||
|
);
|
||||||
println!("rsync equivalent: **none** (rsync cannot revert to a past state)");
|
println!("rsync equivalent: **none** (rsync cannot revert to a past state)");
|
||||||
println!();
|
println!();
|
||||||
|
|
||||||
|
|||||||
@@ -93,12 +93,7 @@ fn parse_serve_addr(line: &str) -> Option<String> {
|
|||||||
/// pipe error when it tries to print subsequent log lines.
|
/// pipe error when it tries to print subsequent log lines.
|
||||||
fn start_server(h5_path: &PathBuf) -> (std::process::Child, String) {
|
fn start_server(h5_path: &PathBuf) -> (std::process::Child, String) {
|
||||||
let mut server = Command::new(BIN)
|
let mut server = Command::new(BIN)
|
||||||
.args([
|
.args(["serve", h5_path.to_str().unwrap(), "--bind", "127.0.0.1:0"])
|
||||||
"serve",
|
|
||||||
h5_path.to_str().unwrap(),
|
|
||||||
"--bind",
|
|
||||||
"127.0.0.1:0",
|
|
||||||
])
|
|
||||||
.stdout(Stdio::piped())
|
.stdout(Stdio::piped())
|
||||||
.stderr(Stdio::null())
|
.stderr(Stdio::null())
|
||||||
.spawn()
|
.spawn()
|
||||||
@@ -213,7 +208,8 @@ fn subprocess_pull_fetches_all_revisions_from_server() {
|
|||||||
|
|
||||||
let reloaded = OnionFile::open(&client_h5).expect("cannot open client onion after pull");
|
let reloaded = OnionFile::open(&client_h5).expect("cannot open client onion after pull");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
reloaded.revision_count(), 4,
|
reloaded.revision_count(),
|
||||||
|
4,
|
||||||
"client should have all 4 pulled revisions"
|
"client should have all 4 pulled revisions"
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -231,7 +227,11 @@ fn subprocess_list_revisions_output_is_correct() {
|
|||||||
.output()
|
.output()
|
||||||
.expect("failed to run clawsync list-revisions");
|
.expect("failed to run clawsync list-revisions");
|
||||||
|
|
||||||
assert!(output.status.success(), "list-revisions exited with {}", output.status);
|
assert!(
|
||||||
|
output.status.success(),
|
||||||
|
"list-revisions exited with {}",
|
||||||
|
output.status
|
||||||
|
);
|
||||||
|
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
assert!(
|
assert!(
|
||||||
@@ -255,7 +255,11 @@ fn subprocess_list_revisions_timestamp_is_iso_date() {
|
|||||||
.output()
|
.output()
|
||||||
.expect("failed to run clawsync list-revisions");
|
.expect("failed to run clawsync list-revisions");
|
||||||
|
|
||||||
assert!(output.status.success(), "list-revisions exited with {}", output.status);
|
assert!(
|
||||||
|
output.status.success(),
|
||||||
|
"list-revisions exited with {}",
|
||||||
|
output.status
|
||||||
|
);
|
||||||
|
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
assert!(
|
assert!(
|
||||||
@@ -311,7 +315,11 @@ fn subprocess_rollback_restores_h5_content_and_preserves_history() {
|
|||||||
|
|
||||||
// Sidecar history must be fully preserved — all 4 revisions still readable.
|
// Sidecar history must be fully preserved — all 4 revisions still readable.
|
||||||
let onion = OnionFile::open(&h5).expect("cannot open onion after rollback");
|
let onion = OnionFile::open(&h5).expect("cannot open onion after rollback");
|
||||||
assert_eq!(onion.revision_count(), 4, "rollback must not delete sidecar history");
|
assert_eq!(
|
||||||
|
onion.revision_count(),
|
||||||
|
4,
|
||||||
|
"rollback must not delete sidecar history"
|
||||||
|
);
|
||||||
|
|
||||||
// The h5 file must reflect the page content at revision 1.
|
// The h5 file must reflect the page content at revision 1.
|
||||||
// make_versioned_h5 writes page 0 = vec![i; 4096] for each revision i,
|
// make_versioned_h5 writes page 0 = vec![i; 4096] for each revision i,
|
||||||
@@ -340,7 +348,11 @@ fn subprocess_gc_keep_last_prunes_old_revisions() {
|
|||||||
assert!(status.success(), "gc exited with {status}");
|
assert!(status.success(), "gc exited with {status}");
|
||||||
|
|
||||||
let onion = OnionFile::open(&h5).expect("cannot open onion after gc");
|
let onion = OnionFile::open(&h5).expect("cannot open onion after gc");
|
||||||
assert_eq!(onion.revision_count(), 2, "gc --keep-last 2 should leave exactly 2 revisions");
|
assert_eq!(
|
||||||
|
onion.revision_count(),
|
||||||
|
2,
|
||||||
|
"gc --keep-last 2 should leave exactly 2 revisions"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `clawsync export-revision`: exports a specific revision to a new file
|
/// `clawsync export-revision`: exports a specific revision to a new file
|
||||||
@@ -369,7 +381,11 @@ fn subprocess_export_revision_creates_standalone_file() {
|
|||||||
|
|
||||||
// Source sidecar must be unchanged — still 3 revisions.
|
// Source sidecar must be unchanged — still 3 revisions.
|
||||||
let onion = OnionFile::open(&h5).expect("cannot open source onion after export");
|
let onion = OnionFile::open(&h5).expect("cannot open source onion after export");
|
||||||
assert_eq!(onion.revision_count(), 3, "source sidecar must not be modified by export");
|
assert_eq!(
|
||||||
|
onion.revision_count(),
|
||||||
|
3,
|
||||||
|
"source sidecar must not be modified by export"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `clawsync snapshot`: creates a snapshot checkpoint revision, incrementing
|
/// `clawsync snapshot`: creates a snapshot checkpoint revision, incrementing
|
||||||
@@ -379,13 +395,22 @@ fn subprocess_snapshot_increments_revision_count() {
|
|||||||
let (_dir, h5, _) = make_versioned_h5(3);
|
let (_dir, h5, _) = make_versioned_h5(3);
|
||||||
|
|
||||||
let status = Command::new(BIN)
|
let status = Command::new(BIN)
|
||||||
.args(["snapshot", h5.to_str().unwrap(), "--message", "test-snapshot"])
|
.args([
|
||||||
|
"snapshot",
|
||||||
|
h5.to_str().unwrap(),
|
||||||
|
"--message",
|
||||||
|
"test-snapshot",
|
||||||
|
])
|
||||||
.status()
|
.status()
|
||||||
.expect("failed to run clawsync snapshot");
|
.expect("failed to run clawsync snapshot");
|
||||||
assert!(status.success(), "snapshot exited with {status}");
|
assert!(status.success(), "snapshot exited with {status}");
|
||||||
|
|
||||||
let onion = OnionFile::open(&h5).expect("cannot open onion after snapshot");
|
let onion = OnionFile::open(&h5).expect("cannot open onion after snapshot");
|
||||||
assert_eq!(onion.revision_count(), 4, "snapshot should add exactly one revision");
|
assert_eq!(
|
||||||
|
onion.revision_count(),
|
||||||
|
4,
|
||||||
|
"snapshot should add exactly one revision"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `clawsync branch create / list / rename / delete`: full lifecycle in order.
|
/// `clawsync branch create / list / rename / delete`: full lifecycle in order.
|
||||||
@@ -406,7 +431,11 @@ fn subprocess_branch_lifecycle() {
|
|||||||
.args(["branch", "list", h5_str])
|
.args(["branch", "list", h5_str])
|
||||||
.output()
|
.output()
|
||||||
.expect("failed to run branch list");
|
.expect("failed to run branch list");
|
||||||
assert!(output.status.success(), "branch list exited with {}", output.status);
|
assert!(
|
||||||
|
output.status.success(),
|
||||||
|
"branch list exited with {}",
|
||||||
|
output.status
|
||||||
|
);
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
assert!(
|
assert!(
|
||||||
stdout.contains("experiment"),
|
stdout.contains("experiment"),
|
||||||
@@ -427,8 +456,14 @@ fn subprocess_branch_lifecycle() {
|
|||||||
.expect("failed to run branch list after rename");
|
.expect("failed to run branch list after rename");
|
||||||
assert!(output.status.success());
|
assert!(output.status.success());
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
assert!(stdout.contains("feature-x"), "expected 'feature-x' after rename:\n{stdout}");
|
assert!(
|
||||||
assert!(!stdout.contains("experiment"), "old name 'experiment' must be gone:\n{stdout}");
|
stdout.contains("feature-x"),
|
||||||
|
"expected 'feature-x' after rename:\n{stdout}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!stdout.contains("experiment"),
|
||||||
|
"old name 'experiment' must be gone:\n{stdout}"
|
||||||
|
);
|
||||||
|
|
||||||
// Delete "feature-x".
|
// Delete "feature-x".
|
||||||
let status = Command::new(BIN)
|
let status = Command::new(BIN)
|
||||||
@@ -444,7 +479,10 @@ fn subprocess_branch_lifecycle() {
|
|||||||
.expect("failed to run branch list after delete");
|
.expect("failed to run branch list after delete");
|
||||||
assert!(output.status.success());
|
assert!(output.status.success());
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
assert!(!stdout.contains("feature-x"), "deleted branch must not appear in list:\n{stdout}");
|
assert!(
|
||||||
|
!stdout.contains("feature-x"),
|
||||||
|
"deleted branch must not appear in list:\n{stdout}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -483,13 +521,20 @@ fn subprocess_push_delta_transfers_exact_count() {
|
|||||||
.output()
|
.output()
|
||||||
.expect("failed to run clawsync push");
|
.expect("failed to run clawsync push");
|
||||||
|
|
||||||
assert!(output.status.success(), "push exited with {}", output.status);
|
assert!(
|
||||||
|
output.status.success(),
|
||||||
|
"push exited with {}",
|
||||||
|
output.status
|
||||||
|
);
|
||||||
|
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
let transferred = parse_push_revision_count(&stdout)
|
let transferred = parse_push_revision_count(&stdout)
|
||||||
.unwrap_or_else(|| panic!("could not parse push count from:\n{stdout}"));
|
.unwrap_or_else(|| panic!("could not parse push count from:\n{stdout}"));
|
||||||
|
|
||||||
assert_eq!(transferred, 4, "expected exactly 4 delta revisions, got {transferred}");
|
assert_eq!(
|
||||||
|
transferred, 4,
|
||||||
|
"expected exactly 4 delta revisions, got {transferred}"
|
||||||
|
);
|
||||||
|
|
||||||
// Server must eventually reach 7 revisions total.
|
// Server must eventually reach 7 revisions total.
|
||||||
let count = await_revisions(&server_h5, 7);
|
let count = await_revisions(&server_h5, 7);
|
||||||
@@ -515,7 +560,9 @@ fn subprocess_pull_delta_transfers_exact_count() {
|
|||||||
for i in 0u8..2 {
|
for i in 0u8..2 {
|
||||||
let mut s = onion.begin_session(None).unwrap();
|
let mut s = onion.begin_session(None).unwrap();
|
||||||
s.record_page(0, &vec![i; 4096]);
|
s.record_page(0, &vec![i; 4096]);
|
||||||
onion.commit_session(s, Some(&format!("local {i}"))).unwrap();
|
onion
|
||||||
|
.commit_session(s, Some(&format!("local {i}")))
|
||||||
|
.unwrap();
|
||||||
}
|
}
|
||||||
onion.flush().unwrap();
|
onion.flush().unwrap();
|
||||||
}
|
}
|
||||||
@@ -527,20 +574,31 @@ fn subprocess_pull_delta_transfers_exact_count() {
|
|||||||
.output()
|
.output()
|
||||||
.expect("failed to run clawsync pull");
|
.expect("failed to run clawsync pull");
|
||||||
|
|
||||||
assert!(output.status.success(), "pull exited with {}", output.status);
|
assert!(
|
||||||
|
output.status.success(),
|
||||||
|
"pull exited with {}",
|
||||||
|
output.status
|
||||||
|
);
|
||||||
|
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
let received = parse_pull_revision_count(&stdout)
|
let received = parse_pull_revision_count(&stdout)
|
||||||
.unwrap_or_else(|| panic!("could not parse pull count from:\n{stdout}"));
|
.unwrap_or_else(|| panic!("could not parse pull count from:\n{stdout}"));
|
||||||
|
|
||||||
assert_eq!(received, 4, "expected exactly 4 delta revisions, got {received}");
|
assert_eq!(
|
||||||
|
received, 4,
|
||||||
|
"expected exactly 4 delta revisions, got {received}"
|
||||||
|
);
|
||||||
|
|
||||||
server.kill().ok();
|
server.kill().ok();
|
||||||
server.wait().ok();
|
server.wait().ok();
|
||||||
|
|
||||||
// Client must end up with 6 revisions (2 original + 4 pulled).
|
// Client must end up with 6 revisions (2 original + 4 pulled).
|
||||||
let onion = OnionFile::open(&client_h5).expect("cannot open client onion after pull");
|
let onion = OnionFile::open(&client_h5).expect("cannot open client onion after pull");
|
||||||
assert_eq!(onion.revision_count(), 6, "client should have 6 revisions after delta pull");
|
assert_eq!(
|
||||||
|
onion.revision_count(),
|
||||||
|
6,
|
||||||
|
"client should have 6 revisions after delta pull"
|
||||||
|
);
|
||||||
|
|
||||||
let _ = (server_dir, client_dir);
|
let _ = (server_dir, client_dir);
|
||||||
}
|
}
|
||||||
@@ -571,18 +629,28 @@ fn subprocess_push_large_delta_correct_count() {
|
|||||||
.output()
|
.output()
|
||||||
.expect("failed to run clawsync push");
|
.expect("failed to run clawsync push");
|
||||||
|
|
||||||
assert!(output.status.success(), "push exited with {}", output.status);
|
assert!(
|
||||||
|
output.status.success(),
|
||||||
|
"push exited with {}",
|
||||||
|
output.status
|
||||||
|
);
|
||||||
|
|
||||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
let transferred = parse_push_revision_count(&stdout)
|
let transferred = parse_push_revision_count(&stdout)
|
||||||
.unwrap_or_else(|| panic!("could not parse push count from:\n{stdout}"));
|
.unwrap_or_else(|| panic!("could not parse push count from:\n{stdout}"));
|
||||||
|
|
||||||
assert_eq!(transferred, 40, "expected exactly 40 delta revisions, got {transferred}");
|
assert_eq!(
|
||||||
|
transferred, 40,
|
||||||
|
"expected exactly 40 delta revisions, got {transferred}"
|
||||||
|
);
|
||||||
|
|
||||||
let count = await_revisions(&server_h5, 50);
|
let count = await_revisions(&server_h5, 50);
|
||||||
|
|
||||||
server.kill().ok();
|
server.kill().ok();
|
||||||
server.wait().ok();
|
server.wait().ok();
|
||||||
|
|
||||||
assert_eq!(count, 50, "server should have 50 revisions after large delta push");
|
assert_eq!(
|
||||||
|
count, 50,
|
||||||
|
"server should have 50 revisions after large delta push"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ fn float_data(n: usize) -> Vec<u8> {
|
|||||||
let mant0 = (i.wrapping_mul(2654435769) >> 24) as u8;
|
let mant0 = (i.wrapping_mul(2654435769) >> 24) as u8;
|
||||||
let mant1 = (i.wrapping_mul(0x811c9dc5) >> 16) as u8;
|
let mant1 = (i.wrapping_mul(0x811c9dc5) >> 16) as u8;
|
||||||
let mant2 = (i.wrapping_mul(0x01000193)) as u8;
|
let mant2 = (i.wrapping_mul(0x01000193)) as u8;
|
||||||
v.push(mant2); // byte 0: low mantissa (often < 64 → hot)
|
v.push(mant2); // byte 0: low mantissa (often < 64 → hot)
|
||||||
v.push(mant1);
|
v.push(mant1);
|
||||||
v.push(mant0);
|
v.push(mant0);
|
||||||
v.push(exp_byte); // byte 3: exponent (often 0x3F-0x45 ≥ 64 → cold)
|
v.push(exp_byte); // byte 3: exponent (often 0x3F-0x45 ≥ 64 → cold)
|
||||||
@@ -60,8 +60,8 @@ fn cold_data(n: usize) -> Vec<u8> {
|
|||||||
|
|
||||||
fn bench_cdc(c: &mut Criterion) {
|
fn bench_cdc(c: &mut Criterion) {
|
||||||
let datasets: &[(&str, fn(usize) -> Vec<u8>)] = &[
|
let datasets: &[(&str, fn(usize) -> Vec<u8>)] = &[
|
||||||
("random", random_data),
|
("random", random_data),
|
||||||
("float", float_data),
|
("float", float_data),
|
||||||
("cold_only", cold_data),
|
("cold_only", cold_data),
|
||||||
];
|
];
|
||||||
let sizes = [1 << 20, 4 << 20, 16 << 20]; // 1 MB, 4 MB, 16 MB
|
let sizes = [1 << 20, 4 << 20, 16 << 20]; // 1 MB, 4 MB, 16 MB
|
||||||
|
|||||||
@@ -35,12 +35,7 @@ pub fn chunk_data(data: &[u8]) -> Vec<Chunk> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Split `data` into content-defined chunks with explicit size parameters.
|
/// Split `data` into content-defined chunks with explicit size parameters.
|
||||||
pub fn chunk_data_sized(
|
pub fn chunk_data_sized(data: &[u8], min_size: u32, avg_size: u32, max_size: u32) -> Vec<Chunk> {
|
||||||
data: &[u8],
|
|
||||||
min_size: u32,
|
|
||||||
avg_size: u32,
|
|
||||||
max_size: u32,
|
|
||||||
) -> Vec<Chunk> {
|
|
||||||
if data.is_empty() {
|
if data.is_empty() {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
}
|
}
|
||||||
@@ -152,8 +147,14 @@ mod tests {
|
|||||||
for (i, c) in chunks.iter().enumerate() {
|
for (i, c) in chunks.iter().enumerate() {
|
||||||
let is_last = i == chunks.len() - 1;
|
let is_last = i == chunks.len() - 1;
|
||||||
if !is_last {
|
if !is_last {
|
||||||
assert!(c.length >= MIN_CHUNK_SIZE as usize, "chunk below min at {i}");
|
assert!(
|
||||||
assert!(c.length <= MAX_CHUNK_SIZE as usize, "chunk above max at {i}");
|
c.length >= MIN_CHUNK_SIZE as usize,
|
||||||
|
"chunk below min at {i}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
c.length <= MAX_CHUNK_SIZE as usize,
|
||||||
|
"chunk above max at {i}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -189,7 +190,10 @@ mod tests {
|
|||||||
let chunks = chunk_data(&data);
|
let chunks = chunk_data(&data);
|
||||||
let source_set = chunk_hash_set(&chunks);
|
let source_set = chunk_hash_set(&chunks);
|
||||||
let diff = changed_chunks(&source_set, &chunks);
|
let diff = changed_chunks(&source_set, &chunks);
|
||||||
assert!(diff.is_empty(), "identical data should have no changed chunks");
|
assert!(
|
||||||
|
diff.is_empty(),
|
||||||
|
"identical data should have no changed chunks"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -63,10 +63,7 @@ pub fn blake3_hash_large(data: &[u8]) -> [u8; 32] {
|
|||||||
/// order and page offsets must be incorporated (use `hash_pages` instead).
|
/// order and page offsets must be incorporated (use `hash_pages` instead).
|
||||||
pub fn blake3_hash_batch(chunks: &[&[u8]]) -> Vec<[u8; 32]> {
|
pub fn blake3_hash_batch(chunks: &[&[u8]]) -> Vec<[u8; 32]> {
|
||||||
use rayon::prelude::*;
|
use rayon::prelude::*;
|
||||||
chunks
|
chunks.par_iter().map(|c| blake3_hash_large(c)).collect()
|
||||||
.par_iter()
|
|
||||||
.map(|c| blake3_hash_large(c))
|
|
||||||
.collect()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Verify that `data` matches an expected BLAKE3 digest.
|
/// Verify that `data` matches an expected BLAKE3 digest.
|
||||||
@@ -83,7 +80,7 @@ pub fn blake3_hex(data: &[u8]) -> String {
|
|||||||
let hash = blake3_hash(data);
|
let hash = blake3_hash(data);
|
||||||
let mut out = vec![0u8; 64];
|
let mut out = vec![0u8; 64];
|
||||||
for (i, &byte) in hash.iter().enumerate() {
|
for (i, &byte) in hash.iter().enumerate() {
|
||||||
out[i * 2] = HEX[(byte >> 4) as usize];
|
out[i * 2] = HEX[(byte >> 4) as usize];
|
||||||
out[i * 2 + 1] = HEX[(byte & 0xf) as usize];
|
out[i * 2 + 1] = HEX[(byte & 0xf) as usize];
|
||||||
}
|
}
|
||||||
// Every byte is a valid ASCII hex character — from_utf8 cannot fail.
|
// Every byte is a valid ASCII hex character — from_utf8 cannot fail.
|
||||||
@@ -95,7 +92,7 @@ pub fn hash_to_hex(hash: &[u8; 32]) -> String {
|
|||||||
const HEX: &[u8; 16] = b"0123456789abcdef";
|
const HEX: &[u8; 16] = b"0123456789abcdef";
|
||||||
let mut out = vec![0u8; 64];
|
let mut out = vec![0u8; 64];
|
||||||
for (i, &byte) in hash.iter().enumerate() {
|
for (i, &byte) in hash.iter().enumerate() {
|
||||||
out[i * 2] = HEX[(byte >> 4) as usize];
|
out[i * 2] = HEX[(byte >> 4) as usize];
|
||||||
out[i * 2 + 1] = HEX[(byte & 0xf) as usize];
|
out[i * 2 + 1] = HEX[(byte & 0xf) as usize];
|
||||||
}
|
}
|
||||||
String::from_utf8(out).expect("hex output is always valid UTF-8")
|
String::from_utf8(out).expect("hex output is always valid UTF-8")
|
||||||
|
|||||||
@@ -34,8 +34,7 @@ pub fn lz4_compress(data: &[u8]) -> Result<Vec<u8>, CoreError> {
|
|||||||
|
|
||||||
/// Decompress lz4-compressed `data` (expects prepended original size).
|
/// Decompress lz4-compressed `data` (expects prepended original size).
|
||||||
pub fn lz4_decompress(data: &[u8]) -> Result<Vec<u8>, CoreError> {
|
pub fn lz4_decompress(data: &[u8]) -> Result<Vec<u8>, CoreError> {
|
||||||
lz4_flex::decompress_size_prepended(data)
|
lz4_flex::decompress_size_prepended(data).map_err(|e| CoreError::Decompress(e.to_string()))
|
||||||
.map_err(|e| CoreError::Decompress(e.to_string()))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -64,10 +63,7 @@ mod tests {
|
|||||||
fn zstd_compress_reduces_size() {
|
fn zstd_compress_reduces_size() {
|
||||||
let data = vec![0u8; 65536]; // highly compressible
|
let data = vec![0u8; 65536]; // highly compressible
|
||||||
let compressed = zstd_compress(&data, ZSTD_DEFAULT_LEVEL).unwrap();
|
let compressed = zstd_compress(&data, ZSTD_DEFAULT_LEVEL).unwrap();
|
||||||
assert!(
|
assert!(compressed.len() < data.len(), "zstd should compress zeroes");
|
||||||
compressed.len() < data.len(),
|
|
||||||
"zstd should compress zeroes"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -58,7 +58,11 @@ pub enum DeltaOp {
|
|||||||
/// This produces a simple but correct delta — it does not do rolling-hash
|
/// This produces a simple but correct delta — it does not do rolling-hash
|
||||||
/// sliding-window matching (that is CDC's job). For block-aligned page data
|
/// sliding-window matching (that is CDC's job). For block-aligned page data
|
||||||
/// (the primary use case) this is optimal.
|
/// (the primary use case) this is optimal.
|
||||||
pub fn compute_delta(source_hashes: &[BlockHash], target: &[u8], block_size: usize) -> Vec<DeltaOp> {
|
pub fn compute_delta(
|
||||||
|
source_hashes: &[BlockHash],
|
||||||
|
target: &[u8],
|
||||||
|
block_size: usize,
|
||||||
|
) -> Vec<DeltaOp> {
|
||||||
// Build a hash → offset map for O(1) lookups
|
// Build a hash → offset map for O(1) lookups
|
||||||
let lookup: HashMap<u64, usize> = source_hashes
|
let lookup: HashMap<u64, usize> = source_hashes
|
||||||
.iter()
|
.iter()
|
||||||
@@ -74,21 +78,28 @@ pub fn compute_delta(source_hashes: &[BlockHash], target: &[u8], block_size: usi
|
|||||||
if let Some(&src_offset) = lookup.get(&hash) {
|
if let Some(&src_offset) = lookup.get(&hash) {
|
||||||
// Coalesce adjacent copies
|
// Coalesce adjacent copies
|
||||||
if let Some(DeltaOp::Copy { offset, length }) = ops.last_mut() {
|
if let Some(DeltaOp::Copy { offset, length }) = ops.last_mut() {
|
||||||
if *offset + *length == src_offset && target_offset == {
|
if *offset + *length == src_offset
|
||||||
let prev_target_block = i.saturating_sub(1) * block_size;
|
&& target_offset == {
|
||||||
prev_target_block + block_size
|
let prev_target_block = i.saturating_sub(1) * block_size;
|
||||||
} {
|
prev_target_block + block_size
|
||||||
|
}
|
||||||
|
{
|
||||||
*length += chunk.len();
|
*length += chunk.len();
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ops.push(DeltaOp::Copy { offset: src_offset, length: chunk.len() });
|
ops.push(DeltaOp::Copy {
|
||||||
|
offset: src_offset,
|
||||||
|
length: chunk.len(),
|
||||||
|
});
|
||||||
} else {
|
} else {
|
||||||
// Coalesce adjacent inserts
|
// Coalesce adjacent inserts
|
||||||
if let Some(DeltaOp::Insert { data }) = ops.last_mut() {
|
if let Some(DeltaOp::Insert { data }) = ops.last_mut() {
|
||||||
data.extend_from_slice(chunk);
|
data.extend_from_slice(chunk);
|
||||||
} else {
|
} else {
|
||||||
ops.push(DeltaOp::Insert { data: chunk.to_vec() });
|
ops.push(DeltaOp::Insert {
|
||||||
|
data: chunk.to_vec(),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -149,7 +160,10 @@ mod tests {
|
|||||||
"identical data should produce only Copy ops"
|
"identical data should produce only Copy ops"
|
||||||
);
|
);
|
||||||
let transfer = delta_transfer_size(&ops);
|
let transfer = delta_transfer_size(&ops);
|
||||||
assert_eq!(transfer, 0, "no bytes should be transferred for identical data");
|
assert_eq!(
|
||||||
|
transfer, 0,
|
||||||
|
"no bytes should be transferred for identical data"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -10,11 +10,11 @@
|
|||||||
//! - [`compress`]: zstd and lz4 streaming wrappers
|
//! - [`compress`]: zstd and lz4 streaming wrappers
|
||||||
//! - [`error`]: [`CoreError`] type
|
//! - [`error`]: [`CoreError`] type
|
||||||
|
|
||||||
#[forbid(unsafe_code)]
|
|
||||||
pub mod checksum;
|
|
||||||
#[forbid(unsafe_code)]
|
#[forbid(unsafe_code)]
|
||||||
pub mod cdc;
|
pub mod cdc;
|
||||||
#[forbid(unsafe_code)]
|
#[forbid(unsafe_code)]
|
||||||
|
pub mod checksum;
|
||||||
|
#[forbid(unsafe_code)]
|
||||||
pub mod compress;
|
pub mod compress;
|
||||||
#[forbid(unsafe_code)]
|
#[forbid(unsafe_code)]
|
||||||
pub mod delta;
|
pub mod delta;
|
||||||
|
|||||||
@@ -49,7 +49,11 @@ const fn make_gear() -> [u64; 256] {
|
|||||||
state ^= state >> 7;
|
state ^= state >> 7;
|
||||||
state ^= state << 17;
|
state ^= state << 17;
|
||||||
// ensure non-zero so hot bytes always change the hash
|
// ensure non-zero so hot bytes always change the hash
|
||||||
t[i] = if state == 0 { 0x0BAD_CAFE_DEAD_BEEFu64 } else { state };
|
t[i] = if state == 0 {
|
||||||
|
0x0BAD_CAFE_DEAD_BEEFu64
|
||||||
|
} else {
|
||||||
|
state
|
||||||
|
};
|
||||||
i += 1;
|
i += 1;
|
||||||
}
|
}
|
||||||
t
|
t
|
||||||
@@ -82,7 +86,12 @@ pub fn chunk_data_simd(data: &[u8]) -> Vec<Chunk> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Split `data` with explicit size parameters.
|
/// Split `data` with explicit size parameters.
|
||||||
pub fn chunk_data_sized_simd(data: &[u8], min_size: u32, avg_size: u32, max_size: u32) -> Vec<Chunk> {
|
pub fn chunk_data_sized_simd(
|
||||||
|
data: &[u8],
|
||||||
|
min_size: u32,
|
||||||
|
avg_size: u32,
|
||||||
|
max_size: u32,
|
||||||
|
) -> Vec<Chunk> {
|
||||||
let min = min_size as usize;
|
let min = min_size as usize;
|
||||||
let avg = avg_size as usize;
|
let avg = avg_size as usize;
|
||||||
let max = max_size as usize;
|
let max = max_size as usize;
|
||||||
@@ -222,7 +231,17 @@ mod neon {
|
|||||||
// Hot byte present, at/past min_size, or approaching max:
|
// Hot byte present, at/past min_size, or approaching max:
|
||||||
// scalar fallback handles boundaries correctly.
|
// scalar fallback handles boundaries correctly.
|
||||||
let window = &data[pos..pos + 16];
|
let window = &data[pos..pos + 16];
|
||||||
scalar_window(data, window, pos, &mut chunk_start, &mut h, min, max, mask, &mut chunks);
|
scalar_window(
|
||||||
|
data,
|
||||||
|
window,
|
||||||
|
pos,
|
||||||
|
&mut chunk_start,
|
||||||
|
&mut h,
|
||||||
|
min,
|
||||||
|
max,
|
||||||
|
mask,
|
||||||
|
&mut chunks,
|
||||||
|
);
|
||||||
pos += 16;
|
pos += 16;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -230,7 +249,17 @@ mod neon {
|
|||||||
// Tail bytes (< 16 remaining)
|
// Tail bytes (< 16 remaining)
|
||||||
if pos < data.len() {
|
if pos < data.len() {
|
||||||
let window = &data[pos..];
|
let window = &data[pos..];
|
||||||
scalar_window(data, window, pos, &mut chunk_start, &mut h, min, max, mask, &mut chunks);
|
scalar_window(
|
||||||
|
data,
|
||||||
|
window,
|
||||||
|
pos,
|
||||||
|
&mut chunk_start,
|
||||||
|
&mut h,
|
||||||
|
min,
|
||||||
|
max,
|
||||||
|
mask,
|
||||||
|
&mut chunks,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if chunk_start < data.len() {
|
if chunk_start < data.len() {
|
||||||
@@ -292,14 +321,34 @@ mod sse2 {
|
|||||||
pos += 16;
|
pos += 16;
|
||||||
} else {
|
} else {
|
||||||
let window = &data[pos..pos + 16];
|
let window = &data[pos..pos + 16];
|
||||||
scalar_window(data, window, pos, &mut chunk_start, &mut h, min, max, mask, &mut chunks);
|
scalar_window(
|
||||||
|
data,
|
||||||
|
window,
|
||||||
|
pos,
|
||||||
|
&mut chunk_start,
|
||||||
|
&mut h,
|
||||||
|
min,
|
||||||
|
max,
|
||||||
|
mask,
|
||||||
|
&mut chunks,
|
||||||
|
);
|
||||||
pos += 16;
|
pos += 16;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if pos < data.len() {
|
if pos < data.len() {
|
||||||
let window = &data[pos..];
|
let window = &data[pos..];
|
||||||
scalar_window(data, window, pos, &mut chunk_start, &mut h, min, max, mask, &mut chunks);
|
scalar_window(
|
||||||
|
data,
|
||||||
|
window,
|
||||||
|
pos,
|
||||||
|
&mut chunk_start,
|
||||||
|
&mut h,
|
||||||
|
min,
|
||||||
|
max,
|
||||||
|
mask,
|
||||||
|
&mut chunks,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
if chunk_start < data.len() {
|
if chunk_start < data.len() {
|
||||||
@@ -410,12 +459,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn chunk_sizes_respect_min_max_except_last() {
|
fn chunk_sizes_respect_min_max_except_last() {
|
||||||
let data = lcg_data(500_000, 7);
|
let data = lcg_data(500_000, 7);
|
||||||
let chunks = chunk_data_sized_simd(
|
let chunks = chunk_data_sized_simd(&data, MIN_CHUNK_SIZE, AVG_CHUNK_SIZE, MAX_CHUNK_SIZE);
|
||||||
&data,
|
|
||||||
MIN_CHUNK_SIZE,
|
|
||||||
AVG_CHUNK_SIZE,
|
|
||||||
MAX_CHUNK_SIZE,
|
|
||||||
);
|
|
||||||
for (i, c) in chunks.iter().enumerate() {
|
for (i, c) in chunks.iter().enumerate() {
|
||||||
let is_last = i == chunks.len() - 1;
|
let is_last = i == chunks.len() - 1;
|
||||||
if !is_last {
|
if !is_last {
|
||||||
@@ -511,7 +555,11 @@ mod tests {
|
|||||||
let data = lcg_data(200_000, 77);
|
let data = lcg_data(200_000, 77);
|
||||||
for c in chunk_data_simd(&data) {
|
for c in chunk_data_simd(&data) {
|
||||||
let expected = crate::checksum::xxh3_block(&data[c.offset..c.offset + c.length]);
|
let expected = crate::checksum::xxh3_block(&data[c.offset..c.offset + c.length]);
|
||||||
assert_eq!(c.hash, expected, "chunk hash mismatch at offset {}", c.offset);
|
assert_eq!(
|
||||||
|
c.hash, expected,
|
||||||
|
"chunk hash mismatch at offset {}",
|
||||||
|
c.offset
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,9 +9,9 @@
|
|||||||
|
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
|
|
||||||
use clawsync_core::cdc::{chunk_data, Chunk};
|
use clawsync_core::cdc::{Chunk, chunk_data};
|
||||||
use clawsync_core::checksum::blake3_hash_large;
|
use clawsync_core::checksum::blake3_hash_large;
|
||||||
use clawsync_core::compress::{zstd_compress, zstd_decompress, ZSTD_DEFAULT_LEVEL};
|
use clawsync_core::compress::{ZSTD_DEFAULT_LEVEL, zstd_compress, zstd_decompress};
|
||||||
use clawsync_transport::protocol::{FsChunk, FsChunkHash};
|
use clawsync_transport::protocol::{FsChunk, FsChunkHash};
|
||||||
|
|
||||||
use crate::error::FsSyncError;
|
use crate::error::FsSyncError;
|
||||||
@@ -40,7 +40,10 @@ pub fn chunk_file_for_request(data: &[u8]) -> (Vec<Chunk>, Vec<FsChunkHash>) {
|
|||||||
let chunks = chunk_data(data);
|
let chunks = chunk_data(data);
|
||||||
let hashes = chunks
|
let hashes = chunks
|
||||||
.iter()
|
.iter()
|
||||||
.map(|c| FsChunkHash { hash: c.hash, length: c.length as u32 })
|
.map(|c| FsChunkHash {
|
||||||
|
hash: c.hash,
|
||||||
|
length: c.length as u32,
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
(chunks, hashes)
|
(chunks, hashes)
|
||||||
}
|
}
|
||||||
@@ -72,11 +75,7 @@ pub fn compute_needed_indices(
|
|||||||
///
|
///
|
||||||
/// Each chunk is zstd-compressed; raw bytes are used instead when the
|
/// Each chunk is zstd-compressed; raw bytes are used instead when the
|
||||||
/// compressed output is not smaller.
|
/// compressed output is not smaller.
|
||||||
pub fn build_chunk_data(
|
pub fn build_chunk_data(data: &[u8], all_chunks: &[Chunk], needed_indices: &[u32]) -> Vec<FsChunk> {
|
||||||
data: &[u8],
|
|
||||||
all_chunks: &[Chunk],
|
|
||||||
needed_indices: &[u32],
|
|
||||||
) -> Vec<FsChunk> {
|
|
||||||
needed_indices
|
needed_indices
|
||||||
.iter()
|
.iter()
|
||||||
.map(|&idx| {
|
.map(|&idx| {
|
||||||
@@ -90,7 +89,11 @@ pub fn build_chunk_data(
|
|||||||
_ => (raw.to_vec(), false),
|
_ => (raw.to_vec(), false),
|
||||||
};
|
};
|
||||||
|
|
||||||
FsChunk { hash: c.hash, data: payload, compressed }
|
FsChunk {
|
||||||
|
hash: c.hash,
|
||||||
|
data: payload,
|
||||||
|
compressed,
|
||||||
|
}
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
@@ -127,9 +130,14 @@ pub fn reconstruct_file(
|
|||||||
|
|
||||||
for desc in client_chunk_order {
|
for desc in client_chunk_order {
|
||||||
if let Some(&src_offset) = server_chunk_map.get(&desc.hash) {
|
if let Some(&src_offset) = server_chunk_map.get(&desc.hash) {
|
||||||
plan.push(ReconstructOp::LocalCopy { src_offset, length: desc.length });
|
plan.push(ReconstructOp::LocalCopy {
|
||||||
|
src_offset,
|
||||||
|
length: desc.length,
|
||||||
|
});
|
||||||
} else {
|
} else {
|
||||||
plan.push(ReconstructOp::Literal { chunk_index: literal_idx });
|
plan.push(ReconstructOp::Literal {
|
||||||
|
chunk_index: literal_idx,
|
||||||
|
});
|
||||||
literal_idx += 1;
|
literal_idx += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -206,8 +214,8 @@ mod tests {
|
|||||||
|
|
||||||
fn pseudo_random(size: usize, seed: u8) -> Vec<u8> {
|
fn pseudo_random(size: usize, seed: u8) -> Vec<u8> {
|
||||||
// Xorshift64 — high entropy, genuinely incompressible output.
|
// Xorshift64 — high entropy, genuinely incompressible output.
|
||||||
let mut s: u64 = 0xDEAD_CAFE_0000_0001u64
|
let mut s: u64 =
|
||||||
^ (seed as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15);
|
0xDEAD_CAFE_0000_0001u64 ^ (seed as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15);
|
||||||
(0..size)
|
(0..size)
|
||||||
.map(|_| {
|
.map(|_| {
|
||||||
s ^= s << 13;
|
s ^= s << 13;
|
||||||
@@ -220,10 +228,7 @@ mod tests {
|
|||||||
|
|
||||||
/// Run the full CDC delta pipeline: chunk client file, compute needs,
|
/// Run the full CDC delta pipeline: chunk client file, compute needs,
|
||||||
/// build literal data, reconstruct, verify BLAKE3.
|
/// build literal data, reconstruct, verify BLAKE3.
|
||||||
fn full_roundtrip(
|
fn full_roundtrip(server_data: &[u8], client_data: &[u8]) -> (Vec<u8>, usize) {
|
||||||
server_data: &[u8],
|
|
||||||
client_data: &[u8],
|
|
||||||
) -> (Vec<u8>, usize) {
|
|
||||||
let expected = blake3_hash_large(client_data);
|
let expected = blake3_hash_large(client_data);
|
||||||
|
|
||||||
let (client_chunks, client_hashes) = chunk_file_for_request(client_data);
|
let (client_chunks, client_hashes) = chunk_file_for_request(client_data);
|
||||||
@@ -252,7 +257,10 @@ mod tests {
|
|||||||
fn roundtrip_identical_file_sends_zero_chunk_bytes() {
|
fn roundtrip_identical_file_sends_zero_chunk_bytes() {
|
||||||
let data = pseudo_random(200_000, 1);
|
let data = pseudo_random(200_000, 1);
|
||||||
let (_, literal_bytes) = full_roundtrip(&data, &data);
|
let (_, literal_bytes) = full_roundtrip(&data, &data);
|
||||||
assert_eq!(literal_bytes, 0, "identical file should send 0 literal bytes");
|
assert_eq!(
|
||||||
|
literal_bytes, 0,
|
||||||
|
"identical file should send 0 literal bytes"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -316,7 +324,10 @@ mod tests {
|
|||||||
let (chunks, _) = chunk_file_for_request(&data);
|
let (chunks, _) = chunk_file_for_request(&data);
|
||||||
let chunk_data_built = build_chunk_data(&data, &chunks, &[0]);
|
let chunk_data_built = build_chunk_data(&data, &chunks, &[0]);
|
||||||
// At least one chunk should be uncompressed (raw) since data is incompressible.
|
// At least one chunk should be uncompressed (raw) since data is incompressible.
|
||||||
assert!(!chunk_data_built[0].compressed, "incompressible chunk should be sent raw");
|
assert!(
|
||||||
|
!chunk_data_built[0].compressed,
|
||||||
|
"incompressible chunk should be sent raw"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -39,8 +39,10 @@ impl FileDiff {
|
|||||||
///
|
///
|
||||||
/// Result is sorted by path for deterministic ordering.
|
/// Result is sorted by path for deterministic ordering.
|
||||||
pub fn diff_manifests(local: &[LocalEntry], remote: &[FsManifestEntry]) -> Vec<FileDiff> {
|
pub fn diff_manifests(local: &[LocalEntry], remote: &[FsManifestEntry]) -> Vec<FileDiff> {
|
||||||
let local_map: HashMap<&str, [u8; 32]> =
|
let local_map: HashMap<&str, [u8; 32]> = local
|
||||||
local.iter().map(|e| (e.rel_path.as_str(), e.blake3)).collect();
|
.iter()
|
||||||
|
.map(|e| (e.rel_path.as_str(), e.blake3))
|
||||||
|
.collect();
|
||||||
let remote_map: HashMap<&str, [u8; 32]> =
|
let remote_map: HashMap<&str, [u8; 32]> =
|
||||||
remote.iter().map(|e| (e.path.as_str(), e.blake3)).collect();
|
remote.iter().map(|e| (e.path.as_str(), e.blake3)).collect();
|
||||||
|
|
||||||
@@ -51,10 +53,10 @@ pub fn diff_manifests(local: &[LocalEntry], remote: &[FsManifestEntry]) -> Vec<F
|
|||||||
all_paths
|
all_paths
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|path| match (local_map.get(path), remote_map.get(path)) {
|
.map(|path| match (local_map.get(path), remote_map.get(path)) {
|
||||||
(Some(_), None) => FileDiff::Added(path.to_string()),
|
(Some(_), None) => FileDiff::Added(path.to_string()),
|
||||||
(None, Some(_)) => FileDiff::Removed(path.to_string()),
|
(None, Some(_)) => FileDiff::Removed(path.to_string()),
|
||||||
(Some(lh), Some(rh)) if lh != rh => FileDiff::Modified(path.to_string()),
|
(Some(lh), Some(rh)) if lh != rh => FileDiff::Modified(path.to_string()),
|
||||||
_ => FileDiff::Unchanged(path.to_string()),
|
_ => FileDiff::Unchanged(path.to_string()),
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
@@ -147,18 +149,38 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn mixed_diff_counts_correct() {
|
fn mixed_diff_counts_correct() {
|
||||||
let l = vec![local("keep.bin", 1), local("new.bin", 3), local("mod.bin", 4)];
|
let l = vec![
|
||||||
let r = vec![remote("keep.bin", 1), remote("old.bin", 9), remote("mod.bin", 5)];
|
local("keep.bin", 1),
|
||||||
|
local("new.bin", 3),
|
||||||
|
local("mod.bin", 4),
|
||||||
|
];
|
||||||
|
let r = vec![
|
||||||
|
remote("keep.bin", 1),
|
||||||
|
remote("old.bin", 9),
|
||||||
|
remote("mod.bin", 5),
|
||||||
|
];
|
||||||
let diffs = diff_manifests(&l, &r);
|
let diffs = diff_manifests(&l, &r);
|
||||||
|
|
||||||
let added: Vec<_> = diffs.iter().filter(|d| matches!(d, FileDiff::Added(_))).collect();
|
let added: Vec<_> = diffs
|
||||||
let modified: Vec<_> = diffs.iter().filter(|d| matches!(d, FileDiff::Modified(_))).collect();
|
.iter()
|
||||||
let removed: Vec<_> = diffs.iter().filter(|d| matches!(d, FileDiff::Removed(_))).collect();
|
.filter(|d| matches!(d, FileDiff::Added(_)))
|
||||||
let unchanged: Vec<_> = diffs.iter().filter(|d| matches!(d, FileDiff::Unchanged(_))).collect();
|
.collect();
|
||||||
|
let modified: Vec<_> = diffs
|
||||||
|
.iter()
|
||||||
|
.filter(|d| matches!(d, FileDiff::Modified(_)))
|
||||||
|
.collect();
|
||||||
|
let removed: Vec<_> = diffs
|
||||||
|
.iter()
|
||||||
|
.filter(|d| matches!(d, FileDiff::Removed(_)))
|
||||||
|
.collect();
|
||||||
|
let unchanged: Vec<_> = diffs
|
||||||
|
.iter()
|
||||||
|
.filter(|d| matches!(d, FileDiff::Unchanged(_)))
|
||||||
|
.collect();
|
||||||
|
|
||||||
assert_eq!(added.len(), 1); // new.bin
|
assert_eq!(added.len(), 1); // new.bin
|
||||||
assert_eq!(modified.len(), 1); // mod.bin
|
assert_eq!(modified.len(), 1); // mod.bin
|
||||||
assert_eq!(removed.len(), 1); // old.bin
|
assert_eq!(removed.len(), 1); // old.bin
|
||||||
assert_eq!(unchanged.len(), 1); // keep.bin
|
assert_eq!(unchanged.len(), 1); // keep.bin
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -51,7 +51,9 @@ impl FsManifest {
|
|||||||
let abs = entry.path().to_path_buf();
|
let abs = entry.path().to_path_buf();
|
||||||
let rel = abs
|
let rel = abs
|
||||||
.strip_prefix(root)
|
.strip_prefix(root)
|
||||||
.map_err(|_| FsSyncError::Path(format!("cannot strip prefix from {}", abs.display())))?
|
.map_err(|_| {
|
||||||
|
FsSyncError::Path(format!("cannot strip prefix from {}", abs.display()))
|
||||||
|
})?
|
||||||
.to_string_lossy()
|
.to_string_lossy()
|
||||||
.replace('\\', "/"); // normalise on Windows
|
.replace('\\', "/"); // normalise on Windows
|
||||||
|
|
||||||
@@ -75,7 +77,13 @@ impl FsManifest {
|
|||||||
.and_then(|t| t.duration_since(UNIX_EPOCH).ok())
|
.and_then(|t| t.duration_since(UNIX_EPOCH).ok())
|
||||||
.map(|d| d.as_secs() as i64)
|
.map(|d| d.as_secs() as i64)
|
||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
Ok(LocalEntry { rel_path: rel, abs_path: abs, blake3, size, mtime })
|
Ok(LocalEntry {
|
||||||
|
rel_path: rel,
|
||||||
|
abs_path: abs,
|
||||||
|
blake3,
|
||||||
|
size,
|
||||||
|
mtime,
|
||||||
|
})
|
||||||
})
|
})
|
||||||
.collect::<Result<Vec<_>, _>>()?;
|
.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
|
||||||
@@ -88,19 +96,17 @@ impl FsManifest {
|
|||||||
self.entries
|
self.entries
|
||||||
.iter()
|
.iter()
|
||||||
.map(|e| FsManifestEntry {
|
.map(|e| FsManifestEntry {
|
||||||
path: e.rel_path.clone(),
|
path: e.rel_path.clone(),
|
||||||
blake3: e.blake3,
|
blake3: e.blake3,
|
||||||
size: e.size,
|
size: e.size,
|
||||||
mtime: e.mtime,
|
mtime: e.mtime,
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Look up an entry by its relative path.
|
/// Look up an entry by its relative path.
|
||||||
pub fn get(&self, rel_path: &str) -> Option<&LocalEntry> {
|
pub fn get(&self, rel_path: &str) -> Option<&LocalEntry> {
|
||||||
self.entries
|
self.entries.iter().find(|e| e.rel_path == rel_path)
|
||||||
.iter()
|
|
||||||
.find(|e| e.rel_path == rel_path)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -162,7 +168,11 @@ mod tests {
|
|||||||
fs::write(dir.path().join("sub").join("nested.bin"), b"nested").unwrap();
|
fs::write(dir.path().join("sub").join("nested.bin"), b"nested").unwrap();
|
||||||
|
|
||||||
let manifest = FsManifest::build(dir.path(), &empty_excludes()).unwrap();
|
let manifest = FsManifest::build(dir.path(), &empty_excludes()).unwrap();
|
||||||
let paths: Vec<&str> = manifest.entries.iter().map(|e| e.rel_path.as_str()).collect();
|
let paths: Vec<&str> = manifest
|
||||||
|
.entries
|
||||||
|
.iter()
|
||||||
|
.map(|e| e.rel_path.as_str())
|
||||||
|
.collect();
|
||||||
assert!(paths.contains(&"root.bin"));
|
assert!(paths.contains(&"root.bin"));
|
||||||
assert!(paths.contains(&"sub/nested.bin"));
|
assert!(paths.contains(&"sub/nested.bin"));
|
||||||
}
|
}
|
||||||
@@ -175,7 +185,11 @@ mod tests {
|
|||||||
fs::write(dir.path().join("m.bin"), b"m").unwrap();
|
fs::write(dir.path().join("m.bin"), b"m").unwrap();
|
||||||
|
|
||||||
let manifest = FsManifest::build(dir.path(), &empty_excludes()).unwrap();
|
let manifest = FsManifest::build(dir.path(), &empty_excludes()).unwrap();
|
||||||
let paths: Vec<&str> = manifest.entries.iter().map(|e| e.rel_path.as_str()).collect();
|
let paths: Vec<&str> = manifest
|
||||||
|
.entries
|
||||||
|
.iter()
|
||||||
|
.map(|e| e.rel_path.as_str())
|
||||||
|
.collect();
|
||||||
assert_eq!(paths, vec!["a.bin", "m.bin", "z.bin"]);
|
assert_eq!(paths, vec!["a.bin", "m.bin", "z.bin"]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,9 +32,9 @@ const PIPELINE_WINDOW: usize = 16;
|
|||||||
/// Aggregate statistics from one completed sync session.
|
/// Aggregate statistics from one completed sync session.
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
pub struct SyncStats {
|
pub struct SyncStats {
|
||||||
pub files_added: u32,
|
pub files_added: u32,
|
||||||
pub files_modified: u32,
|
pub files_modified: u32,
|
||||||
pub files_removed: u32,
|
pub files_removed: u32,
|
||||||
pub bytes_transferred: u64,
|
pub bytes_transferred: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -52,7 +52,12 @@ pub struct FsSyncClient {
|
|||||||
|
|
||||||
impl FsSyncClient {
|
impl FsSyncClient {
|
||||||
pub fn new(conn: SyncPeer, local_root: PathBuf, excludes: GlobSet, delete: bool) -> Self {
|
pub fn new(conn: SyncPeer, local_root: PathBuf, excludes: GlobSet, delete: bool) -> Self {
|
||||||
Self { conn, local_root, excludes, _delete: delete }
|
Self {
|
||||||
|
conn,
|
||||||
|
local_root,
|
||||||
|
excludes,
|
||||||
|
_delete: delete,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Execute the full directory sync protocol.
|
/// Execute the full directory sync protocol.
|
||||||
@@ -60,9 +65,10 @@ impl FsSyncClient {
|
|||||||
// ── RTT 1: build local manifest and send it ──────────────────────────
|
// ── RTT 1: build local manifest and send it ──────────────────────────
|
||||||
let local_root = self.local_root.clone();
|
let local_root = self.local_root.clone();
|
||||||
let excludes = self.excludes.clone();
|
let excludes = self.excludes.clone();
|
||||||
let manifest = tokio::task::spawn_blocking(move || FsManifest::build(&local_root, &excludes))
|
let manifest =
|
||||||
.await
|
tokio::task::spawn_blocking(move || FsManifest::build(&local_root, &excludes))
|
||||||
.map_err(|e| FsSyncError::Join(e.to_string()))??;
|
.await
|
||||||
|
.map_err(|e| FsSyncError::Join(e.to_string()))??;
|
||||||
|
|
||||||
self.conn
|
self.conn
|
||||||
.send(&SyncMessage::FsDirManifest {
|
.send(&SyncMessage::FsDirManifest {
|
||||||
@@ -72,17 +78,37 @@ impl FsSyncClient {
|
|||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let (needed_files, to_delete) = match self.conn.recv().await? {
|
let (needed_files, to_delete) = match self.conn.recv().await? {
|
||||||
SyncMessage::FsDirNeed { needed_files, to_delete } => (needed_files, to_delete),
|
SyncMessage::FsDirNeed {
|
||||||
|
needed_files,
|
||||||
|
to_delete,
|
||||||
|
} => (needed_files, to_delete),
|
||||||
SyncMessage::Error { message } => return Err(FsSyncError::Protocol(message)),
|
SyncMessage::Error { message } => return Err(FsSyncError::Protocol(message)),
|
||||||
other => return Err(FsSyncError::Protocol(format!("expected FsDirNeed, got {other:?}"))),
|
other => {
|
||||||
|
return Err(FsSyncError::Protocol(format!(
|
||||||
|
"expected FsDirNeed, got {other:?}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Warm no-op: nothing to do on either side.
|
// Warm no-op: nothing to do on either side.
|
||||||
if needed_files.is_empty() && to_delete.is_empty() {
|
if needed_files.is_empty() && to_delete.is_empty() {
|
||||||
let final_stats = match self.conn.recv().await? {
|
let final_stats = match self.conn.recv().await? {
|
||||||
SyncMessage::FsDirComplete { files_added, files_modified, files_removed, bytes_transferred } =>
|
SyncMessage::FsDirComplete {
|
||||||
SyncStats { files_added, files_modified, files_removed, bytes_transferred },
|
files_added,
|
||||||
other => return Err(FsSyncError::Protocol(format!("expected FsDirComplete, got {other:?}"))),
|
files_modified,
|
||||||
|
files_removed,
|
||||||
|
bytes_transferred,
|
||||||
|
} => SyncStats {
|
||||||
|
files_added,
|
||||||
|
files_modified,
|
||||||
|
files_removed,
|
||||||
|
bytes_transferred,
|
||||||
|
},
|
||||||
|
other => {
|
||||||
|
return Err(FsSyncError::Protocol(format!(
|
||||||
|
"expected FsDirComplete, got {other:?}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
};
|
};
|
||||||
self.conn.shutdown().await?;
|
self.conn.shutdown().await?;
|
||||||
return Ok(final_stats);
|
return Ok(final_stats);
|
||||||
@@ -112,45 +138,47 @@ impl FsSyncClient {
|
|||||||
|
|
||||||
let sem_writer = semaphore.clone();
|
let sem_writer = semaphore.clone();
|
||||||
|
|
||||||
let writer_task: tokio::task::JoinHandle<
|
let writer_task: tokio::task::JoinHandle<Result<PipeWriteHalf, FsSyncError>> =
|
||||||
Result<PipeWriteHalf, FsSyncError>,
|
tokio::spawn(async move {
|
||||||
> = tokio::spawn(async move {
|
for file_need in needed_files {
|
||||||
for file_need in needed_files {
|
let abs_path = path_map.get(&file_need.path).cloned().ok_or_else(|| {
|
||||||
let abs_path = path_map
|
FsSyncError::Path(format!("local path not found: {}", file_need.path))
|
||||||
.get(&file_need.path)
|
})?;
|
||||||
.cloned()
|
|
||||||
.ok_or_else(|| FsSyncError::Path(format!("local path not found: {}", file_need.path)))?;
|
|
||||||
|
|
||||||
sem_writer.acquire().await.expect("semaphore closed").forget();
|
sem_writer
|
||||||
|
.acquire()
|
||||||
|
.await
|
||||||
|
.expect("semaphore closed")
|
||||||
|
.forget();
|
||||||
|
|
||||||
let server_chunks_wire = file_need.server_chunks.clone();
|
let server_chunks_wire = file_need.server_chunks.clone();
|
||||||
|
|
||||||
// Read + CDC chunk in a blocking thread.
|
// Read + CDC chunk in a blocking thread.
|
||||||
let (chunk_order, literal_chunks, wire_bytes) =
|
let (chunk_order, literal_chunks, wire_bytes) =
|
||||||
tokio::task::spawn_blocking(move || -> Result<_, FsSyncError> {
|
tokio::task::spawn_blocking(move || -> Result<_, FsSyncError> {
|
||||||
let data = std::fs::read(&abs_path)?;
|
let data = std::fs::read(&abs_path)?;
|
||||||
let (all_chunks, hashes) = chunk_file_for_request(&data);
|
let (all_chunks, hashes) = chunk_file_for_request(&data);
|
||||||
let server_existing: HashSet<u64> =
|
let server_existing: HashSet<u64> =
|
||||||
server_chunks_wire.iter().map(|c| c.hash).collect();
|
server_chunks_wire.iter().map(|c| c.hash).collect();
|
||||||
let needed_idx = compute_needed_indices(&server_existing, &hashes);
|
let needed_idx = compute_needed_indices(&server_existing, &hashes);
|
||||||
let literals = build_chunk_data(&data, &all_chunks, &needed_idx);
|
let literals = build_chunk_data(&data, &all_chunks, &needed_idx);
|
||||||
let bytes: u64 = literals.iter().map(|c| c.data.len() as u64).sum();
|
let bytes: u64 = literals.iter().map(|c| c.data.len() as u64).sum();
|
||||||
Ok((hashes, literals, bytes))
|
Ok((hashes, literals, bytes))
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
.map_err(|e| FsSyncError::Join(e.to_string()))??;
|
.map_err(|e| FsSyncError::Join(e.to_string()))??;
|
||||||
|
|
||||||
meta_tx.send(wire_bytes).expect("meta channel closed");
|
meta_tx.send(wire_bytes).expect("meta channel closed");
|
||||||
write_half
|
write_half
|
||||||
.send(&SyncMessage::FsCdcData {
|
.send(&SyncMessage::FsCdcData {
|
||||||
path: file_need.path,
|
path: file_need.path,
|
||||||
chunk_order,
|
chunk_order,
|
||||||
chunks: literal_chunks,
|
chunks: literal_chunks,
|
||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
Ok(write_half)
|
Ok(write_half)
|
||||||
});
|
});
|
||||||
|
|
||||||
// Reader: collect acks.
|
// Reader: collect acks.
|
||||||
let mut bytes_transferred = 0u64;
|
let mut bytes_transferred = 0u64;
|
||||||
@@ -161,7 +189,11 @@ impl FsSyncClient {
|
|||||||
bytes_transferred += meta_rx.recv().await.expect("meta closed early");
|
bytes_transferred += meta_rx.recv().await.expect("meta closed early");
|
||||||
}
|
}
|
||||||
SyncMessage::Error { message } => return Err(FsSyncError::Protocol(message)),
|
SyncMessage::Error { message } => return Err(FsSyncError::Protocol(message)),
|
||||||
other => return Err(FsSyncError::Protocol(format!("expected FsFileAck, got {other:?}"))),
|
other => {
|
||||||
|
return Err(FsSyncError::Protocol(format!(
|
||||||
|
"expected FsFileAck, got {other:?}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -171,9 +203,22 @@ impl FsSyncClient {
|
|||||||
|
|
||||||
// Receive final FsDirComplete from server.
|
// Receive final FsDirComplete from server.
|
||||||
let stats = match read_half.recv().await? {
|
let stats = match read_half.recv().await? {
|
||||||
SyncMessage::FsDirComplete { files_added, files_modified, files_removed, .. } =>
|
SyncMessage::FsDirComplete {
|
||||||
SyncStats { files_added, files_modified, files_removed, bytes_transferred },
|
files_added,
|
||||||
other => return Err(FsSyncError::Protocol(format!("expected FsDirComplete, got {other:?}"))),
|
files_modified,
|
||||||
|
files_removed,
|
||||||
|
..
|
||||||
|
} => SyncStats {
|
||||||
|
files_added,
|
||||||
|
files_modified,
|
||||||
|
files_removed,
|
||||||
|
bytes_transferred,
|
||||||
|
},
|
||||||
|
other => {
|
||||||
|
return Err(FsSyncError::Protocol(format!(
|
||||||
|
"expected FsDirComplete, got {other:?}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
write_half.shutdown().await?;
|
write_half.shutdown().await?;
|
||||||
@@ -194,13 +239,13 @@ pub struct FsSyncServer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl FsSyncServer {
|
impl FsSyncServer {
|
||||||
pub fn new(
|
pub fn new(conn: SyncPeer, serve_root: PathBuf, excludes: GlobSet, allow_delete: bool) -> Self {
|
||||||
conn: SyncPeer,
|
Self {
|
||||||
serve_root: PathBuf,
|
conn,
|
||||||
excludes: GlobSet,
|
serve_root,
|
||||||
allow_delete: bool,
|
excludes,
|
||||||
) -> Self {
|
allow_delete,
|
||||||
Self { conn, serve_root, excludes, allow_delete }
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handle one complete client session.
|
/// Handle one complete client session.
|
||||||
@@ -208,7 +253,11 @@ impl FsSyncServer {
|
|||||||
// Receive client's directory manifest.
|
// Receive client's directory manifest.
|
||||||
let client_entries = match self.conn.recv().await? {
|
let client_entries = match self.conn.recv().await? {
|
||||||
SyncMessage::FsDirManifest { entries, .. } => entries,
|
SyncMessage::FsDirManifest { entries, .. } => entries,
|
||||||
other => return Err(FsSyncError::Protocol(format!("expected FsDirManifest, got {other:?}"))),
|
other => {
|
||||||
|
return Err(FsSyncError::Protocol(format!(
|
||||||
|
"expected FsDirManifest, got {other:?}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Build server's own manifest.
|
// Build server's own manifest.
|
||||||
@@ -277,9 +326,9 @@ impl FsSyncServer {
|
|||||||
|
|
||||||
for rel_path in &needed_paths {
|
for rel_path in &needed_paths {
|
||||||
let is_add = is_add_set.contains(*rel_path);
|
let is_add = is_add_set.contains(*rel_path);
|
||||||
let abs_path = self.serve_root.join(
|
let abs_path = self
|
||||||
rel_path.replace('/', std::path::MAIN_SEPARATOR_STR)
|
.serve_root
|
||||||
);
|
.join(rel_path.replace('/', std::path::MAIN_SEPARATOR_STR));
|
||||||
|
|
||||||
let server_bytes: Vec<u8> = if is_add {
|
let server_bytes: Vec<u8> = if is_add {
|
||||||
vec![]
|
vec![]
|
||||||
@@ -293,7 +342,10 @@ impl FsSyncServer {
|
|||||||
let server_chunk_map = build_server_chunk_map(&server_bytes);
|
let server_chunk_map = build_server_chunk_map(&server_bytes);
|
||||||
let server_chunk_hashes: Vec<FsChunkHash> = chunk_data(&server_bytes)
|
let server_chunk_hashes: Vec<FsChunkHash> = chunk_data(&server_bytes)
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|c| FsChunkHash { hash: c.hash, length: c.length as u32 })
|
.map(|c| FsChunkHash {
|
||||||
|
hash: c.hash,
|
||||||
|
length: c.length as u32,
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let expected_blake3 = client_entries
|
let expected_blake3 = client_entries
|
||||||
@@ -308,37 +360,50 @@ impl FsSyncServer {
|
|||||||
expected_blake3,
|
expected_blake3,
|
||||||
});
|
});
|
||||||
|
|
||||||
file_ctx.insert(rel_path.to_string(), FileCtx {
|
file_ctx.insert(
|
||||||
server_bytes,
|
rel_path.to_string(),
|
||||||
server_chunk_map,
|
FileCtx {
|
||||||
expected_blake3,
|
server_bytes,
|
||||||
is_add,
|
server_chunk_map,
|
||||||
});
|
expected_blake3,
|
||||||
|
is_add,
|
||||||
|
},
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Send FsDirNeed.
|
// Send FsDirNeed.
|
||||||
self.conn
|
self.conn
|
||||||
.send(&SyncMessage::FsDirNeed { needed_files, to_delete: to_delete.clone() })
|
.send(&SyncMessage::FsDirNeed {
|
||||||
|
needed_files,
|
||||||
|
to_delete: to_delete.clone(),
|
||||||
|
})
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// No-op path (nothing to transfer or delete).
|
// No-op path (nothing to transfer or delete).
|
||||||
if file_ctx.is_empty() && to_delete.is_empty() {
|
if file_ctx.is_empty() && to_delete.is_empty() {
|
||||||
self.conn
|
self.conn
|
||||||
.send(&SyncMessage::FsDirComplete {
|
.send(&SyncMessage::FsDirComplete {
|
||||||
files_added: 0, files_modified: 0, files_removed: 0, bytes_transferred: 0,
|
files_added: 0,
|
||||||
|
files_modified: 0,
|
||||||
|
files_removed: 0,
|
||||||
|
bytes_transferred: 0,
|
||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
return Ok(SyncStats::default());
|
return Ok(SyncStats::default());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Receive FsCdcData for each needed file (pipelined, any order).
|
// Receive FsCdcData for each needed file (pipelined, any order).
|
||||||
let mut files_added = 0u32;
|
let mut files_added = 0u32;
|
||||||
let mut files_modified = 0u32;
|
let mut files_modified = 0u32;
|
||||||
let mut bytes_transferred = 0u64;
|
let mut bytes_transferred = 0u64;
|
||||||
|
|
||||||
for _ in 0..file_ctx.len() {
|
for _ in 0..file_ctx.len() {
|
||||||
match self.conn.recv().await? {
|
match self.conn.recv().await? {
|
||||||
SyncMessage::FsCdcData { path, chunk_order, chunks } => {
|
SyncMessage::FsCdcData {
|
||||||
|
path,
|
||||||
|
chunk_order,
|
||||||
|
chunks,
|
||||||
|
} => {
|
||||||
let ctx = file_ctx.remove(&path).ok_or_else(|| {
|
let ctx = file_ctx.remove(&path).ok_or_else(|| {
|
||||||
FsSyncError::Protocol(format!("FsCdcData for unknown path: {path}"))
|
FsSyncError::Protocol(format!("FsCdcData for unknown path: {path}"))
|
||||||
})?;
|
})?;
|
||||||
@@ -366,9 +431,9 @@ impl FsSyncServer {
|
|||||||
bytes_transferred += output.len() as u64;
|
bytes_transferred += output.len() as u64;
|
||||||
|
|
||||||
// Atomic write: temp file + rename.
|
// Atomic write: temp file + rename.
|
||||||
let abs_path = self.serve_root.join(
|
let abs_path = self
|
||||||
path.replace('/', std::path::MAIN_SEPARATOR_STR)
|
.serve_root
|
||||||
);
|
.join(path.replace('/', std::path::MAIN_SEPARATOR_STR));
|
||||||
// Ensure parent directory exists.
|
// Ensure parent directory exists.
|
||||||
if let Some(parent) = abs_path.parent() {
|
if let Some(parent) = abs_path.parent() {
|
||||||
tokio::task::spawn_blocking({
|
tokio::task::spawn_blocking({
|
||||||
@@ -391,18 +456,28 @@ impl FsSyncServer {
|
|||||||
.await
|
.await
|
||||||
.map_err(|e| FsSyncError::Join(e.to_string()))??;
|
.map_err(|e| FsSyncError::Join(e.to_string()))??;
|
||||||
|
|
||||||
if is_add { files_added += 1; } else { files_modified += 1; }
|
if is_add {
|
||||||
|
files_added += 1;
|
||||||
|
} else {
|
||||||
|
files_modified += 1;
|
||||||
|
}
|
||||||
|
|
||||||
self.conn.send(&SyncMessage::FsFileAck { path }).await?;
|
self.conn.send(&SyncMessage::FsFileAck { path }).await?;
|
||||||
}
|
}
|
||||||
SyncMessage::Error { message } => return Err(FsSyncError::Protocol(message)),
|
SyncMessage::Error { message } => return Err(FsSyncError::Protocol(message)),
|
||||||
other => return Err(FsSyncError::Protocol(format!("expected FsCdcData, got {other:?}"))),
|
other => {
|
||||||
|
return Err(FsSyncError::Protocol(format!(
|
||||||
|
"expected FsCdcData, got {other:?}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Apply deletions after all files reconstructed.
|
// Apply deletions after all files reconstructed.
|
||||||
for rel in &to_delete {
|
for rel in &to_delete {
|
||||||
let abs = self.serve_root.join(rel.replace('/', std::path::MAIN_SEPARATOR_STR));
|
let abs = self
|
||||||
|
.serve_root
|
||||||
|
.join(rel.replace('/', std::path::MAIN_SEPARATOR_STR));
|
||||||
if abs.exists() {
|
if abs.exists() {
|
||||||
let abs_clone = abs.clone();
|
let abs_clone = abs.clone();
|
||||||
tokio::task::spawn_blocking(move || std::fs::remove_file(&abs_clone))
|
tokio::task::spawn_blocking(move || std::fs::remove_file(&abs_clone))
|
||||||
@@ -421,7 +496,12 @@ impl FsSyncServer {
|
|||||||
})
|
})
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
Ok(SyncStats { files_added, files_modified, files_removed, bytes_transferred })
|
Ok(SyncStats {
|
||||||
|
files_added,
|
||||||
|
files_modified,
|
||||||
|
files_removed,
|
||||||
|
bytes_transferred,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -432,11 +512,11 @@ impl FsSyncServer {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use clawsync_transport::tcp::{TcpConnection, TcpServer};
|
||||||
use globset::GlobSetBuilder;
|
use globset::GlobSetBuilder;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use tempfile::TempDir;
|
|
||||||
use clawsync_transport::tcp::{TcpConnection, TcpServer};
|
|
||||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||||
|
use tempfile::TempDir;
|
||||||
|
|
||||||
fn empty_excludes() -> GlobSet {
|
fn empty_excludes() -> GlobSet {
|
||||||
GlobSetBuilder::new().build().unwrap()
|
GlobSetBuilder::new().build().unwrap()
|
||||||
@@ -463,10 +543,15 @@ mod tests {
|
|||||||
});
|
});
|
||||||
|
|
||||||
let conn = TcpConnection::connect(addr).await.unwrap();
|
let conn = TcpConnection::connect(addr).await.unwrap();
|
||||||
let stats = FsSyncClient::new(SyncPeer::Tcp(conn), src_dir.to_path_buf(), empty_excludes(), delete)
|
let stats = FsSyncClient::new(
|
||||||
.run()
|
SyncPeer::Tcp(conn),
|
||||||
.await
|
src_dir.to_path_buf(),
|
||||||
.unwrap();
|
empty_excludes(),
|
||||||
|
delete,
|
||||||
|
)
|
||||||
|
.run()
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
server_task.await.unwrap();
|
server_task.await.unwrap();
|
||||||
stats
|
stats
|
||||||
@@ -486,7 +571,10 @@ mod tests {
|
|||||||
assert_eq!(stats.files_modified, 0);
|
assert_eq!(stats.files_modified, 0);
|
||||||
assert!(dst.path().join("a.bin").exists());
|
assert!(dst.path().join("a.bin").exists());
|
||||||
assert!(dst.path().join("b.bin").exists());
|
assert!(dst.path().join("b.bin").exists());
|
||||||
assert_eq!(fs::read(dst.path().join("a.bin")).unwrap(), vec![0x11u8; 4096]);
|
assert_eq!(
|
||||||
|
fs::read(dst.path().join("a.bin")).unwrap(),
|
||||||
|
vec![0x11u8; 4096]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -102,7 +102,10 @@ pub fn diff_manifests(old: &DatasetManifest, new: &DatasetManifest) -> DiffResul
|
|||||||
ord.then_with(|| a.path.cmp(&b.path))
|
ord.then_with(|| a.path.cmp(&b.path))
|
||||||
});
|
});
|
||||||
|
|
||||||
DiffResult { patches, unchanged_count }
|
DiffResult {
|
||||||
|
patches,
|
||||||
|
unchanged_count,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Given a set of patches keyed by dataset path, look up data bytes in `src`
|
/// Given a set of patches keyed by dataset path, look up data bytes in `src`
|
||||||
@@ -207,7 +210,7 @@ mod tests {
|
|||||||
let new = manifest_from(&[("a", vec![1.0]), ("b", vec![9.9]), ("c", vec![3.0])]);
|
let new = manifest_from(&[("a", vec![1.0]), ("b", vec![9.9]), ("c", vec![3.0])]);
|
||||||
let diff = diff_manifests(&old, &new);
|
let diff = diff_manifests(&old, &new);
|
||||||
assert_eq!(diff.unchanged_count, 2); // a and c
|
assert_eq!(diff.unchanged_count, 2); // a and c
|
||||||
assert_eq!(diff.patches.len(), 1); // only b modified
|
assert_eq!(diff.patches.len(), 1); // only b modified
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -269,7 +272,10 @@ mod tests {
|
|||||||
|
|
||||||
let resolved = resolve_patch_data(&diff.patches, &src_file);
|
let resolved = resolve_patch_data(&diff.patches, &src_file);
|
||||||
assert_eq!(resolved.len(), 1);
|
assert_eq!(resolved.len(), 1);
|
||||||
assert!(resolved[0].is_some(), "Added patch should resolve to Some(bytes)");
|
assert!(
|
||||||
|
resolved[0].is_some(),
|
||||||
|
"Added patch should resolve to Some(bytes)"
|
||||||
|
);
|
||||||
// 3 f64 values = 24 bytes
|
// 3 f64 values = 24 bytes
|
||||||
assert_eq!(resolved[0].as_ref().unwrap().len(), 3 * 8);
|
assert_eq!(resolved[0].as_ref().unwrap().len(), 3 * 8);
|
||||||
}
|
}
|
||||||
@@ -284,7 +290,10 @@ mod tests {
|
|||||||
// The Removed patch has kind=Removed, so resolve should return None.
|
// The Removed patch has kind=Removed, so resolve should return None.
|
||||||
let resolved = resolve_patch_data(&diff.patches, &src_file);
|
let resolved = resolve_patch_data(&diff.patches, &src_file);
|
||||||
assert_eq!(resolved.len(), 1);
|
assert_eq!(resolved.len(), 1);
|
||||||
assert!(resolved[0].is_none(), "Removed patch should resolve to None");
|
assert!(
|
||||||
|
resolved[0].is_none(),
|
||||||
|
"Removed patch should resolve to None"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -14,7 +14,11 @@ pub enum Hdf5SyncError {
|
|||||||
DatasetNotFound(String),
|
DatasetNotFound(String),
|
||||||
|
|
||||||
#[error("patch type mismatch for dataset '{path}': expected {expected}, got {actual}")]
|
#[error("patch type mismatch for dataset '{path}': expected {expected}, got {actual}")]
|
||||||
TypeMismatch { path: String, expected: String, actual: String },
|
TypeMismatch {
|
||||||
|
path: String,
|
||||||
|
expected: String,
|
||||||
|
actual: String,
|
||||||
|
},
|
||||||
|
|
||||||
#[error("manifest serialization error: {0}")]
|
#[error("manifest serialization error: {0}")]
|
||||||
Serialize(String),
|
Serialize(String),
|
||||||
|
|||||||
@@ -24,5 +24,6 @@ pub mod patcher;
|
|||||||
pub use differ::{DatasetPatch, DiffResult, PatchKind, diff_manifests};
|
pub use differ::{DatasetPatch, DiffResult, PatchKind, diff_manifests};
|
||||||
pub use error::Hdf5SyncError;
|
pub use error::Hdf5SyncError;
|
||||||
pub use manifest::{DatasetEntry, DatasetManifest};
|
pub use manifest::{DatasetEntry, DatasetManifest};
|
||||||
pub use patcher::{PatchStats, ReceivedDataset, apply_patches, apply_patches_to_bytes,
|
pub use patcher::{
|
||||||
apply_received_payloads};
|
PatchStats, ReceivedDataset, apply_patches, apply_patches_to_bytes, apply_received_payloads,
|
||||||
|
};
|
||||||
|
|||||||
@@ -48,7 +48,10 @@ impl DatasetManifest {
|
|||||||
let file = File::open(h5_path)?;
|
let file = File::open(h5_path)?;
|
||||||
let mut datasets = HashMap::new();
|
let mut datasets = HashMap::new();
|
||||||
collect_datasets(file.root(), "", &mut datasets)?;
|
collect_datasets(file.root(), "", &mut datasets)?;
|
||||||
Ok(Self { file_blake3, datasets })
|
Ok(Self {
|
||||||
|
file_blake3,
|
||||||
|
datasets,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build a manifest from already-read bytes (avoids double I/O when the
|
/// Build a manifest from already-read bytes (avoids double I/O when the
|
||||||
@@ -58,7 +61,10 @@ impl DatasetManifest {
|
|||||||
let file = File::from_bytes(raw.to_vec())?;
|
let file = File::from_bytes(raw.to_vec())?;
|
||||||
let mut datasets = HashMap::new();
|
let mut datasets = HashMap::new();
|
||||||
collect_datasets(file.root(), "", &mut datasets)?;
|
collect_datasets(file.root(), "", &mut datasets)?;
|
||||||
Ok(Self { file_blake3, datasets })
|
Ok(Self {
|
||||||
|
file_blake3,
|
||||||
|
datasets,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Return paths of all datasets in this manifest.
|
/// Return paths of all datasets in this manifest.
|
||||||
@@ -113,7 +119,16 @@ fn collect_datasets(
|
|||||||
let blake3 = blake3_bytes(&raw);
|
let blake3 = blake3_bytes(&raw);
|
||||||
let byte_len = raw.len() as u64;
|
let byte_len = raw.len() as u64;
|
||||||
|
|
||||||
out.insert(path.clone(), DatasetEntry { path, blake3, shape, dtype, byte_len });
|
out.insert(
|
||||||
|
path.clone(),
|
||||||
|
DatasetEntry {
|
||||||
|
path,
|
||||||
|
blake3,
|
||||||
|
shape,
|
||||||
|
dtype,
|
||||||
|
byte_len,
|
||||||
|
},
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Recurse into sub-groups
|
// Recurse into sub-groups
|
||||||
@@ -167,10 +182,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn manifest_counts_datasets() {
|
fn manifest_counts_datasets() {
|
||||||
let bytes = write_test_h5(&[
|
let bytes = write_test_h5(&[("alpha", vec![1.0, 2.0]), ("beta", vec![3.0, 4.0, 5.0])]);
|
||||||
("alpha", vec![1.0, 2.0]),
|
|
||||||
("beta", vec![3.0, 4.0, 5.0]),
|
|
||||||
]);
|
|
||||||
let m = DatasetManifest::from_bytes(&bytes).unwrap();
|
let m = DatasetManifest::from_bytes(&bytes).unwrap();
|
||||||
assert_eq!(m.len(), 2);
|
assert_eq!(m.len(), 2);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -178,7 +178,9 @@ pub fn apply_patches_to_bytes(
|
|||||||
|
|
||||||
for patch in patches {
|
for patch in patches {
|
||||||
match patch.kind {
|
match patch.kind {
|
||||||
PatchKind::Removed => { stats.datasets_removed += 1; }
|
PatchKind::Removed => {
|
||||||
|
stats.datasets_removed += 1;
|
||||||
|
}
|
||||||
PatchKind::Added => {
|
PatchKind::Added => {
|
||||||
if let Some(data) = patched.get(&patch.path) {
|
if let Some(data) = patched.get(&patch.path) {
|
||||||
stats.bytes_written += data.raw.len() as u64;
|
stats.bytes_written += data.raw.len() as u64;
|
||||||
@@ -261,7 +263,11 @@ pub fn apply_received_payloads(
|
|||||||
// Write received (Added/Modified) datasets.
|
// Write received (Added/Modified) datasets.
|
||||||
for p in payloads {
|
for p in payloads {
|
||||||
let dtype = dtype_from_str(&p.dtype);
|
let dtype = dtype_from_str(&p.dtype);
|
||||||
let raw = DatasetRaw { raw: p.data.clone(), dtype, shape: p.shape.clone() };
|
let raw = DatasetRaw {
|
||||||
|
raw: p.data.clone(),
|
||||||
|
dtype,
|
||||||
|
shape: p.shape.clone(),
|
||||||
|
};
|
||||||
write_dataset(&mut builder, p.path.trim_start_matches('/'), &raw);
|
write_dataset(&mut builder, p.path.trim_start_matches('/'), &raw);
|
||||||
// Classify as added or modified based on whether server had it.
|
// Classify as added or modified based on whether server had it.
|
||||||
if existing.contains_key(&p.path) {
|
if existing.contains_key(&p.path) {
|
||||||
@@ -383,31 +389,41 @@ fn write_dataset(builder: &mut FileBuilder, name: &str, data: &DatasetRaw) {
|
|||||||
let ds = builder.create_dataset(name);
|
let ds = builder.create_dataset(name);
|
||||||
match &data.dtype {
|
match &data.dtype {
|
||||||
DType::F64 => {
|
DType::F64 => {
|
||||||
let vals: Vec<f64> = data.raw.chunks_exact(8)
|
let vals: Vec<f64> = data
|
||||||
|
.raw
|
||||||
|
.chunks_exact(8)
|
||||||
.map(|b| f64::from_le_bytes(b.try_into().unwrap_or([0u8; 8])))
|
.map(|b| f64::from_le_bytes(b.try_into().unwrap_or([0u8; 8])))
|
||||||
.collect();
|
.collect();
|
||||||
ds.with_f64_data(&vals);
|
ds.with_f64_data(&vals);
|
||||||
}
|
}
|
||||||
DType::F32 => {
|
DType::F32 => {
|
||||||
let vals: Vec<f32> = data.raw.chunks_exact(4)
|
let vals: Vec<f32> = data
|
||||||
|
.raw
|
||||||
|
.chunks_exact(4)
|
||||||
.map(|b| f32::from_le_bytes(b.try_into().unwrap_or([0u8; 4])))
|
.map(|b| f32::from_le_bytes(b.try_into().unwrap_or([0u8; 4])))
|
||||||
.collect();
|
.collect();
|
||||||
ds.with_f32_data(&vals);
|
ds.with_f32_data(&vals);
|
||||||
}
|
}
|
||||||
DType::I32 => {
|
DType::I32 => {
|
||||||
let vals: Vec<i32> = data.raw.chunks_exact(4)
|
let vals: Vec<i32> = data
|
||||||
|
.raw
|
||||||
|
.chunks_exact(4)
|
||||||
.map(|b| i32::from_le_bytes(b.try_into().unwrap_or([0u8; 4])))
|
.map(|b| i32::from_le_bytes(b.try_into().unwrap_or([0u8; 4])))
|
||||||
.collect();
|
.collect();
|
||||||
ds.with_i32_data(&vals);
|
ds.with_i32_data(&vals);
|
||||||
}
|
}
|
||||||
DType::I64 => {
|
DType::I64 => {
|
||||||
let vals: Vec<i64> = data.raw.chunks_exact(8)
|
let vals: Vec<i64> = data
|
||||||
|
.raw
|
||||||
|
.chunks_exact(8)
|
||||||
.map(|b| i64::from_le_bytes(b.try_into().unwrap_or([0u8; 8])))
|
.map(|b| i64::from_le_bytes(b.try_into().unwrap_or([0u8; 8])))
|
||||||
.collect();
|
.collect();
|
||||||
ds.with_i64_data(&vals);
|
ds.with_i64_data(&vals);
|
||||||
}
|
}
|
||||||
DType::U64 => {
|
DType::U64 => {
|
||||||
let vals: Vec<u64> = data.raw.chunks_exact(8)
|
let vals: Vec<u64> = data
|
||||||
|
.raw
|
||||||
|
.chunks_exact(8)
|
||||||
.map(|b| u64::from_le_bytes(b.try_into().unwrap_or([0u8; 8])))
|
.map(|b| u64::from_le_bytes(b.try_into().unwrap_or([0u8; 8])))
|
||||||
.collect();
|
.collect();
|
||||||
// Use i64 storage for u64 (bit-preserving)
|
// Use i64 storage for u64 (bit-preserving)
|
||||||
@@ -462,7 +478,10 @@ mod tests {
|
|||||||
|
|
||||||
let result = DatasetManifest::from_bytes(&patched).unwrap();
|
let result = DatasetManifest::from_bytes(&patched).unwrap();
|
||||||
let src_m = DatasetManifest::from_bytes(&src).unwrap();
|
let src_m = DatasetManifest::from_bytes(&src).unwrap();
|
||||||
assert_eq!(result.get("/x").unwrap().blake3, src_m.get("/x").unwrap().blake3);
|
assert_eq!(
|
||||||
|
result.get("/x").unwrap().blake3,
|
||||||
|
src_m.get("/x").unwrap().blake3
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -495,8 +514,14 @@ mod tests {
|
|||||||
|
|
||||||
let orig = DatasetManifest::from_bytes(&bytes).unwrap();
|
let orig = DatasetManifest::from_bytes(&bytes).unwrap();
|
||||||
let result = DatasetManifest::from_bytes(&patched).unwrap();
|
let result = DatasetManifest::from_bytes(&patched).unwrap();
|
||||||
assert_eq!(orig.get("/a").unwrap().blake3, result.get("/a").unwrap().blake3);
|
assert_eq!(
|
||||||
assert_eq!(orig.get("/b").unwrap().blake3, result.get("/b").unwrap().blake3);
|
orig.get("/a").unwrap().blake3,
|
||||||
|
result.get("/a").unwrap().blake3
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
orig.get("/b").unwrap().blake3,
|
||||||
|
result.get("/b").unwrap().blake3
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -511,7 +536,7 @@ mod tests {
|
|||||||
let (_, stats) = apply_patches_to_bytes(&diff.patches, &src, &dst).unwrap();
|
let (_, stats) = apply_patches_to_bytes(&diff.patches, &src, &dst).unwrap();
|
||||||
|
|
||||||
assert_eq!(stats.datasets_modified, 1); // a
|
assert_eq!(stats.datasets_modified, 1); // a
|
||||||
assert_eq!(stats.datasets_removed, 1); // b
|
assert_eq!(stats.datasets_removed, 1); // b
|
||||||
assert_eq!(stats.datasets_unchanged, 1); // c
|
assert_eq!(stats.datasets_unchanged, 1); // c
|
||||||
assert_eq!(stats.datasets_added, 0);
|
assert_eq!(stats.datasets_added, 0);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,9 @@
|
|||||||
//! creating real HDF5 files in memory and on disk as needed.
|
//! creating real HDF5 files in memory and on disk as needed.
|
||||||
|
|
||||||
use clawhdf5::{File, FileBuilder};
|
use clawhdf5::{File, FileBuilder};
|
||||||
use clawsync_hdf5::{DatasetManifest, PatchKind, apply_patches, apply_patches_to_bytes, diff_manifests};
|
use clawsync_hdf5::{
|
||||||
|
DatasetManifest, PatchKind, apply_patches, apply_patches_to_bytes, diff_manifests,
|
||||||
|
};
|
||||||
use tempfile::NamedTempFile;
|
use tempfile::NamedTempFile;
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -37,7 +39,10 @@ fn write_temp_h5(bytes: &[u8]) -> (NamedTempFile, std::path::PathBuf) {
|
|||||||
/// A dataset present in source but absent from target is added after patching.
|
/// A dataset present in source but absent from target is added after patching.
|
||||||
#[test]
|
#[test]
|
||||||
fn patch_adds_new_dataset() {
|
fn patch_adds_new_dataset() {
|
||||||
let src = h5_bytes(&[("temperature", &[20.0, 21.5, 22.0]), ("humidity", &[55.0, 57.0])]);
|
let src = h5_bytes(&[
|
||||||
|
("temperature", &[20.0, 21.5, 22.0]),
|
||||||
|
("humidity", &[55.0, 57.0]),
|
||||||
|
]);
|
||||||
let tgt = h5_bytes(&[("temperature", &[20.0, 21.5, 22.0])]);
|
let tgt = h5_bytes(&[("temperature", &[20.0, 21.5, 22.0])]);
|
||||||
|
|
||||||
let old_m = DatasetManifest::from_bytes(&tgt).unwrap();
|
let old_m = DatasetManifest::from_bytes(&tgt).unwrap();
|
||||||
@@ -51,7 +56,8 @@ fn patch_adds_new_dataset() {
|
|||||||
let result_m = DatasetManifest::from_bytes(&patched).unwrap();
|
let result_m = DatasetManifest::from_bytes(&patched).unwrap();
|
||||||
|
|
||||||
// The new dataset must be present with the correct hash.
|
// The new dataset must be present with the correct hash.
|
||||||
let result_entry = result_m.get("/humidity")
|
let result_entry = result_m
|
||||||
|
.get("/humidity")
|
||||||
.expect("humidity should exist after patching");
|
.expect("humidity should exist after patching");
|
||||||
let src_entry = new_m.get("/humidity").unwrap();
|
let src_entry = new_m.get("/humidity").unwrap();
|
||||||
assert_eq!(result_entry.blake3, src_entry.blake3);
|
assert_eq!(result_entry.blake3, src_entry.blake3);
|
||||||
@@ -107,13 +113,22 @@ fn patch_removes_dataset() {
|
|||||||
let diff = diff_manifests(&old_m, &new_m);
|
let diff = diff_manifests(&old_m, &new_m);
|
||||||
|
|
||||||
assert_eq!(diff.of_kind(&PatchKind::Removed).count(), 1);
|
assert_eq!(diff.of_kind(&PatchKind::Removed).count(), 1);
|
||||||
assert_eq!(diff.of_kind(&PatchKind::Removed).next().unwrap().path, "/current");
|
assert_eq!(
|
||||||
|
diff.of_kind(&PatchKind::Removed).next().unwrap().path,
|
||||||
|
"/current"
|
||||||
|
);
|
||||||
|
|
||||||
let (patched, _) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap();
|
let (patched, _) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap();
|
||||||
let result_m = DatasetManifest::from_bytes(&patched).unwrap();
|
let result_m = DatasetManifest::from_bytes(&patched).unwrap();
|
||||||
|
|
||||||
assert!(result_m.get("/current").is_none(), "current should have been removed");
|
assert!(
|
||||||
assert!(result_m.get("/voltage").is_some(), "voltage should be retained");
|
result_m.get("/current").is_none(),
|
||||||
|
"current should have been removed"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
result_m.get("/voltage").is_some(),
|
||||||
|
"voltage should be retained"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -148,13 +163,13 @@ fn patch_noop_when_identical() {
|
|||||||
fn patch_multiple_changes() {
|
fn patch_multiple_changes() {
|
||||||
// source: "kept" (same), "updated" (changed), "added" (new)
|
// source: "kept" (same), "updated" (changed), "added" (new)
|
||||||
let src = h5_bytes(&[
|
let src = h5_bytes(&[
|
||||||
("kept", &[1.0, 2.0]),
|
("kept", &[1.0, 2.0]),
|
||||||
("updated", &[99.0, 100.0]),
|
("updated", &[99.0, 100.0]),
|
||||||
("added", &[7.0, 8.0, 9.0]),
|
("added", &[7.0, 8.0, 9.0]),
|
||||||
]);
|
]);
|
||||||
// target: "kept" (same), "updated" (old values), "removed" (extra)
|
// target: "kept" (same), "updated" (old values), "removed" (extra)
|
||||||
let tgt = h5_bytes(&[
|
let tgt = h5_bytes(&[
|
||||||
("kept", &[1.0, 2.0]),
|
("kept", &[1.0, 2.0]),
|
||||||
("updated", &[0.0, 0.0]),
|
("updated", &[0.0, 0.0]),
|
||||||
("removed", &[42.0]),
|
("removed", &[42.0]),
|
||||||
]);
|
]);
|
||||||
@@ -163,9 +178,9 @@ fn patch_multiple_changes() {
|
|||||||
let new_m = DatasetManifest::from_bytes(&src).unwrap();
|
let new_m = DatasetManifest::from_bytes(&src).unwrap();
|
||||||
let diff = diff_manifests(&old_m, &new_m);
|
let diff = diff_manifests(&old_m, &new_m);
|
||||||
|
|
||||||
assert_eq!(diff.of_kind(&PatchKind::Added).count(), 1);
|
assert_eq!(diff.of_kind(&PatchKind::Added).count(), 1);
|
||||||
assert_eq!(diff.of_kind(&PatchKind::Modified).count(), 1);
|
assert_eq!(diff.of_kind(&PatchKind::Modified).count(), 1);
|
||||||
assert_eq!(diff.of_kind(&PatchKind::Removed).count(), 1);
|
assert_eq!(diff.of_kind(&PatchKind::Removed).count(), 1);
|
||||||
|
|
||||||
let (patched, _) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap();
|
let (patched, _) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap();
|
||||||
let result_m = DatasetManifest::from_bytes(&patched).unwrap();
|
let result_m = DatasetManifest::from_bytes(&patched).unwrap();
|
||||||
@@ -194,26 +209,18 @@ fn patch_multiple_changes() {
|
|||||||
#[test]
|
#[test]
|
||||||
fn stats_counts_correct() {
|
fn stats_counts_correct() {
|
||||||
// source: "a" (modified), "c" (unchanged), "d" (new)
|
// source: "a" (modified), "c" (unchanged), "d" (new)
|
||||||
let src = h5_bytes(&[
|
let src = h5_bytes(&[("a", &[9.0, 9.0]), ("c", &[3.0]), ("d", &[4.0, 5.0])]);
|
||||||
("a", &[9.0, 9.0]),
|
|
||||||
("c", &[3.0]),
|
|
||||||
("d", &[4.0, 5.0]),
|
|
||||||
]);
|
|
||||||
// target: "a" (old values), "b" (to be removed), "c" (same as source)
|
// target: "a" (old values), "b" (to be removed), "c" (same as source)
|
||||||
let tgt = h5_bytes(&[
|
let tgt = h5_bytes(&[("a", &[1.0, 2.0]), ("b", &[0.0]), ("c", &[3.0])]);
|
||||||
("a", &[1.0, 2.0]),
|
|
||||||
("b", &[0.0]),
|
|
||||||
("c", &[3.0]),
|
|
||||||
]);
|
|
||||||
|
|
||||||
let old_m = DatasetManifest::from_bytes(&tgt).unwrap();
|
let old_m = DatasetManifest::from_bytes(&tgt).unwrap();
|
||||||
let new_m = DatasetManifest::from_bytes(&src).unwrap();
|
let new_m = DatasetManifest::from_bytes(&src).unwrap();
|
||||||
let diff = diff_manifests(&old_m, &new_m);
|
let diff = diff_manifests(&old_m, &new_m);
|
||||||
let (_, stats) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap();
|
let (_, stats) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap();
|
||||||
|
|
||||||
assert_eq!(stats.datasets_added, 1, "d was added");
|
assert_eq!(stats.datasets_added, 1, "d was added");
|
||||||
assert_eq!(stats.datasets_modified, 1, "a was modified");
|
assert_eq!(stats.datasets_modified, 1, "a was modified");
|
||||||
assert_eq!(stats.datasets_removed, 1, "b was removed");
|
assert_eq!(stats.datasets_removed, 1, "b was removed");
|
||||||
assert_eq!(stats.datasets_unchanged, 1, "c was unchanged");
|
assert_eq!(stats.datasets_unchanged, 1, "c was unchanged");
|
||||||
// bytes_written must be positive (added + modified + unchanged datasets).
|
// bytes_written must be positive (added + modified + unchanged datasets).
|
||||||
assert!(stats.bytes_written > 0);
|
assert!(stats.bytes_written > 0);
|
||||||
@@ -227,13 +234,8 @@ fn stats_counts_correct() {
|
|||||||
/// manifest read back from disk matches the source manifest.
|
/// manifest read back from disk matches the source manifest.
|
||||||
#[test]
|
#[test]
|
||||||
fn apply_patches_file_path_api() {
|
fn apply_patches_file_path_api() {
|
||||||
let src_bytes = h5_bytes(&[
|
let src_bytes = h5_bytes(&[("chan_x", &[1.1, 2.2, 3.3]), ("chan_y", &[4.4, 5.5])]);
|
||||||
("chan_x", &[1.1, 2.2, 3.3]),
|
let tgt_bytes = h5_bytes(&[("chan_x", &[0.0, 0.0, 0.0])]);
|
||||||
("chan_y", &[4.4, 5.5]),
|
|
||||||
]);
|
|
||||||
let tgt_bytes = h5_bytes(&[
|
|
||||||
("chan_x", &[0.0, 0.0, 0.0]),
|
|
||||||
]);
|
|
||||||
|
|
||||||
// Write both files to disk using NamedTempFile.
|
// Write both files to disk using NamedTempFile.
|
||||||
let (_src_guard, src_path) = write_temp_h5(&src_bytes);
|
let (_src_guard, src_path) = write_temp_h5(&src_bytes);
|
||||||
@@ -248,8 +250,11 @@ fn apply_patches_file_path_api() {
|
|||||||
// Use the file-path API (the one under test).
|
// Use the file-path API (the one under test).
|
||||||
let stats = apply_patches(&diff.patches, &src_path, &tgt_path).unwrap();
|
let stats = apply_patches(&diff.patches, &src_path, &tgt_path).unwrap();
|
||||||
|
|
||||||
assert!(stats.datasets_added >= 1, "chan_y should have been added");
|
assert!(stats.datasets_added >= 1, "chan_y should have been added");
|
||||||
assert!(stats.datasets_modified >= 1, "chan_x should have been modified");
|
assert!(
|
||||||
|
stats.datasets_modified >= 1,
|
||||||
|
"chan_x should have been modified"
|
||||||
|
);
|
||||||
|
|
||||||
// Read the patched file back from disk and verify it matches source.
|
// Read the patched file back from disk and verify it matches source.
|
||||||
let result_m = DatasetManifest::from_path(&tgt_path).unwrap();
|
let result_m = DatasetManifest::from_path(&tgt_path).unwrap();
|
||||||
|
|||||||
@@ -9,7 +9,7 @@
|
|||||||
//! Run:
|
//! Run:
|
||||||
//! cargo bench -p clawsync-onion -- iblt_bench
|
//! cargo bench -p clawsync-onion -- iblt_bench
|
||||||
|
|
||||||
use clawsync_onion::iblt::{IbltSketch, DEFAULT_HASH_COUNT, MIN_CELLS};
|
use clawsync_onion::iblt::{DEFAULT_HASH_COUNT, IbltSketch, MIN_CELLS};
|
||||||
use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
|
use criterion::{BenchmarkId, Criterion, black_box, criterion_group, criterion_main};
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -28,8 +28,12 @@ fn diff_sketches(n: usize, d: usize) -> (IbltSketch, Vec<u8>) {
|
|||||||
let m = ((d * 4).max(MIN_CELLS)).max(IbltSketch::recommended_cells(n));
|
let m = ((d * 4).max(MIN_CELLS)).max(IbltSketch::recommended_cells(n));
|
||||||
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||||
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||||
for &k in &a_keys { a.insert(k); }
|
for &k in &a_keys {
|
||||||
for &k in &b_keys { b.insert(k); }
|
a.insert(k);
|
||||||
|
}
|
||||||
|
for &k in &b_keys {
|
||||||
|
b.insert(k);
|
||||||
|
}
|
||||||
let b_bytes = b.to_bytes();
|
let b_bytes = b.to_bytes();
|
||||||
a.subtract(&b);
|
a.subtract(&b);
|
||||||
(a, b_bytes)
|
(a, b_bytes)
|
||||||
@@ -79,8 +83,12 @@ fn bench_iblt(c: &mut Criterion) {
|
|||||||
let m = (d * 4).max(MIN_CELLS).max(IbltSketch::recommended_cells(n));
|
let m = (d * 4).max(MIN_CELLS).max(IbltSketch::recommended_cells(n));
|
||||||
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||||
let mut b_sk = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
let mut b_sk = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||||
for &k in black_box(ak) { a.insert(k); }
|
for &k in black_box(ak) {
|
||||||
for &k in black_box(bk) { b_sk.insert(k); }
|
a.insert(k);
|
||||||
|
}
|
||||||
|
for &k in black_box(bk) {
|
||||||
|
b_sk.insert(k);
|
||||||
|
}
|
||||||
a.subtract(&b_sk);
|
a.subtract(&b_sk);
|
||||||
black_box(a.decode())
|
black_box(a.decode())
|
||||||
});
|
});
|
||||||
@@ -92,14 +100,20 @@ fn bench_iblt(c: &mut Criterion) {
|
|||||||
|
|
||||||
// ── Wire size table (not timed) ───────────────────────────────────────────
|
// ── Wire size table (not timed) ───────────────────────────────────────────
|
||||||
println!("\n=== IBLT sketch size vs flat manifest (N × 60 B) ===");
|
println!("\n=== IBLT sketch size vs flat manifest (N × 60 B) ===");
|
||||||
println!("{:>8} {:>10} {:>12} {:>10}", "N", "IBLT (B)", "Flat (B)", "ratio");
|
println!(
|
||||||
|
"{:>8} {:>10} {:>12} {:>10}",
|
||||||
|
"N", "IBLT (B)", "Flat (B)", "ratio"
|
||||||
|
);
|
||||||
for &n in &[10usize, 100, 1_000, 10_000, 100_000] {
|
for &n in &[10usize, 100, 1_000, 10_000, 100_000] {
|
||||||
let ks = keys(n);
|
let ks = keys(n);
|
||||||
let sz = IbltSketch::from_keys(&ks, SEED).to_bytes().len();
|
let sz = IbltSketch::from_keys(&ks, SEED).to_bytes().len();
|
||||||
let flat = n * 60;
|
let flat = n * 60;
|
||||||
println!(
|
println!(
|
||||||
"{:>8} {:>10} {:>12} {:>9.1}×",
|
"{:>8} {:>10} {:>12} {:>9.1}×",
|
||||||
n, sz, flat, flat as f64 / sz as f64
|
n,
|
||||||
|
sz,
|
||||||
|
flat,
|
||||||
|
flat as f64 / sz as f64
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,16 +49,21 @@ pub fn diff_revisions(
|
|||||||
|
|
||||||
let pages: Vec<OnionPage> = raw_pages
|
let pages: Vec<OnionPage> = raw_pages
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|(h5_offset, data, codec, orig_size)| OnionPage { h5_offset, data, codec, orig_size })
|
.map(|(h5_offset, data, codec, orig_size)| OnionPage {
|
||||||
|
h5_offset,
|
||||||
|
data,
|
||||||
|
codec,
|
||||||
|
orig_size,
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
// Parse the blake3 hex back to bytes
|
// Parse the blake3 hex back to bytes
|
||||||
let blake3_root = hex_to_bytes32(&summary.blake3_hex)
|
let blake3_root = hex_to_bytes32(&summary.blake3_hex).ok_or_else(|| {
|
||||||
.ok_or_else(|| {
|
SyncOnionError::Onion(clawhdf5_onion::OnionError::Malformed(format!(
|
||||||
SyncOnionError::Onion(clawhdf5_onion::OnionError::Malformed(
|
"invalid blake3_hex for rev {}",
|
||||||
format!("invalid blake3_hex for rev {}", summary.revision),
|
summary.revision
|
||||||
))
|
)))
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
packets.push(OnionLayerPacket {
|
packets.push(OnionLayerPacket {
|
||||||
revision: summary.revision,
|
revision: summary.revision,
|
||||||
@@ -103,13 +108,18 @@ pub fn packets_for_revisions(
|
|||||||
let raw_pages = local.revision_pages_raw(rev)?;
|
let raw_pages = local.revision_pages_raw(rev)?;
|
||||||
let pages: Vec<OnionPage> = raw_pages
|
let pages: Vec<OnionPage> = raw_pages
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|(h5_offset, data, codec, orig_size)| OnionPage { h5_offset, data, codec, orig_size })
|
.map(|(h5_offset, data, codec, orig_size)| OnionPage {
|
||||||
|
h5_offset,
|
||||||
|
data,
|
||||||
|
codec,
|
||||||
|
orig_size,
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let blake3_root = hex_to_bytes32(&summary.blake3_hex).ok_or_else(|| {
|
let blake3_root = hex_to_bytes32(&summary.blake3_hex).ok_or_else(|| {
|
||||||
SyncOnionError::Onion(clawhdf5_onion::OnionError::Malformed(
|
SyncOnionError::Onion(clawhdf5_onion::OnionError::Malformed(format!(
|
||||||
format!("invalid blake3_hex for rev {rev}"),
|
"invalid blake3_hex for rev {rev}"
|
||||||
))
|
)))
|
||||||
})?;
|
})?;
|
||||||
|
|
||||||
packets.push(OnionLayerPacket {
|
packets.push(OnionLayerPacket {
|
||||||
@@ -198,7 +208,12 @@ pub fn diff_revisions_merkle(
|
|||||||
let raw_pages = local.revision_pages_raw(rev)?;
|
let raw_pages = local.revision_pages_raw(rev)?;
|
||||||
let pages: Vec<OnionPage> = raw_pages
|
let pages: Vec<OnionPage> = raw_pages
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|(h5_offset, data, codec, orig_size)| OnionPage { h5_offset, data, codec, orig_size })
|
.map(|(h5_offset, data, codec, orig_size)| OnionPage {
|
||||||
|
h5_offset,
|
||||||
|
data,
|
||||||
|
codec,
|
||||||
|
orig_size,
|
||||||
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let blake3_root = hex_to_bytes32(&summary.blake3_hex).ok_or_else(|| {
|
let blake3_root = hex_to_bytes32(&summary.blake3_hex).ok_or_else(|| {
|
||||||
@@ -413,11 +428,11 @@ mod tests {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let merkle_packets = diff_revisions_merkle(&local, &remote_tree.serialise()).unwrap();
|
let merkle_packets = diff_revisions_merkle(&local, &remote_tree.serialise()).unwrap();
|
||||||
let flat_packets = diff_revisions(&local, 1 /* remote head = rev 1 */).unwrap();
|
let flat_packets = diff_revisions(&local, 1 /* remote head = rev 1 */).unwrap();
|
||||||
|
|
||||||
// Revision numbers should match
|
// Revision numbers should match
|
||||||
let merkle_revs: Vec<u64> = merkle_packets.iter().map(|p| p.revision).collect();
|
let merkle_revs: Vec<u64> = merkle_packets.iter().map(|p| p.revision).collect();
|
||||||
let flat_revs: Vec<u64> = flat_packets.iter().map(|p| p.revision).collect();
|
let flat_revs: Vec<u64> = flat_packets.iter().map(|p| p.revision).collect();
|
||||||
assert_eq!(merkle_revs, flat_revs);
|
assert_eq!(merkle_revs, flat_revs);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -139,7 +139,9 @@ impl IbltSketch {
|
|||||||
pub fn from_keys(keys: &[u64], seed: u64) -> Self {
|
pub fn from_keys(keys: &[u64], seed: u64) -> Self {
|
||||||
let m = Self::recommended_cells(keys.len());
|
let m = Self::recommended_cells(keys.len());
|
||||||
let mut sketch = Self::new(m, DEFAULT_HASH_COUNT, seed);
|
let mut sketch = Self::new(m, DEFAULT_HASH_COUNT, seed);
|
||||||
for &k in keys { sketch.insert(k); }
|
for &k in keys {
|
||||||
|
sketch.insert(k);
|
||||||
|
}
|
||||||
sketch
|
sketch
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -182,12 +184,13 @@ impl IbltSketch {
|
|||||||
/// Panics if `self` and `other` have different cell counts.
|
/// Panics if `self` and `other` have different cell counts.
|
||||||
pub fn subtract(&mut self, other: &IbltSketch) {
|
pub fn subtract(&mut self, other: &IbltSketch) {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
self.cells.len(), other.cells.len(),
|
self.cells.len(),
|
||||||
|
other.cells.len(),
|
||||||
"cannot subtract sketches of different sizes"
|
"cannot subtract sketches of different sizes"
|
||||||
);
|
);
|
||||||
for (a, b) in self.cells.iter_mut().zip(&other.cells) {
|
for (a, b) in self.cells.iter_mut().zip(&other.cells) {
|
||||||
a.count -= b.count;
|
a.count -= b.count;
|
||||||
a.id_sum ^= b.id_sum;
|
a.id_sum ^= b.id_sum;
|
||||||
a.hash_sum ^= b.hash_sum;
|
a.hash_sum ^= b.hash_sum;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -213,9 +216,14 @@ impl IbltSketch {
|
|||||||
|
|
||||||
let Some(pos) = pure_pos else {
|
let Some(pos) = pure_pos else {
|
||||||
// No pure cell found.
|
// No pure cell found.
|
||||||
let done = cells.iter().all(|c| c.count == 0 && c.id_sum == 0 && c.hash_sum == 0);
|
let done = cells
|
||||||
|
.iter()
|
||||||
|
.all(|c| c.count == 0 && c.id_sum == 0 && c.hash_sum == 0);
|
||||||
return if done {
|
return if done {
|
||||||
let mut diff = IbltDiff { only_in_a, only_in_b };
|
let mut diff = IbltDiff {
|
||||||
|
only_in_a,
|
||||||
|
only_in_b,
|
||||||
|
};
|
||||||
diff.only_in_a.sort_unstable();
|
diff.only_in_a.sort_unstable();
|
||||||
diff.only_in_b.sort_unstable();
|
diff.only_in_b.sort_unstable();
|
||||||
IbltDecodeResult::Complete(diff)
|
IbltDecodeResult::Complete(diff)
|
||||||
@@ -227,15 +235,19 @@ impl IbltSketch {
|
|||||||
let key = cells[pos].id_sum;
|
let key = cells[pos].id_sum;
|
||||||
let positive = cells[pos].count == 1;
|
let positive = cells[pos].count == 1;
|
||||||
|
|
||||||
if positive { only_in_a.push(key); } else { only_in_b.push(key); }
|
if positive {
|
||||||
|
only_in_a.push(key);
|
||||||
|
} else {
|
||||||
|
only_in_b.push(key);
|
||||||
|
}
|
||||||
|
|
||||||
// Peel this key from all its mapped cells.
|
// Peel this key from all its mapped cells.
|
||||||
let sign: i32 = if positive { 1 } else { -1 };
|
let sign: i32 = if positive { 1 } else { -1 };
|
||||||
let h_key = xxh3_key(key);
|
let h_key = xxh3_key(key);
|
||||||
for h in 0..k {
|
for h in 0..k {
|
||||||
let idx = cell_index(key, h, seed, m);
|
let idx = cell_index(key, h, seed, m);
|
||||||
cells[idx].count -= sign;
|
cells[idx].count -= sign;
|
||||||
cells[idx].id_sum ^= key;
|
cells[idx].id_sum ^= key;
|
||||||
cells[idx].hash_sum ^= h_key;
|
cells[idx].hash_sum ^= h_key;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -287,13 +299,21 @@ impl IbltSketch {
|
|||||||
let mut cells = Vec::with_capacity(m);
|
let mut cells = Vec::with_capacity(m);
|
||||||
let mut off = 21usize;
|
let mut off = 21usize;
|
||||||
for _ in 0..m {
|
for _ in 0..m {
|
||||||
let count = i32::from_le_bytes(data[off..off + 4].try_into().unwrap());
|
let count = i32::from_le_bytes(data[off..off + 4].try_into().unwrap());
|
||||||
let id_sum = u64::from_le_bytes(data[off + 4..off + 12].try_into().unwrap());
|
let id_sum = u64::from_le_bytes(data[off + 4..off + 12].try_into().unwrap());
|
||||||
let hash_sum = u64::from_le_bytes(data[off + 12..off + 20].try_into().unwrap());
|
let hash_sum = u64::from_le_bytes(data[off + 12..off + 20].try_into().unwrap());
|
||||||
cells.push(IbltCell { count, id_sum, hash_sum });
|
cells.push(IbltCell {
|
||||||
|
count,
|
||||||
|
id_sum,
|
||||||
|
hash_sum,
|
||||||
|
});
|
||||||
off += 20;
|
off += 20;
|
||||||
}
|
}
|
||||||
Ok(Self { cells, hash_count: k, seed })
|
Ok(Self {
|
||||||
|
cells,
|
||||||
|
hash_count: k,
|
||||||
|
seed,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Private helpers ───────────────────────────────────────────────────────
|
// ── Private helpers ───────────────────────────────────────────────────────
|
||||||
@@ -303,8 +323,8 @@ impl IbltSketch {
|
|||||||
let m = self.cells.len();
|
let m = self.cells.len();
|
||||||
for h in 0..self.hash_count {
|
for h in 0..self.hash_count {
|
||||||
let idx = cell_index(key, h, self.seed, m);
|
let idx = cell_index(key, h, self.seed, m);
|
||||||
self.cells[idx].count += delta;
|
self.cells[idx].count += delta;
|
||||||
self.cells[idx].id_sum ^= key;
|
self.cells[idx].id_sum ^= key;
|
||||||
self.cells[idx].hash_sum ^= h_key;
|
self.cells[idx].hash_sum ^= h_key;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -349,12 +369,16 @@ pub fn reconcile(
|
|||||||
seed: u64,
|
seed: u64,
|
||||||
) -> Result<(IbltDiff, IbltSketch), IbltError> {
|
) -> Result<(IbltDiff, IbltSketch), IbltError> {
|
||||||
let b_sketch = IbltSketch::from_bytes(b_sketch_bytes)?;
|
let b_sketch = IbltSketch::from_bytes(b_sketch_bytes)?;
|
||||||
let m_start = b_sketch.cell_count().max(IbltSketch::recommended_cells(a_keys.len()));
|
let m_start = b_sketch
|
||||||
|
.cell_count()
|
||||||
|
.max(IbltSketch::recommended_cells(a_keys.len()));
|
||||||
|
|
||||||
let mut m = m_start;
|
let mut m = m_start;
|
||||||
loop {
|
loop {
|
||||||
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, seed);
|
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, seed);
|
||||||
for &k in a_keys { a.insert(k); }
|
for &k in a_keys {
|
||||||
|
a.insert(k);
|
||||||
|
}
|
||||||
|
|
||||||
let mut diff_sketch = a.clone();
|
let mut diff_sketch = a.clone();
|
||||||
let b_padded = pad_or_trim_sketch(&b_sketch, m, seed);
|
let b_padded = pad_or_trim_sketch(&b_sketch, m, seed);
|
||||||
@@ -372,7 +396,11 @@ pub fn reconcile(
|
|||||||
fn pad_or_trim_sketch(src: &IbltSketch, target_m: usize, seed: u64) -> IbltSketch {
|
fn pad_or_trim_sketch(src: &IbltSketch, target_m: usize, seed: u64) -> IbltSketch {
|
||||||
let mut cells = src.cells.clone();
|
let mut cells = src.cells.clone();
|
||||||
cells.resize(target_m, IbltCell::default());
|
cells.resize(target_m, IbltCell::default());
|
||||||
IbltSketch { cells, hash_count: src.hash_count, seed }
|
IbltSketch {
|
||||||
|
cells,
|
||||||
|
hash_count: src.hash_count,
|
||||||
|
seed,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -397,7 +425,11 @@ mod tests {
|
|||||||
let mut s = IbltSketch::new(64, 3, SEED);
|
let mut s = IbltSketch::new(64, 3, SEED);
|
||||||
s.insert(42);
|
s.insert(42);
|
||||||
s.remove(42);
|
s.remove(42);
|
||||||
assert!(s.cells.iter().all(|c| c.count == 0 && c.id_sum == 0 && c.hash_sum == 0));
|
assert!(
|
||||||
|
s.cells
|
||||||
|
.iter()
|
||||||
|
.all(|c| c.count == 0 && c.id_sum == 0 && c.hash_sum == 0)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── decode: empty diff ────────────────────────────────────────────────────
|
// ── decode: empty diff ────────────────────────────────────────────────────
|
||||||
@@ -409,9 +441,13 @@ mod tests {
|
|||||||
let b = sketch_of(&keys);
|
let b = sketch_of(&keys);
|
||||||
a.subtract(&b);
|
a.subtract(&b);
|
||||||
let result = a.decode();
|
let result = a.decode();
|
||||||
assert_eq!(result, IbltDecodeResult::Complete(IbltDiff {
|
assert_eq!(
|
||||||
only_in_a: vec![], only_in_b: vec![],
|
result,
|
||||||
}));
|
IbltDecodeResult::Complete(IbltDiff {
|
||||||
|
only_in_a: vec![],
|
||||||
|
only_in_b: vec![],
|
||||||
|
})
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── decode: one-sided diff ────────────────────────────────────────────────
|
// ── decode: one-sided diff ────────────────────────────────────────────────
|
||||||
@@ -420,14 +456,22 @@ mod tests {
|
|||||||
fn decode_one_sided_diff_a_has_extra() {
|
fn decode_one_sided_diff_a_has_extra() {
|
||||||
let base: Vec<u64> = (0..20).collect();
|
let base: Vec<u64> = (0..20).collect();
|
||||||
let extra = 9999u64;
|
let extra = 9999u64;
|
||||||
let a_keys: Vec<u64> = base.iter().chain(std::iter::once(&extra)).copied().collect();
|
let a_keys: Vec<u64> = base
|
||||||
|
.iter()
|
||||||
|
.chain(std::iter::once(&extra))
|
||||||
|
.copied()
|
||||||
|
.collect();
|
||||||
let b_keys = base.clone();
|
let b_keys = base.clone();
|
||||||
|
|
||||||
let m = IbltSketch::recommended_cells(a_keys.len()) * 4; // generous m
|
let m = IbltSketch::recommended_cells(a_keys.len()) * 4; // generous m
|
||||||
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||||
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||||
for &k in &a_keys { a.insert(k); }
|
for &k in &a_keys {
|
||||||
for &k in &b_keys { b.insert(k); }
|
a.insert(k);
|
||||||
|
}
|
||||||
|
for &k in &b_keys {
|
||||||
|
b.insert(k);
|
||||||
|
}
|
||||||
a.subtract(&b);
|
a.subtract(&b);
|
||||||
|
|
||||||
match a.decode() {
|
match a.decode() {
|
||||||
@@ -444,13 +488,21 @@ mod tests {
|
|||||||
let base: Vec<u64> = (0..20).collect();
|
let base: Vec<u64> = (0..20).collect();
|
||||||
let extra = 8888u64;
|
let extra = 8888u64;
|
||||||
let a_keys = base.clone();
|
let a_keys = base.clone();
|
||||||
let b_keys: Vec<u64> = base.iter().chain(std::iter::once(&extra)).copied().collect();
|
let b_keys: Vec<u64> = base
|
||||||
|
.iter()
|
||||||
|
.chain(std::iter::once(&extra))
|
||||||
|
.copied()
|
||||||
|
.collect();
|
||||||
|
|
||||||
let m = IbltSketch::recommended_cells(b_keys.len()) * 4;
|
let m = IbltSketch::recommended_cells(b_keys.len()) * 4;
|
||||||
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||||
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||||
for &k in &a_keys { a.insert(k); }
|
for &k in &a_keys {
|
||||||
for &k in &b_keys { b.insert(k); }
|
a.insert(k);
|
||||||
|
}
|
||||||
|
for &k in &b_keys {
|
||||||
|
b.insert(k);
|
||||||
|
}
|
||||||
a.subtract(&b);
|
a.subtract(&b);
|
||||||
|
|
||||||
match a.decode() {
|
match a.decode() {
|
||||||
@@ -476,8 +528,12 @@ mod tests {
|
|||||||
let m = IbltSketch::recommended_cells(a_keys.len().max(b_keys.len())) * 8;
|
let m = IbltSketch::recommended_cells(a_keys.len().max(b_keys.len())) * 8;
|
||||||
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||||
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||||
for &k in &a_keys { a.insert(k); }
|
for &k in &a_keys {
|
||||||
for &k in &b_keys { b.insert(k); }
|
a.insert(k);
|
||||||
|
}
|
||||||
|
for &k in &b_keys {
|
||||||
|
b.insert(k);
|
||||||
|
}
|
||||||
a.subtract(&b);
|
a.subtract(&b);
|
||||||
|
|
||||||
match a.decode() {
|
match a.decode() {
|
||||||
@@ -500,8 +556,12 @@ mod tests {
|
|||||||
let m = 1; // deliberately too small
|
let m = 1; // deliberately too small
|
||||||
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||||
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
let mut b = IbltSketch::new(m, DEFAULT_HASH_COUNT, SEED);
|
||||||
for &k in &a_keys { a.insert(k); }
|
for &k in &a_keys {
|
||||||
for &k in &b_keys { b.insert(k); }
|
a.insert(k);
|
||||||
|
}
|
||||||
|
for &k in &b_keys {
|
||||||
|
b.insert(k);
|
||||||
|
}
|
||||||
a.subtract(&b);
|
a.subtract(&b);
|
||||||
|
|
||||||
// Must not panic — returns NeedMoreCells
|
// Must not panic — returns NeedMoreCells
|
||||||
|
|||||||
@@ -19,9 +19,11 @@ pub mod merger;
|
|||||||
pub mod packet;
|
pub mod packet;
|
||||||
pub mod selector;
|
pub mod selector;
|
||||||
|
|
||||||
pub use error::SyncOnionError;
|
|
||||||
pub use iblt::{IbltDecodeResult, IbltDiff, IbltSketch, DEFAULT_HASH_COUNT, IBLT_SYNC_SEED, MIN_CELLS};
|
|
||||||
pub use manifest::{ClawSyncManifest, IbltManifest};
|
|
||||||
pub use differ::packets_for_revisions;
|
pub use differ::packets_for_revisions;
|
||||||
|
pub use error::SyncOnionError;
|
||||||
|
pub use iblt::{
|
||||||
|
DEFAULT_HASH_COUNT, IBLT_SYNC_SEED, IbltDecodeResult, IbltDiff, IbltSketch, MIN_CELLS,
|
||||||
|
};
|
||||||
|
pub use manifest::{ClawSyncManifest, IbltManifest};
|
||||||
pub use merger::MergeStats;
|
pub use merger::MergeStats;
|
||||||
pub use packet::{OnionLayerPacket, OnionPage};
|
pub use packet::{OnionLayerPacket, OnionPage};
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ use rkyv::{Archive, Deserialize, Serialize};
|
|||||||
|
|
||||||
use clawhdf5_onion::writer::OnionFile;
|
use clawhdf5_onion::writer::OnionFile;
|
||||||
|
|
||||||
use crate::iblt::{IbltDiff, IbltDecodeResult, IbltSketch, DEFAULT_HASH_COUNT};
|
use crate::iblt::{DEFAULT_HASH_COUNT, IbltDecodeResult, IbltDiff, IbltSketch};
|
||||||
|
|
||||||
/// A compact summary of one revision — used in the manifest.
|
/// A compact summary of one revision — used in the manifest.
|
||||||
#[derive(Archive, Deserialize, Serialize, Debug, Clone, PartialEq)]
|
#[derive(Archive, Deserialize, Serialize, Debug, Clone, PartialEq)]
|
||||||
@@ -60,10 +60,7 @@ impl ClawSyncManifest {
|
|||||||
})
|
})
|
||||||
.unwrap_or((0, [0u8; 32]));
|
.unwrap_or((0, [0u8; 32]));
|
||||||
|
|
||||||
let last_write = summaries
|
let last_write = summaries.last().map(|s| s.timestamp).unwrap_or(0.0);
|
||||||
.last()
|
|
||||||
.map(|s| s.timestamp)
|
|
||||||
.unwrap_or(0.0);
|
|
||||||
|
|
||||||
let revisions = summaries
|
let revisions = summaries
|
||||||
.into_iter()
|
.into_iter()
|
||||||
@@ -122,16 +119,16 @@ impl ClawSyncManifest {
|
|||||||
/// For N=1 000: ~3 KB vs ~60 KB for `ClawSyncManifest`.
|
/// For N=1 000: ~3 KB vs ~60 KB for `ClawSyncManifest`.
|
||||||
#[derive(Archive, Deserialize, Serialize, Debug, Clone, PartialEq)]
|
#[derive(Archive, Deserialize, Serialize, Debug, Clone, PartialEq)]
|
||||||
pub struct IbltManifest {
|
pub struct IbltManifest {
|
||||||
pub agent_id: String,
|
pub agent_id: String,
|
||||||
pub file_blake3: [u8; 32],
|
pub file_blake3: [u8; 32],
|
||||||
pub revision_count: u64,
|
pub revision_count: u64,
|
||||||
pub head_revision: u64,
|
pub head_revision: u64,
|
||||||
pub head_blake3: [u8; 32],
|
pub head_blake3: [u8; 32],
|
||||||
pub last_write: f64,
|
pub last_write: f64,
|
||||||
/// Serialised [`IbltSketch`] bytes.
|
/// Serialised [`IbltSketch`] bytes.
|
||||||
pub sketch: Vec<u8>,
|
pub sketch: Vec<u8>,
|
||||||
/// Number of cells in the sketch (m).
|
/// Number of cells in the sketch (m).
|
||||||
pub sketch_cells: u32,
|
pub sketch_cells: u32,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl IbltManifest {
|
impl IbltManifest {
|
||||||
@@ -144,7 +141,12 @@ impl IbltManifest {
|
|||||||
|
|
||||||
let (head_revision, head_blake3) = summaries
|
let (head_revision, head_blake3) = summaries
|
||||||
.last()
|
.last()
|
||||||
.map(|s| (s.revision, hex_to_bytes32(&s.blake3_hex).unwrap_or([0u8; 32])))
|
.map(|s| {
|
||||||
|
(
|
||||||
|
s.revision,
|
||||||
|
hex_to_bytes32(&s.blake3_hex).unwrap_or([0u8; 32]),
|
||||||
|
)
|
||||||
|
})
|
||||||
.unwrap_or((0, [0u8; 32]));
|
.unwrap_or((0, [0u8; 32]));
|
||||||
|
|
||||||
let last_write = summaries.last().map(|s| s.timestamp).unwrap_or(0.0);
|
let last_write = summaries.last().map(|s| s.timestamp).unwrap_or(0.0);
|
||||||
@@ -174,10 +176,7 @@ impl IbltManifest {
|
|||||||
///
|
///
|
||||||
/// Returns `Ok(IbltDiff)` or `Err` if the sketch cannot be decoded (too
|
/// Returns `Ok(IbltDiff)` or `Err` if the sketch cannot be decoded (too
|
||||||
/// small or malformed).
|
/// small or malformed).
|
||||||
pub fn diff_against(
|
pub fn diff_against(&self, local_keys: &[u64]) -> Result<IbltDiff, &'static str> {
|
||||||
&self,
|
|
||||||
local_keys: &[u64],
|
|
||||||
) -> Result<IbltDiff, &'static str> {
|
|
||||||
let remote_sketch = IbltSketch::from_bytes(&self.sketch)
|
let remote_sketch = IbltSketch::from_bytes(&self.sketch)
|
||||||
.map_err(|_| "cannot deserialise remote IBLT sketch")?;
|
.map_err(|_| "cannot deserialise remote IBLT sketch")?;
|
||||||
|
|
||||||
@@ -189,7 +188,9 @@ impl IbltManifest {
|
|||||||
.max(IbltSketch::recommended_cells(local_keys.len()));
|
.max(IbltSketch::recommended_cells(local_keys.len()));
|
||||||
|
|
||||||
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, seed);
|
let mut a = IbltSketch::new(m, DEFAULT_HASH_COUNT, seed);
|
||||||
for &k in local_keys { a.insert(k); }
|
for &k in local_keys {
|
||||||
|
a.insert(k);
|
||||||
|
}
|
||||||
|
|
||||||
// Pad remote to same m if needed.
|
// Pad remote to same m if needed.
|
||||||
let b = if remote_sketch.cell_count() == m {
|
let b = if remote_sketch.cell_count() == m {
|
||||||
@@ -202,7 +203,7 @@ impl IbltManifest {
|
|||||||
a.subtract(&b);
|
a.subtract(&b);
|
||||||
match a.decode() {
|
match a.decode() {
|
||||||
IbltDecodeResult::Complete(diff) => Ok(diff),
|
IbltDecodeResult::Complete(diff) => Ok(diff),
|
||||||
IbltDecodeResult::NeedMoreCells => Err("IBLT sketch too small; retry with larger m"),
|
IbltDecodeResult::NeedMoreCells => Err("IBLT sketch too small; retry with larger m"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -217,8 +218,7 @@ impl IbltManifest {
|
|||||||
pub fn from_bytes(bytes: &[u8]) -> Result<Self, String> {
|
pub fn from_bytes(bytes: &[u8]) -> Result<Self, String> {
|
||||||
let mut aligned = rkyv::util::AlignedVec::<16>::with_capacity(bytes.len());
|
let mut aligned = rkyv::util::AlignedVec::<16>::with_capacity(bytes.len());
|
||||||
aligned.extend_from_slice(bytes);
|
aligned.extend_from_slice(bytes);
|
||||||
rkyv::from_bytes::<IbltManifest, rkyv::rancor::Error>(&aligned)
|
rkyv::from_bytes::<IbltManifest, rkyv::rancor::Error>(&aligned).map_err(|e| e.to_string())
|
||||||
.map_err(|e| e.to_string())
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -63,8 +63,7 @@ pub fn merge_packets(
|
|||||||
session.record_page(page.h5_offset, &page.data);
|
session.record_page(page.h5_offset, &page.data);
|
||||||
total_bytes += page.data.len() as u64;
|
total_bytes += page.data.len() as u64;
|
||||||
} else {
|
} else {
|
||||||
let codec = Codec::from_u8(page.codec)
|
let codec = Codec::from_u8(page.codec).map_err(SyncOnionError::Onion)?;
|
||||||
.map_err(SyncOnionError::Onion)?;
|
|
||||||
let raw = decompress_page(&page.data, codec, page.orig_size)
|
let raw = decompress_page(&page.data, codec, page.orig_size)
|
||||||
.map_err(SyncOnionError::Onion)?;
|
.map_err(SyncOnionError::Onion)?;
|
||||||
total_bytes += raw.len() as u64;
|
total_bytes += raw.len() as u64;
|
||||||
@@ -101,8 +100,7 @@ fn verify_packet_hash(packet: &OnionLayerPacket) -> Result<(), SyncOnionError> {
|
|||||||
if page.codec == 0 {
|
if page.codec == 0 {
|
||||||
hasher.update(&page.data);
|
hasher.update(&page.data);
|
||||||
} else {
|
} else {
|
||||||
let codec = Codec::from_u8(page.codec)
|
let codec = Codec::from_u8(page.codec).map_err(SyncOnionError::Onion)?;
|
||||||
.map_err(SyncOnionError::Onion)?;
|
|
||||||
let raw = decompress_page(&page.data, codec, page.orig_size)
|
let raw = decompress_page(&page.data, codec, page.orig_size)
|
||||||
.map_err(SyncOnionError::Onion)?;
|
.map_err(SyncOnionError::Onion)?;
|
||||||
hasher.update(&raw);
|
hasher.update(&raw);
|
||||||
@@ -204,7 +202,12 @@ mod tests {
|
|||||||
timestamp: 0.0,
|
timestamp: 0.0,
|
||||||
annotation: None,
|
annotation: None,
|
||||||
blake3_root: [0xDEu8; 32], // wrong hash
|
blake3_root: [0xDEu8; 32], // wrong hash
|
||||||
pages: vec![OnionPage { h5_offset: 0, data: page_data.clone(), codec: 0, orig_size: page_data.len() as u32 }],
|
pages: vec![OnionPage {
|
||||||
|
h5_offset: 0,
|
||||||
|
data: page_data.clone(),
|
||||||
|
codec: 0,
|
||||||
|
orig_size: page_data.len() as u32,
|
||||||
|
}],
|
||||||
};
|
};
|
||||||
bad_packet.blake3_root = [0xDE; 32]; // deliberately wrong
|
bad_packet.blake3_root = [0xDE; 32]; // deliberately wrong
|
||||||
|
|
||||||
@@ -229,7 +232,12 @@ mod tests {
|
|||||||
timestamp: 0.0,
|
timestamp: 0.0,
|
||||||
annotation: None,
|
annotation: None,
|
||||||
blake3_root: hash,
|
blake3_root: hash,
|
||||||
pages: vec![OnionPage { h5_offset: 0, data: page_data.clone(), codec: 0, orig_size: page_data.len() as u32 }],
|
pages: vec![OnionPage {
|
||||||
|
h5_offset: 0,
|
||||||
|
data: page_data.clone(),
|
||||||
|
codec: 0,
|
||||||
|
orig_size: page_data.len() as u32,
|
||||||
|
}],
|
||||||
};
|
};
|
||||||
|
|
||||||
let (_h5, mut dst) = make_empty_onion();
|
let (_h5, mut dst) = make_empty_onion();
|
||||||
|
|||||||
@@ -113,8 +113,18 @@ mod tests {
|
|||||||
annotation: Some(format!("rev {revision}")),
|
annotation: Some(format!("rev {revision}")),
|
||||||
blake3_root: [0xABu8; 32],
|
blake3_root: [0xABu8; 32],
|
||||||
pages: vec![
|
pages: vec![
|
||||||
OnionPage { h5_offset: 0, data: vec![0xAAu8; 4096], codec: 0, orig_size: 4096 },
|
OnionPage {
|
||||||
OnionPage { h5_offset: 4096, data: vec![0xBBu8; 4096], codec: 0, orig_size: 4096 },
|
h5_offset: 0,
|
||||||
|
data: vec![0xAAu8; 4096],
|
||||||
|
codec: 0,
|
||||||
|
orig_size: 4096,
|
||||||
|
},
|
||||||
|
OnionPage {
|
||||||
|
h5_offset: 4096,
|
||||||
|
data: vec![0xBBu8; 4096],
|
||||||
|
codec: 0,
|
||||||
|
orig_size: 4096,
|
||||||
|
},
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -193,11 +203,16 @@ mod tests {
|
|||||||
blake3_root: [0u8; 32],
|
blake3_root: [0u8; 32],
|
||||||
pages: vec![
|
pages: vec![
|
||||||
// "compressed" to 1600 bytes, original 4096
|
// "compressed" to 1600 bytes, original 4096
|
||||||
OnionPage { h5_offset: 0, data: vec![0xCCu8; 1600], codec: 1, orig_size: 4096 },
|
OnionPage {
|
||||||
|
h5_offset: 0,
|
||||||
|
data: vec![0xCCu8; 1600],
|
||||||
|
codec: 1,
|
||||||
|
orig_size: 4096,
|
||||||
|
},
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
assert_eq!(pkt.page_data_size(), 4096); // logical (uncompressed)
|
assert_eq!(pkt.page_data_size(), 4096); // logical (uncompressed)
|
||||||
assert_eq!(pkt.page_wire_size(), 1600); // wire (compressed)
|
assert_eq!(pkt.page_wire_size(), 1600); // wire (compressed)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -63,10 +63,7 @@ pub fn filter_packets(
|
|||||||
/// Compute the remote HEAD for a given selector on the local `OnionFile`.
|
/// Compute the remote HEAD for a given selector on the local `OnionFile`.
|
||||||
///
|
///
|
||||||
/// Returns [`NO_PARENT`] if no revisions match (full sync needed).
|
/// Returns [`NO_PARENT`] if no revisions match (full sync needed).
|
||||||
pub fn remote_head_for_selector(
|
pub fn remote_head_for_selector(_selector: &SyncSelector, remote_revision_count: u64) -> u64 {
|
||||||
_selector: &SyncSelector,
|
|
||||||
remote_revision_count: u64,
|
|
||||||
) -> u64 {
|
|
||||||
// For a simple linear sync, remote HEAD = remote_revision_count - 1
|
// For a simple linear sync, remote HEAD = remote_revision_count - 1
|
||||||
// (or NO_PARENT if empty).
|
// (or NO_PARENT if empty).
|
||||||
if remote_revision_count == 0 {
|
if remote_revision_count == 0 {
|
||||||
@@ -131,8 +128,11 @@ mod tests {
|
|||||||
fn selector_branch_nonexistent_errors() {
|
fn selector_branch_nonexistent_errors() {
|
||||||
let onion = make_onion(2);
|
let onion = make_onion(2);
|
||||||
let packets = diff_revisions(&onion, NO_PARENT).unwrap();
|
let packets = diff_revisions(&onion, NO_PARENT).unwrap();
|
||||||
let result =
|
let result = filter_packets(
|
||||||
filter_packets(packets, &SyncSelector::Branch("no-such".to_string()), &onion);
|
packets,
|
||||||
|
&SyncSelector::Branch("no-such".to_string()),
|
||||||
|
&onion,
|
||||||
|
);
|
||||||
assert!(matches!(result, Err(SyncOnionError::BranchNotFound(_))));
|
assert!(matches!(result, Err(SyncOnionError::BranchNotFound(_))));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ use clawhdf5_onion::format::NO_PARENT;
|
|||||||
use clawhdf5_onion::writer::OnionFile;
|
use clawhdf5_onion::writer::OnionFile;
|
||||||
use clawsync_onion::differ::diff_revisions;
|
use clawsync_onion::differ::diff_revisions;
|
||||||
use clawsync_onion::merger::merge_packets;
|
use clawsync_onion::merger::merge_packets;
|
||||||
use clawsync_onion::selector::{filter_packets, remote_head_for_selector, SyncSelector};
|
use clawsync_onion::selector::{SyncSelector, filter_packets, remote_head_for_selector};
|
||||||
use tempfile::NamedTempFile;
|
use tempfile::NamedTempFile;
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -309,7 +309,11 @@ fn merge_auto_registers_unknown_branch() {
|
|||||||
assert_eq!(stats.revisions_merged, 4);
|
assert_eq!(stats.revisions_merged, 4);
|
||||||
// The remote should have both branches registered
|
// The remote should have both branches registered
|
||||||
let branches = dst.list_branches();
|
let branches = dst.list_branches();
|
||||||
assert!(branches.len() >= 2, "expected at least 2 branches, got {:?}", branches.len());
|
assert!(
|
||||||
|
branches.len() >= 2,
|
||||||
|
"expected at least 2 branches, got {:?}",
|
||||||
|
branches.len()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -327,7 +331,10 @@ fn can_branch_on_remote_after_branch_selective_sync() {
|
|||||||
|
|
||||||
// The remote should now be able to fork its own branch
|
// The remote should now be able to fork its own branch
|
||||||
let result = dst.create_branch("experiment", "main");
|
let result = dst.create_branch("experiment", "main");
|
||||||
assert!(result.is_ok(), "should be able to create branch after sync: {result:?}");
|
assert!(
|
||||||
|
result.is_ok(),
|
||||||
|
"should be able to create branch after sync: {result:?}"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -340,14 +347,12 @@ fn branch_filtered_merge_is_idempotent() {
|
|||||||
let mut dst = empty_onion();
|
let mut dst = empty_onion();
|
||||||
|
|
||||||
let packets1 = diff_revisions(&src, NO_PARENT).unwrap();
|
let packets1 = diff_revisions(&src, NO_PARENT).unwrap();
|
||||||
let filtered1 =
|
let filtered1 = filter_packets(packets1, &SyncSelector::Branch("main".into()), &src).unwrap();
|
||||||
filter_packets(packets1, &SyncSelector::Branch("main".into()), &src).unwrap();
|
|
||||||
merge_packets(&mut dst, filtered1, false).unwrap();
|
merge_packets(&mut dst, filtered1, false).unwrap();
|
||||||
|
|
||||||
// Apply again — should skip all
|
// Apply again — should skip all
|
||||||
let packets2 = diff_revisions(&src, NO_PARENT).unwrap();
|
let packets2 = diff_revisions(&src, NO_PARENT).unwrap();
|
||||||
let filtered2 =
|
let filtered2 = filter_packets(packets2, &SyncSelector::Branch("main".into()), &src).unwrap();
|
||||||
filter_packets(packets2, &SyncSelector::Branch("main".into()), &src).unwrap();
|
|
||||||
let stats2 = merge_packets(&mut dst, filtered2, false).unwrap();
|
let stats2 = merge_packets(&mut dst, filtered2, false).unwrap();
|
||||||
|
|
||||||
assert_eq!(stats2.revisions_merged, 0);
|
assert_eq!(stats2.revisions_merged, 0);
|
||||||
|
|||||||
@@ -16,8 +16,8 @@ pub mod quic;
|
|||||||
pub mod tcp;
|
pub mod tcp;
|
||||||
|
|
||||||
pub use error::TransportError;
|
pub use error::TransportError;
|
||||||
pub use mmap::{MmapChannel, DEFAULT_CAPACITY};
|
pub use mmap::{DEFAULT_CAPACITY, MmapChannel};
|
||||||
pub use peer::{PipeReadHalf, PipeWriteHalf, SyncPeer};
|
pub use peer::{PipeReadHalf, PipeWriteHalf, SyncPeer};
|
||||||
pub use protocol::SyncMessage;
|
pub use protocol::SyncMessage;
|
||||||
pub use tcp::{TcpConnection, TcpServer};
|
|
||||||
pub use quic::{QuicConfig, QuicConnection, QuicServer, quic_connect};
|
pub use quic::{QuicConfig, QuicConnection, QuicServer, quic_connect};
|
||||||
|
pub use tcp::{TcpConnection, TcpServer};
|
||||||
|
|||||||
@@ -26,12 +26,12 @@
|
|||||||
//! - Capacity is fixed at creation time.
|
//! - Capacity is fixed at creation time.
|
||||||
//! - No cross-machine support (file must be on a shared filesystem).
|
//! - No cross-machine support (file must be on a shared filesystem).
|
||||||
|
|
||||||
|
use memmap2::MmapMut;
|
||||||
use std::fs::OpenOptions;
|
use std::fs::OpenOptions;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use memmap2::MmapMut;
|
|
||||||
|
|
||||||
use crate::error::TransportError;
|
use crate::error::TransportError;
|
||||||
use crate::protocol::{SyncMessage, MAX_FRAME_SIZE};
|
use crate::protocol::{MAX_FRAME_SIZE, SyncMessage};
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// Constants
|
// Constants
|
||||||
@@ -92,7 +92,11 @@ impl MmapChannel {
|
|||||||
write_u64(&mut mmap, 24, capacity as u64); // capacity
|
write_u64(&mut mmap, 24, capacity as u64); // capacity
|
||||||
mmap.flush().map_err(TransportError::Io)?;
|
mmap.flush().map_err(TransportError::Io)?;
|
||||||
|
|
||||||
Ok(Self { path: path.to_owned(), mmap, capacity })
|
Ok(Self {
|
||||||
|
path: path.to_owned(),
|
||||||
|
mmap,
|
||||||
|
capacity,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Open an existing channel file.
|
/// Open an existing channel file.
|
||||||
@@ -105,10 +109,16 @@ impl MmapChannel {
|
|||||||
let mmap = unsafe { MmapMut::map_mut(&file).map_err(TransportError::Io)? };
|
let mmap = unsafe { MmapMut::map_mut(&file).map_err(TransportError::Io)? };
|
||||||
|
|
||||||
if &mmap[..8] != MAGIC {
|
if &mmap[..8] != MAGIC {
|
||||||
return Err(TransportError::Protocol("invalid mmap channel magic".into()));
|
return Err(TransportError::Protocol(
|
||||||
|
"invalid mmap channel magic".into(),
|
||||||
|
));
|
||||||
}
|
}
|
||||||
let capacity = read_u64(&mmap, 24) as usize;
|
let capacity = read_u64(&mmap, 24) as usize;
|
||||||
Ok(Self { path: path.to_owned(), mmap, capacity })
|
Ok(Self {
|
||||||
|
path: path.to_owned(),
|
||||||
|
mmap,
|
||||||
|
capacity,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Send a message into the ring buffer.
|
/// Send a message into the ring buffer.
|
||||||
@@ -194,8 +204,7 @@ impl MmapChannel {
|
|||||||
write_u64(&mut self.mmap, 16, new_rh);
|
write_u64(&mut self.mmap, 16, new_rh);
|
||||||
self.mmap.flush_range(0, 32).map_err(TransportError::Io)?;
|
self.mmap.flush_range(0, 32).map_err(TransportError::Io)?;
|
||||||
|
|
||||||
SyncMessage::from_bytes(&body)
|
SyncMessage::from_bytes(&body).map_err(TransportError::Protocol)
|
||||||
.map_err(TransportError::Protocol)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Path to the backing file.
|
/// Path to the backing file.
|
||||||
@@ -231,14 +240,20 @@ impl MmapChannel {
|
|||||||
impl MmapSender {
|
impl MmapSender {
|
||||||
pub fn open(path: &Path) -> Result<Self, TransportError> {
|
pub fn open(path: &Path) -> Result<Self, TransportError> {
|
||||||
let ch = MmapChannel::open(path)?;
|
let ch = MmapChannel::open(path)?;
|
||||||
Ok(Self { _path: path.to_owned(), _capacity: ch.capacity })
|
Ok(Self {
|
||||||
|
_path: path.to_owned(),
|
||||||
|
_capacity: ch.capacity,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MmapReceiver {
|
impl MmapReceiver {
|
||||||
pub fn open(path: &Path) -> Result<Self, TransportError> {
|
pub fn open(path: &Path) -> Result<Self, TransportError> {
|
||||||
let ch = MmapChannel::open(path)?;
|
let ch = MmapChannel::open(path)?;
|
||||||
Ok(Self { _path: path.to_owned(), _capacity: ch.capacity })
|
Ok(Self {
|
||||||
|
_path: path.to_owned(),
|
||||||
|
_capacity: ch.capacity,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -310,7 +325,9 @@ mod tests {
|
|||||||
let (_f, path) = tmp_path();
|
let (_f, path) = tmp_path();
|
||||||
let mut ch = MmapChannel::create(&path, DEFAULT_CAPACITY).unwrap();
|
let mut ch = MmapChannel::create(&path, DEFAULT_CAPACITY).unwrap();
|
||||||
|
|
||||||
let msg = SyncMessage::Error { message: "oops".into() };
|
let msg = SyncMessage::Error {
|
||||||
|
message: "oops".into(),
|
||||||
|
};
|
||||||
ch.send(&msg).unwrap();
|
ch.send(&msg).unwrap();
|
||||||
let recv = ch.recv().unwrap();
|
let recv = ch.recv().unwrap();
|
||||||
assert!(matches!(recv, SyncMessage::Error { .. }));
|
assert!(matches!(recv, SyncMessage::Error { .. }));
|
||||||
@@ -324,9 +341,13 @@ mod tests {
|
|||||||
ch.send(&SyncMessage::SyncComplete {
|
ch.send(&SyncMessage::SyncComplete {
|
||||||
revisions_transferred: 100,
|
revisions_transferred: 100,
|
||||||
bytes_transferred: 4096 * 100,
|
bytes_transferred: 4096 * 100,
|
||||||
}).unwrap();
|
})
|
||||||
|
.unwrap();
|
||||||
match ch.recv().unwrap() {
|
match ch.recv().unwrap() {
|
||||||
SyncMessage::SyncComplete { revisions_transferred, .. } => {
|
SyncMessage::SyncComplete {
|
||||||
|
revisions_transferred,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
assert_eq!(revisions_transferred, 100);
|
assert_eq!(revisions_transferred, 100);
|
||||||
}
|
}
|
||||||
other => panic!("unexpected: {other:?}"),
|
other => panic!("unexpected: {other:?}"),
|
||||||
|
|||||||
@@ -63,9 +63,7 @@ impl SyncPeer {
|
|||||||
let (r, w) = conn.into_split();
|
let (r, w) = conn.into_split();
|
||||||
(PipeReadHalf::Tcp(r), PipeWriteHalf::Tcp(w))
|
(PipeReadHalf::Tcp(r), PipeWriteHalf::Tcp(w))
|
||||||
}
|
}
|
||||||
SyncPeer::Quic(arc) => {
|
SyncPeer::Quic(arc) => (PipeReadHalf::Quic(arc.clone()), PipeWriteHalf::Quic(arc)),
|
||||||
(PipeReadHalf::Quic(arc.clone()), PipeWriteHalf::Quic(arc))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -156,8 +156,8 @@ pub enum SyncMessage {
|
|||||||
/// lacks (what to push). The client decodes `sketch` against its own
|
/// lacks (what to push). The client decodes `sketch` against its own
|
||||||
/// sketch to find revisions it needs to pull.
|
/// sketch to find revisions it needs to pull.
|
||||||
IbltResponse {
|
IbltResponse {
|
||||||
sketch: IbltManifest,
|
sketch: IbltManifest,
|
||||||
missing_from_remote: Vec<u64>,
|
missing_from_remote: Vec<u64>,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// An error occurred; connection will be closed after this message.
|
/// An error occurred; connection will be closed after this message.
|
||||||
@@ -167,7 +167,6 @@ pub enum SyncMessage {
|
|||||||
//
|
//
|
||||||
// IMPORTANT: rkyv assigns discriminants positionally. These variants MUST
|
// IMPORTANT: rkyv assigns discriminants positionally. These variants MUST
|
||||||
// remain appended after `Error` (index 8). Never insert or reorder.
|
// remain appended after `Error` (index 8). Never insert or reorder.
|
||||||
|
|
||||||
/// Client announces it wants to sync a single file using CDC delta.
|
/// Client announces it wants to sync a single file using CDC delta.
|
||||||
///
|
///
|
||||||
/// Carries the ordered CDC chunk descriptors of the client's version of
|
/// Carries the ordered CDC chunk descriptors of the client's version of
|
||||||
@@ -205,7 +204,6 @@ pub enum SyncMessage {
|
|||||||
FsFileAck { path: String },
|
FsFileAck { path: String },
|
||||||
|
|
||||||
// ── FS directory tree sync (variants 13–16) ───────────────────────────────
|
// ── FS directory tree sync (variants 13–16) ───────────────────────────────
|
||||||
|
|
||||||
/// Client sends a per-file manifest for an entire directory tree.
|
/// Client sends a per-file manifest for an entire directory tree.
|
||||||
FsDirManifest {
|
FsDirManifest {
|
||||||
/// Root path on the client (informational).
|
/// Root path on the client (informational).
|
||||||
@@ -227,15 +225,15 @@ pub enum SyncMessage {
|
|||||||
|
|
||||||
/// Server reports that all changes have been applied.
|
/// Server reports that all changes have been applied.
|
||||||
FsDirComplete {
|
FsDirComplete {
|
||||||
files_added: u32,
|
files_added: u32,
|
||||||
files_modified: u32,
|
files_modified: u32,
|
||||||
files_removed: u32,
|
files_removed: u32,
|
||||||
bytes_transferred: u64,
|
bytes_transferred: u64,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Dry-run: reports what *would* change without applying anything.
|
/// Dry-run: reports what *would* change without applying anything.
|
||||||
FsDirDryRun {
|
FsDirDryRun {
|
||||||
would_add: Vec<String>,
|
would_add: Vec<String>,
|
||||||
would_modify: Vec<String>,
|
would_modify: Vec<String>,
|
||||||
would_remove: Vec<String>,
|
would_remove: Vec<String>,
|
||||||
},
|
},
|
||||||
@@ -244,7 +242,6 @@ pub enum SyncMessage {
|
|||||||
//
|
//
|
||||||
// IMPORTANT: rkyv assigns discriminants positionally. These variants MUST
|
// IMPORTANT: rkyv assigns discriminants positionally. These variants MUST
|
||||||
// remain appended after `FsDirDryRun` (index 16). Never insert or reorder.
|
// remain appended after `FsDirDryRun` (index 16). Never insert or reorder.
|
||||||
|
|
||||||
/// Client sends its per-dataset manifest and the remote h5 path to sync.
|
/// Client sends its per-dataset manifest and the remote h5 path to sync.
|
||||||
///
|
///
|
||||||
/// The server compares this manifest against its own to find which datasets
|
/// The server compares this manifest against its own to find which datasets
|
||||||
@@ -288,9 +285,9 @@ pub enum SyncMessage {
|
|||||||
|
|
||||||
/// Server reports that the HDF5 sync is complete.
|
/// Server reports that the HDF5 sync is complete.
|
||||||
Hdf5SyncComplete {
|
Hdf5SyncComplete {
|
||||||
datasets_added: u32,
|
datasets_added: u32,
|
||||||
datasets_modified: u32,
|
datasets_modified: u32,
|
||||||
datasets_removed: u32,
|
datasets_removed: u32,
|
||||||
bytes_transferred: u64,
|
bytes_transferred: u64,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -310,8 +307,7 @@ impl SyncMessage {
|
|||||||
pub fn from_bytes(bytes: &[u8]) -> Result<Self, String> {
|
pub fn from_bytes(bytes: &[u8]) -> Result<Self, String> {
|
||||||
let mut aligned = rkyv::util::AlignedVec::<16>::with_capacity(bytes.len());
|
let mut aligned = rkyv::util::AlignedVec::<16>::with_capacity(bytes.len());
|
||||||
aligned.extend_from_slice(bytes);
|
aligned.extend_from_slice(bytes);
|
||||||
rkyv::from_bytes::<SyncMessage, rkyv::rancor::Error>(&aligned)
|
rkyv::from_bytes::<SyncMessage, rkyv::rancor::Error>(&aligned).map_err(|e| e.to_string())
|
||||||
.map_err(|e| e.to_string())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Encode to a length-prefixed frame: `[u32 LE length][message bytes]`.
|
/// Encode to a length-prefixed frame: `[u32 LE length][message bytes]`.
|
||||||
@@ -333,10 +329,16 @@ impl SyncMessage {
|
|||||||
}
|
}
|
||||||
let len = u32::from_le_bytes(data[..4].try_into().unwrap()) as usize;
|
let len = u32::from_le_bytes(data[..4].try_into().unwrap()) as usize;
|
||||||
if len > MAX_FRAME_SIZE {
|
if len > MAX_FRAME_SIZE {
|
||||||
return Err(format!("frame too large: {len} bytes (max {MAX_FRAME_SIZE})"));
|
return Err(format!(
|
||||||
|
"frame too large: {len} bytes (max {MAX_FRAME_SIZE})"
|
||||||
|
));
|
||||||
}
|
}
|
||||||
if data.len() < 4 + len {
|
if data.len() < 4 + len {
|
||||||
return Err(format!("incomplete frame: have {} bytes, need {}", data.len(), 4 + len));
|
return Err(format!(
|
||||||
|
"incomplete frame: have {} bytes, need {}",
|
||||||
|
data.len(),
|
||||||
|
4 + len
|
||||||
|
));
|
||||||
}
|
}
|
||||||
let msg = Self::from_bytes(&data[4..4 + len])?;
|
let msg = Self::from_bytes(&data[4..4 + len])?;
|
||||||
Ok((msg, 4 + len))
|
Ok((msg, 4 + len))
|
||||||
@@ -377,7 +379,11 @@ mod tests {
|
|||||||
let bytes = msg.to_bytes().unwrap();
|
let bytes = msg.to_bytes().unwrap();
|
||||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||||
match recovered {
|
match recovered {
|
||||||
SyncMessage::ManifestRequest { agent_id, head_revision, revision_count } => {
|
SyncMessage::ManifestRequest {
|
||||||
|
agent_id,
|
||||||
|
head_revision,
|
||||||
|
revision_count,
|
||||||
|
} => {
|
||||||
assert_eq!(agent_id, "test-agent");
|
assert_eq!(agent_id, "test-agent");
|
||||||
assert_eq!(head_revision, 5);
|
assert_eq!(head_revision, 5);
|
||||||
assert_eq!(revision_count, 6);
|
assert_eq!(revision_count, 6);
|
||||||
@@ -419,7 +425,10 @@ mod tests {
|
|||||||
let bytes = msg.to_bytes().unwrap();
|
let bytes = msg.to_bytes().unwrap();
|
||||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||||
match recovered {
|
match recovered {
|
||||||
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => {
|
SyncMessage::SyncComplete {
|
||||||
|
revisions_transferred,
|
||||||
|
bytes_transferred,
|
||||||
|
} => {
|
||||||
assert_eq!(revisions_transferred, 10);
|
assert_eq!(revisions_transferred, 10);
|
||||||
assert_eq!(bytes_transferred, 40960);
|
assert_eq!(bytes_transferred, 40960);
|
||||||
}
|
}
|
||||||
@@ -429,7 +438,9 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn error_message_roundtrip() {
|
fn error_message_roundtrip() {
|
||||||
let msg = SyncMessage::Error { message: "something went wrong".to_string() };
|
let msg = SyncMessage::Error {
|
||||||
|
message: "something went wrong".to_string(),
|
||||||
|
};
|
||||||
let bytes = msg.to_bytes().unwrap();
|
let bytes = msg.to_bytes().unwrap();
|
||||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||||
match recovered {
|
match recovered {
|
||||||
@@ -440,21 +451,23 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn iblt_request_roundtrip() {
|
fn iblt_request_roundtrip() {
|
||||||
use clawsync_onion::manifest::IbltManifest;
|
|
||||||
use clawsync_onion::iblt::IbltSketch;
|
use clawsync_onion::iblt::IbltSketch;
|
||||||
|
use clawsync_onion::manifest::IbltManifest;
|
||||||
// Build a minimal IbltManifest
|
// Build a minimal IbltManifest
|
||||||
let sketch = IbltSketch::from_keys(&[0u64, 1, 2], 0xABCD);
|
let sketch = IbltSketch::from_keys(&[0u64, 1, 2], 0xABCD);
|
||||||
let iblt_manifest = IbltManifest {
|
let iblt_manifest = IbltManifest {
|
||||||
agent_id: "test".to_string(),
|
agent_id: "test".to_string(),
|
||||||
file_blake3: [0u8; 32],
|
file_blake3: [0u8; 32],
|
||||||
revision_count: 3,
|
revision_count: 3,
|
||||||
head_revision: 2,
|
head_revision: 2,
|
||||||
head_blake3: [0u8; 32],
|
head_blake3: [0u8; 32],
|
||||||
last_write: 0.0,
|
last_write: 0.0,
|
||||||
sketch: sketch.to_bytes(),
|
sketch: sketch.to_bytes(),
|
||||||
sketch_cells: sketch.cell_count() as u32,
|
sketch_cells: sketch.cell_count() as u32,
|
||||||
|
};
|
||||||
|
let msg = SyncMessage::IbltRequest {
|
||||||
|
sketch: iblt_manifest,
|
||||||
};
|
};
|
||||||
let msg = SyncMessage::IbltRequest { sketch: iblt_manifest };
|
|
||||||
let bytes = msg.to_bytes().unwrap();
|
let bytes = msg.to_bytes().unwrap();
|
||||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||||
assert!(matches!(recovered, SyncMessage::IbltRequest { .. }));
|
assert!(matches!(recovered, SyncMessage::IbltRequest { .. }));
|
||||||
@@ -462,18 +475,18 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn iblt_response_roundtrip() {
|
fn iblt_response_roundtrip() {
|
||||||
use clawsync_onion::manifest::IbltManifest;
|
|
||||||
use clawsync_onion::iblt::IbltSketch;
|
use clawsync_onion::iblt::IbltSketch;
|
||||||
|
use clawsync_onion::manifest::IbltManifest;
|
||||||
let sketch = IbltSketch::from_keys(&[5u64, 6], 0xDEAD);
|
let sketch = IbltSketch::from_keys(&[5u64, 6], 0xDEAD);
|
||||||
let iblt_manifest = IbltManifest {
|
let iblt_manifest = IbltManifest {
|
||||||
agent_id: "srv".to_string(),
|
agent_id: "srv".to_string(),
|
||||||
file_blake3: [1u8; 32],
|
file_blake3: [1u8; 32],
|
||||||
revision_count: 2,
|
revision_count: 2,
|
||||||
head_revision: 6,
|
head_revision: 6,
|
||||||
head_blake3: [2u8; 32],
|
head_blake3: [2u8; 32],
|
||||||
last_write: 1.0,
|
last_write: 1.0,
|
||||||
sketch: sketch.to_bytes(),
|
sketch: sketch.to_bytes(),
|
||||||
sketch_cells: sketch.cell_count() as u32,
|
sketch_cells: sketch.cell_count() as u32,
|
||||||
};
|
};
|
||||||
let msg = SyncMessage::IbltResponse {
|
let msg = SyncMessage::IbltResponse {
|
||||||
sketch: iblt_manifest,
|
sketch: iblt_manifest,
|
||||||
@@ -482,7 +495,10 @@ mod tests {
|
|||||||
let bytes = msg.to_bytes().unwrap();
|
let bytes = msg.to_bytes().unwrap();
|
||||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||||
match recovered {
|
match recovered {
|
||||||
SyncMessage::IbltResponse { missing_from_remote, .. } => {
|
SyncMessage::IbltResponse {
|
||||||
|
missing_from_remote,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
assert_eq!(missing_from_remote, vec![7, 8, 9]);
|
assert_eq!(missing_from_remote, vec![7, 8, 9]);
|
||||||
}
|
}
|
||||||
_ => panic!("wrong variant"),
|
_ => panic!("wrong variant"),
|
||||||
@@ -496,9 +512,18 @@ mod tests {
|
|||||||
let msg = SyncMessage::FsCdcRequest {
|
let msg = SyncMessage::FsCdcRequest {
|
||||||
path: "data/weights.bin".to_string(),
|
path: "data/weights.bin".to_string(),
|
||||||
chunk_hashes: vec![
|
chunk_hashes: vec![
|
||||||
FsChunkHash { hash: 0xDEAD, length: 65536 },
|
FsChunkHash {
|
||||||
FsChunkHash { hash: 0xBEEF, length: 32768 },
|
hash: 0xDEAD,
|
||||||
FsChunkHash { hash: 0xCAFE, length: 65536 },
|
length: 65536,
|
||||||
|
},
|
||||||
|
FsChunkHash {
|
||||||
|
hash: 0xBEEF,
|
||||||
|
length: 32768,
|
||||||
|
},
|
||||||
|
FsChunkHash {
|
||||||
|
hash: 0xCAFE,
|
||||||
|
length: 65536,
|
||||||
|
},
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
let bytes = msg.to_bytes().unwrap();
|
let bytes = msg.to_bytes().unwrap();
|
||||||
@@ -522,7 +547,10 @@ mod tests {
|
|||||||
let bytes = msg.to_bytes().unwrap();
|
let bytes = msg.to_bytes().unwrap();
|
||||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||||
match recovered {
|
match recovered {
|
||||||
SyncMessage::FsCdcNeed { path, needed_indices } => {
|
SyncMessage::FsCdcNeed {
|
||||||
|
path,
|
||||||
|
needed_indices,
|
||||||
|
} => {
|
||||||
assert_eq!(path, "model.bin");
|
assert_eq!(path, "model.bin");
|
||||||
assert_eq!(needed_indices, vec![0, 2]);
|
assert_eq!(needed_indices, vec![0, 2]);
|
||||||
}
|
}
|
||||||
@@ -535,18 +563,36 @@ mod tests {
|
|||||||
let msg = SyncMessage::FsCdcData {
|
let msg = SyncMessage::FsCdcData {
|
||||||
path: "cfg.bin".to_string(),
|
path: "cfg.bin".to_string(),
|
||||||
chunk_order: vec![
|
chunk_order: vec![
|
||||||
FsChunkHash { hash: 0x11, length: 65536 },
|
FsChunkHash {
|
||||||
FsChunkHash { hash: 0x22, length: 65536 },
|
hash: 0x11,
|
||||||
|
length: 65536,
|
||||||
|
},
|
||||||
|
FsChunkHash {
|
||||||
|
hash: 0x22,
|
||||||
|
length: 65536,
|
||||||
|
},
|
||||||
],
|
],
|
||||||
chunks: vec![
|
chunks: vec![
|
||||||
FsChunk { hash: 0x11, data: vec![0xAAu8; 100], compressed: true },
|
FsChunk {
|
||||||
FsChunk { hash: 0x22, data: vec![0xBBu8; 200], compressed: false },
|
hash: 0x11,
|
||||||
|
data: vec![0xAAu8; 100],
|
||||||
|
compressed: true,
|
||||||
|
},
|
||||||
|
FsChunk {
|
||||||
|
hash: 0x22,
|
||||||
|
data: vec![0xBBu8; 200],
|
||||||
|
compressed: false,
|
||||||
|
},
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
let bytes = msg.to_bytes().unwrap();
|
let bytes = msg.to_bytes().unwrap();
|
||||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||||
match recovered {
|
match recovered {
|
||||||
SyncMessage::FsCdcData { path, chunk_order, chunks } => {
|
SyncMessage::FsCdcData {
|
||||||
|
path,
|
||||||
|
chunk_order,
|
||||||
|
chunks,
|
||||||
|
} => {
|
||||||
assert_eq!(path, "cfg.bin");
|
assert_eq!(path, "cfg.bin");
|
||||||
assert_eq!(chunk_order.len(), 2);
|
assert_eq!(chunk_order.len(), 2);
|
||||||
assert_eq!(chunks.len(), 2);
|
assert_eq!(chunks.len(), 2);
|
||||||
@@ -560,10 +606,14 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn fs_file_ack_roundtrip() {
|
fn fs_file_ack_roundtrip() {
|
||||||
let msg = SyncMessage::FsFileAck { path: "embeddings.bin".to_string() };
|
let msg = SyncMessage::FsFileAck {
|
||||||
|
path: "embeddings.bin".to_string(),
|
||||||
|
};
|
||||||
let bytes = msg.to_bytes().unwrap();
|
let bytes = msg.to_bytes().unwrap();
|
||||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||||
assert!(matches!(recovered, SyncMessage::FsFileAck { ref path } if path == "embeddings.bin"));
|
assert!(
|
||||||
|
matches!(recovered, SyncMessage::FsFileAck { ref path } if path == "embeddings.bin")
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -571,8 +621,18 @@ mod tests {
|
|||||||
let msg = SyncMessage::FsDirManifest {
|
let msg = SyncMessage::FsDirManifest {
|
||||||
local_root: "/home/user/data".to_string(),
|
local_root: "/home/user/data".to_string(),
|
||||||
entries: vec![
|
entries: vec![
|
||||||
FsManifestEntry { path: "a.bin".to_string(), blake3: [1u8; 32], size: 4096, mtime: 1000 },
|
FsManifestEntry {
|
||||||
FsManifestEntry { path: "b.bin".to_string(), blake3: [2u8; 32], size: 8192, mtime: 2000 },
|
path: "a.bin".to_string(),
|
||||||
|
blake3: [1u8; 32],
|
||||||
|
size: 4096,
|
||||||
|
mtime: 1000,
|
||||||
|
},
|
||||||
|
FsManifestEntry {
|
||||||
|
path: "b.bin".to_string(),
|
||||||
|
blake3: [2u8; 32],
|
||||||
|
size: 8192,
|
||||||
|
mtime: 2000,
|
||||||
|
},
|
||||||
],
|
],
|
||||||
};
|
};
|
||||||
let bytes = msg.to_bytes().unwrap();
|
let bytes = msg.to_bytes().unwrap();
|
||||||
@@ -590,19 +650,23 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn fs_dir_need_roundtrip() {
|
fn fs_dir_need_roundtrip() {
|
||||||
let msg = SyncMessage::FsDirNeed {
|
let msg = SyncMessage::FsDirNeed {
|
||||||
needed_files: vec![
|
needed_files: vec![FsFileNeed {
|
||||||
FsFileNeed {
|
path: "weights.bin".to_string(),
|
||||||
path: "weights.bin".to_string(),
|
server_chunks: vec![FsChunkHash {
|
||||||
server_chunks: vec![FsChunkHash { hash: 0xAA, length: 65536 }],
|
hash: 0xAA,
|
||||||
expected_blake3: [0xABu8; 32],
|
length: 65536,
|
||||||
},
|
}],
|
||||||
],
|
expected_blake3: [0xABu8; 32],
|
||||||
|
}],
|
||||||
to_delete: vec!["old_model.bin".to_string()],
|
to_delete: vec!["old_model.bin".to_string()],
|
||||||
};
|
};
|
||||||
let bytes = msg.to_bytes().unwrap();
|
let bytes = msg.to_bytes().unwrap();
|
||||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||||
match recovered {
|
match recovered {
|
||||||
SyncMessage::FsDirNeed { needed_files, to_delete } => {
|
SyncMessage::FsDirNeed {
|
||||||
|
needed_files,
|
||||||
|
to_delete,
|
||||||
|
} => {
|
||||||
assert_eq!(needed_files.len(), 1);
|
assert_eq!(needed_files.len(), 1);
|
||||||
assert_eq!(needed_files[0].path, "weights.bin");
|
assert_eq!(needed_files[0].path, "weights.bin");
|
||||||
assert_eq!(to_delete, vec!["old_model.bin"]);
|
assert_eq!(to_delete, vec!["old_model.bin"]);
|
||||||
@@ -614,12 +678,19 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn fs_dir_complete_roundtrip() {
|
fn fs_dir_complete_roundtrip() {
|
||||||
let msg = SyncMessage::FsDirComplete {
|
let msg = SyncMessage::FsDirComplete {
|
||||||
files_added: 2, files_modified: 3, files_removed: 1, bytes_transferred: 1_048_576,
|
files_added: 2,
|
||||||
|
files_modified: 3,
|
||||||
|
files_removed: 1,
|
||||||
|
bytes_transferred: 1_048_576,
|
||||||
};
|
};
|
||||||
let bytes = msg.to_bytes().unwrap();
|
let bytes = msg.to_bytes().unwrap();
|
||||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||||
match recovered {
|
match recovered {
|
||||||
SyncMessage::FsDirComplete { files_added, bytes_transferred, .. } => {
|
SyncMessage::FsDirComplete {
|
||||||
|
files_added,
|
||||||
|
bytes_transferred,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
assert_eq!(files_added, 2);
|
assert_eq!(files_added, 2);
|
||||||
assert_eq!(bytes_transferred, 1_048_576);
|
assert_eq!(bytes_transferred, 1_048_576);
|
||||||
}
|
}
|
||||||
@@ -637,8 +708,10 @@ mod tests {
|
|||||||
// We just verify the roundtrip still produces the exact same ManifestRequest —
|
// We just verify the roundtrip still produces the exact same ManifestRequest —
|
||||||
// any reordering would cause a deserialization error here.
|
// any reordering would cause a deserialization error here.
|
||||||
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
let recovered = SyncMessage::from_bytes(&bytes).unwrap();
|
||||||
assert!(matches!(recovered, SyncMessage::ManifestRequest { .. }),
|
assert!(
|
||||||
"ManifestRequest discriminant changed — rkyv variant ordering was modified");
|
matches!(recovered, SyncMessage::ManifestRequest { .. }),
|
||||||
|
"ManifestRequest discriminant changed — rkyv variant ordering was modified"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -49,11 +49,8 @@ impl QuicConfig {
|
|||||||
.map_err(|e| TransportError::Tls(e.to_string()))?;
|
.map_err(|e| TransportError::Tls(e.to_string()))?;
|
||||||
|
|
||||||
// Server config
|
// Server config
|
||||||
let server_config = ServerConfig::with_single_cert(
|
let server_config = ServerConfig::with_single_cert(vec![cert_der.clone()], key_der)
|
||||||
vec![cert_der.clone()],
|
.map_err(|e| TransportError::Tls(e.to_string()))?;
|
||||||
key_der,
|
|
||||||
)
|
|
||||||
.map_err(|e| TransportError::Tls(e.to_string()))?;
|
|
||||||
|
|
||||||
// Client config: trust only our self-signed cert
|
// Client config: trust only our self-signed cert
|
||||||
let mut roots = rustls::RootCertStore::empty();
|
let mut roots = rustls::RootCertStore::empty();
|
||||||
@@ -63,7 +60,10 @@ impl QuicConfig {
|
|||||||
let client_config = ClientConfig::with_root_certificates(Arc::new(roots))
|
let client_config = ClientConfig::with_root_certificates(Arc::new(roots))
|
||||||
.map_err(|e| TransportError::Tls(e.to_string()))?;
|
.map_err(|e| TransportError::Tls(e.to_string()))?;
|
||||||
|
|
||||||
Ok(Self { server_config, client_config })
|
Ok(Self {
|
||||||
|
server_config,
|
||||||
|
client_config,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -83,9 +83,7 @@ impl QuicConnection {
|
|||||||
|
|
||||||
/// Send a `SyncMessage` on a new unidirectional QUIC stream.
|
/// Send a `SyncMessage` on a new unidirectional QUIC stream.
|
||||||
pub async fn send(&self, msg: &SyncMessage) -> Result<(), TransportError> {
|
pub async fn send(&self, msg: &SyncMessage) -> Result<(), TransportError> {
|
||||||
let body = msg
|
let body = msg.to_bytes().map_err(TransportError::Serialization)?;
|
||||||
.to_bytes()
|
|
||||||
.map_err(TransportError::Serialization)?;
|
|
||||||
|
|
||||||
let mut send = self
|
let mut send = self
|
||||||
.conn
|
.conn
|
||||||
@@ -101,7 +99,8 @@ impl QuicConnection {
|
|||||||
send.write_all(&body)
|
send.write_all(&body)
|
||||||
.await
|
.await
|
||||||
.map_err(|e| TransportError::Io(e.into()))?;
|
.map_err(|e| TransportError::Io(e.into()))?;
|
||||||
send.finish().map_err(|e| TransportError::Quic(e.to_string()))?;
|
send.finish()
|
||||||
|
.map_err(|e| TransportError::Quic(e.to_string()))?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -147,10 +146,12 @@ pub struct QuicServer {
|
|||||||
impl QuicServer {
|
impl QuicServer {
|
||||||
/// Bind a QUIC server endpoint.
|
/// Bind a QUIC server endpoint.
|
||||||
pub async fn bind(addr: SocketAddr, config: QuicConfig) -> Result<Self, TransportError> {
|
pub async fn bind(addr: SocketAddr, config: QuicConfig) -> Result<Self, TransportError> {
|
||||||
let endpoint = Endpoint::server(config.server_config, addr)
|
let endpoint = Endpoint::server(config.server_config, addr).map_err(TransportError::Io)?;
|
||||||
.map_err(TransportError::Io)?;
|
|
||||||
let local_addr = endpoint.local_addr()?;
|
let local_addr = endpoint.local_addr()?;
|
||||||
Ok(Self { endpoint, local_addr })
|
Ok(Self {
|
||||||
|
endpoint,
|
||||||
|
local_addr,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Accept the next incoming QUIC connection.
|
/// Accept the next incoming QUIC connection.
|
||||||
@@ -177,8 +178,8 @@ pub async fn quic_connect(
|
|||||||
server_name: &str,
|
server_name: &str,
|
||||||
config: QuicConfig,
|
config: QuicConfig,
|
||||||
) -> Result<QuicConnection, TransportError> {
|
) -> Result<QuicConnection, TransportError> {
|
||||||
let mut endpoint = Endpoint::client("0.0.0.0:0".parse().unwrap())
|
let mut endpoint =
|
||||||
.map_err(TransportError::Io)?;
|
Endpoint::client("0.0.0.0:0".parse().unwrap()).map_err(TransportError::Io)?;
|
||||||
endpoint.set_default_client_config(config.client_config);
|
endpoint.set_default_client_config(config.client_config);
|
||||||
|
|
||||||
let conn = endpoint
|
let conn = endpoint
|
||||||
@@ -217,10 +218,15 @@ mod tests {
|
|||||||
roots.add(cert_der).unwrap();
|
roots.add(cert_der).unwrap();
|
||||||
let cli_cfg = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap();
|
let cli_cfg = ClientConfig::with_root_certificates(Arc::new(roots)).unwrap();
|
||||||
|
|
||||||
let server = QuicServer::bind(any_addr(), QuicConfig {
|
let server = QuicServer::bind(
|
||||||
server_config: srv_cfg,
|
any_addr(),
|
||||||
client_config: cli_cfg.clone(),
|
QuicConfig {
|
||||||
}).await.unwrap();
|
server_config: srv_cfg,
|
||||||
|
client_config: cli_cfg.clone(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
let addr = server.local_addr;
|
let addr = server.local_addr;
|
||||||
|
|
||||||
let server_task = tokio::spawn(async move {
|
let server_task = tokio::spawn(async move {
|
||||||
|
|||||||
@@ -7,11 +7,11 @@
|
|||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
|
|
||||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
use tokio::net::{TcpListener, TcpStream};
|
|
||||||
use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf};
|
use tokio::net::tcp::{OwnedReadHalf, OwnedWriteHalf};
|
||||||
|
use tokio::net::{TcpListener, TcpStream};
|
||||||
|
|
||||||
use crate::error::TransportError;
|
use crate::error::TransportError;
|
||||||
use crate::protocol::{SyncMessage, MAX_FRAME_SIZE};
|
use crate::protocol::{MAX_FRAME_SIZE, SyncMessage};
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
// TcpConnection — async read/write of SyncMessages
|
// TcpConnection — async read/write of SyncMessages
|
||||||
@@ -37,9 +37,7 @@ impl TcpConnection {
|
|||||||
|
|
||||||
/// Send a `SyncMessage` over the connection.
|
/// Send a `SyncMessage` over the connection.
|
||||||
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
||||||
let body = msg
|
let body = msg.to_bytes().map_err(TransportError::Serialization)?;
|
||||||
.to_bytes()
|
|
||||||
.map_err(TransportError::Serialization)?;
|
|
||||||
let len = body.len() as u32;
|
let len = body.len() as u32;
|
||||||
self.stream.write_all(&len.to_le_bytes()).await?;
|
self.stream.write_all(&len.to_le_bytes()).await?;
|
||||||
self.stream.write_all(&body).await?;
|
self.stream.write_all(&body).await?;
|
||||||
@@ -65,8 +63,7 @@ impl TcpConnection {
|
|||||||
let mut body = vec![0u8; len];
|
let mut body = vec![0u8; len];
|
||||||
self.stream.read_exact(&mut body).await?;
|
self.stream.read_exact(&mut body).await?;
|
||||||
|
|
||||||
SyncMessage::from_bytes(&body)
|
SyncMessage::from_bytes(&body).map_err(TransportError::Deserialization)
|
||||||
.map_err(TransportError::Deserialization)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Gracefully shut down the write half of the connection.
|
/// Gracefully shut down the write half of the connection.
|
||||||
@@ -81,7 +78,10 @@ impl TcpConnection {
|
|||||||
/// pipelined send/receive (sliding window) without holding a single lock.
|
/// pipelined send/receive (sliding window) without holding a single lock.
|
||||||
pub fn into_split(self) -> (TcpReadHalf, TcpWriteHalf) {
|
pub fn into_split(self) -> (TcpReadHalf, TcpWriteHalf) {
|
||||||
let (read_half, write_half) = self.stream.into_split();
|
let (read_half, write_half) = self.stream.into_split();
|
||||||
(TcpReadHalf { half: read_half }, TcpWriteHalf { half: write_half })
|
(
|
||||||
|
TcpReadHalf { half: read_half },
|
||||||
|
TcpWriteHalf { half: write_half },
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -102,7 +102,10 @@ impl TcpReadHalf {
|
|||||||
let len = u32::from_le_bytes(len_buf) as usize;
|
let len = u32::from_le_bytes(len_buf) as usize;
|
||||||
|
|
||||||
if len > MAX_FRAME_SIZE {
|
if len > MAX_FRAME_SIZE {
|
||||||
return Err(TransportError::FrameTooLarge { size: len, max: MAX_FRAME_SIZE });
|
return Err(TransportError::FrameTooLarge {
|
||||||
|
size: len,
|
||||||
|
max: MAX_FRAME_SIZE,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut body = vec![0u8; len];
|
let mut body = vec![0u8; len];
|
||||||
@@ -149,7 +152,10 @@ impl TcpServer {
|
|||||||
pub async fn bind(addr: SocketAddr) -> Result<Self, TransportError> {
|
pub async fn bind(addr: SocketAddr) -> Result<Self, TransportError> {
|
||||||
let listener = TcpListener::bind(addr).await?;
|
let listener = TcpListener::bind(addr).await?;
|
||||||
let local_addr = listener.local_addr()?;
|
let local_addr = listener.local_addr()?;
|
||||||
Ok(Self { listener, local_addr })
|
Ok(Self {
|
||||||
|
listener,
|
||||||
|
local_addr,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Accept the next incoming connection.
|
/// Accept the next incoming connection.
|
||||||
@@ -186,7 +192,10 @@ mod tests {
|
|||||||
});
|
});
|
||||||
|
|
||||||
let mut client = TcpConnection::connect(addr).await.unwrap();
|
let mut client = TcpConnection::connect(addr).await.unwrap();
|
||||||
client.send(&SyncMessage::Ack { revision: 42 }).await.unwrap();
|
client
|
||||||
|
.send(&SyncMessage::Ack { revision: 42 })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
let reply = client.recv().await.unwrap();
|
let reply = client.recv().await.unwrap();
|
||||||
assert!(matches!(reply, SyncMessage::Ack { revision: 42 }));
|
assert!(matches!(reply, SyncMessage::Ack { revision: 42 }));
|
||||||
|
|
||||||
@@ -208,7 +217,10 @@ mod tests {
|
|||||||
|
|
||||||
let mut client = TcpConnection::connect(addr).await.unwrap();
|
let mut client = TcpConnection::connect(addr).await.unwrap();
|
||||||
for rev in 0u64..5 {
|
for rev in 0u64..5 {
|
||||||
client.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
client
|
||||||
|
.send(&SyncMessage::Ack { revision: rev })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
}
|
}
|
||||||
server_task.await.unwrap();
|
server_task.await.unwrap();
|
||||||
}
|
}
|
||||||
@@ -233,7 +245,10 @@ mod tests {
|
|||||||
client.send(&msg).await.unwrap();
|
client.send(&msg).await.unwrap();
|
||||||
let reply = client.recv().await.unwrap();
|
let reply = client.recv().await.unwrap();
|
||||||
match reply {
|
match reply {
|
||||||
SyncMessage::SyncComplete { revisions_transferred, bytes_transferred } => {
|
SyncMessage::SyncComplete {
|
||||||
|
revisions_transferred,
|
||||||
|
bytes_transferred,
|
||||||
|
} => {
|
||||||
assert_eq!(revisions_transferred, 7);
|
assert_eq!(revisions_transferred, 7);
|
||||||
assert_eq!(bytes_transferred, 28672);
|
assert_eq!(bytes_transferred, 28672);
|
||||||
}
|
}
|
||||||
@@ -263,7 +278,11 @@ mod tests {
|
|||||||
client.send(&msg).await.unwrap();
|
client.send(&msg).await.unwrap();
|
||||||
let reply = client.recv().await.unwrap();
|
let reply = client.recv().await.unwrap();
|
||||||
match reply {
|
match reply {
|
||||||
SyncMessage::ManifestRequest { agent_id, head_revision, .. } => {
|
SyncMessage::ManifestRequest {
|
||||||
|
agent_id,
|
||||||
|
head_revision,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
assert_eq!(agent_id, "my-agent");
|
assert_eq!(agent_id, "my-agent");
|
||||||
assert_eq!(head_revision, 3);
|
assert_eq!(head_revision, 3);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ use clawhdf5_onion::writer::OnionFile;
|
|||||||
use clawsync_onion::{differ::diff_revisions, manifest::ClawSyncManifest, merger::merge_packets};
|
use clawsync_onion::{differ::diff_revisions, manifest::ClawSyncManifest, merger::merge_packets};
|
||||||
use clawsync_transport::{
|
use clawsync_transport::{
|
||||||
protocol::SyncMessage,
|
protocol::SyncMessage,
|
||||||
quic::{quic_connect, QuicConfig, QuicServer},
|
quic::{QuicConfig, QuicServer, quic_connect},
|
||||||
};
|
};
|
||||||
use tempfile::NamedTempFile;
|
use tempfile::NamedTempFile;
|
||||||
|
|
||||||
@@ -62,8 +62,7 @@ fn make_quic_configs() -> (QuicConfig, QuicConfig) {
|
|||||||
|
|
||||||
let mut roots = rustls::RootCertStore::empty();
|
let mut roots = rustls::RootCertStore::empty();
|
||||||
roots.add(srv_cert_der).unwrap();
|
roots.add(srv_cert_der).unwrap();
|
||||||
let client_config =
|
let client_config = quinn::ClientConfig::with_root_certificates(Arc::new(roots)).unwrap();
|
||||||
quinn::ClientConfig::with_root_certificates(Arc::new(roots)).unwrap();
|
|
||||||
|
|
||||||
// Client-side QuicConfig requires a server_config field; generate a fresh
|
// Client-side QuicConfig requires a server_config field; generate a fresh
|
||||||
// cert+key pair so there is no key/cert mismatch.
|
// cert+key pair so there is no key/cert mismatch.
|
||||||
@@ -71,8 +70,14 @@ fn make_quic_configs() -> (QuicConfig, QuicConfig) {
|
|||||||
let cli_server_config =
|
let cli_server_config =
|
||||||
quinn::ServerConfig::with_single_cert(vec![cli_cert_der], cli_key_der).unwrap();
|
quinn::ServerConfig::with_single_cert(vec![cli_cert_der], cli_key_der).unwrap();
|
||||||
|
|
||||||
let srv_cfg = QuicConfig { server_config: srv_server_config, client_config: client_config.clone() };
|
let srv_cfg = QuicConfig {
|
||||||
let cli_cfg = QuicConfig { server_config: cli_server_config, client_config };
|
server_config: srv_server_config,
|
||||||
|
client_config: client_config.clone(),
|
||||||
|
};
|
||||||
|
let cli_cfg = QuicConfig {
|
||||||
|
server_config: cli_server_config,
|
||||||
|
client_config,
|
||||||
|
};
|
||||||
(srv_cfg, cli_cfg)
|
(srv_cfg, cli_cfg)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -99,16 +104,20 @@ async fn run_quic_push(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
|||||||
match msg {
|
match msg {
|
||||||
SyncMessage::ManifestRequest { .. } => {
|
SyncMessage::ManifestRequest { .. } => {
|
||||||
let dst_manifest = ClawSyncManifest::from_onion("dst", &dst, H5_BASE);
|
let dst_manifest = ClawSyncManifest::from_onion("dst", &dst, H5_BASE);
|
||||||
conn.send(&SyncMessage::ManifestResponse { manifest: dst_manifest })
|
conn.send(&SyncMessage::ManifestResponse {
|
||||||
.await
|
manifest: dst_manifest,
|
||||||
.unwrap();
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
SyncMessage::LayerPacket { packet } => {
|
SyncMessage::LayerPacket { packet } => {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
merge_packets(&mut dst, vec![packet], false).unwrap();
|
merge_packets(&mut dst, vec![packet], false).unwrap();
|
||||||
packets_received += 1;
|
packets_received += 1;
|
||||||
conn.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
conn.send(&SyncMessage::Ack { revision: rev })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
SyncMessage::SyncComplete { .. } => {
|
SyncMessage::SyncComplete { .. } => {
|
||||||
@@ -125,7 +134,9 @@ async fn run_quic_push(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// ── Client logic ─────────────────────────────────────────────────────────
|
// ── Client logic ─────────────────────────────────────────────────────────
|
||||||
let client = quic_connect(server_addr, "localhost", cli_cfg).await.unwrap();
|
let client = quic_connect(server_addr, "localhost", cli_cfg)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
client
|
client
|
||||||
.send(&SyncMessage::ManifestRequest {
|
.send(&SyncMessage::ManifestRequest {
|
||||||
@@ -152,7 +163,10 @@ async fn run_quic_push(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
|||||||
|
|
||||||
for packet in packets {
|
for packet in packets {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
client.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
client
|
||||||
|
.send(&SyncMessage::LayerPacket { packet })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
match client.recv().await.unwrap() {
|
match client.recv().await.unwrap() {
|
||||||
SyncMessage::Ack { revision } => assert_eq!(revision, rev),
|
SyncMessage::Ack { revision } => assert_eq!(revision, rev),
|
||||||
other => panic!("client: expected Ack, got {other:?}"),
|
other => panic!("client: expected Ack, got {other:?}"),
|
||||||
@@ -228,7 +242,11 @@ async fn quic_revision_hashes_match_after_sync() {
|
|||||||
|
|
||||||
assert_eq!(src_revisions.len(), dst_revisions.len());
|
assert_eq!(src_revisions.len(), dst_revisions.len());
|
||||||
for (s, d) in src_revisions.iter().zip(dst_revisions.iter()) {
|
for (s, d) in src_revisions.iter().zip(dst_revisions.iter()) {
|
||||||
assert_eq!(s.blake3_hex, d.blake3_hex, "rev {} hash mismatch", s.revision);
|
assert_eq!(
|
||||||
|
s.blake3_hex, d.blake3_hex,
|
||||||
|
"rev {} hash mismatch",
|
||||||
|
s.revision
|
||||||
|
);
|
||||||
assert_eq!(s.annotation, d.annotation);
|
assert_eq!(s.annotation, d.annotation);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -269,9 +287,11 @@ async fn run_quic_pull(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let srv_manifest = ClawSyncManifest::from_onion("srv", &src, H5_BASE);
|
let srv_manifest = ClawSyncManifest::from_onion("srv", &src, H5_BASE);
|
||||||
conn.send(&SyncMessage::ManifestResponse { manifest: srv_manifest })
|
conn.send(&SyncMessage::ManifestResponse {
|
||||||
.await
|
manifest: srv_manifest,
|
||||||
.unwrap();
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
if src.revision_count() > client_rev_count {
|
if src.revision_count() > client_rev_count {
|
||||||
let remote_head = if client_rev_count == 0 {
|
let remote_head = if client_rev_count == 0 {
|
||||||
@@ -284,7 +304,9 @@ async fn run_quic_pull(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
|||||||
let mut bytes_sent = 0u64;
|
let mut bytes_sent = 0u64;
|
||||||
for packet in packets {
|
for packet in packets {
|
||||||
bytes_sent += packet.page_data_size() as u64;
|
bytes_sent += packet.page_data_size() as u64;
|
||||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
conn.send(&SyncMessage::LayerPacket { packet })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::Ack { .. } => {}
|
SyncMessage::Ack { .. } => {}
|
||||||
other => panic!("server: expected Ack, got {other:?}"),
|
other => panic!("server: expected Ack, got {other:?}"),
|
||||||
@@ -310,7 +332,9 @@ async fn run_quic_pull(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
|||||||
});
|
});
|
||||||
|
|
||||||
// ── Client logic ─────────────────────────────────────────────────────────
|
// ── Client logic ─────────────────────────────────────────────────────────
|
||||||
let client = quic_connect(server_addr, "localhost", cli_cfg).await.unwrap();
|
let client = quic_connect(server_addr, "localhost", cli_cfg)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
let local_rev_count = dst.revision_count();
|
let local_rev_count = dst.revision_count();
|
||||||
|
|
||||||
client
|
client
|
||||||
@@ -344,10 +368,16 @@ async fn run_quic_pull(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
|||||||
match client.recv().await.unwrap() {
|
match client.recv().await.unwrap() {
|
||||||
SyncMessage::LayerPacket { packet } => {
|
SyncMessage::LayerPacket { packet } => {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
client.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
client
|
||||||
|
.send(&SyncMessage::Ack { revision: rev })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
packets.push(packet);
|
packets.push(packet);
|
||||||
}
|
}
|
||||||
SyncMessage::SyncComplete { revisions_transferred, .. } => {
|
SyncMessage::SyncComplete {
|
||||||
|
revisions_transferred,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
merge_packets(&mut dst, packets, true).unwrap();
|
merge_packets(&mut dst, packets, true).unwrap();
|
||||||
server_handle.await.unwrap();
|
server_handle.await.unwrap();
|
||||||
return (revisions_transferred, dst);
|
return (revisions_transferred, dst);
|
||||||
@@ -412,7 +442,11 @@ async fn quic_pull_revision_hashes_match() {
|
|||||||
|
|
||||||
assert_eq!(srv_revisions.len(), cli_revisions.len());
|
assert_eq!(srv_revisions.len(), cli_revisions.len());
|
||||||
for (s, c) in srv_revisions.iter().zip(cli_revisions.iter()) {
|
for (s, c) in srv_revisions.iter().zip(cli_revisions.iter()) {
|
||||||
assert_eq!(s.blake3_hex, c.blake3_hex, "rev {} hash mismatch", s.revision);
|
assert_eq!(
|
||||||
|
s.blake3_hex, c.blake3_hex,
|
||||||
|
"rev {} hash mismatch",
|
||||||
|
s.revision
|
||||||
|
);
|
||||||
assert_eq!(s.annotation, c.annotation);
|
assert_eq!(s.annotation, c.annotation);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,11 +18,7 @@
|
|||||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||||
|
|
||||||
use clawhdf5_onion::writer::OnionFile;
|
use clawhdf5_onion::writer::OnionFile;
|
||||||
use clawsync_onion::{
|
use clawsync_onion::{differ::diff_revisions, manifest::ClawSyncManifest, merger::merge_packets};
|
||||||
differ::diff_revisions,
|
|
||||||
manifest::ClawSyncManifest,
|
|
||||||
merger::merge_packets,
|
|
||||||
};
|
|
||||||
use clawsync_transport::{
|
use clawsync_transport::{
|
||||||
protocol::SyncMessage,
|
protocol::SyncMessage,
|
||||||
tcp::{TcpConnection, TcpServer},
|
tcp::{TcpConnection, TcpServer},
|
||||||
@@ -45,9 +41,7 @@ fn make_onion(n: u8) -> (NamedTempFile, OnionFile) {
|
|||||||
for i in 0..n {
|
for i in 0..n {
|
||||||
let mut s = onion.begin_session(None).unwrap();
|
let mut s = onion.begin_session(None).unwrap();
|
||||||
s.record_page(0, &vec![i; 4096]);
|
s.record_page(0, &vec![i; 4096]);
|
||||||
onion
|
onion.commit_session(s, Some(&format!("rev {i}"))).unwrap();
|
||||||
.commit_session(s, Some(&format!("rev {i}")))
|
|
||||||
.unwrap();
|
|
||||||
}
|
}
|
||||||
(tmp, onion)
|
(tmp, onion)
|
||||||
}
|
}
|
||||||
@@ -57,10 +51,7 @@ fn make_onion(n: u8) -> (NamedTempFile, OnionFile) {
|
|||||||
// number of LayerPackets actually transferred.
|
// number of LayerPackets actually transferred.
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
async fn run_push(
|
async fn run_push(src: OnionFile, mut dst: OnionFile) -> (u64, OnionFile) {
|
||||||
src: OnionFile,
|
|
||||||
mut dst: OnionFile,
|
|
||||||
) -> (u64, OnionFile) {
|
|
||||||
// Bind the server; the OS picks a free port.
|
// Bind the server; the OS picks a free port.
|
||||||
let server = TcpServer::bind(localhost_any()).await.unwrap();
|
let server = TcpServer::bind(localhost_any()).await.unwrap();
|
||||||
let server_addr = server.local_addr;
|
let server_addr = server.local_addr;
|
||||||
@@ -79,8 +70,7 @@ async fn run_push(
|
|||||||
match msg {
|
match msg {
|
||||||
SyncMessage::ManifestRequest { head_revision, .. } => {
|
SyncMessage::ManifestRequest { head_revision, .. } => {
|
||||||
// Build our local manifest and send it back.
|
// Build our local manifest and send it back.
|
||||||
let dst_manifest =
|
let dst_manifest = ClawSyncManifest::from_onion("dst", &dst, H5_BASE);
|
||||||
ClawSyncManifest::from_onion("dst", &dst, H5_BASE);
|
|
||||||
conn.send(&SyncMessage::ManifestResponse {
|
conn.send(&SyncMessage::ManifestResponse {
|
||||||
manifest: dst_manifest,
|
manifest: dst_manifest,
|
||||||
})
|
})
|
||||||
@@ -163,7 +153,10 @@ async fn run_push(
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let (server_received, dst_final) = server_handle.await.unwrap();
|
let (server_received, dst_final) = server_handle.await.unwrap();
|
||||||
assert_eq!(server_received, total_sent, "server count must match client send count");
|
assert_eq!(
|
||||||
|
server_received, total_sent,
|
||||||
|
"server count must match client send count"
|
||||||
|
);
|
||||||
|
|
||||||
(total_sent, dst_final)
|
(total_sent, dst_final)
|
||||||
}
|
}
|
||||||
@@ -287,7 +280,7 @@ async fn large_push_correct_packet_count() {
|
|||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
async fn run_pull(
|
async fn run_pull(
|
||||||
src: OnionFile, // server-side onion (richer)
|
src: OnionFile, // server-side onion (richer)
|
||||||
mut dst: OnionFile, // client-side onion (missing revisions)
|
mut dst: OnionFile, // client-side onion (missing revisions)
|
||||||
) -> (u64, OnionFile) {
|
) -> (u64, OnionFile) {
|
||||||
let server = TcpServer::bind(localhost_any()).await.unwrap();
|
let server = TcpServer::bind(localhost_any()).await.unwrap();
|
||||||
@@ -305,9 +298,11 @@ async fn run_pull(
|
|||||||
|
|
||||||
// 2. Send back the server manifest.
|
// 2. Send back the server manifest.
|
||||||
let srv_manifest = ClawSyncManifest::from_onion("srv", &src, H5_BASE);
|
let srv_manifest = ClawSyncManifest::from_onion("srv", &src, H5_BASE);
|
||||||
conn.send(&SyncMessage::ManifestResponse { manifest: srv_manifest })
|
conn.send(&SyncMessage::ManifestResponse {
|
||||||
.await
|
manifest: srv_manifest,
|
||||||
.unwrap();
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
// 3. Server has more revisions — push the delta to the client.
|
// 3. Server has more revisions — push the delta to the client.
|
||||||
if src.revision_count() > client_rev_count {
|
if src.revision_count() > client_rev_count {
|
||||||
@@ -321,7 +316,9 @@ async fn run_pull(
|
|||||||
let mut bytes_sent = 0u64;
|
let mut bytes_sent = 0u64;
|
||||||
for packet in packets {
|
for packet in packets {
|
||||||
bytes_sent += packet.page_data_size() as u64;
|
bytes_sent += packet.page_data_size() as u64;
|
||||||
conn.send(&SyncMessage::LayerPacket { packet }).await.unwrap();
|
conn.send(&SyncMessage::LayerPacket { packet })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
match conn.recv().await.unwrap() {
|
match conn.recv().await.unwrap() {
|
||||||
SyncMessage::Ack { .. } => {}
|
SyncMessage::Ack { .. } => {}
|
||||||
other => panic!("server: expected Ack, got {other:?}"),
|
other => panic!("server: expected Ack, got {other:?}"),
|
||||||
@@ -381,10 +378,16 @@ async fn run_pull(
|
|||||||
match client.recv().await.unwrap() {
|
match client.recv().await.unwrap() {
|
||||||
SyncMessage::LayerPacket { packet } => {
|
SyncMessage::LayerPacket { packet } => {
|
||||||
let rev = packet.revision;
|
let rev = packet.revision;
|
||||||
client.send(&SyncMessage::Ack { revision: rev }).await.unwrap();
|
client
|
||||||
|
.send(&SyncMessage::Ack { revision: rev })
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
packets.push(packet);
|
packets.push(packet);
|
||||||
}
|
}
|
||||||
SyncMessage::SyncComplete { revisions_transferred, .. } => {
|
SyncMessage::SyncComplete {
|
||||||
|
revisions_transferred,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
merge_packets(&mut dst, packets, true).unwrap();
|
merge_packets(&mut dst, packets, true).unwrap();
|
||||||
server_handle.await.unwrap();
|
server_handle.await.unwrap();
|
||||||
return (revisions_transferred, dst);
|
return (revisions_transferred, dst);
|
||||||
@@ -449,7 +452,11 @@ async fn pull_revision_hashes_match() {
|
|||||||
|
|
||||||
assert_eq!(srv_revisions.len(), cli_revisions.len());
|
assert_eq!(srv_revisions.len(), cli_revisions.len());
|
||||||
for (s, c) in srv_revisions.iter().zip(cli_revisions.iter()) {
|
for (s, c) in srv_revisions.iter().zip(cli_revisions.iter()) {
|
||||||
assert_eq!(s.blake3_hex, c.blake3_hex, "rev {} hash mismatch", s.revision);
|
assert_eq!(
|
||||||
|
s.blake3_hex, c.blake3_hex,
|
||||||
|
"rev {} hash mismatch",
|
||||||
|
s.revision
|
||||||
|
);
|
||||||
assert_eq!(s.annotation, c.annotation);
|
assert_eq!(s.annotation, c.annotation);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user