Initial commit: ClawSync v0.1.0

8-crate pure-Rust workspace for revision-aware HDF5 sync. ## Crates - clawhdf5-onion: ClawOnion VFD — page-level versioned HDF5 storage, binary format, writer/reader, branch DAG, GC, snapshots, provenance - clawsync-core: BLAKE3, xxHash3, FastCDC (+ SIMD NEON), zstd/lz4 - clawsync-onion: IBLT sketch, Merkle tree differ, packet differ/merger, ClawSyncManifest, SyncSelector - clawsync-hdf5: dataset-level manifest, differ, patcher, wire payload reconstruction (apply_received_payloads) - clawsync-transport: TCP, QUIC (quinn 0.11/TLS 1.3), SyncPeer abstraction, length-prefixed rkyv wire protocol (21 SyncMessage variants) - clawsync-agent: OnionMemory, SyncScheduler, TcpSyncBackend, PeerCapabilities negotiation - clawsync-fs: CDC-based delta sync for any file type; FsSyncClient/Server, W=16 pipelining, atomic writes - clawsync-cli: push/pull/serve/hdf5-sync/serve-hdf5/sync/serve-fs + all local management commands; --quic on all network commands ## Key features - IBLT pre-flight: O(revision count) vs rsync's O(file size) - W=16 sliding-window push: 13–15x speedup over stop-and-wait at WAN RTT - Dataset-granular HDF5 sync: only modified datasets transferred - CDC delta for any file type: insertion-stable chunk boundaries - Full revision DAG: branch, merge, rollback, export, snapshot, GC - QUIC transport: TLS 1.3, per-message streams via quinn 0.11 ## Tests ~573 passing (default features); ~589 with --features simd-cdc ## Performance (Apple Silicon) - Reconstruct rev=100: 68 µs (target ≤ 1 ms) - BLAKE3 Rayon 1 MB: 10.3 GiB/s (target ≥ 5 GB/s) - GC 500 revisions: 20.6 µs (target ≤ 2 s) - W=16 vs W=1 at 5 ms RTT: 14.8x speedup - No-op pre-flight at 16 MB: 4 ms vs rsync 35 ms (7.8x) Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
2026-04-04 18:41:22 -05:00
commit 260e15f5b6
102 changed files with 30098 additions and 0 deletions
@@ -0,0 +1,314 @@
+//! Integration tests for `clawsync-hdf5`.
+//!
+//! These tests exercise the public surface — `DatasetManifest`, `diff_manifests`,
+//! `apply_patches` (file-path API), and `apply_patches_to_bytes` — end-to-end,
+//! creating real HDF5 files in memory and on disk as needed.
+
+use clawhdf5::{File, FileBuilder};
+use clawsync_hdf5::{DatasetManifest, PatchKind, apply_patches, apply_patches_to_bytes, diff_manifests};
+use tempfile::NamedTempFile;
+
+// ─────────────────────────────────────────────────────────────────────────────
+// Helpers
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// Build HDF5 bytes from a list of (dataset_path, values) pairs.
+fn h5_bytes(datasets: &[(&str, &[f64])]) -> Vec<u8> {
+    let mut b = FileBuilder::new();
+    for (name, data) in datasets {
+        b.create_dataset(name).with_f64_data(data);
+    }
+    b.finish().unwrap()
+}
+
+/// Write `bytes` to a `NamedTempFile` with an `.h5` extension and return both
+/// the guard (keeps the file alive) and the `.h5` path.
+fn write_temp_h5(bytes: &[u8]) -> (NamedTempFile, std::path::PathBuf) {
+    let guard = NamedTempFile::new().unwrap();
+    let path = guard.path().with_extension("h5");
+    std::fs::write(&path, bytes).unwrap();
+    (guard, path)
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 1. patch_adds_new_dataset
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// A dataset present in source but absent from target is added after patching.
+#[test]
+fn patch_adds_new_dataset() {
+    let src = h5_bytes(&[("temperature", &[20.0, 21.5, 22.0]), ("humidity", &[55.0, 57.0])]);
+    let tgt = h5_bytes(&[("temperature", &[20.0, 21.5, 22.0])]);
+
+    let old_m = DatasetManifest::from_bytes(&tgt).unwrap();
+    let new_m = DatasetManifest::from_bytes(&src).unwrap();
+    let diff = diff_manifests(&old_m, &new_m);
+
+    assert!(!diff.is_empty());
+    assert_eq!(diff.of_kind(&PatchKind::Added).count(), 1);
+
+    let (patched, _) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap();
+    let result_m = DatasetManifest::from_bytes(&patched).unwrap();
+
+    // The new dataset must be present with the correct hash.
+    let result_entry = result_m.get("/humidity")
+        .expect("humidity should exist after patching");
+    let src_entry = new_m.get("/humidity").unwrap();
+    assert_eq!(result_entry.blake3, src_entry.blake3);
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 2. patch_modifies_dataset
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// A dataset with updated values in source replaces the stale target version.
+#[test]
+fn patch_modifies_dataset() {
+    let src = h5_bytes(&[("pressure", &[1013.25, 1014.0, 1012.5])]);
+    let tgt = h5_bytes(&[("pressure", &[1000.0, 1001.0, 999.5])]);
+
+    let old_m = DatasetManifest::from_bytes(&tgt).unwrap();
+    let new_m = DatasetManifest::from_bytes(&src).unwrap();
+    let diff = diff_manifests(&old_m, &new_m);
+
+    assert_eq!(diff.of_kind(&PatchKind::Modified).count(), 1);
+
+    let (patched, _) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap();
+    let result_m = DatasetManifest::from_bytes(&patched).unwrap();
+
+    // Verify the dataset in the result matches source, not old target.
+    assert_eq!(
+        result_m.get("/pressure").unwrap().blake3,
+        new_m.get("/pressure").unwrap().blake3,
+    );
+    assert_ne!(
+        result_m.get("/pressure").unwrap().blake3,
+        old_m.get("/pressure").unwrap().blake3,
+    );
+
+    // Also verify the actual values round-trip correctly.
+    let file = File::from_bytes(patched).unwrap();
+    let vals = file.dataset("pressure").unwrap().read_f64().unwrap();
+    assert_eq!(vals, vec![1013.25, 1014.0, 1012.5]);
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 3. patch_removes_dataset
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// A dataset absent from source but present in target is removed after patching.
+#[test]
+fn patch_removes_dataset() {
+    let src = h5_bytes(&[("voltage", &[3.3, 5.0])]);
+    let tgt = h5_bytes(&[("voltage", &[3.3, 5.0]), ("current", &[0.1, 0.2, 0.3])]);
+
+    let old_m = DatasetManifest::from_bytes(&tgt).unwrap();
+    let new_m = DatasetManifest::from_bytes(&src).unwrap();
+    let diff = diff_manifests(&old_m, &new_m);
+
+    assert_eq!(diff.of_kind(&PatchKind::Removed).count(), 1);
+    assert_eq!(diff.of_kind(&PatchKind::Removed).next().unwrap().path, "/current");
+
+    let (patched, _) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap();
+    let result_m = DatasetManifest::from_bytes(&patched).unwrap();
+
+    assert!(result_m.get("/current").is_none(), "current should have been removed");
+    assert!(result_m.get("/voltage").is_some(), "voltage should be retained");
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 4. patch_noop_when_identical
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// When source and target are identical the diff is empty and stats are zeroed.
+#[test]
+fn patch_noop_when_identical() {
+    let bytes = h5_bytes(&[("alpha", &[1.0, 2.0, 3.0]), ("beta", &[10.0, 20.0])]);
+
+    let manifest = DatasetManifest::from_bytes(&bytes).unwrap();
+    let diff = diff_manifests(&manifest, &manifest);
+
+    assert!(diff.is_empty(), "diff of identical files must be empty");
+
+    // apply_patches with an empty patch list returns default stats.
+    let (out, stats) = apply_patches_to_bytes(&diff.patches, &bytes, &bytes).unwrap();
+    assert_eq!(out, bytes);
+    assert_eq!(stats.datasets_added, 0);
+    assert_eq!(stats.datasets_modified, 0);
+    assert_eq!(stats.datasets_removed, 0);
+    assert_eq!(stats.bytes_written, 0);
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 5. patch_multiple_changes
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// A single diff/patch handles additions, modifications, and removals together.
+#[test]
+fn patch_multiple_changes() {
+    // source: "kept" (same), "updated" (changed), "added" (new)
+    let src = h5_bytes(&[
+        ("kept",    &[1.0, 2.0]),
+        ("updated", &[99.0, 100.0]),
+        ("added",   &[7.0, 8.0, 9.0]),
+    ]);
+    // target: "kept" (same), "updated" (old values), "removed" (extra)
+    let tgt = h5_bytes(&[
+        ("kept",    &[1.0, 2.0]),
+        ("updated", &[0.0, 0.0]),
+        ("removed", &[42.0]),
+    ]);
+
+    let old_m = DatasetManifest::from_bytes(&tgt).unwrap();
+    let new_m = DatasetManifest::from_bytes(&src).unwrap();
+    let diff = diff_manifests(&old_m, &new_m);
+
+    assert_eq!(diff.of_kind(&PatchKind::Added).count(),    1);
+    assert_eq!(diff.of_kind(&PatchKind::Modified).count(), 1);
+    assert_eq!(diff.of_kind(&PatchKind::Removed).count(),  1);
+
+    let (patched, _) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap();
+    let result_m = DatasetManifest::from_bytes(&patched).unwrap();
+
+    // "kept" should still exist and be unchanged.
+    assert_eq!(
+        result_m.get("/kept").unwrap().blake3,
+        new_m.get("/kept").unwrap().blake3,
+    );
+    // "updated" should now match source.
+    assert_eq!(
+        result_m.get("/updated").unwrap().blake3,
+        new_m.get("/updated").unwrap().blake3,
+    );
+    // "added" should now exist.
+    assert!(result_m.get("/added").is_some());
+    // "removed" should be gone.
+    assert!(result_m.get("/removed").is_none());
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 6. stats_counts_correct
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// `PatchStats` fields accurately reflect the number of each operation kind.
+#[test]
+fn stats_counts_correct() {
+    // source: "a" (modified), "c" (unchanged), "d" (new)
+    let src = h5_bytes(&[
+        ("a", &[9.0, 9.0]),
+        ("c", &[3.0]),
+        ("d", &[4.0, 5.0]),
+    ]);
+    // target: "a" (old values), "b" (to be removed), "c" (same as source)
+    let tgt = h5_bytes(&[
+        ("a", &[1.0, 2.0]),
+        ("b", &[0.0]),
+        ("c", &[3.0]),
+    ]);
+
+    let old_m = DatasetManifest::from_bytes(&tgt).unwrap();
+    let new_m = DatasetManifest::from_bytes(&src).unwrap();
+    let diff = diff_manifests(&old_m, &new_m);
+    let (_, stats) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap();
+
+    assert_eq!(stats.datasets_added,    1, "d was added");
+    assert_eq!(stats.datasets_modified, 1, "a was modified");
+    assert_eq!(stats.datasets_removed,  1, "b was removed");
+    assert_eq!(stats.datasets_unchanged, 1, "c was unchanged");
+    // bytes_written must be positive (added + modified + unchanged datasets).
+    assert!(stats.bytes_written > 0);
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 7. apply_patches_file_path_api
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// The file-path variant of `apply_patches` writes the result to disk and the
+/// manifest read back from disk matches the source manifest.
+#[test]
+fn apply_patches_file_path_api() {
+    let src_bytes = h5_bytes(&[
+        ("chan_x", &[1.1, 2.2, 3.3]),
+        ("chan_y", &[4.4, 5.5]),
+    ]);
+    let tgt_bytes = h5_bytes(&[
+        ("chan_x", &[0.0, 0.0, 0.0]),
+    ]);
+
+    // Write both files to disk using NamedTempFile.
+    let (_src_guard, src_path) = write_temp_h5(&src_bytes);
+    let (_tgt_guard, tgt_path) = write_temp_h5(&tgt_bytes);
+
+    let old_m = DatasetManifest::from_path(&tgt_path).unwrap();
+    let new_m = DatasetManifest::from_path(&src_path).unwrap();
+    let diff = diff_manifests(&old_m, &new_m);
+
+    assert!(!diff.is_empty());
+
+    // Use the file-path API (the one under test).
+    let stats = apply_patches(&diff.patches, &src_path, &tgt_path).unwrap();
+
+    assert!(stats.datasets_added >= 1,    "chan_y should have been added");
+    assert!(stats.datasets_modified >= 1, "chan_x should have been modified");
+
+    // Read the patched file back from disk and verify it matches source.
+    let result_m = DatasetManifest::from_path(&tgt_path).unwrap();
+
+    assert_eq!(
+        result_m.get("/chan_x").unwrap().blake3,
+        new_m.get("/chan_x").unwrap().blake3,
+        "chan_x should match source after patch",
+    );
+    assert!(
+        result_m.get("/chan_y").is_some(),
+        "chan_y should exist after patch",
+    );
+
+    // Verify values by reading the dataset directly.
+    let file = File::open(&tgt_path).unwrap();
+    let x_vals = file.dataset("chan_x").unwrap().read_f64().unwrap();
+    assert_eq!(x_vals, vec![1.1, 2.2, 3.3]);
+
+    let y_vals = file.dataset("chan_y").unwrap().read_f64().unwrap();
+    assert_eq!(y_vals, vec![4.4, 5.5]);
+}
+
+// ─────────────────────────────────────────────────────────────────────────────
+// 8. manifest_blake3_changes_on_modification
+// ─────────────────────────────────────────────────────────────────────────────
+
+/// After patching, the BLAKE3 hash of a modified dataset in the result manifest
+/// matches the source manifest — i.e. the hash is faithfully transported.
+#[test]
+fn manifest_blake3_changes_on_modification() {
+    let src = h5_bytes(&[("signal", &[0.1, 0.2, 0.3, 0.4, 0.5])]);
+    let tgt = h5_bytes(&[("signal", &[1.0, 1.0, 1.0, 1.0, 1.0])]);
+
+    let old_m = DatasetManifest::from_bytes(&tgt).unwrap();
+    let new_m = DatasetManifest::from_bytes(&src).unwrap();
+
+    // Sanity: hashes differ before patching.
+    assert_ne!(
+        old_m.get("/signal").unwrap().blake3,
+        new_m.get("/signal").unwrap().blake3,
+        "pre-patch: source and target hashes must differ",
+    );
+
+    let diff = diff_manifests(&old_m, &new_m);
+    assert_eq!(diff.of_kind(&PatchKind::Modified).count(), 1);
+
+    let (patched, _) = apply_patches_to_bytes(&diff.patches, &src, &tgt).unwrap();
+    let result_m = DatasetManifest::from_bytes(&patched).unwrap();
+
+    // Post-patch: result hash must match source, not old target.
+    assert_eq!(
+        result_m.get("/signal").unwrap().blake3,
+        new_m.get("/signal").unwrap().blake3,
+        "post-patch: result hash must match source manifest",
+    );
+    assert_ne!(
+        result_m.get("/signal").unwrap().blake3,
+        old_m.get("/signal").unwrap().blake3,
+        "post-patch: result hash must not match old target manifest",
+    );
+}