Initial commit: ClawSync v0.1.0

8-crate pure-Rust workspace for revision-aware HDF5 sync.

## Crates
- clawhdf5-onion: ClawOnion VFD — page-level versioned HDF5 storage,
  binary format, writer/reader, branch DAG, GC, snapshots, provenance
- clawsync-core: BLAKE3, xxHash3, FastCDC (+ SIMD NEON), zstd/lz4
- clawsync-onion: IBLT sketch, Merkle tree differ, packet differ/merger,
  ClawSyncManifest, SyncSelector
- clawsync-hdf5: dataset-level manifest, differ, patcher, wire payload
  reconstruction (apply_received_payloads)
- clawsync-transport: TCP, QUIC (quinn 0.11/TLS 1.3), SyncPeer abstraction,
  length-prefixed rkyv wire protocol (21 SyncMessage variants)
- clawsync-agent: OnionMemory, SyncScheduler, TcpSyncBackend,
  PeerCapabilities negotiation
- clawsync-fs: CDC-based delta sync for any file type; FsSyncClient/Server,
  W=16 pipelining, atomic writes
- clawsync-cli: push/pull/serve/hdf5-sync/serve-hdf5/sync/serve-fs +
  all local management commands; --quic on all network commands

## Key features
- IBLT pre-flight: O(revision count) vs rsync's O(file size)
- W=16 sliding-window push: 13–15x speedup over stop-and-wait at WAN RTT
- Dataset-granular HDF5 sync: only modified datasets transferred
- CDC delta for any file type: insertion-stable chunk boundaries
- Full revision DAG: branch, merge, rollback, export, snapshot, GC
- QUIC transport: TLS 1.3, per-message streams via quinn 0.11

## Tests
~573 passing (default features); ~589 with --features simd-cdc

## Performance (Apple Silicon)
- Reconstruct rev=100: 68 µs (target ≤ 1 ms)
- BLAKE3 Rayon 1 MB: 10.3 GiB/s (target ≥ 5 GB/s)
- GC 500 revisions: 20.6 µs (target ≤ 2 s)
- W=16 vs W=1 at 5 ms RTT: 14.8x speedup
- No-op pre-flight at 16 MB: 4 ms vs rsync 35 ms (7.8x)

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
osobh
2026-04-04 18:41:22 -05:00
co-authored by Claude Sonnet 4.6
commit 260e15f5b6
102 changed files with 30098 additions and 0 deletions
+455
View File
@@ -0,0 +1,455 @@
//! Golden-file test for ClawOnion v1 binary format.
//!
//! Asserts that a minimal `.onion` file serialises to exact byte values at
//! every spec-defined offset. This test exists solely to catch **silent
//! `zerocopy` layout regressions** — if any struct field shifts by even one
//! byte, at least one assertion below will fire.
//!
//! The test vector matches §13 of `CLAWONION_SPEC.md`:
//! - 1 revision (rev 0) on branch `main`
//! - 1 page at `h5_offset = 0`, 4 096 bytes, filled with `0xAB`
//! - No revision annotation
//!
//! All offsets are little-endian (LE) as per the spec.
use clawhdf5_onion::format::Codec;
use clawhdf5_onion::writer::OnionFile;
use tempfile::NamedTempFile;
// ─────────────────────────────────────────────────────────────────────────────
// Layout constants (must match CLAWONION_SPEC.md §2 + §3)
// ─────────────────────────────────────────────────────────────────────────────
const HEADER_SIZE: usize = 128;
const REV_ENTRY_SIZE: usize = 112;
const BRANCH_ENTRY_SIZE: usize = 40;
const PAGE_ENTRY_SIZE: usize = 32;
const PAGE_SIZE: u32 = 4096;
const INDEX_OFFSET: usize = HEADER_SIZE; // 128
const BRANCH_OFFSET: usize = INDEX_OFFSET + REV_ENTRY_SIZE; // 240
const PT_OFFSET: usize = BRANCH_OFFSET + BRANCH_ENTRY_SIZE; // 280
const PD_OFFSET: usize = PT_OFFSET + PAGE_ENTRY_SIZE; // 312
// ─────────────────────────────────────────────────────────────────────────────
// Helpers
// ─────────────────────────────────────────────────────────────────────────────
fn le64(v: u64) -> [u8; 8] { v.to_le_bytes() }
fn le32(v: u32) -> [u8; 4] { v.to_le_bytes() }
/// Build the expected BLAKE3 for one 4096-byte page of 0xAB at h5_offset = 0.
///
/// Algorithm from spec §5:
/// hasher.update(h5_offset.to_le_bytes())
/// hasher.update(uncompressed_page_data)
fn expected_blake3(page_bytes: &[u8]) -> [u8; 32] {
let mut h = blake3::Hasher::new();
h.update(&0u64.to_le_bytes()); // h5_offset = 0
h.update(page_bytes);
*h.finalize().as_bytes()
}
// ─────────────────────────────────────────────────────────────────────────────
// Golden test
// ─────────────────────────────────────────────────────────────────────────────
#[test]
fn golden_minimal_onion_byte_layout() {
// ── Build the test vector ────────────────────────────────────────────────
let _tmp = NamedTempFile::new().unwrap();
let h5_path = _tmp.path().with_extension("h5");
let base_bytes = b"\x89HDF\r\n\x1a\n".to_vec();
std::fs::write(&h5_path, &base_bytes).unwrap();
let mut onion = OnionFile::create(&h5_path, PAGE_SIZE).unwrap();
// Use Codec::None so data_size == orig_size and page data is unambiguous.
// This also matches the §13 spec test vector which specifies codec = None.
onion.set_codec(Codec::None);
let page_bytes = vec![0xABu8; PAGE_SIZE as usize];
let mut session = onion.begin_session(None).unwrap();
session.record_page(0, &page_bytes);
onion.commit_session(session, None).unwrap();
let bytes = onion.to_bytes().unwrap();
// ── Verify total file size ───────────────────────────────────────────────
// Header(128) + RevisionEntry(112) + BranchEntry(40) + PageTableEntry(32)
// + PageData(4096) + AnnotationHeap(9)
let ann_heap: &[u8] = &[0x00, 0x04, 0x00, 0x00, 0x00, b'm', b'a', b'i', b'n'];
let expected_total = PD_OFFSET + PAGE_SIZE as usize + ann_heap.len(); // 4417
assert_eq!(
bytes.len(), expected_total,
"total file size mismatch (got {}, expected {expected_total})",
bytes.len()
);
// ══════════════════════════════════════════════════════════════════════════
// §3.1 OnionHeader — 128 bytes @ offset 0
// ══════════════════════════════════════════════════════════════════════════
// [0..9] magic
assert_eq!(&bytes[0..9], b"CLAWONION", "magic mismatch");
// [9] format_version
assert_eq!(bytes[9], 0x01, "format_version");
// [10..16] _pad_align — must be zero
assert_eq!(&bytes[10..16], &[0u8; 6], "_pad_align must be zero");
// [16..24] feature_flags = 0x07 (COMPRESSION | BRANCHING | PROVENANCE)
assert_eq!(&bytes[16..24], &le64(0x07), "feature_flags");
// [24..28] page_size = 4096
assert_eq!(&bytes[24..28], &le32(4096), "page_size");
// [28..32] _pad_ps — must be zero
assert_eq!(&bytes[28..32], &[0u8; 4], "_pad_ps must be zero");
// [32..40] revision_count = 1
assert_eq!(&bytes[32..40], &le64(1), "revision_count");
// [40..44] branch_count = 1
assert_eq!(&bytes[40..44], &le32(1), "branch_count");
// [44..48] _pad_bc — must be zero
assert_eq!(&bytes[44..48], &[0u8; 4], "_pad_bc must be zero");
// [48..56] index_offset = 128
assert_eq!(&bytes[48..56], &le64(128), "index_offset");
// [56..64] branch_offset = 240
assert_eq!(&bytes[56..64], &le64(240), "branch_offset");
// [64..72] created_at — f64, time-dependent; skip exact value check,
// but assert it's non-zero (a real Unix timestamp).
let created_at = f64::from_le_bytes(bytes[64..72].try_into().unwrap());
assert!(created_at > 0.0, "created_at should be a positive Unix timestamp");
// [72..128] reserved — 56 bytes, must all be zero
assert_eq!(&bytes[72..128], &[0u8; 56], "reserved header bytes must be zero");
// ══════════════════════════════════════════════════════════════════════════
// §3.2 RevisionEntry — 112 bytes @ offset 128
// ══════════════════════════════════════════════════════════════════════════
let re = INDEX_OFFSET; // 128
// [re+0..re+8] revision = 0
assert_eq!(&bytes[re..re+8], &le64(0), "revision number");
// [re+8..re+12] branch_id = 0 (main)
assert_eq!(&bytes[re+8..re+12], &le32(0), "branch_id");
// [re+12..re+16] _pad_bi — must be zero
assert_eq!(&bytes[re+12..re+16], &[0u8; 4], "_pad_bi must be zero");
// [re+16..re+24] parent_rev = u64::MAX (root has no parent)
assert_eq!(&bytes[re+16..re+24], &le64(u64::MAX), "parent_rev (NO_PARENT sentinel)");
// [re+24..re+28] page_count = 1
assert_eq!(&bytes[re+24..re+28], &le32(1), "page_count");
// [re+28..re+32] _pad_pc — must be zero
assert_eq!(&bytes[re+28..re+32], &[0u8; 4], "_pad_pc must be zero");
// [re+32..re+40] page_table_off = 280 (PT_OFFSET)
assert_eq!(&bytes[re+32..re+40], &le64(PT_OFFSET as u64), "page_table_off");
// [re+40..re+48] timestamp — f64, time-dependent; assert positive
let ts = f64::from_le_bytes(bytes[re+40..re+48].try_into().unwrap());
assert!(ts > 0.0, "revision timestamp should be positive");
// [re+48..re+80] blake3 — verify against spec §5 algorithm
let computed_blake3 = expected_blake3(&page_bytes);
assert_eq!(&bytes[re+48..re+80], &computed_blake3, "BLAKE3 hash mismatch");
// [re+80..re+96] session_uuid — 16 bytes UUIDv7; must be non-zero
assert_ne!(&bytes[re+80..re+96], &[0u8; 16], "session_uuid must not be all zeros");
// [re+96..re+104] annotation_off = 0 (no annotation)
assert_eq!(&bytes[re+96..re+104], &le64(0), "annotation_off must be 0 (no annotation)");
// [re+104] flags = 0 (not a snapshot)
assert_eq!(bytes[re+104], 0x00, "revision flags");
// [re+105..re+112] _pad_flags — must be zero
assert_eq!(&bytes[re+105..re+112], &[0u8; 7], "_pad_flags must be zero");
// ══════════════════════════════════════════════════════════════════════════
// §3.3 BranchEntry — 40 bytes @ offset 240
// ══════════════════════════════════════════════════════════════════════════
let be = BRANCH_OFFSET; // 240
// [be+0..be+4] id = 0 (main)
assert_eq!(&bytes[be..be+4], &le32(0), "branch id");
// [be+4..be+8] _pad_id — must be zero
assert_eq!(&bytes[be+4..be+8], &[0u8; 4], "_pad_id must be zero");
// [be+8..be+16] name_off = 1 (heap-relative; first string after reserved null)
assert_eq!(&bytes[be+8..be+16], &le64(1), "name_off for main branch");
// [be+16..be+24] head_rev = 0 (updated after commit)
assert_eq!(&bytes[be+16..be+24], &le64(0), "head_rev after first commit");
// [be+24..be+32] fork_rev = u64::MAX (main is not forked)
assert_eq!(&bytes[be+24..be+32], &le64(u64::MAX), "fork_rev (NO_PARENT for main)");
// [be+32..be+40] created_at — f64, time-dependent; assert positive
let branch_ts = f64::from_le_bytes(bytes[be+32..be+40].try_into().unwrap());
assert!(branch_ts > 0.0, "branch created_at should be positive");
// ══════════════════════════════════════════════════════════════════════════
// §3.4 PageTableEntry — 32 bytes @ offset 280
// ══════════════════════════════════════════════════════════════════════════
let pt = PT_OFFSET; // 280
// [pt+0..pt+8] h5_offset = 0
assert_eq!(&bytes[pt..pt+8], &le64(0), "h5_offset");
// [pt+8..pt+16] data_offset = 312 (PD_OFFSET, absolute file position)
assert_eq!(&bytes[pt+8..pt+16], &le64(PD_OFFSET as u64), "data_offset (absolute)");
// [pt+16..pt+20] orig_size = 4096
assert_eq!(&bytes[pt+16..pt+20], &le32(4096), "orig_size");
// [pt+20..pt+24] data_size = 4096 (codec=None → no compression)
assert_eq!(&bytes[pt+20..pt+24], &le32(4096), "data_size (codec=None, uncompressed)");
// [pt+24] codec = 0 (None)
assert_eq!(bytes[pt+24], 0x00, "codec = None (0)");
// [pt+25..pt+32] _pad — must be zero
assert_eq!(&bytes[pt+25..pt+32], &[0u8; 7], "_pad must be zero");
// ══════════════════════════════════════════════════════════════════════════
// PageData — 4096 bytes @ offset 312
// ══════════════════════════════════════════════════════════════════════════
assert_eq!(&bytes[PD_OFFSET..PD_OFFSET + 4096], page_bytes.as_slice(),
"page data must equal the uncompressed original bytes");
// ══════════════════════════════════════════════════════════════════════════
// §3.5 AnnotationHeap — tail of file
// ══════════════════════════════════════════════════════════════════════════
// Last 9 bytes: [0x00 reserved] [u32 LE length=4] ["main"]
assert_eq!(&bytes[bytes.len()-9..], ann_heap, "annotation heap content");
// ══════════════════════════════════════════════════════════════════════════
// §7 Reconstruction — verify round-trip
// ══════════════════════════════════════════════════════════════════════════
let reconstructed = onion.reconstruct_revision(0, &base_bytes).unwrap();
assert_eq!(
&reconstructed[..PAGE_SIZE as usize],
page_bytes.as_slice(),
"reconstruct_revision must reproduce the original page"
);
}
// ─────────────────────────────────────────────────────────────────────────────
// Byte-exact format freeze — §13 canonical test vector
// ─────────────────────────────────────────────────────────────────────────────
//
// This test builds the COMPLETE expected byte buffer from pure spec knowledge,
// then compares it against the actual serialised output byte-for-byte.
//
// Dynamic fields (wall-clock timestamps, UUIDs) are zeroed in BOTH buffers
// before the comparison — this preserves the freeze property for all layout
// fields while remaining deterministic.
//
// This complements `golden_minimal_onion_byte_layout` (which checks individual
// fields) by catching any bytes in between that neither test checks explicitly.
#[test]
fn byte_exact_format_freeze() {
let re = INDEX_OFFSET; // 128 — RevisionEntry start
let be = BRANCH_OFFSET; // 240 — BranchEntry start
let pt = PT_OFFSET; // 280 — PageTableEntry start
let pd = PD_OFFSET; // 312 — PageData start
// ── Build the actual serialised file ────────────────────────────────────
let _tmp = NamedTempFile::new().unwrap();
let h5_path = _tmp.path().with_extension("h5");
std::fs::write(&h5_path, b"\x89HDF\r\n\x1a\n").unwrap();
let mut onion = OnionFile::create(&h5_path, PAGE_SIZE).unwrap();
onion.set_codec(Codec::None);
let page_bytes = vec![0xABu8; PAGE_SIZE as usize];
let mut session = onion.begin_session(None).unwrap();
session.record_page(0, &page_bytes);
onion.commit_session(session, None).unwrap();
let mut actual = onion.to_bytes().unwrap();
// ── Build the expected buffer from spec ──────────────────────────────────
let ann_heap: &[u8] = &[0x00, 0x04, 0x00, 0x00, 0x00, b'm', b'a', b'i', b'n'];
let total = pd + PAGE_SIZE as usize + ann_heap.len(); // 4417
let mut expected = vec![0u8; total];
// §3.1 OnionHeader (128 bytes)
expected[0..9].copy_from_slice(b"CLAWONION"); // magic
expected[9] = 0x01; // format_version
// [10..16] = 0 (_pad_align)
expected[16..24].copy_from_slice(&le64(0x07)); // feature_flags
expected[24..28].copy_from_slice(&le32(4096)); // page_size
// [28..32] = 0 (_pad_ps)
expected[32..40].copy_from_slice(&le64(1)); // revision_count = 1
expected[40..44].copy_from_slice(&le32(1)); // branch_count = 1
// [44..48] = 0 (_pad_bc)
expected[48..56].copy_from_slice(&le64(128)); // index_offset
expected[56..64].copy_from_slice(&le64(240)); // branch_offset
// [64..72] = 0 (created_at — zeroed, dynamic)
// [72..128] = 0 (reserved)
// §3.2 RevisionEntry (112 bytes @ 128)
expected[re..re+8] .copy_from_slice(&le64(0)); // revision = 0
expected[re+8..re+12].copy_from_slice(&le32(0)); // branch_id = 0
// [re+12..re+16] = 0 (_pad_bi)
expected[re+16..re+24].copy_from_slice(&le64(u64::MAX)); // parent_rev = NO_PARENT
expected[re+24..re+28].copy_from_slice(&le32(1)); // page_count = 1
// [re+28..re+32] = 0 (_pad_pc)
expected[re+32..re+40].copy_from_slice(&le64(pt as u64)); // page_table_off
// [re+40..re+48] = 0 (timestamp — zeroed, dynamic)
expected[re+48..re+80].copy_from_slice(&expected_blake3(&page_bytes)); // blake3
// [re+80..re+96] = 0 (session_uuid — zeroed, dynamic)
// [re+96..re+104] = 0 (annotation_off = 0, no annotation)
// [re+104] = 0 (flags = 0, not a snapshot)
// [re+105..re+112] = 0 (_pad_flags)
// §3.3 BranchEntry (40 bytes @ 240)
expected[be..be+4] .copy_from_slice(&le32(0)); // id = 0 (main)
// [be+4..be+8] = 0 (_pad_id)
expected[be+8..be+16].copy_from_slice(&le64(1)); // name_off = 1 (heap offset of "main")
expected[be+16..be+24].copy_from_slice(&le64(0)); // head_rev = 0
expected[be+24..be+32].copy_from_slice(&le64(u64::MAX)); // fork_rev = NO_PARENT
// [be+32..be+40] = 0 (created_at — zeroed, dynamic)
// §3.4 PageTableEntry (32 bytes @ 280)
expected[pt..pt+8] .copy_from_slice(&le64(0)); // h5_offset = 0
expected[pt+8..pt+16].copy_from_slice(&le64(pd as u64)); // data_offset
expected[pt+16..pt+20].copy_from_slice(&le32(4096)); // orig_size
expected[pt+20..pt+24].copy_from_slice(&le32(4096)); // data_size (no compression)
// [pt+24] = 0 (codec = None)
// [pt+25..pt+32] = 0 (_pad)
// PageData (4096 bytes @ 312)
expected[pd..pd+4096].fill(0xAB);
// §3.5 AnnotationHeap (9 bytes at end)
expected[pd+4096..].copy_from_slice(ann_heap);
// ── Zero dynamic fields in both buffers ──────────────────────────────────
// OnionHeader.created_at
actual[64..72].fill(0);
// RevisionEntry.timestamp
actual[re+40..re+48].fill(0);
// RevisionEntry.session_uuid
actual[re+80..re+96].fill(0);
// BranchEntry.created_at
actual[be+32..be+40].fill(0);
// ── Compare byte-for-byte ────────────────────────────────────────────────
assert_eq!(actual.len(), expected.len(),
"total file size mismatch: actual={} expected={}",
actual.len(), expected.len());
// Find the first differing byte for a useful failure message.
if actual != expected {
for i in 0..actual.len() {
if actual[i] != expected[i] {
panic!(
"byte mismatch at offset 0x{i:03X} ({i}): \
actual=0x{:02X} expected=0x{:02X}\n\
(fields: header=0..128, rev_entry=128..240, branch=240..280, \
page_table=280..312, page_data=312..4408, ann_heap=4408..4417)",
actual[i], expected[i]
);
}
}
}
}
// ─────────────────────────────────────────────────────────────────────────────
// Struct size guards — compile-time; here as runtime assertions too so that
// a zerocopy version bump that somehow bypasses the const asserts still fires.
// ─────────────────────────────────────────────────────────────────────────────
#[test]
fn struct_sizes_match_spec() {
use std::mem::size_of;
use clawhdf5_onion::format::{OnionHeader, RevisionEntry, BranchEntry, PageTableEntry};
assert_eq!(size_of::<OnionHeader>(), 128, "OnionHeader must be 128 bytes (§3.1)");
assert_eq!(size_of::<RevisionEntry>(), 112, "RevisionEntry must be 112 bytes (§3.2)");
assert_eq!(size_of::<BranchEntry>(), 40, "BranchEntry must be 40 bytes (§3.3)");
assert_eq!(size_of::<PageTableEntry>(), 32, "PageTableEntry must be 32 bytes (§3.4)");
}
// ─────────────────────────────────────────────────────────────────────────────
// Magic / version rejection — §10 conformance
// ─────────────────────────────────────────────────────────────────────────────
#[test]
fn bad_magic_is_rejected() {
let _tmp = NamedTempFile::new().unwrap();
let h5_path = _tmp.path().with_extension("h5");
std::fs::write(&h5_path, b"\x89HDF\r\n\x1a\n").unwrap();
let mut onion = OnionFile::create(&h5_path, 4096).unwrap();
onion.set_codec(Codec::None);
let mut s = onion.begin_session(None).unwrap();
s.record_page(0, &vec![0u8; 4096]);
onion.commit_session(s, None).unwrap();
let mut bytes = onion.to_bytes().unwrap();
// Corrupt the magic
bytes[0] = b'X';
let sidecar = h5_path.with_extension("h5.onion");
std::fs::write(&sidecar, &bytes).unwrap();
let err = OnionFile::open(&h5_path).unwrap_err();
assert!(
matches!(err, clawhdf5_onion::error::OnionError::InvalidMagic),
"expected InvalidMagic, got {err:?}"
);
}
#[test]
fn unknown_version_is_rejected() {
let _tmp = NamedTempFile::new().unwrap();
let h5_path = _tmp.path().with_extension("h5");
std::fs::write(&h5_path, b"\x89HDF\r\n\x1a\n").unwrap();
let mut onion = OnionFile::create(&h5_path, 4096).unwrap();
onion.set_codec(Codec::None);
let mut s = onion.begin_session(None).unwrap();
s.record_page(0, &vec![0u8; 4096]);
onion.commit_session(s, None).unwrap();
let mut bytes = onion.to_bytes().unwrap();
// Bump format_version to something unknown
bytes[9] = 0xFF;
let sidecar = h5_path.with_extension("h5.onion");
std::fs::write(&sidecar, &bytes).unwrap();
let err = OnionFile::open(&h5_path).unwrap_err();
assert!(
matches!(err, clawhdf5_onion::error::OnionError::UnknownVersion(0xFF)),
"expected UnknownVersion(0xFF), got {err:?}"
);
}