Fix intra-doc link warnings; add clawsync-fs README and metadata

Doc fixes (20 warnings → 1 benign name-collision warning):
- clawsync-transport/quic.rs: QuicConfig::with_cert doesn't exist → backtick
- clawsync-onion/iblt.rs: bare `insert` link → backtick
- clawsync-core/lib.rs: simd_cdc is feature-gated → backtick
- clawhdf5-onion/annotation.rs: RevisionEntry/BranchEntry are in external
  clawhdf5-format crate, not re-exported → backtick
- clawhdf5-onion/gc.rs: compact_dead_epoch_revisions is private; flush links
  broken → backtick
- clawhdf5-onion/provenance.rs: RevisionEntry from external crate → backtick
- clawhdf5-onion/reader.rs: reconstruct_revision/revision_pages → Self:: prefix
- clawhdf5-onion/writer.rs: REV_FLAG_SNAPSHOT → crate::format:: path;
  reconstruct_revision → Self:: prefix
Remaining warning is `format` module/macro name collision — not a broken link.

clawsync-fs crate metadata:
- Add readme, keywords, categories to Cargo.toml
- Write README.md (CDC protocol diagram, module overview, usage examples)

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
osobh
2026-04-04 20:29:28 -05:00
co-authored by Claude Sonnet 4.6
parent 2e423d2682
commit 3d524e2d63
10 changed files with 126 additions and 16 deletions
+3 -3
View File
@@ -2,7 +2,7 @@
//! //!
//! Stores revision annotations and branch names as length-prefixed //! Stores revision annotations and branch names as length-prefixed
//! (`u32` LE + UTF-8 bytes) strings. An offset of `0` in a //! (`u32` LE + UTF-8 bytes) strings. An offset of `0` in a
//! [`RevisionEntry`] or [`BranchEntry`] means "no annotation". //! `RevisionEntry` or `BranchEntry` means "no annotation".
//! //!
//! The offset `0` is reserved. The heap always begins with a single //! The offset `0` is reserved. The heap always begins with a single
//! null byte so that offset 0 is unambiguously "absent". //! null byte so that offset 0 is unambiguously "absent".
@@ -39,8 +39,8 @@ impl AnnotationHeap {
/// Append a UTF-8 string and return its byte offset within the heap. /// Append a UTF-8 string and return its byte offset within the heap.
/// ///
/// The returned offset can be stored in a [`RevisionEntry::annotation_off`] /// The returned offset can be stored in a `RevisionEntry::annotation_off`
/// or [`BranchEntry::name_off`]. /// or `BranchEntry::name_off`.
pub fn push(&mut self, s: &str) -> u64 { pub fn push(&mut self, s: &str) -> u64 {
let offset = self.data.len() as u64; let offset = self.data.len() as u64;
let len = s.len() as u32; let len = s.len() as u32;
+4 -4
View File
@@ -24,8 +24,8 @@ pub enum GcPolicy {
/// Computes the dead set using `inner`, marks those entries with /// Computes the dead set using `inner`, marks those entries with
/// [`EPOCH_DEAD`] in their padding bytes, and returns immediately /// [`EPOCH_DEAD`] in their padding bytes, and returns immediately
/// without touching `page_data`. The actual compaction is deferred to /// without touching `page_data`. The actual compaction is deferred to
/// the next [`OnionFile::flush`] call, which runs /// the next `OnionFile::flush` call, which runs
/// [`OnionFile::compact_dead_epoch_revisions`] before writing. /// `OnionFile::compact_dead_epoch_revisions` before writing.
/// ///
/// Use this when you want GC to be non-blocking: the mark pass is /// Use this when you want GC to be non-blocking: the mark pass is
/// O(revisions) with no I/O; the compaction is amortised into the next /// O(revisions) with no I/O; the compaction is amortised into the next
@@ -47,12 +47,12 @@ impl OnionFile {
/// ///
/// For all policies except [`GcPolicy::EpochFlip`]: /// For all policies except [`GcPolicy::EpochFlip`]:
/// - The `RevisionIndex` is updated immediately. /// - The `RevisionIndex` is updated immediately.
/// - Page data is compacted in-memory; call [`flush`] to persist. /// - Page data is compacted in-memory; call `OnionFile::flush` to persist.
/// ///
/// For [`GcPolicy::EpochFlip`]: /// For [`GcPolicy::EpochFlip`]:
/// - Dead revisions are *marked* with [`EPOCH_DEAD`] in O(revisions). /// - Dead revisions are *marked* with [`EPOCH_DEAD`] in O(revisions).
/// - Page data is **not** touched; compaction is deferred to the next /// - Page data is **not** touched; compaction is deferred to the next
/// [`flush`] call. This makes the GC call itself non-blocking. /// `OnionFile::flush` call. This makes the GC call itself non-blocking.
/// ///
/// **Note:** Immediate GC is irreversible. Epoch-flip GC can be /// **Note:** Immediate GC is irreversible. Epoch-flip GC can be
/// cancelled by calling `flush_wal()` without `flush()`, but only /// cancelled by calling `flush_wal()` without `flush()`, but only
+2 -2
View File
@@ -9,7 +9,7 @@ const RAYON_PAGE_THRESHOLD: usize = 128 * 1024; // 128 KB
/// Compute the BLAKE3 hash over a set of `(h5_offset, page_bytes)` pairs, /// Compute the BLAKE3 hash over a set of `(h5_offset, page_bytes)` pairs,
/// processed in ascending `h5_offset` order. /// processed in ascending `h5_offset` order.
/// ///
/// This is the canonical per-revision hash stored in [`RevisionEntry::blake3`]. /// This is the canonical per-revision hash stored in `RevisionEntry::blake3`.
/// For pages ≥ 128 KB each, the page data is hashed using Rayon tree /// For pages ≥ 128 KB each, the page data is hashed using Rayon tree
/// parallelism; the offset bytes always use the single-threaded path. /// parallelism; the offset bytes always use the single-threaded path.
pub fn hash_pages(pages: &[(u64, &[u8])]) -> [u8; 32] { pub fn hash_pages(pages: &[(u64, &[u8])]) -> [u8; 32] {
@@ -48,7 +48,7 @@ impl SessionId {
Self(*Uuid::now_v7().as_bytes()) Self(*Uuid::now_v7().as_bytes())
} }
/// Construct from raw bytes (e.g., loaded from a [`RevisionEntry`]). /// Construct from raw bytes (e.g., loaded from a `RevisionEntry`).
pub fn from_bytes(bytes: [u8; 16]) -> Self { pub fn from_bytes(bytes: [u8; 16]) -> Self {
Self(bytes) Self(bytes)
} }
+2 -2
View File
@@ -165,7 +165,7 @@ impl OnionFile {
} }
/// Return the decompressed pages that were changed *only in revision `rev`* /// Return the decompressed pages that were changed *only in revision `rev`*
/// (not the accumulated file state — use [`reconstruct_revision`] for that). /// (not the accumulated file state — use [`Self::reconstruct_revision`] for that).
/// ///
/// Each element is `(h5_offset, uncompressed_page_bytes)`. /// Each element is `(h5_offset, uncompressed_page_bytes)`.
/// This is used by `clawsync-onion` to build transfer packets. /// This is used by `clawsync-onion` to build transfer packets.
@@ -196,7 +196,7 @@ impl OnionFile {
/// Return the raw **compressed** page bytes for a revision together with /// Return the raw **compressed** page bytes for a revision together with
/// the codec and original (uncompressed) size. /// the codec and original (uncompressed) size.
/// ///
/// Compared to [`revision_pages`] this skips the decompression step, so /// Compared to [`Self::revision_pages`] this skips the decompression step, so
/// callers that intend to send the data over the network can ship the /// callers that intend to send the data over the network can ship the
/// compressed bytes directly and let the receiver decompress. /// compressed bytes directly and let the receiver decompress.
/// ///
+2 -2
View File
@@ -312,8 +312,8 @@ impl OnionFile {
/// Commit a full-state snapshot revision. /// Commit a full-state snapshot revision.
/// ///
/// A snapshot records every page of the current HEAD state (not just the /// A snapshot records every page of the current HEAD state (not just the
/// diff) and is flagged with [`REV_FLAG_SNAPSHOT`]. Subsequent calls to /// diff) and is flagged with [`crate::format::REV_FLAG_SNAPSHOT`]. Subsequent calls to
/// [`reconstruct_revision`] will use the nearest snapshot as their starting /// [`Self::reconstruct_revision`] will use the nearest snapshot as their starting
/// point, bounding reconstruction depth to `O(N_since_snapshot · P)`. /// point, bounding reconstruction depth to `O(N_since_snapshot · P)`.
/// ///
/// `h5_base` is the raw bytes of the base `.h5` file (needed to resolve /// `h5_base` is the raw bytes of the base `.h5` file (needed to resolve
+1 -1
View File
@@ -5,7 +5,7 @@
//! //!
//! - [`checksum`]: xxHash3-64 block hashing + BLAKE3 integrity verification //! - [`checksum`]: xxHash3-64 block hashing + BLAKE3 integrity verification
//! - [`cdc`]: FastCDC content-defined chunking //! - [`cdc`]: FastCDC content-defined chunking
//! - [`simd_cdc`]: SIMD-accelerated Gear hash CDC (feature `simd-cdc`) //! - `simd_cdc`: SIMD-accelerated Gear hash CDC (feature `simd-cdc`)
//! - [`delta`]: Fixed-block delta (copy/insert ops, apply, transfer size) //! - [`delta`]: Fixed-block delta (copy/insert ops, apply, transfer size)
//! - [`compress`]: zstd and lz4 streaming wrappers //! - [`compress`]: zstd and lz4 streaming wrappers
//! - [`error`]: [`CoreError`] type //! - [`error`]: [`CoreError`] type
+3
View File
@@ -6,6 +6,9 @@ edition.workspace = true
rust-version.workspace = true rust-version.workspace = true
license.workspace = true license.workspace = true
repository.workspace = true repository.workspace = true
readme = "README.md"
keywords = ["sync", "rsync", "cdc", "delta", "filesystem"]
categories = ["filesystem", "network-programming"]
[dependencies] [dependencies]
clawsync-core = { workspace = true } clawsync-core = { workspace = true }
+107
View File
@@ -0,0 +1,107 @@
# clawsync-fs
CDC-based delta sync for any file type — the general-purpose sync engine behind
`clawsync sync` and `clawsync serve-fs`.
## Overview
`clawsync-fs` closes the gap between ClawSync's HDF5-specific revision sync and
rsync's general-purpose file sync. It handles arbitrary file types and directory
trees using **Content-Defined Chunking** (FastCDC) so chunk boundaries are
insertion-stable: a 1-byte prefix insertion does not invalidate the rest of the
file's chunks.
## Architecture
```
FsSyncClient FsSyncServer
│ │
├── FsManifest::build (rayon parallel) │
│ BLAKE3 every file in local tree │
│ │
├─── FsDirManifest ──────────────────────▶│
│ │ FsManifest::build (server tree)
│ │ diff_manifests → Added/Modified/Removed
│◀── FsDirNeed ───────────────────────────│
│ (needed_files with server chunks, │
│ to_delete if allow_delete) │
│ │
│ [W=16 pipelined — into_pipe_halves()] │
├─── FsCdcData (/path, chunk_order, ─────▶│ reconstruct_file
│ literal_chunks) │ atomic write (.tmp → rename)
│◀── FsFileAck ───────────────────────────│
│ (repeat for each modified/added │
│ file in any order) │
│◀── FsDirComplete ───────────────────────│
```
**Protocol cost:**
| Scenario | RTTs |
|----------|------|
| Warm no-op | 1 |
| Cold copy | 2 |
| Incremental | 2 |
## Modules
### `manifest`
`FsManifest::build(root, excludes)` — parallel BLAKE3 walk of a directory tree
using `walkdir` + Rayon. Returns a sorted manifest of all regular files with
their BLAKE3 hash, size, and mtime.
```rust
let excludes = GlobSet::empty();
let manifest = FsManifest::build(Path::new("./data"), &excludes)?;
for entry in &manifest.entries {
println!("{} {} bytes", entry.rel_path, entry.size);
}
```
### `differ`
`diff_manifests(local, remote)` — compare two manifest entry lists by path and
BLAKE3 hash. Returns `Vec<FileDiff>` with `Added`, `Modified`, `Removed`, and
`Unchanged` variants.
### `delta`
CDC chunking, transfer-need computation, and file reconstruction.
- `chunk_file_for_request(data)` — chunk bytes and return `Vec<FsChunkHash>`
- `compute_needed_indices(server_existing, client_request)` — which chunks the
server lacks (by xxHash3-64)
- `build_chunk_data(data, chunks, needed_indices)` — pack literal data for
transfer; zstd-compresses each chunk, falls back to raw if compressed ≥ raw
- `reconstruct_file(server_file, client_order, server_map, literals, expected_blake3)`
rebuild the target file from server-local chunks + received literals; verifies
BLAKE3 on output
### `session`
`FsSyncClient` and `FsSyncServer` — async protocol orchestration over a
`SyncPeer` (TCP or QUIC).
```rust
// Client
let stats = FsSyncClient::new(peer, local_root, excludes, delete)
.run()
.await?;
println!("{} added, {} modified, {} removed", stats.files_added, stats.files_modified, stats.files_removed);
// Server (called per accepted connection)
FsSyncServer::new(peer, serve_root, excludes, allow_delete)
.handle()
.await?;
```
## Performance
- **Manifest build:** parallelised with Rayon — scales with core count
- **Chunk transfer:** W=16 pipelined send/recv via `SyncPeer::into_pipe_halves()`
- **CDC boundaries:** FastCDC, immune to insertion staircase that breaks rsync's rolling checksum
- **Compression:** per-chunk zstd; falls back to raw for already-compressed data
## License
MIT — see repository root.
+1 -1
View File
@@ -165,7 +165,7 @@ impl IbltSketch {
/// Remove a key from the sketch. /// Remove a key from the sketch.
/// ///
/// Symmetric with [`insert`]: calling `remove(k)` after `insert(k)` leaves /// Symmetric with `insert`: calling `remove(k)` after `insert(k)` leaves
/// the sketch in its original state. /// the sketch in its original state.
pub fn remove(&mut self, key: u64) { pub fn remove(&mut self, key: u64) {
self.update(key, -1); self.update(key, -1);
+1 -1
View File
@@ -8,7 +8,7 @@
//! //!
//! For development / testing, use [`QuicConfig::self_signed`] which generates //! For development / testing, use [`QuicConfig::self_signed`] which generates
//! a temporary self-signed certificate. Production deployments should use //! a temporary self-signed certificate. Production deployments should use
//! [`QuicConfig::with_cert`] with a properly-signed certificate. //! `QuicConfig::with_cert` with a properly-signed certificate.
//! //!
//! # Status //! # Status
//! //!