Fix intra-doc link warnings; add clawsync-fs README and metadata
Doc fixes (20 warnings → 1 benign name-collision warning): - clawsync-transport/quic.rs: QuicConfig::with_cert doesn't exist → backtick - clawsync-onion/iblt.rs: bare `insert` link → backtick - clawsync-core/lib.rs: simd_cdc is feature-gated → backtick - clawhdf5-onion/annotation.rs: RevisionEntry/BranchEntry are in external clawhdf5-format crate, not re-exported → backtick - clawhdf5-onion/gc.rs: compact_dead_epoch_revisions is private; flush links broken → backtick - clawhdf5-onion/provenance.rs: RevisionEntry from external crate → backtick - clawhdf5-onion/reader.rs: reconstruct_revision/revision_pages → Self:: prefix - clawhdf5-onion/writer.rs: REV_FLAG_SNAPSHOT → crate::format:: path; reconstruct_revision → Self:: prefix Remaining warning is `format` module/macro name collision — not a broken link. clawsync-fs crate metadata: - Add readme, keywords, categories to Cargo.toml - Write README.md (CDC protocol diagram, module overview, usage examples) Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
2e423d2682
commit
3d524e2d63
@@ -2,7 +2,7 @@
|
|||||||
//!
|
//!
|
||||||
//! Stores revision annotations and branch names as length-prefixed
|
//! Stores revision annotations and branch names as length-prefixed
|
||||||
//! (`u32` LE + UTF-8 bytes) strings. An offset of `0` in a
|
//! (`u32` LE + UTF-8 bytes) strings. An offset of `0` in a
|
||||||
//! [`RevisionEntry`] or [`BranchEntry`] means "no annotation".
|
//! `RevisionEntry` or `BranchEntry` means "no annotation".
|
||||||
//!
|
//!
|
||||||
//! The offset `0` is reserved. The heap always begins with a single
|
//! The offset `0` is reserved. The heap always begins with a single
|
||||||
//! null byte so that offset 0 is unambiguously "absent".
|
//! null byte so that offset 0 is unambiguously "absent".
|
||||||
@@ -39,8 +39,8 @@ impl AnnotationHeap {
|
|||||||
|
|
||||||
/// Append a UTF-8 string and return its byte offset within the heap.
|
/// Append a UTF-8 string and return its byte offset within the heap.
|
||||||
///
|
///
|
||||||
/// The returned offset can be stored in a [`RevisionEntry::annotation_off`]
|
/// The returned offset can be stored in a `RevisionEntry::annotation_off`
|
||||||
/// or [`BranchEntry::name_off`].
|
/// or `BranchEntry::name_off`.
|
||||||
pub fn push(&mut self, s: &str) -> u64 {
|
pub fn push(&mut self, s: &str) -> u64 {
|
||||||
let offset = self.data.len() as u64;
|
let offset = self.data.len() as u64;
|
||||||
let len = s.len() as u32;
|
let len = s.len() as u32;
|
||||||
|
|||||||
@@ -24,8 +24,8 @@ pub enum GcPolicy {
|
|||||||
/// Computes the dead set using `inner`, marks those entries with
|
/// Computes the dead set using `inner`, marks those entries with
|
||||||
/// [`EPOCH_DEAD`] in their padding bytes, and returns immediately
|
/// [`EPOCH_DEAD`] in their padding bytes, and returns immediately
|
||||||
/// without touching `page_data`. The actual compaction is deferred to
|
/// without touching `page_data`. The actual compaction is deferred to
|
||||||
/// the next [`OnionFile::flush`] call, which runs
|
/// the next `OnionFile::flush` call, which runs
|
||||||
/// [`OnionFile::compact_dead_epoch_revisions`] before writing.
|
/// `OnionFile::compact_dead_epoch_revisions` before writing.
|
||||||
///
|
///
|
||||||
/// Use this when you want GC to be non-blocking: the mark pass is
|
/// Use this when you want GC to be non-blocking: the mark pass is
|
||||||
/// O(revisions) with no I/O; the compaction is amortised into the next
|
/// O(revisions) with no I/O; the compaction is amortised into the next
|
||||||
@@ -47,12 +47,12 @@ impl OnionFile {
|
|||||||
///
|
///
|
||||||
/// For all policies except [`GcPolicy::EpochFlip`]:
|
/// For all policies except [`GcPolicy::EpochFlip`]:
|
||||||
/// - The `RevisionIndex` is updated immediately.
|
/// - The `RevisionIndex` is updated immediately.
|
||||||
/// - Page data is compacted in-memory; call [`flush`] to persist.
|
/// - Page data is compacted in-memory; call `OnionFile::flush` to persist.
|
||||||
///
|
///
|
||||||
/// For [`GcPolicy::EpochFlip`]:
|
/// For [`GcPolicy::EpochFlip`]:
|
||||||
/// - Dead revisions are *marked* with [`EPOCH_DEAD`] in O(revisions).
|
/// - Dead revisions are *marked* with [`EPOCH_DEAD`] in O(revisions).
|
||||||
/// - Page data is **not** touched; compaction is deferred to the next
|
/// - Page data is **not** touched; compaction is deferred to the next
|
||||||
/// [`flush`] call. This makes the GC call itself non-blocking.
|
/// `OnionFile::flush` call. This makes the GC call itself non-blocking.
|
||||||
///
|
///
|
||||||
/// **Note:** Immediate GC is irreversible. Epoch-flip GC can be
|
/// **Note:** Immediate GC is irreversible. Epoch-flip GC can be
|
||||||
/// cancelled by calling `flush_wal()` without `flush()`, but only
|
/// cancelled by calling `flush_wal()` without `flush()`, but only
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ const RAYON_PAGE_THRESHOLD: usize = 128 * 1024; // 128 KB
|
|||||||
/// Compute the BLAKE3 hash over a set of `(h5_offset, page_bytes)` pairs,
|
/// Compute the BLAKE3 hash over a set of `(h5_offset, page_bytes)` pairs,
|
||||||
/// processed in ascending `h5_offset` order.
|
/// processed in ascending `h5_offset` order.
|
||||||
///
|
///
|
||||||
/// This is the canonical per-revision hash stored in [`RevisionEntry::blake3`].
|
/// This is the canonical per-revision hash stored in `RevisionEntry::blake3`.
|
||||||
/// For pages ≥ 128 KB each, the page data is hashed using Rayon tree
|
/// For pages ≥ 128 KB each, the page data is hashed using Rayon tree
|
||||||
/// parallelism; the offset bytes always use the single-threaded path.
|
/// parallelism; the offset bytes always use the single-threaded path.
|
||||||
pub fn hash_pages(pages: &[(u64, &[u8])]) -> [u8; 32] {
|
pub fn hash_pages(pages: &[(u64, &[u8])]) -> [u8; 32] {
|
||||||
@@ -48,7 +48,7 @@ impl SessionId {
|
|||||||
Self(*Uuid::now_v7().as_bytes())
|
Self(*Uuid::now_v7().as_bytes())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Construct from raw bytes (e.g., loaded from a [`RevisionEntry`]).
|
/// Construct from raw bytes (e.g., loaded from a `RevisionEntry`).
|
||||||
pub fn from_bytes(bytes: [u8; 16]) -> Self {
|
pub fn from_bytes(bytes: [u8; 16]) -> Self {
|
||||||
Self(bytes)
|
Self(bytes)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -165,7 +165,7 @@ impl OnionFile {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Return the decompressed pages that were changed *only in revision `rev`*
|
/// Return the decompressed pages that were changed *only in revision `rev`*
|
||||||
/// (not the accumulated file state — use [`reconstruct_revision`] for that).
|
/// (not the accumulated file state — use [`Self::reconstruct_revision`] for that).
|
||||||
///
|
///
|
||||||
/// Each element is `(h5_offset, uncompressed_page_bytes)`.
|
/// Each element is `(h5_offset, uncompressed_page_bytes)`.
|
||||||
/// This is used by `clawsync-onion` to build transfer packets.
|
/// This is used by `clawsync-onion` to build transfer packets.
|
||||||
@@ -196,7 +196,7 @@ impl OnionFile {
|
|||||||
/// Return the raw **compressed** page bytes for a revision together with
|
/// Return the raw **compressed** page bytes for a revision together with
|
||||||
/// the codec and original (uncompressed) size.
|
/// the codec and original (uncompressed) size.
|
||||||
///
|
///
|
||||||
/// Compared to [`revision_pages`] this skips the decompression step, so
|
/// Compared to [`Self::revision_pages`] this skips the decompression step, so
|
||||||
/// callers that intend to send the data over the network can ship the
|
/// callers that intend to send the data over the network can ship the
|
||||||
/// compressed bytes directly and let the receiver decompress.
|
/// compressed bytes directly and let the receiver decompress.
|
||||||
///
|
///
|
||||||
|
|||||||
@@ -312,8 +312,8 @@ impl OnionFile {
|
|||||||
/// Commit a full-state snapshot revision.
|
/// Commit a full-state snapshot revision.
|
||||||
///
|
///
|
||||||
/// A snapshot records every page of the current HEAD state (not just the
|
/// A snapshot records every page of the current HEAD state (not just the
|
||||||
/// diff) and is flagged with [`REV_FLAG_SNAPSHOT`]. Subsequent calls to
|
/// diff) and is flagged with [`crate::format::REV_FLAG_SNAPSHOT`]. Subsequent calls to
|
||||||
/// [`reconstruct_revision`] will use the nearest snapshot as their starting
|
/// [`Self::reconstruct_revision`] will use the nearest snapshot as their starting
|
||||||
/// point, bounding reconstruction depth to `O(N_since_snapshot · P)`.
|
/// point, bounding reconstruction depth to `O(N_since_snapshot · P)`.
|
||||||
///
|
///
|
||||||
/// `h5_base` is the raw bytes of the base `.h5` file (needed to resolve
|
/// `h5_base` is the raw bytes of the base `.h5` file (needed to resolve
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
//!
|
//!
|
||||||
//! - [`checksum`]: xxHash3-64 block hashing + BLAKE3 integrity verification
|
//! - [`checksum`]: xxHash3-64 block hashing + BLAKE3 integrity verification
|
||||||
//! - [`cdc`]: FastCDC content-defined chunking
|
//! - [`cdc`]: FastCDC content-defined chunking
|
||||||
//! - [`simd_cdc`]: SIMD-accelerated Gear hash CDC (feature `simd-cdc`)
|
//! - `simd_cdc`: SIMD-accelerated Gear hash CDC (feature `simd-cdc`)
|
||||||
//! - [`delta`]: Fixed-block delta (copy/insert ops, apply, transfer size)
|
//! - [`delta`]: Fixed-block delta (copy/insert ops, apply, transfer size)
|
||||||
//! - [`compress`]: zstd and lz4 streaming wrappers
|
//! - [`compress`]: zstd and lz4 streaming wrappers
|
||||||
//! - [`error`]: [`CoreError`] type
|
//! - [`error`]: [`CoreError`] type
|
||||||
|
|||||||
@@ -6,6 +6,9 @@ edition.workspace = true
|
|||||||
rust-version.workspace = true
|
rust-version.workspace = true
|
||||||
license.workspace = true
|
license.workspace = true
|
||||||
repository.workspace = true
|
repository.workspace = true
|
||||||
|
readme = "README.md"
|
||||||
|
keywords = ["sync", "rsync", "cdc", "delta", "filesystem"]
|
||||||
|
categories = ["filesystem", "network-programming"]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
clawsync-core = { workspace = true }
|
clawsync-core = { workspace = true }
|
||||||
|
|||||||
@@ -0,0 +1,107 @@
|
|||||||
|
# clawsync-fs
|
||||||
|
|
||||||
|
CDC-based delta sync for any file type — the general-purpose sync engine behind
|
||||||
|
`clawsync sync` and `clawsync serve-fs`.
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
`clawsync-fs` closes the gap between ClawSync's HDF5-specific revision sync and
|
||||||
|
rsync's general-purpose file sync. It handles arbitrary file types and directory
|
||||||
|
trees using **Content-Defined Chunking** (FastCDC) so chunk boundaries are
|
||||||
|
insertion-stable: a 1-byte prefix insertion does not invalidate the rest of the
|
||||||
|
file's chunks.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
FsSyncClient FsSyncServer
|
||||||
|
│ │
|
||||||
|
├── FsManifest::build (rayon parallel) │
|
||||||
|
│ BLAKE3 every file in local tree │
|
||||||
|
│ │
|
||||||
|
├─── FsDirManifest ──────────────────────▶│
|
||||||
|
│ │ FsManifest::build (server tree)
|
||||||
|
│ │ diff_manifests → Added/Modified/Removed
|
||||||
|
│◀── FsDirNeed ───────────────────────────│
|
||||||
|
│ (needed_files with server chunks, │
|
||||||
|
│ to_delete if allow_delete) │
|
||||||
|
│ │
|
||||||
|
│ [W=16 pipelined — into_pipe_halves()] │
|
||||||
|
├─── FsCdcData (/path, chunk_order, ─────▶│ reconstruct_file
|
||||||
|
│ literal_chunks) │ atomic write (.tmp → rename)
|
||||||
|
│◀── FsFileAck ───────────────────────────│
|
||||||
|
│ (repeat for each modified/added │
|
||||||
|
│ file in any order) │
|
||||||
|
│◀── FsDirComplete ───────────────────────│
|
||||||
|
```
|
||||||
|
|
||||||
|
**Protocol cost:**
|
||||||
|
| Scenario | RTTs |
|
||||||
|
|----------|------|
|
||||||
|
| Warm no-op | 1 |
|
||||||
|
| Cold copy | 2 |
|
||||||
|
| Incremental | 2 |
|
||||||
|
|
||||||
|
## Modules
|
||||||
|
|
||||||
|
### `manifest`
|
||||||
|
|
||||||
|
`FsManifest::build(root, excludes)` — parallel BLAKE3 walk of a directory tree
|
||||||
|
using `walkdir` + Rayon. Returns a sorted manifest of all regular files with
|
||||||
|
their BLAKE3 hash, size, and mtime.
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let excludes = GlobSet::empty();
|
||||||
|
let manifest = FsManifest::build(Path::new("./data"), &excludes)?;
|
||||||
|
for entry in &manifest.entries {
|
||||||
|
println!("{} {} bytes", entry.rel_path, entry.size);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### `differ`
|
||||||
|
|
||||||
|
`diff_manifests(local, remote)` — compare two manifest entry lists by path and
|
||||||
|
BLAKE3 hash. Returns `Vec<FileDiff>` with `Added`, `Modified`, `Removed`, and
|
||||||
|
`Unchanged` variants.
|
||||||
|
|
||||||
|
### `delta`
|
||||||
|
|
||||||
|
CDC chunking, transfer-need computation, and file reconstruction.
|
||||||
|
|
||||||
|
- `chunk_file_for_request(data)` — chunk bytes and return `Vec<FsChunkHash>`
|
||||||
|
- `compute_needed_indices(server_existing, client_request)` — which chunks the
|
||||||
|
server lacks (by xxHash3-64)
|
||||||
|
- `build_chunk_data(data, chunks, needed_indices)` — pack literal data for
|
||||||
|
transfer; zstd-compresses each chunk, falls back to raw if compressed ≥ raw
|
||||||
|
- `reconstruct_file(server_file, client_order, server_map, literals, expected_blake3)` —
|
||||||
|
rebuild the target file from server-local chunks + received literals; verifies
|
||||||
|
BLAKE3 on output
|
||||||
|
|
||||||
|
### `session`
|
||||||
|
|
||||||
|
`FsSyncClient` and `FsSyncServer` — async protocol orchestration over a
|
||||||
|
`SyncPeer` (TCP or QUIC).
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Client
|
||||||
|
let stats = FsSyncClient::new(peer, local_root, excludes, delete)
|
||||||
|
.run()
|
||||||
|
.await?;
|
||||||
|
println!("{} added, {} modified, {} removed", stats.files_added, stats.files_modified, stats.files_removed);
|
||||||
|
|
||||||
|
// Server (called per accepted connection)
|
||||||
|
FsSyncServer::new(peer, serve_root, excludes, allow_delete)
|
||||||
|
.handle()
|
||||||
|
.await?;
|
||||||
|
```
|
||||||
|
|
||||||
|
## Performance
|
||||||
|
|
||||||
|
- **Manifest build:** parallelised with Rayon — scales with core count
|
||||||
|
- **Chunk transfer:** W=16 pipelined send/recv via `SyncPeer::into_pipe_halves()`
|
||||||
|
- **CDC boundaries:** FastCDC, immune to insertion staircase that breaks rsync's rolling checksum
|
||||||
|
- **Compression:** per-chunk zstd; falls back to raw for already-compressed data
|
||||||
|
|
||||||
|
## License
|
||||||
|
|
||||||
|
MIT — see repository root.
|
||||||
@@ -165,7 +165,7 @@ impl IbltSketch {
|
|||||||
|
|
||||||
/// Remove a key from the sketch.
|
/// Remove a key from the sketch.
|
||||||
///
|
///
|
||||||
/// Symmetric with [`insert`]: calling `remove(k)` after `insert(k)` leaves
|
/// Symmetric with `insert`: calling `remove(k)` after `insert(k)` leaves
|
||||||
/// the sketch in its original state.
|
/// the sketch in its original state.
|
||||||
pub fn remove(&mut self, key: u64) {
|
pub fn remove(&mut self, key: u64) {
|
||||||
self.update(key, -1);
|
self.update(key, -1);
|
||||||
|
|||||||
@@ -8,7 +8,7 @@
|
|||||||
//!
|
//!
|
||||||
//! For development / testing, use [`QuicConfig::self_signed`] which generates
|
//! For development / testing, use [`QuicConfig::self_signed`] which generates
|
||||||
//! a temporary self-signed certificate. Production deployments should use
|
//! a temporary self-signed certificate. Production deployments should use
|
||||||
//! [`QuicConfig::with_cert`] with a properly-signed certificate.
|
//! `QuicConfig::with_cert` with a properly-signed certificate.
|
||||||
//!
|
//!
|
||||||
//! # Status
|
//! # Status
|
||||||
//!
|
//!
|
||||||
|
|||||||
Reference in New Issue
Block a user