Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d54482ae60 |
@@ -73,8 +73,7 @@ fn bench_compress(data: &[u8], codec: Codec) -> usize {
|
||||
fn bench_tdt_compress(c: &mut Criterion) {
|
||||
let sizes = [4096usize, 65536];
|
||||
|
||||
type DataGen = fn(usize) -> Vec<u8>;
|
||||
let datasets: &[(&str, DataGen, usize)] = &[
|
||||
let datasets: &[(&str, fn(usize) -> Vec<u8>, usize)] = &[
|
||||
("f32_smooth", make_f32_smooth, 4),
|
||||
("f32_random", make_f32_random, 4),
|
||||
("int32_random", make_int32_random, 4),
|
||||
|
||||
@@ -33,9 +33,14 @@ pub enum MergeStrategy {
|
||||
/// Public summary of a branch.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BranchInfo {
|
||||
/// Unique numeric branch identifier (`0` = `main`).
|
||||
pub id: u32,
|
||||
/// Human-readable branch name.
|
||||
pub name: String,
|
||||
/// Revision number of the current HEAD on this branch.
|
||||
pub head_rev: u64,
|
||||
/// Revision at which this branch was forked from its parent;
|
||||
/// [`crate::format::NO_PARENT`] for the root (`main`) branch.
|
||||
pub fork_rev: u64,
|
||||
}
|
||||
|
||||
|
||||
@@ -32,8 +32,11 @@ pub enum OnionError {
|
||||
/// Page data BLAKE3 hash does not match stored hash — integrity failure.
|
||||
#[error("BLAKE3 hash mismatch on revision {revision}: stored {stored}, computed {computed}")]
|
||||
HashMismatch {
|
||||
/// The revision number whose page data failed the integrity check.
|
||||
revision: u64,
|
||||
/// The BLAKE3 hex string recorded in the revision index.
|
||||
stored: String,
|
||||
/// The BLAKE3 hex string computed from the actual page bytes.
|
||||
computed: String,
|
||||
},
|
||||
|
||||
@@ -47,7 +50,12 @@ pub enum OnionError {
|
||||
|
||||
/// A merge was attempted on a branch with no common ancestor.
|
||||
#[error("no common ancestor found between branches {a:?} and {b:?}")]
|
||||
NoCommonAncestor { a: String, b: String },
|
||||
NoCommonAncestor {
|
||||
/// Name of the first branch in the attempted merge.
|
||||
a: String,
|
||||
/// Name of the second branch in the attempted merge.
|
||||
b: String,
|
||||
},
|
||||
|
||||
/// Compression or decompression failed.
|
||||
#[error("compression error: {0}")]
|
||||
|
||||
@@ -47,13 +47,25 @@ pub const HEADER_SIZE: usize = 128;
|
||||
// Feature flags
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Bit-flag constants for `OnionHeader::feature_flags`.
|
||||
///
|
||||
/// Each flag is a single bit; combine with `|` when constructing a header.
|
||||
pub mod feature_flags {
|
||||
/// Page data may be stored with a non-`None` [`super::Codec`] (zstd, lz4, etc.).
|
||||
pub const COMPRESSION: u64 = 1 << 0;
|
||||
/// The file uses multi-branch history (`BranchEntry` table is populated).
|
||||
pub const BRANCHING: u64 = 1 << 1;
|
||||
/// `RevisionEntry.session_uuid` and `annotation_off` fields are meaningful.
|
||||
pub const PROVENANCE: u64 = 1 << 2;
|
||||
/// Full-file snapshots are stored in the revision index
|
||||
/// (i.e., at least one `RevisionEntry` has `REV_FLAG_SNAPSHOT` set).
|
||||
pub const SNAPSHOTS: u64 = 1 << 3;
|
||||
}
|
||||
|
||||
/// Feature flags enabled by default when creating a new `.onion` file.
|
||||
///
|
||||
/// Includes [`feature_flags::COMPRESSION`], [`feature_flags::BRANCHING`], and
|
||||
/// [`feature_flags::PROVENANCE`]. [`feature_flags::SNAPSHOTS`] is opt-in.
|
||||
pub const DEFAULT_FEATURE_FLAGS: u64 =
|
||||
feature_flags::COMPRESSION | feature_flags::BRANCHING | feature_flags::PROVENANCE;
|
||||
|
||||
@@ -61,12 +73,20 @@ pub const DEFAULT_FEATURE_FLAGS: u64 =
|
||||
// Codec
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Compression codec used for page data storage.
|
||||
///
|
||||
/// Stored as a single byte (`u8`) in each [`PageTableEntry`]. Unknown values
|
||||
/// cause [`OnionError::UnknownCodec`] when reading.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[repr(u8)]
|
||||
pub enum Codec {
|
||||
/// No compression — page bytes are stored verbatim.
|
||||
None = 0,
|
||||
/// [Zstandard](https://facebook.github.io/zstd/) compression.
|
||||
Zstd = 1,
|
||||
/// [LZ4](https://lz4.github.io/lz4/) compression.
|
||||
Lz4 = 2,
|
||||
/// [Brotli](https://github.com/google/brotli) compression.
|
||||
Brotli = 3,
|
||||
/// zstd applied after TDT byte-interleaving transform (arXiv:2506.18062).
|
||||
/// Improves compression ratio ~16% for `f32`/`f16` numeric pages.
|
||||
@@ -74,6 +94,22 @@ pub enum Codec {
|
||||
}
|
||||
|
||||
impl Codec {
|
||||
/// Parse a raw codec byte from a [`PageTableEntry`] into a [`Codec`].
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`OnionError::UnknownCodec`] if `v` does not correspond to a
|
||||
/// known codec variant.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use clawhdf5_onion::format::Codec;
|
||||
///
|
||||
/// assert_eq!(Codec::from_u8(0).unwrap(), Codec::None);
|
||||
/// assert_eq!(Codec::from_u8(1).unwrap(), Codec::Zstd);
|
||||
/// assert!(Codec::from_u8(99).is_err());
|
||||
/// ```
|
||||
pub fn from_u8(v: u8) -> Result<Self, OnionError> {
|
||||
match v {
|
||||
0 => Ok(Codec::None),
|
||||
@@ -105,27 +141,52 @@ impl Codec {
|
||||
// [72..128) reserved [u8; 56]
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Fixed 128-byte file header at offset 0 of every `.onion` sidecar.
|
||||
///
|
||||
/// All fields are little-endian. Readers must call [`OnionHeader::validate`]
|
||||
/// before trusting any other field.
|
||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
||||
#[repr(C)]
|
||||
pub struct OnionHeader {
|
||||
/// Magic bytes — must equal [`MAGIC`] (`b"CLAWONION"`).
|
||||
pub magic: [u8; 9],
|
||||
/// Format version — must equal [`FORMAT_VERSION`] (`1`).
|
||||
pub format_version: u8,
|
||||
/// Alignment padding to the next `u64` boundary; always `[0u8; 6]`.
|
||||
pub _pad_align: [u8; 6],
|
||||
/// Bitmask of enabled features; see [`feature_flags`].
|
||||
pub feature_flags: u64,
|
||||
/// Page size in bytes used for all page data in this file; must be a
|
||||
/// non-zero power of two.
|
||||
pub page_size: u32,
|
||||
/// Padding after `page_size` to keep `revision_count` at a `u64` boundary.
|
||||
pub _pad_ps: [u8; 4],
|
||||
/// Total number of `RevisionEntry` records in the index.
|
||||
pub revision_count: u64,
|
||||
/// Number of `BranchEntry` records; `0` when the file predates branching.
|
||||
pub branch_count: u32,
|
||||
/// Padding after `branch_count` to keep `index_offset` at a `u64` boundary.
|
||||
pub _pad_bc: [u8; 4],
|
||||
/// Byte offset of the first `RevisionEntry` in the file.
|
||||
pub index_offset: u64,
|
||||
/// Byte offset of the first `BranchEntry`; `0` when `branch_count == 0`.
|
||||
pub branch_offset: u64,
|
||||
/// Unix timestamp (seconds since epoch, as `f64`) when the file was created.
|
||||
pub created_at: f64,
|
||||
/// Reserved bytes, always zero; available for future format extensions.
|
||||
pub reserved: [u8; 56],
|
||||
}
|
||||
|
||||
const _: () = assert!(size_of::<OnionHeader>() == HEADER_SIZE);
|
||||
|
||||
impl OnionHeader {
|
||||
/// Validate the header's magic bytes and format version.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns [`OnionError::InvalidMagic`] if the magic bytes do not match
|
||||
/// [`MAGIC`], or [`OnionError::UnknownVersion`] if `format_version` is not
|
||||
/// equal to [`FORMAT_VERSION`].
|
||||
pub fn validate(&self) -> Result<(), OnionError> {
|
||||
if &self.magic != MAGIC {
|
||||
return Err(OnionError::InvalidMagic);
|
||||
@@ -136,10 +197,15 @@ impl OnionHeader {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Return `true` if the feature bit `flag` is set in `feature_flags`.
|
||||
///
|
||||
/// Use the constants in [`feature_flags`] as arguments.
|
||||
pub fn has_feature(&self, flag: u64) -> bool {
|
||||
self.feature_flags & flag != 0
|
||||
}
|
||||
|
||||
/// Construct a new header with zero revision/branch counts and
|
||||
/// `index_offset` pointing immediately after the header.
|
||||
pub fn new(page_size: u32, feature_flags: u64, created_at: f64) -> Self {
|
||||
Self {
|
||||
magic: *MAGIC,
|
||||
@@ -177,24 +243,48 @@ impl OnionHeader {
|
||||
// [105..112) _pad_flags [u8; 7]
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Fixed 112-byte record describing one committed revision.
|
||||
///
|
||||
/// Stored in the revision index array starting at [`OnionHeader::index_offset`].
|
||||
/// Sorted by `revision` in ascending order.
|
||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
||||
#[repr(C)]
|
||||
pub struct RevisionEntry {
|
||||
/// Monotonically increasing revision number (0-based).
|
||||
pub revision: u64,
|
||||
/// ID of the branch this revision belongs to; `0` = `main`.
|
||||
pub branch_id: u32,
|
||||
/// Padding after `branch_id` to keep `parent_rev` at a `u64` boundary.
|
||||
pub _pad_bi: [u8; 4],
|
||||
/// Revision number of the direct parent; [`NO_PARENT`] for the initial commit.
|
||||
pub parent_rev: u64,
|
||||
/// Number of [`PageTableEntry`] records for this revision.
|
||||
pub page_count: u32,
|
||||
/// Padding after `page_count` to keep `page_table_off` at a `u64` boundary.
|
||||
pub _pad_pc: [u8; 4],
|
||||
/// Byte offset of the first [`PageTableEntry`] for this revision.
|
||||
pub page_table_off: u64,
|
||||
/// Unix timestamp (seconds since epoch, as `f64`) when this revision was committed.
|
||||
pub timestamp: f64,
|
||||
/// BLAKE3 hash of all page data in this revision (concatenated before hashing).
|
||||
pub blake3: [u8; 32],
|
||||
/// UUID (16 bytes) identifying the session that produced this revision.
|
||||
/// Zero-filled if [`feature_flags::PROVENANCE`] is not set.
|
||||
pub session_uuid: [u8; 16],
|
||||
/// Byte offset into the annotation heap for this revision's annotation string;
|
||||
/// `0` means no annotation.
|
||||
pub annotation_off: u64,
|
||||
/// Per-revision bit flags; see `REV_FLAG_*` constants.
|
||||
pub flags: u8,
|
||||
/// Padding after `flags`, also used by GC to store an epoch tag in bytes `[0..4]`.
|
||||
pub _pad_flags: [u8; 7],
|
||||
}
|
||||
|
||||
/// Flag bit indicating that this revision is a full-state snapshot checkpoint.
|
||||
///
|
||||
/// Set in `RevisionEntry::flags`. When this bit is set the revision stores a
|
||||
/// complete copy of all HDF5 pages (not just deltas), making it a fast restore
|
||||
/// point that does not require replaying earlier revisions.
|
||||
pub const REV_FLAG_SNAPSHOT: u8 = 1 << 0;
|
||||
|
||||
/// Sentinel epoch value written into `RevisionEntry._pad_flags[0..4]` by
|
||||
@@ -230,14 +320,25 @@ impl RevisionEntry {
|
||||
// [32..40) created_at f64
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Fixed 40-byte record describing one named branch.
|
||||
///
|
||||
/// Stored in the branch table starting at [`OnionHeader::branch_offset`].
|
||||
/// Branch `0` is always `main`.
|
||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
||||
#[repr(C)]
|
||||
pub struct BranchEntry {
|
||||
/// Unique branch identifier; `0` = `main`.
|
||||
pub id: u32,
|
||||
/// Padding after `id` to keep `name_off` at a `u64` boundary.
|
||||
pub _pad_id: [u8; 4],
|
||||
/// Byte offset into the annotation heap for the branch name string.
|
||||
pub name_off: u64,
|
||||
/// Revision number of the current HEAD on this branch.
|
||||
pub head_rev: u64,
|
||||
/// Revision at which this branch was forked from its parent;
|
||||
/// [`NO_PARENT`] for the root (`main`) branch.
|
||||
pub fork_rev: u64,
|
||||
/// Unix timestamp (seconds since epoch, as `f64`) when the branch was created.
|
||||
pub created_at: f64,
|
||||
}
|
||||
|
||||
@@ -252,14 +353,25 @@ pub struct BranchEntry {
|
||||
// [25..32) _pad [u8; 7]
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Fixed 32-byte record mapping one HDF5 page to its stored data.
|
||||
///
|
||||
/// Each [`RevisionEntry`] is followed by `page_count` of these records in the
|
||||
/// page table. Pages are ordered by `h5_offset` (ascending).
|
||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
||||
#[repr(C)]
|
||||
pub struct PageTableEntry {
|
||||
/// Byte offset within the logical HDF5 file that this page covers.
|
||||
pub h5_offset: u64,
|
||||
/// Byte offset within the `.onion` file where the (possibly compressed)
|
||||
/// page data begins.
|
||||
pub data_offset: u64,
|
||||
/// Original (uncompressed) page size in bytes.
|
||||
pub orig_size: u32,
|
||||
/// Stored (possibly compressed) data size in bytes.
|
||||
pub data_size: u32,
|
||||
/// Compression codec byte; parse with [`Codec::from_u8`].
|
||||
pub codec: u8,
|
||||
/// Alignment padding; always zero.
|
||||
pub _pad: [u8; 7],
|
||||
}
|
||||
|
||||
|
||||
@@ -125,8 +125,11 @@ impl<'a> Iterator for AncestorIter<'a> {
|
||||
/// Summary of branch head state, used by public query APIs.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BranchHeadInfo {
|
||||
/// Unique numeric branch identifier (`0` = `main`).
|
||||
pub branch_id: u32,
|
||||
/// Revision number of the current HEAD on this branch.
|
||||
pub head_rev: u64,
|
||||
/// Total number of revisions on this branch.
|
||||
pub revision_count: usize,
|
||||
}
|
||||
|
||||
|
||||
@@ -66,11 +66,24 @@ pub struct MerkleNode {
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum WalkStep {
|
||||
/// Subtree [rev_lo, rev_hi] is identical on both sides — skip.
|
||||
Skip { rev_lo: u64, rev_hi: u64 },
|
||||
Skip {
|
||||
/// Lowest revision number covered by this identical subtree.
|
||||
rev_lo: u64,
|
||||
/// Highest revision number covered by this identical subtree.
|
||||
rev_hi: u64,
|
||||
},
|
||||
/// Subtree differs; descend into children.
|
||||
Descend { node: MerkleNode },
|
||||
Descend {
|
||||
/// The tree node whose children should be diffed next.
|
||||
node: MerkleNode,
|
||||
},
|
||||
/// Leaf differs; this revision needs to be transferred.
|
||||
Leaf { revision: u64, hash: [u8; 32] },
|
||||
Leaf {
|
||||
/// The revision number that differs between the two trees.
|
||||
revision: u64,
|
||||
/// BLAKE3 hash of this revision's pages on the local side.
|
||||
hash: [u8; 32],
|
||||
},
|
||||
}
|
||||
|
||||
/// Balanced binary Merkle tree over OnionFile revision BLAKE3 hashes.
|
||||
@@ -339,17 +352,27 @@ fn node_hash(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] {
|
||||
// Error type
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Errors from Merkle tree serialisation/deserialisation.
|
||||
/// Errors from Merkle tree serialisation/deserialisation.
|
||||
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
|
||||
pub enum MerkleError {
|
||||
/// The byte slice is too short to contain even the 13-byte header.
|
||||
#[error("truncated header (< 13 bytes)")]
|
||||
TruncatedHeader,
|
||||
/// The first 4 bytes do not match the expected magic sequence.
|
||||
#[error("bad magic bytes")]
|
||||
BadMagic,
|
||||
/// The format version byte is not supported by this reader.
|
||||
#[error("unknown format version {0}")]
|
||||
UnknownVersion(u8),
|
||||
/// The payload is shorter than the length declared in the header.
|
||||
#[error("truncated payload: expected {expected} bytes, got {got}")]
|
||||
TruncatedPayload { expected: usize, got: usize },
|
||||
TruncatedPayload {
|
||||
/// Number of payload bytes declared in the header.
|
||||
expected: usize,
|
||||
/// Number of bytes actually available.
|
||||
got: usize,
|
||||
},
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -540,7 +563,7 @@ mod tests {
|
||||
let all_entries: Vec<(u64, [u8; 32])> = (0..total as u64)
|
||||
.map(|r| (r, rev_hash(r)))
|
||||
.collect();
|
||||
let keep_count = (total as f64 * keep_frac) as usize;
|
||||
let keep_count = ((total as f64 * keep_frac) as usize).max(0);
|
||||
let remote_entries = &all_entries[..keep_count];
|
||||
|
||||
let local = RevisionMerkleTree::build(&all_entries);
|
||||
|
||||
@@ -446,6 +446,9 @@ impl OnionFile {
|
||||
self.branches.iter().find(|b| b.id == id)
|
||||
}
|
||||
|
||||
/// Look up a `BranchEntry` by its human-readable name.
|
||||
///
|
||||
/// Returns `None` if no branch with that name exists.
|
||||
pub fn branch_by_name(&self, name: &str) -> Option<&BranchEntry> {
|
||||
self.branches
|
||||
.iter()
|
||||
@@ -674,12 +677,19 @@ impl OnionFile {
|
||||
/// Human-readable summary of a single revision, returned by [`OnionFile::list_revisions`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RevisionSummary {
|
||||
/// Monotonically increasing revision number (0-based).
|
||||
pub revision: u64,
|
||||
/// ID of the branch this revision belongs to; `0` = `main`.
|
||||
pub branch_id: u32,
|
||||
/// Revision number of the direct parent; `u64::MAX` for the initial commit.
|
||||
pub parent_rev: u64,
|
||||
/// Unix timestamp (seconds since epoch, as `f64`) when the revision was committed.
|
||||
pub timestamp: f64,
|
||||
/// Number of page entries recorded for this revision.
|
||||
pub page_count: u32,
|
||||
/// Optional human-readable annotation stored with the revision.
|
||||
pub annotation: Option<String>,
|
||||
/// BLAKE3 hex digest of this revision's page data.
|
||||
pub blake3_hex: String,
|
||||
/// `true` if this revision is a full-state snapshot checkpoint.
|
||||
pub is_snapshot: bool,
|
||||
|
||||
@@ -34,8 +34,11 @@ use crate::error::AgentSyncError;
|
||||
/// Statistics from a push or pull operation.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct SyncStats {
|
||||
/// Number of revisions actually sent or received.
|
||||
pub revisions_transferred: u64,
|
||||
/// Total compressed bytes sent or received.
|
||||
pub bytes_transferred: u64,
|
||||
/// Revisions that were already present on the remote and therefore skipped.
|
||||
pub revisions_skipped: u64,
|
||||
}
|
||||
|
||||
@@ -95,6 +98,10 @@ pub struct TcpSyncBackend {
|
||||
}
|
||||
|
||||
impl TcpSyncBackend {
|
||||
/// Create a new backend targeting `remote_addr`.
|
||||
///
|
||||
/// `agent_id` is embedded in all protocol messages to identify the local
|
||||
/// agent to the remote server.
|
||||
pub fn new(remote_addr: SocketAddr, agent_id: impl Into<String>) -> Self {
|
||||
Self {
|
||||
remote_addr,
|
||||
|
||||
@@ -2,29 +2,38 @@
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// All errors that can occur in the `clawsync-agent` autonomous sync layer.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum AgentSyncError {
|
||||
/// An error from the `clawhdf5-onion` VFD (e.g., opening or writing the sidecar).
|
||||
#[error("onion VFD error: {0}")]
|
||||
Onion(#[from] clawhdf5_onion::OnionError),
|
||||
|
||||
/// An error propagated from the `clawsync-onion` diffing/merging layer.
|
||||
#[error("sync-onion error: {0}")]
|
||||
SyncOnion(#[from] clawsync_onion::error::SyncOnionError),
|
||||
|
||||
/// An error propagated from the `clawsync-transport` layer.
|
||||
#[error("transport error: {0}")]
|
||||
Transport(#[from] clawsync_transport::error::TransportError),
|
||||
|
||||
/// An underlying I/O error.
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
/// The remote peer reported an application-level error.
|
||||
#[error("remote error: {0}")]
|
||||
Remote(String),
|
||||
|
||||
/// The remote peer violated the sync protocol.
|
||||
#[error("protocol error: {0}")]
|
||||
Protocol(String),
|
||||
|
||||
/// An error from the `clawhdf5-agent` memory layer (save, compact, WAL flush).
|
||||
#[error("agent memory error: {0}")]
|
||||
AgentMemory(String),
|
||||
|
||||
/// A restore was requested for a revision that does not exist in the sidecar.
|
||||
#[error("no revisions to restore at revision {0}")]
|
||||
RevisionNotFound(u64),
|
||||
}
|
||||
|
||||
@@ -37,9 +37,13 @@ pub struct OnionMemory {
|
||||
/// Summary of a historical snapshot.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MemorySnapshot {
|
||||
/// Revision number within the `.onion` sidecar.
|
||||
pub revision: u64,
|
||||
/// Optional human-readable annotation stored when the snapshot was committed.
|
||||
pub annotation: Option<String>,
|
||||
/// Unix timestamp (seconds since epoch, as `f64`) when the snapshot was committed.
|
||||
pub timestamp: f64,
|
||||
/// BLAKE3 hex digest of the snapshotted HDF5 page data.
|
||||
pub blake3_hex: String,
|
||||
}
|
||||
|
||||
|
||||
@@ -58,10 +58,8 @@ fn cold_data(n: usize) -> Vec<u8> {
|
||||
// Benchmark
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
type DataGen = fn(usize) -> Vec<u8>;
|
||||
|
||||
fn bench_cdc(c: &mut Criterion) {
|
||||
let datasets: &[(&str, DataGen)] = &[
|
||||
let datasets: &[(&str, fn(usize) -> Vec<u8>)] = &[
|
||||
("random", random_data),
|
||||
("float", float_data),
|
||||
("cold_only", cold_data),
|
||||
|
||||
@@ -44,9 +44,17 @@ pub fn compute_block_hashes(data: &[u8], block_size: usize) -> Vec<BlockHash> {
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum DeltaOp {
|
||||
/// Copy `length` bytes starting at `offset` from the source.
|
||||
Copy { offset: usize, length: usize },
|
||||
Copy {
|
||||
/// Byte offset in the source buffer to copy from.
|
||||
offset: usize,
|
||||
/// Number of bytes to copy.
|
||||
length: usize,
|
||||
},
|
||||
/// Insert the given literal bytes (no corresponding source region).
|
||||
Insert { data: Vec<u8> },
|
||||
Insert {
|
||||
/// Literal bytes to insert at the current output position.
|
||||
data: Vec<u8>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Compute the delta needed to transform `source` into `target`.
|
||||
|
||||
@@ -2,14 +2,24 @@
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// All errors that can occur in the `clawsync-core` low-level layer.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum CoreError {
|
||||
/// A compression operation (zstd, lz4, brotli, or ZstdTdt) failed.
|
||||
#[error("compression failed: {0}")]
|
||||
Compress(String),
|
||||
/// A decompression operation failed (malformed or truncated input).
|
||||
#[error("decompression failed: {0}")]
|
||||
Decompress(String),
|
||||
/// The BLAKE3 hash of a block does not match the stored/expected value.
|
||||
#[error("checksum mismatch: expected {expected}, got {actual}")]
|
||||
ChecksumMismatch { expected: String, actual: String },
|
||||
ChecksumMismatch {
|
||||
/// The BLAKE3 hex string that was expected.
|
||||
expected: String,
|
||||
/// The BLAKE3 hex string that was computed.
|
||||
actual: String,
|
||||
},
|
||||
/// An underlying I/O error.
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
}
|
||||
|
||||
@@ -114,6 +114,11 @@ pub fn chunk_data_sized_simd(
|
||||
// Scalar fallback
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Scalar (non-SIMD) content-defined chunking fallback.
|
||||
///
|
||||
/// Splits `data` into variable-size chunks using a rolling hash with the
|
||||
/// Rabin fingerprint approach. `min`, `avg`, and `max` are the minimum,
|
||||
/// target average, and maximum chunk sizes in bytes.
|
||||
pub fn chunk_scalar(data: &[u8], min: usize, avg: usize, max: usize) -> Vec<Chunk> {
|
||||
if data.is_empty() {
|
||||
return Vec::new();
|
||||
@@ -151,9 +156,6 @@ pub fn chunk_scalar(data: &[u8], min: usize, avg: usize, max: usize) -> Vec<Chun
|
||||
/// Scalar processing for a single 16-byte (or shorter) slice, folding the
|
||||
/// updated hash back out. Used by both SIMD paths when a hot byte is detected.
|
||||
#[inline(always)]
|
||||
// Hot SIMD inner-loop helper: each arg is a distinct cursor/limit; bundling
|
||||
// into a struct would add per-call overhead in the chunker fast path.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn scalar_window(
|
||||
data: &[u8],
|
||||
window: &[u8],
|
||||
@@ -545,7 +547,7 @@ mod tests {
|
||||
let original = chunk_data_simd(&data);
|
||||
// Flip a hot byte in the middle to force a different boundary
|
||||
let mid = 150_000;
|
||||
data[mid] &= 0x3F; // ensure it's a hot byte (< 64)
|
||||
data[mid] = data[mid] & 0x3F; // ensure it's a hot byte (< 64)
|
||||
data[mid] ^= 0x11;
|
||||
let mutated = chunk_data_simd(&data);
|
||||
assert_ne!(original, mutated, "mutation should change chunk boundaries");
|
||||
|
||||
@@ -20,6 +20,7 @@ pub enum FileDiff {
|
||||
}
|
||||
|
||||
impl FileDiff {
|
||||
/// Return the relative path for any variant.
|
||||
pub fn path(&self) -> &str {
|
||||
match self {
|
||||
Self::Added(p) | Self::Modified(p) | Self::Removed(p) | Self::Unchanged(p) => p,
|
||||
|
||||
@@ -1,29 +1,47 @@
|
||||
//! Error types for `clawsync-fs`.
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// All errors that can occur in the `clawsync-fs` filesystem sync layer.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum FsSyncError {
|
||||
/// An underlying I/O error (file read/write, directory traversal, etc.).
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
/// An error propagated from the `clawsync-transport` layer.
|
||||
#[error("transport error: {0}")]
|
||||
Transport(#[from] clawsync_transport::TransportError),
|
||||
|
||||
/// A file or directory path is invalid or could not be resolved.
|
||||
#[error("path error: {0}")]
|
||||
Path(String),
|
||||
|
||||
/// The remote peer sent an unexpected message type during an fs-sync exchange.
|
||||
#[error("protocol error: unexpected message: {0}")]
|
||||
Protocol(String),
|
||||
|
||||
/// The BLAKE3 checksum of a received file does not match the expected value.
|
||||
#[error("checksum mismatch for {path}: expected {expected}, got {actual}")]
|
||||
ChecksumMismatch {
|
||||
/// The file path whose checksum failed verification.
|
||||
path: String,
|
||||
/// The BLAKE3 hex string that was expected.
|
||||
expected: String,
|
||||
/// The BLAKE3 hex string computed from the received bytes.
|
||||
actual: String,
|
||||
},
|
||||
|
||||
/// A file could not be reassembled from the received chunks.
|
||||
#[error("reconstruction failed for {path}: {reason}")]
|
||||
Reconstruction { path: String, reason: String },
|
||||
Reconstruction {
|
||||
/// The file path that failed to reconstruct.
|
||||
path: String,
|
||||
/// A human-readable description of the reconstruction failure.
|
||||
reason: String,
|
||||
},
|
||||
|
||||
/// A Tokio task panicked or was cancelled during an async fs-sync operation.
|
||||
#[error("task join error: {0}")]
|
||||
Join(String),
|
||||
}
|
||||
|
||||
@@ -158,6 +158,7 @@ pub struct LocalEntry {
|
||||
|
||||
/// In-memory manifest of a local directory tree, sorted by `rel_path`.
|
||||
pub struct FsManifest {
|
||||
/// All file entries in the directory tree, sorted by `rel_path`.
|
||||
pub entries: Vec<LocalEntry>,
|
||||
}
|
||||
|
||||
|
||||
@@ -56,8 +56,11 @@ pub enum ProgressEvent {
|
||||
/// fields are `None`.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct SyncStats {
|
||||
/// Number of files newly created on the destination.
|
||||
pub files_added: u32,
|
||||
/// Number of files overwritten on the destination.
|
||||
pub files_modified: u32,
|
||||
/// Number of files deleted from the destination.
|
||||
pub files_removed: u32,
|
||||
/// Actual bytes sent over the wire (compressed literal chunks only).
|
||||
pub bytes_transferred: u64,
|
||||
@@ -373,6 +376,10 @@ pub struct FsSyncServer {
|
||||
}
|
||||
|
||||
impl FsSyncServer {
|
||||
/// Create a new push server.
|
||||
///
|
||||
/// `allow_delete` controls whether the server will honour delete requests
|
||||
/// from the client (i.e., remove files absent from the client's manifest).
|
||||
pub fn new(conn: SyncPeer, serve_root: PathBuf, excludes: GlobSet, allow_delete: bool) -> Self {
|
||||
Self {
|
||||
conn,
|
||||
@@ -756,6 +763,10 @@ pub struct FsSyncPullClient {
|
||||
}
|
||||
|
||||
impl FsSyncPullClient {
|
||||
/// Create a new pull client.
|
||||
///
|
||||
/// `delete` controls whether local files absent from the server's manifest
|
||||
/// are removed after the pull completes.
|
||||
pub fn new(conn: SyncPeer, local_root: PathBuf, excludes: GlobSet, delete: bool) -> Self {
|
||||
Self {
|
||||
conn,
|
||||
@@ -972,6 +983,10 @@ pub struct FsSyncPullServer {
|
||||
}
|
||||
|
||||
impl FsSyncPullServer {
|
||||
/// Create a new pull server.
|
||||
///
|
||||
/// `allow_delete` controls whether the server will accept client requests
|
||||
/// to delete files from the server's serve root during a pull session.
|
||||
pub fn new(conn: SyncPeer, serve_root: PathBuf, excludes: GlobSet, allow_delete: bool) -> Self {
|
||||
Self {
|
||||
conn,
|
||||
|
||||
@@ -33,6 +33,7 @@ pub struct DatasetPatch {
|
||||
/// Result of diffing two manifests.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct DiffResult {
|
||||
/// Ordered list of patches (Added, Modified, Removed) to apply to the target.
|
||||
pub patches: Vec<DatasetPatch>,
|
||||
/// Number of datasets that are identical in both manifests.
|
||||
pub unchanged_count: usize,
|
||||
|
||||
@@ -2,24 +2,33 @@
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// All errors that can occur in the `clawsync-hdf5` HDF5 sync layer.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Hdf5SyncError {
|
||||
/// An error from the `clawhdf5` HDF5 parser or builder.
|
||||
#[error("HDF5 error: {0}")]
|
||||
Hdf5(#[from] clawhdf5::Error),
|
||||
|
||||
/// An underlying I/O error (file read/write, temp-file rename, etc.).
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
/// A dataset path referenced in a patch does not exist in the target file.
|
||||
#[error("dataset not found: {0}")]
|
||||
DatasetNotFound(String),
|
||||
|
||||
/// A patch attempted to overwrite a dataset with data of a different dtype.
|
||||
#[error("patch type mismatch for dataset '{path}': expected {expected}, got {actual}")]
|
||||
TypeMismatch {
|
||||
/// HDF5 path of the mismatched dataset (e.g., `/group/weights`).
|
||||
path: String,
|
||||
/// The dtype present in the target file.
|
||||
expected: String,
|
||||
/// The dtype carried by the incoming patch.
|
||||
actual: String,
|
||||
},
|
||||
|
||||
/// A `DatasetManifest` could not be serialized or deserialized.
|
||||
#[error("manifest serialization error: {0}")]
|
||||
Serialize(String),
|
||||
}
|
||||
|
||||
@@ -24,10 +24,15 @@ use crate::error::Hdf5SyncError;
|
||||
/// Statistics from a patch operation.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct PatchStats {
|
||||
/// Number of new datasets added to the target file.
|
||||
pub datasets_added: u32,
|
||||
/// Number of existing datasets overwritten with new data.
|
||||
pub datasets_modified: u32,
|
||||
/// Number of datasets deleted from the target file.
|
||||
pub datasets_removed: u32,
|
||||
/// Number of datasets copied unchanged from the old target file.
|
||||
pub datasets_unchanged: u32,
|
||||
/// Total bytes written to the rebuilt target file.
|
||||
pub bytes_written: u64,
|
||||
}
|
||||
|
||||
|
||||
@@ -2,36 +2,54 @@
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// All errors that can occur in the `clawsync-onion` sync layer.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum SyncOnionError {
|
||||
/// An error propagated from the underlying `clawhdf5-onion` VFD.
|
||||
#[error("onion error: {0}")]
|
||||
Onion(#[from] clawhdf5_onion::OnionError),
|
||||
|
||||
/// An error propagated from `clawsync-core` (compression, checksums, I/O).
|
||||
#[error("core error: {0}")]
|
||||
Core(#[from] clawsync_core::CoreError),
|
||||
|
||||
/// An underlying I/O error.
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
/// The requested revision number does not exist in the source `.onion` file.
|
||||
#[error("revision {0} not found in source")]
|
||||
RevisionNotFound(u64),
|
||||
|
||||
/// The named branch does not exist in the source `.onion` file.
|
||||
#[error("branch not found: {0}")]
|
||||
BranchNotFound(String),
|
||||
|
||||
/// The BLAKE3 hash of a received revision's pages does not match the expected value.
|
||||
#[error("BLAKE3 mismatch on revision {revision}: expected {expected}, got {actual}")]
|
||||
HashMismatch {
|
||||
/// The revision number whose pages failed the integrity check.
|
||||
revision: u64,
|
||||
/// The BLAKE3 hex string that was expected.
|
||||
expected: String,
|
||||
/// The BLAKE3 hex string that was computed from the received pages.
|
||||
actual: String,
|
||||
},
|
||||
|
||||
/// A `LayerPacket` could not be serialized or deserialized.
|
||||
#[error("packet serialization error: {0}")]
|
||||
Serialization(String),
|
||||
|
||||
/// A merge was attempted with an empty set of packets.
|
||||
#[error("empty packet set — nothing to merge")]
|
||||
EmptyPackets,
|
||||
|
||||
/// Revisions arrived out of order during a merge — a gap was detected.
|
||||
#[error("revision gap: received {got}, expected {expected}")]
|
||||
RevisionGap { expected: u64, got: u64 },
|
||||
RevisionGap {
|
||||
/// The next consecutive revision that was expected.
|
||||
expected: u64,
|
||||
/// The revision number that was actually received.
|
||||
got: u64,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -100,12 +100,16 @@ pub enum IbltDecodeResult {
|
||||
// Error type
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Errors from IBLT sketch serialisation/deserialisation.
|
||||
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
|
||||
pub enum IbltError {
|
||||
/// The byte slice is too short to contain a valid sketch header or payload.
|
||||
#[error("serialised data too short")]
|
||||
Truncated,
|
||||
/// The first bytes do not match the expected IBLT magic sequence.
|
||||
#[error("bad magic bytes")]
|
||||
BadMagic,
|
||||
/// The format version byte is not supported by this reader.
|
||||
#[error("unknown version {0}")]
|
||||
UnknownVersion(u8),
|
||||
}
|
||||
|
||||
@@ -15,12 +15,17 @@ use crate::iblt::{DEFAULT_HASH_COUNT, IbltDecodeResult, IbltDiff, IbltSketch};
|
||||
/// A compact summary of one revision — used in the manifest.
|
||||
#[derive(Archive, Deserialize, Serialize, Debug, Clone, PartialEq)]
|
||||
pub struct RevisionSummaryPacket {
|
||||
/// Monotonically increasing revision number (0-based).
|
||||
pub revision: u64,
|
||||
/// ID of the branch this revision belongs to; `0` = `main`.
|
||||
pub branch_id: u32,
|
||||
/// Revision number of the direct parent; `u64::MAX` for the initial commit.
|
||||
pub parent_rev: u64,
|
||||
/// Unix timestamp (seconds since epoch, as `f64`) when the revision was committed.
|
||||
pub timestamp: f64,
|
||||
/// BLAKE3 hash of the pages in this revision.
|
||||
pub blake3: [u8; 32],
|
||||
/// Optional human-readable annotation stored with the revision.
|
||||
pub annotation: Option<String>,
|
||||
}
|
||||
|
||||
@@ -119,11 +124,17 @@ impl ClawSyncManifest {
|
||||
/// For N=1 000: ~3 KB vs ~60 KB for `ClawSyncManifest`.
|
||||
#[derive(Archive, Deserialize, Serialize, Debug, Clone, PartialEq)]
|
||||
pub struct IbltManifest {
|
||||
/// Logical agent / file identifier (same as in `ClawSyncManifest`).
|
||||
pub agent_id: String,
|
||||
/// BLAKE3 hash of the raw `.h5` base file; zero-filled during pre-flight.
|
||||
pub file_blake3: [u8; 32],
|
||||
/// Total number of revisions in the sender's sidecar.
|
||||
pub revision_count: u64,
|
||||
/// HEAD revision number on the sender's default branch.
|
||||
pub head_revision: u64,
|
||||
/// BLAKE3 hash of the HEAD revision's pages; zero-filled during pre-flight.
|
||||
pub head_blake3: [u8; 32],
|
||||
/// Unix timestamp (seconds since epoch, as `f64`) of the last committed revision.
|
||||
pub last_write: f64,
|
||||
/// Serialised [`IbltSketch`] bytes.
|
||||
pub sketch: Vec<u8>,
|
||||
|
||||
@@ -2,32 +2,48 @@
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// All errors that can occur in the `clawsync-transport` layer.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum TransportError {
|
||||
/// An underlying I/O error (socket read/write, bind, etc.).
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
/// The remote peer closed the connection before the exchange completed.
|
||||
#[error("connection closed unexpectedly")]
|
||||
ConnectionClosed,
|
||||
|
||||
/// A received frame's length prefix exceeds the protocol maximum.
|
||||
#[error("frame too large: {size} bytes (max {max})")]
|
||||
FrameTooLarge { size: usize, max: usize },
|
||||
FrameTooLarge {
|
||||
/// The length the remote declared in the 4-byte LE prefix.
|
||||
size: usize,
|
||||
/// The maximum frame size permitted by the protocol.
|
||||
max: usize,
|
||||
},
|
||||
|
||||
/// A `SyncMessage` could not be serialized to bytes.
|
||||
#[error("serialization error: {0}")]
|
||||
Serialization(String),
|
||||
|
||||
/// Received bytes could not be deserialized into a `SyncMessage`.
|
||||
#[error("deserialization error: {0}")]
|
||||
Deserialization(String),
|
||||
|
||||
/// An error from the underlying QUIC stack (e.g., `quinn`).
|
||||
#[error("QUIC error: {0}")]
|
||||
Quic(String),
|
||||
|
||||
/// A TLS configuration or handshake error.
|
||||
#[error("TLS error: {0}")]
|
||||
Tls(String),
|
||||
|
||||
/// The remote peer violated the sync protocol (unexpected message type
|
||||
/// or message sequencing error).
|
||||
#[error("protocol error: {0}")]
|
||||
Protocol(String),
|
||||
|
||||
/// An error propagated from the `clawsync-onion` sync layer.
|
||||
#[error("sync onion error: {0}")]
|
||||
SyncOnion(#[from] clawsync_onion::SyncOnionError),
|
||||
}
|
||||
|
||||
@@ -24,10 +24,12 @@ pub struct FramedReader<R: AsyncRead + Unpin + Send> {
|
||||
}
|
||||
|
||||
impl<R: AsyncRead + Unpin + Send> FramedReader<R> {
|
||||
/// Wrap `inner` in a framed reader.
|
||||
pub fn new(inner: R) -> Self {
|
||||
Self { inner }
|
||||
}
|
||||
|
||||
/// Read and deserialize the next length-prefixed `SyncMessage` frame.
|
||||
pub async fn recv(&mut self) -> Result<SyncMessage, TransportError> {
|
||||
let mut len_buf = [0u8; 4];
|
||||
self.inner.read_exact(&mut len_buf).await?;
|
||||
@@ -52,10 +54,12 @@ pub struct FramedWriter<W: AsyncWrite + Unpin + Send> {
|
||||
}
|
||||
|
||||
impl<W: AsyncWrite + Unpin + Send> FramedWriter<W> {
|
||||
/// Wrap `inner` in a framed writer.
|
||||
pub fn new(inner: W) -> Self {
|
||||
Self { inner }
|
||||
}
|
||||
|
||||
/// Serialize `msg` and write it as a length-prefixed frame.
|
||||
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
||||
let body = msg.to_bytes().map_err(TransportError::Serialization)?;
|
||||
let len = body.len() as u32;
|
||||
@@ -65,6 +69,7 @@ impl<W: AsyncWrite + Unpin + Send> FramedWriter<W> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush pending writes and shut down the write half.
|
||||
pub async fn shutdown(&mut self) -> Result<(), TransportError> {
|
||||
self.inner.shutdown().await?;
|
||||
Ok(())
|
||||
@@ -112,14 +117,17 @@ impl StreamPeer {
|
||||
}
|
||||
}
|
||||
|
||||
/// Send a `SyncMessage` through the peer's write half.
|
||||
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
||||
self.writer.send(msg).await
|
||||
}
|
||||
|
||||
/// Receive the next `SyncMessage` from the peer's read half.
|
||||
pub async fn recv(&mut self) -> Result<SyncMessage, TransportError> {
|
||||
self.reader.recv().await
|
||||
}
|
||||
|
||||
/// Flush and shut down the write half.
|
||||
pub async fn shutdown(&mut self) -> Result<(), TransportError> {
|
||||
self.writer.shutdown().await
|
||||
}
|
||||
@@ -134,21 +142,26 @@ impl StreamPeer {
|
||||
// Split halves
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Read half produced by [`StreamPeer::into_split`].
|
||||
pub struct StreamReadHalf(pub(crate) FramedReader<BoxReader>);
|
||||
|
||||
impl StreamReadHalf {
|
||||
/// Receive the next `SyncMessage` from the boxed reader.
|
||||
pub async fn recv(&mut self) -> Result<SyncMessage, TransportError> {
|
||||
self.0.recv().await
|
||||
}
|
||||
}
|
||||
|
||||
/// Write half produced by [`StreamPeer::into_split`].
|
||||
pub struct StreamWriteHalf(pub(crate) FramedWriter<BoxWriter>);
|
||||
|
||||
impl StreamWriteHalf {
|
||||
/// Send a `SyncMessage` through the boxed writer.
|
||||
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
||||
self.0.send(msg).await
|
||||
}
|
||||
|
||||
/// Flush and shut down the boxed writer.
|
||||
pub async fn shutdown(&mut self) -> Result<(), TransportError> {
|
||||
self.0.shutdown().await
|
||||
}
|
||||
|
||||
@@ -27,13 +27,17 @@ use crate::tcp::{TcpConnection, TcpReadHalf, TcpWriteHalf};
|
||||
|
||||
/// A unified connection handle for a single sync session (TCP, QUIC, mmap, or stream).
|
||||
pub enum SyncPeer {
|
||||
/// An established async TCP connection.
|
||||
Tcp(TcpConnection),
|
||||
/// A QUIC connection shared via `Arc` (send + recv are `&self`).
|
||||
Quic(Arc<QuicConnection>),
|
||||
/// Local same-node transport backed by two memory-mapped ring buffers —
|
||||
/// one per direction. `send_ch` is the outgoing channel; `recv_ch` is
|
||||
/// the incoming channel.
|
||||
Mmap {
|
||||
/// Outgoing mmap ring-buffer sender.
|
||||
send_ch: MmapSender,
|
||||
/// Incoming mmap ring-buffer receiver.
|
||||
recv_ch: MmapReceiver,
|
||||
},
|
||||
/// Generic boxed-I/O peer — covers SSH child-process pipes and `--stdio`
|
||||
@@ -42,6 +46,11 @@ pub enum SyncPeer {
|
||||
}
|
||||
|
||||
impl SyncPeer {
|
||||
/// Send a `SyncMessage` to the remote peer.
|
||||
///
|
||||
/// For TCP and Stream peers this performs a framed write (4-byte LE length
|
||||
/// + body). For QUIC, each call opens a new unidirectional stream. For
|
||||
/// Mmap, the call blocks until the ring buffer has space.
|
||||
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
||||
match self {
|
||||
SyncPeer::Tcp(c) => c.send(msg).await,
|
||||
@@ -51,6 +60,11 @@ impl SyncPeer {
|
||||
}
|
||||
}
|
||||
|
||||
/// Receive the next `SyncMessage` from the remote peer.
|
||||
///
|
||||
/// Blocks until a complete message is available. Returns
|
||||
/// [`TransportError::ConnectionClosed`] when the peer has cleanly closed
|
||||
/// the connection.
|
||||
pub async fn recv(&mut self) -> Result<SyncMessage, TransportError> {
|
||||
match self {
|
||||
SyncPeer::Tcp(c) => c.recv().await,
|
||||
@@ -60,6 +74,7 @@ impl SyncPeer {
|
||||
}
|
||||
}
|
||||
|
||||
/// Gracefully shut down the connection, flushing any pending writes.
|
||||
pub async fn shutdown(self) -> Result<(), TransportError> {
|
||||
match self {
|
||||
SyncPeer::Tcp(mut c) => c.shutdown().await,
|
||||
@@ -136,13 +151,18 @@ impl SyncPeer {
|
||||
|
||||
/// Write half for the sliding-window pipeline.
|
||||
pub enum PipeWriteHalf {
|
||||
/// Write half of a split `TcpConnection`.
|
||||
Tcp(TcpWriteHalf),
|
||||
/// QUIC connection `Arc` clone used for sending (QUIC send is `&self`).
|
||||
Quic(Arc<QuicConnection>),
|
||||
/// Mmap ring-buffer sender.
|
||||
Mmap(MmapSender),
|
||||
/// Boxed-I/O write half (SSH / stdio).
|
||||
Stream(StreamWriteHalf),
|
||||
}
|
||||
|
||||
impl PipeWriteHalf {
|
||||
/// Send a `SyncMessage` through the write half.
|
||||
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
||||
match self {
|
||||
PipeWriteHalf::Tcp(h) => h.send(msg).await,
|
||||
@@ -152,6 +172,7 @@ impl PipeWriteHalf {
|
||||
}
|
||||
}
|
||||
|
||||
/// Gracefully shut down the write half.
|
||||
pub async fn shutdown(self) -> Result<(), TransportError> {
|
||||
match self {
|
||||
PipeWriteHalf::Tcp(mut h) => h.shutdown().await,
|
||||
@@ -184,13 +205,18 @@ impl PipeWriteHalf {
|
||||
|
||||
/// Read half for the sliding-window pipeline.
|
||||
pub enum PipeReadHalf {
|
||||
/// Read half of a split `TcpConnection`.
|
||||
Tcp(TcpReadHalf),
|
||||
/// QUIC connection `Arc` clone used for receiving.
|
||||
Quic(Arc<QuicConnection>),
|
||||
/// Mmap ring-buffer receiver.
|
||||
Mmap(MmapReceiver),
|
||||
/// Boxed-I/O read half (SSH / stdio).
|
||||
Stream(StreamReadHalf),
|
||||
}
|
||||
|
||||
impl PipeReadHalf {
|
||||
/// Receive the next `SyncMessage` from the read half.
|
||||
pub async fn recv(&mut self) -> Result<SyncMessage, TransportError> {
|
||||
match self {
|
||||
PipeReadHalf::Tcp(h) => h.recv().await,
|
||||
|
||||
@@ -120,27 +120,45 @@ pub const MAX_FRAME_SIZE: usize = 256 * 1024 * 1024;
|
||||
pub enum SyncMessage {
|
||||
/// Client announces itself and its current HEAD revision.
|
||||
ManifestRequest {
|
||||
/// Logical agent / file identifier (used by the server to look up the sidecar).
|
||||
agent_id: String,
|
||||
/// HEAD revision number the client currently holds.
|
||||
head_revision: u64,
|
||||
/// Revision count (so the server can compute the delta).
|
||||
/// Total revision count (so the server can compute the delta).
|
||||
revision_count: u64,
|
||||
},
|
||||
|
||||
/// Server responds with its own manifest.
|
||||
ManifestResponse { manifest: ClawSyncManifest },
|
||||
ManifestResponse {
|
||||
/// Full revision manifest of the server's sidecar.
|
||||
manifest: ClawSyncManifest,
|
||||
},
|
||||
|
||||
/// One revision packet (sent by either side, depending on push/pull).
|
||||
LayerPacket { packet: OnionLayerPacket },
|
||||
LayerPacket {
|
||||
/// The serialized revision data (pages, metadata, BLAKE3).
|
||||
packet: OnionLayerPacket,
|
||||
},
|
||||
|
||||
/// Acknowledgement of a successfully received + verified revision.
|
||||
Ack { revision: u64 },
|
||||
Ack {
|
||||
/// The revision number that was successfully applied.
|
||||
revision: u64,
|
||||
},
|
||||
|
||||
/// Request to retransmit a revision (hash verification failed).
|
||||
RetryRequest { revision: u64, reason: String },
|
||||
RetryRequest {
|
||||
/// The revision number that failed integrity verification.
|
||||
revision: u64,
|
||||
/// Human-readable reason for the retry request.
|
||||
reason: String,
|
||||
},
|
||||
|
||||
/// Transfer is complete.
|
||||
SyncComplete {
|
||||
/// Total number of revisions successfully transferred in this session.
|
||||
revisions_transferred: u64,
|
||||
/// Total compressed bytes transferred in this session.
|
||||
bytes_transferred: u64,
|
||||
},
|
||||
|
||||
@@ -148,7 +166,10 @@ pub enum SyncMessage {
|
||||
///
|
||||
/// If the remote supports IBLT it responds with `IbltResponse`; otherwise
|
||||
/// it responds with `Error` and the client falls back to `ManifestRequest`.
|
||||
IbltRequest { sketch: IbltManifest },
|
||||
IbltRequest {
|
||||
/// The client's IBLT sketch of its local revision set.
|
||||
sketch: IbltManifest,
|
||||
},
|
||||
|
||||
/// IBLT pre-flight response: server's sketch + decoded missing revisions.
|
||||
///
|
||||
@@ -156,12 +177,17 @@ pub enum SyncMessage {
|
||||
/// lacks (what to push). The client decodes `sketch` against its own
|
||||
/// sketch to find revisions it needs to pull.
|
||||
IbltResponse {
|
||||
/// The server's IBLT sketch (for the client to decode what it should pull).
|
||||
sketch: IbltManifest,
|
||||
/// Revision numbers the server lacks (client should push these).
|
||||
missing_from_remote: Vec<u64>,
|
||||
},
|
||||
|
||||
/// An error occurred; connection will be closed after this message.
|
||||
Error { message: String },
|
||||
Error {
|
||||
/// Human-readable error description.
|
||||
message: String,
|
||||
},
|
||||
|
||||
// ── FS CDC file delta (variants 9–12) ────────────────────────────────────
|
||||
//
|
||||
@@ -179,7 +205,9 @@ pub enum SyncMessage {
|
||||
/// Originally: client announces a single-file CDC sync, carrying its ordered
|
||||
/// chunk descriptors. Superseded by the server chunk embedding in `FsDirNeed`.
|
||||
FsCdcRequest {
|
||||
/// Relative path of the file being synced.
|
||||
path: String,
|
||||
/// Client's ordered CDC chunk descriptors (hash + length per chunk).
|
||||
chunk_hashes: Vec<FsChunkHash>,
|
||||
},
|
||||
|
||||
@@ -188,7 +216,9 @@ pub enum SyncMessage {
|
||||
/// Originally: server replies with the indices it needs transferred.
|
||||
/// Superseded by the server chunk embedding in `FsDirNeed`.
|
||||
FsCdcNeed {
|
||||
/// Relative path of the file being synced.
|
||||
path: String,
|
||||
/// Indices (into the client's chunk list) that the server needs.
|
||||
needed_indices: Vec<u32>,
|
||||
},
|
||||
|
||||
@@ -202,6 +232,7 @@ pub enum SyncMessage {
|
||||
/// sync path, while remaining self-contained for single-file use too.
|
||||
/// Wire overhead is ~12 bytes × N_chunks (negligible relative to data).
|
||||
FsCdcData {
|
||||
/// Relative path of the file being transferred.
|
||||
path: String,
|
||||
/// Full ordered chunk layout of the client's file.
|
||||
chunk_order: Vec<FsChunkHash>,
|
||||
@@ -210,7 +241,10 @@ pub enum SyncMessage {
|
||||
},
|
||||
|
||||
/// Server confirms it has reconstructed and written the file at `path`.
|
||||
FsFileAck { path: String },
|
||||
FsFileAck {
|
||||
/// Relative path of the file that was successfully written.
|
||||
path: String,
|
||||
},
|
||||
|
||||
// ── FS directory tree sync (variants 13–16) ───────────────────────────────
|
||||
/// Client sends a per-file manifest for an entire directory tree.
|
||||
@@ -237,16 +271,23 @@ pub enum SyncMessage {
|
||||
|
||||
/// Server reports that all changes have been applied.
|
||||
FsDirComplete {
|
||||
/// Number of files newly created on the server.
|
||||
files_added: u32,
|
||||
/// Number of files overwritten on the server.
|
||||
files_modified: u32,
|
||||
/// Number of files deleted from the server.
|
||||
files_removed: u32,
|
||||
/// Total compressed bytes transferred in this session.
|
||||
bytes_transferred: u64,
|
||||
},
|
||||
|
||||
/// Dry-run: reports what *would* change without applying anything.
|
||||
FsDirDryRun {
|
||||
/// Paths that would be created on the server.
|
||||
would_add: Vec<String>,
|
||||
/// Paths that would be overwritten on the server.
|
||||
would_modify: Vec<String>,
|
||||
/// Paths that would be deleted from the server.
|
||||
would_remove: Vec<String>,
|
||||
},
|
||||
|
||||
@@ -284,7 +325,9 @@ pub enum SyncMessage {
|
||||
/// `dtype` is the `{:?}` format of `clawhdf5::DType` (e.g. `"F64"`).
|
||||
/// `shape` is the dataset's dimension sizes.
|
||||
Hdf5DataPayload {
|
||||
/// HDF5 path of the dataset within the file (e.g., `/group/embeddings`).
|
||||
path: String,
|
||||
/// Raw little-endian bytes of the dataset.
|
||||
data: Vec<u8>,
|
||||
/// HDF5 DType debug string, e.g. `"F64"`, `"F32"`, `"I32"`.
|
||||
dtype: String,
|
||||
@@ -293,13 +336,20 @@ pub enum SyncMessage {
|
||||
},
|
||||
|
||||
/// Server acknowledges successful reconstruction of a dataset.
|
||||
Hdf5DataAck { path: String },
|
||||
Hdf5DataAck {
|
||||
/// HDF5 path of the dataset that was successfully applied.
|
||||
path: String,
|
||||
},
|
||||
|
||||
/// Server reports that the HDF5 sync is complete.
|
||||
Hdf5SyncComplete {
|
||||
/// Number of datasets newly added to the server's file.
|
||||
datasets_added: u32,
|
||||
/// Number of datasets overwritten on the server.
|
||||
datasets_modified: u32,
|
||||
/// Number of datasets removed from the server's file.
|
||||
datasets_removed: u32,
|
||||
/// Total compressed bytes transferred in this session.
|
||||
bytes_transferred: u64,
|
||||
},
|
||||
|
||||
|
||||
@@ -32,7 +32,9 @@ use crate::protocol::SyncMessage;
|
||||
|
||||
/// QUIC endpoint configuration.
|
||||
pub struct QuicConfig {
|
||||
/// TLS server configuration for the QUIC server endpoint.
|
||||
pub server_config: ServerConfig,
|
||||
/// TLS client configuration for outgoing QUIC connections.
|
||||
pub client_config: ClientConfig,
|
||||
}
|
||||
|
||||
@@ -453,6 +455,10 @@ impl QuicConnection {
|
||||
/// A QUIC server endpoint.
|
||||
pub struct QuicServer {
|
||||
endpoint: Endpoint,
|
||||
/// The local `SocketAddr` this server is bound to.
|
||||
///
|
||||
/// Useful when the server was bound to port `0` and the assigned ephemeral
|
||||
/// port must be communicated to clients.
|
||||
pub local_addr: SocketAddr,
|
||||
}
|
||||
|
||||
|
||||
@@ -144,6 +144,10 @@ impl TcpWriteHalf {
|
||||
/// A listening TCP server that accepts `TcpConnection`s.
|
||||
pub struct TcpServer {
|
||||
listener: TcpListener,
|
||||
/// The local `SocketAddr` this server is bound to.
|
||||
///
|
||||
/// Useful for callers that bind to port `0` and need to discover the
|
||||
/// assigned ephemeral port after binding.
|
||||
pub local_addr: SocketAddr,
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user