Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d54482ae60 |
@@ -73,8 +73,7 @@ fn bench_compress(data: &[u8], codec: Codec) -> usize {
|
|||||||
fn bench_tdt_compress(c: &mut Criterion) {
|
fn bench_tdt_compress(c: &mut Criterion) {
|
||||||
let sizes = [4096usize, 65536];
|
let sizes = [4096usize, 65536];
|
||||||
|
|
||||||
type DataGen = fn(usize) -> Vec<u8>;
|
let datasets: &[(&str, fn(usize) -> Vec<u8>, usize)] = &[
|
||||||
let datasets: &[(&str, DataGen, usize)] = &[
|
|
||||||
("f32_smooth", make_f32_smooth, 4),
|
("f32_smooth", make_f32_smooth, 4),
|
||||||
("f32_random", make_f32_random, 4),
|
("f32_random", make_f32_random, 4),
|
||||||
("int32_random", make_int32_random, 4),
|
("int32_random", make_int32_random, 4),
|
||||||
|
|||||||
@@ -33,9 +33,14 @@ pub enum MergeStrategy {
|
|||||||
/// Public summary of a branch.
|
/// Public summary of a branch.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct BranchInfo {
|
pub struct BranchInfo {
|
||||||
|
/// Unique numeric branch identifier (`0` = `main`).
|
||||||
pub id: u32,
|
pub id: u32,
|
||||||
|
/// Human-readable branch name.
|
||||||
pub name: String,
|
pub name: String,
|
||||||
|
/// Revision number of the current HEAD on this branch.
|
||||||
pub head_rev: u64,
|
pub head_rev: u64,
|
||||||
|
/// Revision at which this branch was forked from its parent;
|
||||||
|
/// [`crate::format::NO_PARENT`] for the root (`main`) branch.
|
||||||
pub fork_rev: u64,
|
pub fork_rev: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -32,8 +32,11 @@ pub enum OnionError {
|
|||||||
/// Page data BLAKE3 hash does not match stored hash — integrity failure.
|
/// Page data BLAKE3 hash does not match stored hash — integrity failure.
|
||||||
#[error("BLAKE3 hash mismatch on revision {revision}: stored {stored}, computed {computed}")]
|
#[error("BLAKE3 hash mismatch on revision {revision}: stored {stored}, computed {computed}")]
|
||||||
HashMismatch {
|
HashMismatch {
|
||||||
|
/// The revision number whose page data failed the integrity check.
|
||||||
revision: u64,
|
revision: u64,
|
||||||
|
/// The BLAKE3 hex string recorded in the revision index.
|
||||||
stored: String,
|
stored: String,
|
||||||
|
/// The BLAKE3 hex string computed from the actual page bytes.
|
||||||
computed: String,
|
computed: String,
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -47,7 +50,12 @@ pub enum OnionError {
|
|||||||
|
|
||||||
/// A merge was attempted on a branch with no common ancestor.
|
/// A merge was attempted on a branch with no common ancestor.
|
||||||
#[error("no common ancestor found between branches {a:?} and {b:?}")]
|
#[error("no common ancestor found between branches {a:?} and {b:?}")]
|
||||||
NoCommonAncestor { a: String, b: String },
|
NoCommonAncestor {
|
||||||
|
/// Name of the first branch in the attempted merge.
|
||||||
|
a: String,
|
||||||
|
/// Name of the second branch in the attempted merge.
|
||||||
|
b: String,
|
||||||
|
},
|
||||||
|
|
||||||
/// Compression or decompression failed.
|
/// Compression or decompression failed.
|
||||||
#[error("compression error: {0}")]
|
#[error("compression error: {0}")]
|
||||||
|
|||||||
@@ -47,13 +47,25 @@ pub const HEADER_SIZE: usize = 128;
|
|||||||
// Feature flags
|
// Feature flags
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Bit-flag constants for `OnionHeader::feature_flags`.
|
||||||
|
///
|
||||||
|
/// Each flag is a single bit; combine with `|` when constructing a header.
|
||||||
pub mod feature_flags {
|
pub mod feature_flags {
|
||||||
|
/// Page data may be stored with a non-`None` [`super::Codec`] (zstd, lz4, etc.).
|
||||||
pub const COMPRESSION: u64 = 1 << 0;
|
pub const COMPRESSION: u64 = 1 << 0;
|
||||||
|
/// The file uses multi-branch history (`BranchEntry` table is populated).
|
||||||
pub const BRANCHING: u64 = 1 << 1;
|
pub const BRANCHING: u64 = 1 << 1;
|
||||||
|
/// `RevisionEntry.session_uuid` and `annotation_off` fields are meaningful.
|
||||||
pub const PROVENANCE: u64 = 1 << 2;
|
pub const PROVENANCE: u64 = 1 << 2;
|
||||||
|
/// Full-file snapshots are stored in the revision index
|
||||||
|
/// (i.e., at least one `RevisionEntry` has `REV_FLAG_SNAPSHOT` set).
|
||||||
pub const SNAPSHOTS: u64 = 1 << 3;
|
pub const SNAPSHOTS: u64 = 1 << 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Feature flags enabled by default when creating a new `.onion` file.
|
||||||
|
///
|
||||||
|
/// Includes [`feature_flags::COMPRESSION`], [`feature_flags::BRANCHING`], and
|
||||||
|
/// [`feature_flags::PROVENANCE`]. [`feature_flags::SNAPSHOTS`] is opt-in.
|
||||||
pub const DEFAULT_FEATURE_FLAGS: u64 =
|
pub const DEFAULT_FEATURE_FLAGS: u64 =
|
||||||
feature_flags::COMPRESSION | feature_flags::BRANCHING | feature_flags::PROVENANCE;
|
feature_flags::COMPRESSION | feature_flags::BRANCHING | feature_flags::PROVENANCE;
|
||||||
|
|
||||||
@@ -61,12 +73,20 @@ pub const DEFAULT_FEATURE_FLAGS: u64 =
|
|||||||
// Codec
|
// Codec
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Compression codec used for page data storage.
|
||||||
|
///
|
||||||
|
/// Stored as a single byte (`u8`) in each [`PageTableEntry`]. Unknown values
|
||||||
|
/// cause [`OnionError::UnknownCodec`] when reading.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
#[repr(u8)]
|
#[repr(u8)]
|
||||||
pub enum Codec {
|
pub enum Codec {
|
||||||
|
/// No compression — page bytes are stored verbatim.
|
||||||
None = 0,
|
None = 0,
|
||||||
|
/// [Zstandard](https://facebook.github.io/zstd/) compression.
|
||||||
Zstd = 1,
|
Zstd = 1,
|
||||||
|
/// [LZ4](https://lz4.github.io/lz4/) compression.
|
||||||
Lz4 = 2,
|
Lz4 = 2,
|
||||||
|
/// [Brotli](https://github.com/google/brotli) compression.
|
||||||
Brotli = 3,
|
Brotli = 3,
|
||||||
/// zstd applied after TDT byte-interleaving transform (arXiv:2506.18062).
|
/// zstd applied after TDT byte-interleaving transform (arXiv:2506.18062).
|
||||||
/// Improves compression ratio ~16% for `f32`/`f16` numeric pages.
|
/// Improves compression ratio ~16% for `f32`/`f16` numeric pages.
|
||||||
@@ -74,6 +94,22 @@ pub enum Codec {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl Codec {
|
impl Codec {
|
||||||
|
/// Parse a raw codec byte from a [`PageTableEntry`] into a [`Codec`].
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
///
|
||||||
|
/// Returns [`OnionError::UnknownCodec`] if `v` does not correspond to a
|
||||||
|
/// known codec variant.
|
||||||
|
///
|
||||||
|
/// # Examples
|
||||||
|
///
|
||||||
|
/// ```
|
||||||
|
/// use clawhdf5_onion::format::Codec;
|
||||||
|
///
|
||||||
|
/// assert_eq!(Codec::from_u8(0).unwrap(), Codec::None);
|
||||||
|
/// assert_eq!(Codec::from_u8(1).unwrap(), Codec::Zstd);
|
||||||
|
/// assert!(Codec::from_u8(99).is_err());
|
||||||
|
/// ```
|
||||||
pub fn from_u8(v: u8) -> Result<Self, OnionError> {
|
pub fn from_u8(v: u8) -> Result<Self, OnionError> {
|
||||||
match v {
|
match v {
|
||||||
0 => Ok(Codec::None),
|
0 => Ok(Codec::None),
|
||||||
@@ -105,27 +141,52 @@ impl Codec {
|
|||||||
// [72..128) reserved [u8; 56]
|
// [72..128) reserved [u8; 56]
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Fixed 128-byte file header at offset 0 of every `.onion` sidecar.
|
||||||
|
///
|
||||||
|
/// All fields are little-endian. Readers must call [`OnionHeader::validate`]
|
||||||
|
/// before trusting any other field.
|
||||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
||||||
#[repr(C)]
|
#[repr(C)]
|
||||||
pub struct OnionHeader {
|
pub struct OnionHeader {
|
||||||
|
/// Magic bytes — must equal [`MAGIC`] (`b"CLAWONION"`).
|
||||||
pub magic: [u8; 9],
|
pub magic: [u8; 9],
|
||||||
|
/// Format version — must equal [`FORMAT_VERSION`] (`1`).
|
||||||
pub format_version: u8,
|
pub format_version: u8,
|
||||||
|
/// Alignment padding to the next `u64` boundary; always `[0u8; 6]`.
|
||||||
pub _pad_align: [u8; 6],
|
pub _pad_align: [u8; 6],
|
||||||
|
/// Bitmask of enabled features; see [`feature_flags`].
|
||||||
pub feature_flags: u64,
|
pub feature_flags: u64,
|
||||||
|
/// Page size in bytes used for all page data in this file; must be a
|
||||||
|
/// non-zero power of two.
|
||||||
pub page_size: u32,
|
pub page_size: u32,
|
||||||
|
/// Padding after `page_size` to keep `revision_count` at a `u64` boundary.
|
||||||
pub _pad_ps: [u8; 4],
|
pub _pad_ps: [u8; 4],
|
||||||
|
/// Total number of `RevisionEntry` records in the index.
|
||||||
pub revision_count: u64,
|
pub revision_count: u64,
|
||||||
|
/// Number of `BranchEntry` records; `0` when the file predates branching.
|
||||||
pub branch_count: u32,
|
pub branch_count: u32,
|
||||||
|
/// Padding after `branch_count` to keep `index_offset` at a `u64` boundary.
|
||||||
pub _pad_bc: [u8; 4],
|
pub _pad_bc: [u8; 4],
|
||||||
|
/// Byte offset of the first `RevisionEntry` in the file.
|
||||||
pub index_offset: u64,
|
pub index_offset: u64,
|
||||||
|
/// Byte offset of the first `BranchEntry`; `0` when `branch_count == 0`.
|
||||||
pub branch_offset: u64,
|
pub branch_offset: u64,
|
||||||
|
/// Unix timestamp (seconds since epoch, as `f64`) when the file was created.
|
||||||
pub created_at: f64,
|
pub created_at: f64,
|
||||||
|
/// Reserved bytes, always zero; available for future format extensions.
|
||||||
pub reserved: [u8; 56],
|
pub reserved: [u8; 56],
|
||||||
}
|
}
|
||||||
|
|
||||||
const _: () = assert!(size_of::<OnionHeader>() == HEADER_SIZE);
|
const _: () = assert!(size_of::<OnionHeader>() == HEADER_SIZE);
|
||||||
|
|
||||||
impl OnionHeader {
|
impl OnionHeader {
|
||||||
|
/// Validate the header's magic bytes and format version.
|
||||||
|
///
|
||||||
|
/// # Errors
|
||||||
|
///
|
||||||
|
/// Returns [`OnionError::InvalidMagic`] if the magic bytes do not match
|
||||||
|
/// [`MAGIC`], or [`OnionError::UnknownVersion`] if `format_version` is not
|
||||||
|
/// equal to [`FORMAT_VERSION`].
|
||||||
pub fn validate(&self) -> Result<(), OnionError> {
|
pub fn validate(&self) -> Result<(), OnionError> {
|
||||||
if &self.magic != MAGIC {
|
if &self.magic != MAGIC {
|
||||||
return Err(OnionError::InvalidMagic);
|
return Err(OnionError::InvalidMagic);
|
||||||
@@ -136,10 +197,15 @@ impl OnionHeader {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Return `true` if the feature bit `flag` is set in `feature_flags`.
|
||||||
|
///
|
||||||
|
/// Use the constants in [`feature_flags`] as arguments.
|
||||||
pub fn has_feature(&self, flag: u64) -> bool {
|
pub fn has_feature(&self, flag: u64) -> bool {
|
||||||
self.feature_flags & flag != 0
|
self.feature_flags & flag != 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Construct a new header with zero revision/branch counts and
|
||||||
|
/// `index_offset` pointing immediately after the header.
|
||||||
pub fn new(page_size: u32, feature_flags: u64, created_at: f64) -> Self {
|
pub fn new(page_size: u32, feature_flags: u64, created_at: f64) -> Self {
|
||||||
Self {
|
Self {
|
||||||
magic: *MAGIC,
|
magic: *MAGIC,
|
||||||
@@ -177,24 +243,48 @@ impl OnionHeader {
|
|||||||
// [105..112) _pad_flags [u8; 7]
|
// [105..112) _pad_flags [u8; 7]
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Fixed 112-byte record describing one committed revision.
|
||||||
|
///
|
||||||
|
/// Stored in the revision index array starting at [`OnionHeader::index_offset`].
|
||||||
|
/// Sorted by `revision` in ascending order.
|
||||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
||||||
#[repr(C)]
|
#[repr(C)]
|
||||||
pub struct RevisionEntry {
|
pub struct RevisionEntry {
|
||||||
|
/// Monotonically increasing revision number (0-based).
|
||||||
pub revision: u64,
|
pub revision: u64,
|
||||||
|
/// ID of the branch this revision belongs to; `0` = `main`.
|
||||||
pub branch_id: u32,
|
pub branch_id: u32,
|
||||||
|
/// Padding after `branch_id` to keep `parent_rev` at a `u64` boundary.
|
||||||
pub _pad_bi: [u8; 4],
|
pub _pad_bi: [u8; 4],
|
||||||
|
/// Revision number of the direct parent; [`NO_PARENT`] for the initial commit.
|
||||||
pub parent_rev: u64,
|
pub parent_rev: u64,
|
||||||
|
/// Number of [`PageTableEntry`] records for this revision.
|
||||||
pub page_count: u32,
|
pub page_count: u32,
|
||||||
|
/// Padding after `page_count` to keep `page_table_off` at a `u64` boundary.
|
||||||
pub _pad_pc: [u8; 4],
|
pub _pad_pc: [u8; 4],
|
||||||
|
/// Byte offset of the first [`PageTableEntry`] for this revision.
|
||||||
pub page_table_off: u64,
|
pub page_table_off: u64,
|
||||||
|
/// Unix timestamp (seconds since epoch, as `f64`) when this revision was committed.
|
||||||
pub timestamp: f64,
|
pub timestamp: f64,
|
||||||
|
/// BLAKE3 hash of all page data in this revision (concatenated before hashing).
|
||||||
pub blake3: [u8; 32],
|
pub blake3: [u8; 32],
|
||||||
|
/// UUID (16 bytes) identifying the session that produced this revision.
|
||||||
|
/// Zero-filled if [`feature_flags::PROVENANCE`] is not set.
|
||||||
pub session_uuid: [u8; 16],
|
pub session_uuid: [u8; 16],
|
||||||
|
/// Byte offset into the annotation heap for this revision's annotation string;
|
||||||
|
/// `0` means no annotation.
|
||||||
pub annotation_off: u64,
|
pub annotation_off: u64,
|
||||||
|
/// Per-revision bit flags; see `REV_FLAG_*` constants.
|
||||||
pub flags: u8,
|
pub flags: u8,
|
||||||
|
/// Padding after `flags`, also used by GC to store an epoch tag in bytes `[0..4]`.
|
||||||
pub _pad_flags: [u8; 7],
|
pub _pad_flags: [u8; 7],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Flag bit indicating that this revision is a full-state snapshot checkpoint.
|
||||||
|
///
|
||||||
|
/// Set in `RevisionEntry::flags`. When this bit is set the revision stores a
|
||||||
|
/// complete copy of all HDF5 pages (not just deltas), making it a fast restore
|
||||||
|
/// point that does not require replaying earlier revisions.
|
||||||
pub const REV_FLAG_SNAPSHOT: u8 = 1 << 0;
|
pub const REV_FLAG_SNAPSHOT: u8 = 1 << 0;
|
||||||
|
|
||||||
/// Sentinel epoch value written into `RevisionEntry._pad_flags[0..4]` by
|
/// Sentinel epoch value written into `RevisionEntry._pad_flags[0..4]` by
|
||||||
@@ -230,14 +320,25 @@ impl RevisionEntry {
|
|||||||
// [32..40) created_at f64
|
// [32..40) created_at f64
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Fixed 40-byte record describing one named branch.
|
||||||
|
///
|
||||||
|
/// Stored in the branch table starting at [`OnionHeader::branch_offset`].
|
||||||
|
/// Branch `0` is always `main`.
|
||||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
||||||
#[repr(C)]
|
#[repr(C)]
|
||||||
pub struct BranchEntry {
|
pub struct BranchEntry {
|
||||||
|
/// Unique branch identifier; `0` = `main`.
|
||||||
pub id: u32,
|
pub id: u32,
|
||||||
|
/// Padding after `id` to keep `name_off` at a `u64` boundary.
|
||||||
pub _pad_id: [u8; 4],
|
pub _pad_id: [u8; 4],
|
||||||
|
/// Byte offset into the annotation heap for the branch name string.
|
||||||
pub name_off: u64,
|
pub name_off: u64,
|
||||||
|
/// Revision number of the current HEAD on this branch.
|
||||||
pub head_rev: u64,
|
pub head_rev: u64,
|
||||||
|
/// Revision at which this branch was forked from its parent;
|
||||||
|
/// [`NO_PARENT`] for the root (`main`) branch.
|
||||||
pub fork_rev: u64,
|
pub fork_rev: u64,
|
||||||
|
/// Unix timestamp (seconds since epoch, as `f64`) when the branch was created.
|
||||||
pub created_at: f64,
|
pub created_at: f64,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -252,14 +353,25 @@ pub struct BranchEntry {
|
|||||||
// [25..32) _pad [u8; 7]
|
// [25..32) _pad [u8; 7]
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Fixed 32-byte record mapping one HDF5 page to its stored data.
|
||||||
|
///
|
||||||
|
/// Each [`RevisionEntry`] is followed by `page_count` of these records in the
|
||||||
|
/// page table. Pages are ordered by `h5_offset` (ascending).
|
||||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
||||||
#[repr(C)]
|
#[repr(C)]
|
||||||
pub struct PageTableEntry {
|
pub struct PageTableEntry {
|
||||||
|
/// Byte offset within the logical HDF5 file that this page covers.
|
||||||
pub h5_offset: u64,
|
pub h5_offset: u64,
|
||||||
|
/// Byte offset within the `.onion` file where the (possibly compressed)
|
||||||
|
/// page data begins.
|
||||||
pub data_offset: u64,
|
pub data_offset: u64,
|
||||||
|
/// Original (uncompressed) page size in bytes.
|
||||||
pub orig_size: u32,
|
pub orig_size: u32,
|
||||||
|
/// Stored (possibly compressed) data size in bytes.
|
||||||
pub data_size: u32,
|
pub data_size: u32,
|
||||||
|
/// Compression codec byte; parse with [`Codec::from_u8`].
|
||||||
pub codec: u8,
|
pub codec: u8,
|
||||||
|
/// Alignment padding; always zero.
|
||||||
pub _pad: [u8; 7],
|
pub _pad: [u8; 7],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -125,8 +125,11 @@ impl<'a> Iterator for AncestorIter<'a> {
|
|||||||
/// Summary of branch head state, used by public query APIs.
|
/// Summary of branch head state, used by public query APIs.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct BranchHeadInfo {
|
pub struct BranchHeadInfo {
|
||||||
|
/// Unique numeric branch identifier (`0` = `main`).
|
||||||
pub branch_id: u32,
|
pub branch_id: u32,
|
||||||
|
/// Revision number of the current HEAD on this branch.
|
||||||
pub head_rev: u64,
|
pub head_rev: u64,
|
||||||
|
/// Total number of revisions on this branch.
|
||||||
pub revision_count: usize,
|
pub revision_count: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -66,11 +66,24 @@ pub struct MerkleNode {
|
|||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
pub enum WalkStep {
|
pub enum WalkStep {
|
||||||
/// Subtree [rev_lo, rev_hi] is identical on both sides — skip.
|
/// Subtree [rev_lo, rev_hi] is identical on both sides — skip.
|
||||||
Skip { rev_lo: u64, rev_hi: u64 },
|
Skip {
|
||||||
|
/// Lowest revision number covered by this identical subtree.
|
||||||
|
rev_lo: u64,
|
||||||
|
/// Highest revision number covered by this identical subtree.
|
||||||
|
rev_hi: u64,
|
||||||
|
},
|
||||||
/// Subtree differs; descend into children.
|
/// Subtree differs; descend into children.
|
||||||
Descend { node: MerkleNode },
|
Descend {
|
||||||
|
/// The tree node whose children should be diffed next.
|
||||||
|
node: MerkleNode,
|
||||||
|
},
|
||||||
/// Leaf differs; this revision needs to be transferred.
|
/// Leaf differs; this revision needs to be transferred.
|
||||||
Leaf { revision: u64, hash: [u8; 32] },
|
Leaf {
|
||||||
|
/// The revision number that differs between the two trees.
|
||||||
|
revision: u64,
|
||||||
|
/// BLAKE3 hash of this revision's pages on the local side.
|
||||||
|
hash: [u8; 32],
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Balanced binary Merkle tree over OnionFile revision BLAKE3 hashes.
|
/// Balanced binary Merkle tree over OnionFile revision BLAKE3 hashes.
|
||||||
@@ -339,17 +352,27 @@ fn node_hash(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] {
|
|||||||
// Error type
|
// Error type
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Errors from Merkle tree serialisation/deserialisation.
|
||||||
/// Errors from Merkle tree serialisation/deserialisation.
|
/// Errors from Merkle tree serialisation/deserialisation.
|
||||||
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
|
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
|
||||||
pub enum MerkleError {
|
pub enum MerkleError {
|
||||||
|
/// The byte slice is too short to contain even the 13-byte header.
|
||||||
#[error("truncated header (< 13 bytes)")]
|
#[error("truncated header (< 13 bytes)")]
|
||||||
TruncatedHeader,
|
TruncatedHeader,
|
||||||
|
/// The first 4 bytes do not match the expected magic sequence.
|
||||||
#[error("bad magic bytes")]
|
#[error("bad magic bytes")]
|
||||||
BadMagic,
|
BadMagic,
|
||||||
|
/// The format version byte is not supported by this reader.
|
||||||
#[error("unknown format version {0}")]
|
#[error("unknown format version {0}")]
|
||||||
UnknownVersion(u8),
|
UnknownVersion(u8),
|
||||||
|
/// The payload is shorter than the length declared in the header.
|
||||||
#[error("truncated payload: expected {expected} bytes, got {got}")]
|
#[error("truncated payload: expected {expected} bytes, got {got}")]
|
||||||
TruncatedPayload { expected: usize, got: usize },
|
TruncatedPayload {
|
||||||
|
/// Number of payload bytes declared in the header.
|
||||||
|
expected: usize,
|
||||||
|
/// Number of bytes actually available.
|
||||||
|
got: usize,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
@@ -540,7 +563,7 @@ mod tests {
|
|||||||
let all_entries: Vec<(u64, [u8; 32])> = (0..total as u64)
|
let all_entries: Vec<(u64, [u8; 32])> = (0..total as u64)
|
||||||
.map(|r| (r, rev_hash(r)))
|
.map(|r| (r, rev_hash(r)))
|
||||||
.collect();
|
.collect();
|
||||||
let keep_count = (total as f64 * keep_frac) as usize;
|
let keep_count = ((total as f64 * keep_frac) as usize).max(0);
|
||||||
let remote_entries = &all_entries[..keep_count];
|
let remote_entries = &all_entries[..keep_count];
|
||||||
|
|
||||||
let local = RevisionMerkleTree::build(&all_entries);
|
let local = RevisionMerkleTree::build(&all_entries);
|
||||||
|
|||||||
@@ -446,6 +446,9 @@ impl OnionFile {
|
|||||||
self.branches.iter().find(|b| b.id == id)
|
self.branches.iter().find(|b| b.id == id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Look up a `BranchEntry` by its human-readable name.
|
||||||
|
///
|
||||||
|
/// Returns `None` if no branch with that name exists.
|
||||||
pub fn branch_by_name(&self, name: &str) -> Option<&BranchEntry> {
|
pub fn branch_by_name(&self, name: &str) -> Option<&BranchEntry> {
|
||||||
self.branches
|
self.branches
|
||||||
.iter()
|
.iter()
|
||||||
@@ -674,12 +677,19 @@ impl OnionFile {
|
|||||||
/// Human-readable summary of a single revision, returned by [`OnionFile::list_revisions`].
|
/// Human-readable summary of a single revision, returned by [`OnionFile::list_revisions`].
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct RevisionSummary {
|
pub struct RevisionSummary {
|
||||||
|
/// Monotonically increasing revision number (0-based).
|
||||||
pub revision: u64,
|
pub revision: u64,
|
||||||
|
/// ID of the branch this revision belongs to; `0` = `main`.
|
||||||
pub branch_id: u32,
|
pub branch_id: u32,
|
||||||
|
/// Revision number of the direct parent; `u64::MAX` for the initial commit.
|
||||||
pub parent_rev: u64,
|
pub parent_rev: u64,
|
||||||
|
/// Unix timestamp (seconds since epoch, as `f64`) when the revision was committed.
|
||||||
pub timestamp: f64,
|
pub timestamp: f64,
|
||||||
|
/// Number of page entries recorded for this revision.
|
||||||
pub page_count: u32,
|
pub page_count: u32,
|
||||||
|
/// Optional human-readable annotation stored with the revision.
|
||||||
pub annotation: Option<String>,
|
pub annotation: Option<String>,
|
||||||
|
/// BLAKE3 hex digest of this revision's page data.
|
||||||
pub blake3_hex: String,
|
pub blake3_hex: String,
|
||||||
/// `true` if this revision is a full-state snapshot checkpoint.
|
/// `true` if this revision is a full-state snapshot checkpoint.
|
||||||
pub is_snapshot: bool,
|
pub is_snapshot: bool,
|
||||||
|
|||||||
@@ -34,8 +34,11 @@ use crate::error::AgentSyncError;
|
|||||||
/// Statistics from a push or pull operation.
|
/// Statistics from a push or pull operation.
|
||||||
#[derive(Debug, Default, Clone)]
|
#[derive(Debug, Default, Clone)]
|
||||||
pub struct SyncStats {
|
pub struct SyncStats {
|
||||||
|
/// Number of revisions actually sent or received.
|
||||||
pub revisions_transferred: u64,
|
pub revisions_transferred: u64,
|
||||||
|
/// Total compressed bytes sent or received.
|
||||||
pub bytes_transferred: u64,
|
pub bytes_transferred: u64,
|
||||||
|
/// Revisions that were already present on the remote and therefore skipped.
|
||||||
pub revisions_skipped: u64,
|
pub revisions_skipped: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -95,6 +98,10 @@ pub struct TcpSyncBackend {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl TcpSyncBackend {
|
impl TcpSyncBackend {
|
||||||
|
/// Create a new backend targeting `remote_addr`.
|
||||||
|
///
|
||||||
|
/// `agent_id` is embedded in all protocol messages to identify the local
|
||||||
|
/// agent to the remote server.
|
||||||
pub fn new(remote_addr: SocketAddr, agent_id: impl Into<String>) -> Self {
|
pub fn new(remote_addr: SocketAddr, agent_id: impl Into<String>) -> Self {
|
||||||
Self {
|
Self {
|
||||||
remote_addr,
|
remote_addr,
|
||||||
|
|||||||
@@ -2,29 +2,38 @@
|
|||||||
|
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
|
/// All errors that can occur in the `clawsync-agent` autonomous sync layer.
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
pub enum AgentSyncError {
|
pub enum AgentSyncError {
|
||||||
|
/// An error from the `clawhdf5-onion` VFD (e.g., opening or writing the sidecar).
|
||||||
#[error("onion VFD error: {0}")]
|
#[error("onion VFD error: {0}")]
|
||||||
Onion(#[from] clawhdf5_onion::OnionError),
|
Onion(#[from] clawhdf5_onion::OnionError),
|
||||||
|
|
||||||
|
/// An error propagated from the `clawsync-onion` diffing/merging layer.
|
||||||
#[error("sync-onion error: {0}")]
|
#[error("sync-onion error: {0}")]
|
||||||
SyncOnion(#[from] clawsync_onion::error::SyncOnionError),
|
SyncOnion(#[from] clawsync_onion::error::SyncOnionError),
|
||||||
|
|
||||||
|
/// An error propagated from the `clawsync-transport` layer.
|
||||||
#[error("transport error: {0}")]
|
#[error("transport error: {0}")]
|
||||||
Transport(#[from] clawsync_transport::error::TransportError),
|
Transport(#[from] clawsync_transport::error::TransportError),
|
||||||
|
|
||||||
|
/// An underlying I/O error.
|
||||||
#[error("I/O error: {0}")]
|
#[error("I/O error: {0}")]
|
||||||
Io(#[from] std::io::Error),
|
Io(#[from] std::io::Error),
|
||||||
|
|
||||||
|
/// The remote peer reported an application-level error.
|
||||||
#[error("remote error: {0}")]
|
#[error("remote error: {0}")]
|
||||||
Remote(String),
|
Remote(String),
|
||||||
|
|
||||||
|
/// The remote peer violated the sync protocol.
|
||||||
#[error("protocol error: {0}")]
|
#[error("protocol error: {0}")]
|
||||||
Protocol(String),
|
Protocol(String),
|
||||||
|
|
||||||
|
/// An error from the `clawhdf5-agent` memory layer (save, compact, WAL flush).
|
||||||
#[error("agent memory error: {0}")]
|
#[error("agent memory error: {0}")]
|
||||||
AgentMemory(String),
|
AgentMemory(String),
|
||||||
|
|
||||||
|
/// A restore was requested for a revision that does not exist in the sidecar.
|
||||||
#[error("no revisions to restore at revision {0}")]
|
#[error("no revisions to restore at revision {0}")]
|
||||||
RevisionNotFound(u64),
|
RevisionNotFound(u64),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -37,9 +37,13 @@ pub struct OnionMemory {
|
|||||||
/// Summary of a historical snapshot.
|
/// Summary of a historical snapshot.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct MemorySnapshot {
|
pub struct MemorySnapshot {
|
||||||
|
/// Revision number within the `.onion` sidecar.
|
||||||
pub revision: u64,
|
pub revision: u64,
|
||||||
|
/// Optional human-readable annotation stored when the snapshot was committed.
|
||||||
pub annotation: Option<String>,
|
pub annotation: Option<String>,
|
||||||
|
/// Unix timestamp (seconds since epoch, as `f64`) when the snapshot was committed.
|
||||||
pub timestamp: f64,
|
pub timestamp: f64,
|
||||||
|
/// BLAKE3 hex digest of the snapshotted HDF5 page data.
|
||||||
pub blake3_hex: String,
|
pub blake3_hex: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -58,10 +58,8 @@ fn cold_data(n: usize) -> Vec<u8> {
|
|||||||
// Benchmark
|
// Benchmark
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
type DataGen = fn(usize) -> Vec<u8>;
|
|
||||||
|
|
||||||
fn bench_cdc(c: &mut Criterion) {
|
fn bench_cdc(c: &mut Criterion) {
|
||||||
let datasets: &[(&str, DataGen)] = &[
|
let datasets: &[(&str, fn(usize) -> Vec<u8>)] = &[
|
||||||
("random", random_data),
|
("random", random_data),
|
||||||
("float", float_data),
|
("float", float_data),
|
||||||
("cold_only", cold_data),
|
("cold_only", cold_data),
|
||||||
|
|||||||
@@ -44,9 +44,17 @@ pub fn compute_block_hashes(data: &[u8], block_size: usize) -> Vec<BlockHash> {
|
|||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
pub enum DeltaOp {
|
pub enum DeltaOp {
|
||||||
/// Copy `length` bytes starting at `offset` from the source.
|
/// Copy `length` bytes starting at `offset` from the source.
|
||||||
Copy { offset: usize, length: usize },
|
Copy {
|
||||||
|
/// Byte offset in the source buffer to copy from.
|
||||||
|
offset: usize,
|
||||||
|
/// Number of bytes to copy.
|
||||||
|
length: usize,
|
||||||
|
},
|
||||||
/// Insert the given literal bytes (no corresponding source region).
|
/// Insert the given literal bytes (no corresponding source region).
|
||||||
Insert { data: Vec<u8> },
|
Insert {
|
||||||
|
/// Literal bytes to insert at the current output position.
|
||||||
|
data: Vec<u8>,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Compute the delta needed to transform `source` into `target`.
|
/// Compute the delta needed to transform `source` into `target`.
|
||||||
|
|||||||
@@ -2,14 +2,24 @@
|
|||||||
|
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
|
/// All errors that can occur in the `clawsync-core` low-level layer.
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
pub enum CoreError {
|
pub enum CoreError {
|
||||||
|
/// A compression operation (zstd, lz4, brotli, or ZstdTdt) failed.
|
||||||
#[error("compression failed: {0}")]
|
#[error("compression failed: {0}")]
|
||||||
Compress(String),
|
Compress(String),
|
||||||
|
/// A decompression operation failed (malformed or truncated input).
|
||||||
#[error("decompression failed: {0}")]
|
#[error("decompression failed: {0}")]
|
||||||
Decompress(String),
|
Decompress(String),
|
||||||
|
/// The BLAKE3 hash of a block does not match the stored/expected value.
|
||||||
#[error("checksum mismatch: expected {expected}, got {actual}")]
|
#[error("checksum mismatch: expected {expected}, got {actual}")]
|
||||||
ChecksumMismatch { expected: String, actual: String },
|
ChecksumMismatch {
|
||||||
|
/// The BLAKE3 hex string that was expected.
|
||||||
|
expected: String,
|
||||||
|
/// The BLAKE3 hex string that was computed.
|
||||||
|
actual: String,
|
||||||
|
},
|
||||||
|
/// An underlying I/O error.
|
||||||
#[error("I/O error: {0}")]
|
#[error("I/O error: {0}")]
|
||||||
Io(#[from] std::io::Error),
|
Io(#[from] std::io::Error),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -114,6 +114,11 @@ pub fn chunk_data_sized_simd(
|
|||||||
// Scalar fallback
|
// Scalar fallback
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Scalar (non-SIMD) content-defined chunking fallback.
|
||||||
|
///
|
||||||
|
/// Splits `data` into variable-size chunks using a rolling hash with the
|
||||||
|
/// Rabin fingerprint approach. `min`, `avg`, and `max` are the minimum,
|
||||||
|
/// target average, and maximum chunk sizes in bytes.
|
||||||
pub fn chunk_scalar(data: &[u8], min: usize, avg: usize, max: usize) -> Vec<Chunk> {
|
pub fn chunk_scalar(data: &[u8], min: usize, avg: usize, max: usize) -> Vec<Chunk> {
|
||||||
if data.is_empty() {
|
if data.is_empty() {
|
||||||
return Vec::new();
|
return Vec::new();
|
||||||
@@ -151,9 +156,6 @@ pub fn chunk_scalar(data: &[u8], min: usize, avg: usize, max: usize) -> Vec<Chun
|
|||||||
/// Scalar processing for a single 16-byte (or shorter) slice, folding the
|
/// Scalar processing for a single 16-byte (or shorter) slice, folding the
|
||||||
/// updated hash back out. Used by both SIMD paths when a hot byte is detected.
|
/// updated hash back out. Used by both SIMD paths when a hot byte is detected.
|
||||||
#[inline(always)]
|
#[inline(always)]
|
||||||
// Hot SIMD inner-loop helper: each arg is a distinct cursor/limit; bundling
|
|
||||||
// into a struct would add per-call overhead in the chunker fast path.
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
|
||||||
fn scalar_window(
|
fn scalar_window(
|
||||||
data: &[u8],
|
data: &[u8],
|
||||||
window: &[u8],
|
window: &[u8],
|
||||||
@@ -545,7 +547,7 @@ mod tests {
|
|||||||
let original = chunk_data_simd(&data);
|
let original = chunk_data_simd(&data);
|
||||||
// Flip a hot byte in the middle to force a different boundary
|
// Flip a hot byte in the middle to force a different boundary
|
||||||
let mid = 150_000;
|
let mid = 150_000;
|
||||||
data[mid] &= 0x3F; // ensure it's a hot byte (< 64)
|
data[mid] = data[mid] & 0x3F; // ensure it's a hot byte (< 64)
|
||||||
data[mid] ^= 0x11;
|
data[mid] ^= 0x11;
|
||||||
let mutated = chunk_data_simd(&data);
|
let mutated = chunk_data_simd(&data);
|
||||||
assert_ne!(original, mutated, "mutation should change chunk boundaries");
|
assert_ne!(original, mutated, "mutation should change chunk boundaries");
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ pub enum FileDiff {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl FileDiff {
|
impl FileDiff {
|
||||||
|
/// Return the relative path for any variant.
|
||||||
pub fn path(&self) -> &str {
|
pub fn path(&self) -> &str {
|
||||||
match self {
|
match self {
|
||||||
Self::Added(p) | Self::Modified(p) | Self::Removed(p) | Self::Unchanged(p) => p,
|
Self::Added(p) | Self::Modified(p) | Self::Removed(p) | Self::Unchanged(p) => p,
|
||||||
|
|||||||
@@ -1,29 +1,47 @@
|
|||||||
|
//! Error types for `clawsync-fs`.
|
||||||
|
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
|
/// All errors that can occur in the `clawsync-fs` filesystem sync layer.
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
pub enum FsSyncError {
|
pub enum FsSyncError {
|
||||||
|
/// An underlying I/O error (file read/write, directory traversal, etc.).
|
||||||
#[error("I/O error: {0}")]
|
#[error("I/O error: {0}")]
|
||||||
Io(#[from] std::io::Error),
|
Io(#[from] std::io::Error),
|
||||||
|
|
||||||
|
/// An error propagated from the `clawsync-transport` layer.
|
||||||
#[error("transport error: {0}")]
|
#[error("transport error: {0}")]
|
||||||
Transport(#[from] clawsync_transport::TransportError),
|
Transport(#[from] clawsync_transport::TransportError),
|
||||||
|
|
||||||
|
/// A file or directory path is invalid or could not be resolved.
|
||||||
#[error("path error: {0}")]
|
#[error("path error: {0}")]
|
||||||
Path(String),
|
Path(String),
|
||||||
|
|
||||||
|
/// The remote peer sent an unexpected message type during an fs-sync exchange.
|
||||||
#[error("protocol error: unexpected message: {0}")]
|
#[error("protocol error: unexpected message: {0}")]
|
||||||
Protocol(String),
|
Protocol(String),
|
||||||
|
|
||||||
|
/// The BLAKE3 checksum of a received file does not match the expected value.
|
||||||
#[error("checksum mismatch for {path}: expected {expected}, got {actual}")]
|
#[error("checksum mismatch for {path}: expected {expected}, got {actual}")]
|
||||||
ChecksumMismatch {
|
ChecksumMismatch {
|
||||||
|
/// The file path whose checksum failed verification.
|
||||||
path: String,
|
path: String,
|
||||||
|
/// The BLAKE3 hex string that was expected.
|
||||||
expected: String,
|
expected: String,
|
||||||
|
/// The BLAKE3 hex string computed from the received bytes.
|
||||||
actual: String,
|
actual: String,
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/// A file could not be reassembled from the received chunks.
|
||||||
#[error("reconstruction failed for {path}: {reason}")]
|
#[error("reconstruction failed for {path}: {reason}")]
|
||||||
Reconstruction { path: String, reason: String },
|
Reconstruction {
|
||||||
|
/// The file path that failed to reconstruct.
|
||||||
|
path: String,
|
||||||
|
/// A human-readable description of the reconstruction failure.
|
||||||
|
reason: String,
|
||||||
|
},
|
||||||
|
|
||||||
|
/// A Tokio task panicked or was cancelled during an async fs-sync operation.
|
||||||
#[error("task join error: {0}")]
|
#[error("task join error: {0}")]
|
||||||
Join(String),
|
Join(String),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -158,6 +158,7 @@ pub struct LocalEntry {
|
|||||||
|
|
||||||
/// In-memory manifest of a local directory tree, sorted by `rel_path`.
|
/// In-memory manifest of a local directory tree, sorted by `rel_path`.
|
||||||
pub struct FsManifest {
|
pub struct FsManifest {
|
||||||
|
/// All file entries in the directory tree, sorted by `rel_path`.
|
||||||
pub entries: Vec<LocalEntry>,
|
pub entries: Vec<LocalEntry>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -56,8 +56,11 @@ pub enum ProgressEvent {
|
|||||||
/// fields are `None`.
|
/// fields are `None`.
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
pub struct SyncStats {
|
pub struct SyncStats {
|
||||||
|
/// Number of files newly created on the destination.
|
||||||
pub files_added: u32,
|
pub files_added: u32,
|
||||||
|
/// Number of files overwritten on the destination.
|
||||||
pub files_modified: u32,
|
pub files_modified: u32,
|
||||||
|
/// Number of files deleted from the destination.
|
||||||
pub files_removed: u32,
|
pub files_removed: u32,
|
||||||
/// Actual bytes sent over the wire (compressed literal chunks only).
|
/// Actual bytes sent over the wire (compressed literal chunks only).
|
||||||
pub bytes_transferred: u64,
|
pub bytes_transferred: u64,
|
||||||
@@ -373,6 +376,10 @@ pub struct FsSyncServer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl FsSyncServer {
|
impl FsSyncServer {
|
||||||
|
/// Create a new push server.
|
||||||
|
///
|
||||||
|
/// `allow_delete` controls whether the server will honour delete requests
|
||||||
|
/// from the client (i.e., remove files absent from the client's manifest).
|
||||||
pub fn new(conn: SyncPeer, serve_root: PathBuf, excludes: GlobSet, allow_delete: bool) -> Self {
|
pub fn new(conn: SyncPeer, serve_root: PathBuf, excludes: GlobSet, allow_delete: bool) -> Self {
|
||||||
Self {
|
Self {
|
||||||
conn,
|
conn,
|
||||||
@@ -756,6 +763,10 @@ pub struct FsSyncPullClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl FsSyncPullClient {
|
impl FsSyncPullClient {
|
||||||
|
/// Create a new pull client.
|
||||||
|
///
|
||||||
|
/// `delete` controls whether local files absent from the server's manifest
|
||||||
|
/// are removed after the pull completes.
|
||||||
pub fn new(conn: SyncPeer, local_root: PathBuf, excludes: GlobSet, delete: bool) -> Self {
|
pub fn new(conn: SyncPeer, local_root: PathBuf, excludes: GlobSet, delete: bool) -> Self {
|
||||||
Self {
|
Self {
|
||||||
conn,
|
conn,
|
||||||
@@ -972,6 +983,10 @@ pub struct FsSyncPullServer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl FsSyncPullServer {
|
impl FsSyncPullServer {
|
||||||
|
/// Create a new pull server.
|
||||||
|
///
|
||||||
|
/// `allow_delete` controls whether the server will accept client requests
|
||||||
|
/// to delete files from the server's serve root during a pull session.
|
||||||
pub fn new(conn: SyncPeer, serve_root: PathBuf, excludes: GlobSet, allow_delete: bool) -> Self {
|
pub fn new(conn: SyncPeer, serve_root: PathBuf, excludes: GlobSet, allow_delete: bool) -> Self {
|
||||||
Self {
|
Self {
|
||||||
conn,
|
conn,
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ pub struct DatasetPatch {
|
|||||||
/// Result of diffing two manifests.
|
/// Result of diffing two manifests.
|
||||||
#[derive(Debug, Default)]
|
#[derive(Debug, Default)]
|
||||||
pub struct DiffResult {
|
pub struct DiffResult {
|
||||||
|
/// Ordered list of patches (Added, Modified, Removed) to apply to the target.
|
||||||
pub patches: Vec<DatasetPatch>,
|
pub patches: Vec<DatasetPatch>,
|
||||||
/// Number of datasets that are identical in both manifests.
|
/// Number of datasets that are identical in both manifests.
|
||||||
pub unchanged_count: usize,
|
pub unchanged_count: usize,
|
||||||
|
|||||||
@@ -2,24 +2,33 @@
|
|||||||
|
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
|
/// All errors that can occur in the `clawsync-hdf5` HDF5 sync layer.
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
pub enum Hdf5SyncError {
|
pub enum Hdf5SyncError {
|
||||||
|
/// An error from the `clawhdf5` HDF5 parser or builder.
|
||||||
#[error("HDF5 error: {0}")]
|
#[error("HDF5 error: {0}")]
|
||||||
Hdf5(#[from] clawhdf5::Error),
|
Hdf5(#[from] clawhdf5::Error),
|
||||||
|
|
||||||
|
/// An underlying I/O error (file read/write, temp-file rename, etc.).
|
||||||
#[error("I/O error: {0}")]
|
#[error("I/O error: {0}")]
|
||||||
Io(#[from] std::io::Error),
|
Io(#[from] std::io::Error),
|
||||||
|
|
||||||
|
/// A dataset path referenced in a patch does not exist in the target file.
|
||||||
#[error("dataset not found: {0}")]
|
#[error("dataset not found: {0}")]
|
||||||
DatasetNotFound(String),
|
DatasetNotFound(String),
|
||||||
|
|
||||||
|
/// A patch attempted to overwrite a dataset with data of a different dtype.
|
||||||
#[error("patch type mismatch for dataset '{path}': expected {expected}, got {actual}")]
|
#[error("patch type mismatch for dataset '{path}': expected {expected}, got {actual}")]
|
||||||
TypeMismatch {
|
TypeMismatch {
|
||||||
|
/// HDF5 path of the mismatched dataset (e.g., `/group/weights`).
|
||||||
path: String,
|
path: String,
|
||||||
|
/// The dtype present in the target file.
|
||||||
expected: String,
|
expected: String,
|
||||||
|
/// The dtype carried by the incoming patch.
|
||||||
actual: String,
|
actual: String,
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/// A `DatasetManifest` could not be serialized or deserialized.
|
||||||
#[error("manifest serialization error: {0}")]
|
#[error("manifest serialization error: {0}")]
|
||||||
Serialize(String),
|
Serialize(String),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,10 +24,15 @@ use crate::error::Hdf5SyncError;
|
|||||||
/// Statistics from a patch operation.
|
/// Statistics from a patch operation.
|
||||||
#[derive(Debug, Default, Clone)]
|
#[derive(Debug, Default, Clone)]
|
||||||
pub struct PatchStats {
|
pub struct PatchStats {
|
||||||
|
/// Number of new datasets added to the target file.
|
||||||
pub datasets_added: u32,
|
pub datasets_added: u32,
|
||||||
|
/// Number of existing datasets overwritten with new data.
|
||||||
pub datasets_modified: u32,
|
pub datasets_modified: u32,
|
||||||
|
/// Number of datasets deleted from the target file.
|
||||||
pub datasets_removed: u32,
|
pub datasets_removed: u32,
|
||||||
|
/// Number of datasets copied unchanged from the old target file.
|
||||||
pub datasets_unchanged: u32,
|
pub datasets_unchanged: u32,
|
||||||
|
/// Total bytes written to the rebuilt target file.
|
||||||
pub bytes_written: u64,
|
pub bytes_written: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,36 +2,54 @@
|
|||||||
|
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
|
/// All errors that can occur in the `clawsync-onion` sync layer.
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
pub enum SyncOnionError {
|
pub enum SyncOnionError {
|
||||||
|
/// An error propagated from the underlying `clawhdf5-onion` VFD.
|
||||||
#[error("onion error: {0}")]
|
#[error("onion error: {0}")]
|
||||||
Onion(#[from] clawhdf5_onion::OnionError),
|
Onion(#[from] clawhdf5_onion::OnionError),
|
||||||
|
|
||||||
|
/// An error propagated from `clawsync-core` (compression, checksums, I/O).
|
||||||
#[error("core error: {0}")]
|
#[error("core error: {0}")]
|
||||||
Core(#[from] clawsync_core::CoreError),
|
Core(#[from] clawsync_core::CoreError),
|
||||||
|
|
||||||
|
/// An underlying I/O error.
|
||||||
#[error("I/O error: {0}")]
|
#[error("I/O error: {0}")]
|
||||||
Io(#[from] std::io::Error),
|
Io(#[from] std::io::Error),
|
||||||
|
|
||||||
|
/// The requested revision number does not exist in the source `.onion` file.
|
||||||
#[error("revision {0} not found in source")]
|
#[error("revision {0} not found in source")]
|
||||||
RevisionNotFound(u64),
|
RevisionNotFound(u64),
|
||||||
|
|
||||||
|
/// The named branch does not exist in the source `.onion` file.
|
||||||
#[error("branch not found: {0}")]
|
#[error("branch not found: {0}")]
|
||||||
BranchNotFound(String),
|
BranchNotFound(String),
|
||||||
|
|
||||||
|
/// The BLAKE3 hash of a received revision's pages does not match the expected value.
|
||||||
#[error("BLAKE3 mismatch on revision {revision}: expected {expected}, got {actual}")]
|
#[error("BLAKE3 mismatch on revision {revision}: expected {expected}, got {actual}")]
|
||||||
HashMismatch {
|
HashMismatch {
|
||||||
|
/// The revision number whose pages failed the integrity check.
|
||||||
revision: u64,
|
revision: u64,
|
||||||
|
/// The BLAKE3 hex string that was expected.
|
||||||
expected: String,
|
expected: String,
|
||||||
|
/// The BLAKE3 hex string that was computed from the received pages.
|
||||||
actual: String,
|
actual: String,
|
||||||
},
|
},
|
||||||
|
|
||||||
|
/// A `LayerPacket` could not be serialized or deserialized.
|
||||||
#[error("packet serialization error: {0}")]
|
#[error("packet serialization error: {0}")]
|
||||||
Serialization(String),
|
Serialization(String),
|
||||||
|
|
||||||
|
/// A merge was attempted with an empty set of packets.
|
||||||
#[error("empty packet set — nothing to merge")]
|
#[error("empty packet set — nothing to merge")]
|
||||||
EmptyPackets,
|
EmptyPackets,
|
||||||
|
|
||||||
|
/// Revisions arrived out of order during a merge — a gap was detected.
|
||||||
#[error("revision gap: received {got}, expected {expected}")]
|
#[error("revision gap: received {got}, expected {expected}")]
|
||||||
RevisionGap { expected: u64, got: u64 },
|
RevisionGap {
|
||||||
|
/// The next consecutive revision that was expected.
|
||||||
|
expected: u64,
|
||||||
|
/// The revision number that was actually received.
|
||||||
|
got: u64,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -100,12 +100,16 @@ pub enum IbltDecodeResult {
|
|||||||
// Error type
|
// Error type
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Errors from IBLT sketch serialisation/deserialisation.
|
||||||
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
|
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
|
||||||
pub enum IbltError {
|
pub enum IbltError {
|
||||||
|
/// The byte slice is too short to contain a valid sketch header or payload.
|
||||||
#[error("serialised data too short")]
|
#[error("serialised data too short")]
|
||||||
Truncated,
|
Truncated,
|
||||||
|
/// The first bytes do not match the expected IBLT magic sequence.
|
||||||
#[error("bad magic bytes")]
|
#[error("bad magic bytes")]
|
||||||
BadMagic,
|
BadMagic,
|
||||||
|
/// The format version byte is not supported by this reader.
|
||||||
#[error("unknown version {0}")]
|
#[error("unknown version {0}")]
|
||||||
UnknownVersion(u8),
|
UnknownVersion(u8),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,12 +15,17 @@ use crate::iblt::{DEFAULT_HASH_COUNT, IbltDecodeResult, IbltDiff, IbltSketch};
|
|||||||
/// A compact summary of one revision — used in the manifest.
|
/// A compact summary of one revision — used in the manifest.
|
||||||
#[derive(Archive, Deserialize, Serialize, Debug, Clone, PartialEq)]
|
#[derive(Archive, Deserialize, Serialize, Debug, Clone, PartialEq)]
|
||||||
pub struct RevisionSummaryPacket {
|
pub struct RevisionSummaryPacket {
|
||||||
|
/// Monotonically increasing revision number (0-based).
|
||||||
pub revision: u64,
|
pub revision: u64,
|
||||||
|
/// ID of the branch this revision belongs to; `0` = `main`.
|
||||||
pub branch_id: u32,
|
pub branch_id: u32,
|
||||||
|
/// Revision number of the direct parent; `u64::MAX` for the initial commit.
|
||||||
pub parent_rev: u64,
|
pub parent_rev: u64,
|
||||||
|
/// Unix timestamp (seconds since epoch, as `f64`) when the revision was committed.
|
||||||
pub timestamp: f64,
|
pub timestamp: f64,
|
||||||
/// BLAKE3 hash of the pages in this revision.
|
/// BLAKE3 hash of the pages in this revision.
|
||||||
pub blake3: [u8; 32],
|
pub blake3: [u8; 32],
|
||||||
|
/// Optional human-readable annotation stored with the revision.
|
||||||
pub annotation: Option<String>,
|
pub annotation: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -119,11 +124,17 @@ impl ClawSyncManifest {
|
|||||||
/// For N=1 000: ~3 KB vs ~60 KB for `ClawSyncManifest`.
|
/// For N=1 000: ~3 KB vs ~60 KB for `ClawSyncManifest`.
|
||||||
#[derive(Archive, Deserialize, Serialize, Debug, Clone, PartialEq)]
|
#[derive(Archive, Deserialize, Serialize, Debug, Clone, PartialEq)]
|
||||||
pub struct IbltManifest {
|
pub struct IbltManifest {
|
||||||
|
/// Logical agent / file identifier (same as in `ClawSyncManifest`).
|
||||||
pub agent_id: String,
|
pub agent_id: String,
|
||||||
|
/// BLAKE3 hash of the raw `.h5` base file; zero-filled during pre-flight.
|
||||||
pub file_blake3: [u8; 32],
|
pub file_blake3: [u8; 32],
|
||||||
|
/// Total number of revisions in the sender's sidecar.
|
||||||
pub revision_count: u64,
|
pub revision_count: u64,
|
||||||
|
/// HEAD revision number on the sender's default branch.
|
||||||
pub head_revision: u64,
|
pub head_revision: u64,
|
||||||
|
/// BLAKE3 hash of the HEAD revision's pages; zero-filled during pre-flight.
|
||||||
pub head_blake3: [u8; 32],
|
pub head_blake3: [u8; 32],
|
||||||
|
/// Unix timestamp (seconds since epoch, as `f64`) of the last committed revision.
|
||||||
pub last_write: f64,
|
pub last_write: f64,
|
||||||
/// Serialised [`IbltSketch`] bytes.
|
/// Serialised [`IbltSketch`] bytes.
|
||||||
pub sketch: Vec<u8>,
|
pub sketch: Vec<u8>,
|
||||||
|
|||||||
@@ -2,32 +2,48 @@
|
|||||||
|
|
||||||
use thiserror::Error;
|
use thiserror::Error;
|
||||||
|
|
||||||
|
/// All errors that can occur in the `clawsync-transport` layer.
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
pub enum TransportError {
|
pub enum TransportError {
|
||||||
|
/// An underlying I/O error (socket read/write, bind, etc.).
|
||||||
#[error("I/O error: {0}")]
|
#[error("I/O error: {0}")]
|
||||||
Io(#[from] std::io::Error),
|
Io(#[from] std::io::Error),
|
||||||
|
|
||||||
|
/// The remote peer closed the connection before the exchange completed.
|
||||||
#[error("connection closed unexpectedly")]
|
#[error("connection closed unexpectedly")]
|
||||||
ConnectionClosed,
|
ConnectionClosed,
|
||||||
|
|
||||||
|
/// A received frame's length prefix exceeds the protocol maximum.
|
||||||
#[error("frame too large: {size} bytes (max {max})")]
|
#[error("frame too large: {size} bytes (max {max})")]
|
||||||
FrameTooLarge { size: usize, max: usize },
|
FrameTooLarge {
|
||||||
|
/// The length the remote declared in the 4-byte LE prefix.
|
||||||
|
size: usize,
|
||||||
|
/// The maximum frame size permitted by the protocol.
|
||||||
|
max: usize,
|
||||||
|
},
|
||||||
|
|
||||||
|
/// A `SyncMessage` could not be serialized to bytes.
|
||||||
#[error("serialization error: {0}")]
|
#[error("serialization error: {0}")]
|
||||||
Serialization(String),
|
Serialization(String),
|
||||||
|
|
||||||
|
/// Received bytes could not be deserialized into a `SyncMessage`.
|
||||||
#[error("deserialization error: {0}")]
|
#[error("deserialization error: {0}")]
|
||||||
Deserialization(String),
|
Deserialization(String),
|
||||||
|
|
||||||
|
/// An error from the underlying QUIC stack (e.g., `quinn`).
|
||||||
#[error("QUIC error: {0}")]
|
#[error("QUIC error: {0}")]
|
||||||
Quic(String),
|
Quic(String),
|
||||||
|
|
||||||
|
/// A TLS configuration or handshake error.
|
||||||
#[error("TLS error: {0}")]
|
#[error("TLS error: {0}")]
|
||||||
Tls(String),
|
Tls(String),
|
||||||
|
|
||||||
|
/// The remote peer violated the sync protocol (unexpected message type
|
||||||
|
/// or message sequencing error).
|
||||||
#[error("protocol error: {0}")]
|
#[error("protocol error: {0}")]
|
||||||
Protocol(String),
|
Protocol(String),
|
||||||
|
|
||||||
|
/// An error propagated from the `clawsync-onion` sync layer.
|
||||||
#[error("sync onion error: {0}")]
|
#[error("sync onion error: {0}")]
|
||||||
SyncOnion(#[from] clawsync_onion::SyncOnionError),
|
SyncOnion(#[from] clawsync_onion::SyncOnionError),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,10 +24,12 @@ pub struct FramedReader<R: AsyncRead + Unpin + Send> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl<R: AsyncRead + Unpin + Send> FramedReader<R> {
|
impl<R: AsyncRead + Unpin + Send> FramedReader<R> {
|
||||||
|
/// Wrap `inner` in a framed reader.
|
||||||
pub fn new(inner: R) -> Self {
|
pub fn new(inner: R) -> Self {
|
||||||
Self { inner }
|
Self { inner }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Read and deserialize the next length-prefixed `SyncMessage` frame.
|
||||||
pub async fn recv(&mut self) -> Result<SyncMessage, TransportError> {
|
pub async fn recv(&mut self) -> Result<SyncMessage, TransportError> {
|
||||||
let mut len_buf = [0u8; 4];
|
let mut len_buf = [0u8; 4];
|
||||||
self.inner.read_exact(&mut len_buf).await?;
|
self.inner.read_exact(&mut len_buf).await?;
|
||||||
@@ -52,10 +54,12 @@ pub struct FramedWriter<W: AsyncWrite + Unpin + Send> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl<W: AsyncWrite + Unpin + Send> FramedWriter<W> {
|
impl<W: AsyncWrite + Unpin + Send> FramedWriter<W> {
|
||||||
|
/// Wrap `inner` in a framed writer.
|
||||||
pub fn new(inner: W) -> Self {
|
pub fn new(inner: W) -> Self {
|
||||||
Self { inner }
|
Self { inner }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Serialize `msg` and write it as a length-prefixed frame.
|
||||||
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
||||||
let body = msg.to_bytes().map_err(TransportError::Serialization)?;
|
let body = msg.to_bytes().map_err(TransportError::Serialization)?;
|
||||||
let len = body.len() as u32;
|
let len = body.len() as u32;
|
||||||
@@ -65,6 +69,7 @@ impl<W: AsyncWrite + Unpin + Send> FramedWriter<W> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Flush pending writes and shut down the write half.
|
||||||
pub async fn shutdown(&mut self) -> Result<(), TransportError> {
|
pub async fn shutdown(&mut self) -> Result<(), TransportError> {
|
||||||
self.inner.shutdown().await?;
|
self.inner.shutdown().await?;
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -112,14 +117,17 @@ impl StreamPeer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Send a `SyncMessage` through the peer's write half.
|
||||||
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
||||||
self.writer.send(msg).await
|
self.writer.send(msg).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Receive the next `SyncMessage` from the peer's read half.
|
||||||
pub async fn recv(&mut self) -> Result<SyncMessage, TransportError> {
|
pub async fn recv(&mut self) -> Result<SyncMessage, TransportError> {
|
||||||
self.reader.recv().await
|
self.reader.recv().await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Flush and shut down the write half.
|
||||||
pub async fn shutdown(&mut self) -> Result<(), TransportError> {
|
pub async fn shutdown(&mut self) -> Result<(), TransportError> {
|
||||||
self.writer.shutdown().await
|
self.writer.shutdown().await
|
||||||
}
|
}
|
||||||
@@ -134,21 +142,26 @@ impl StreamPeer {
|
|||||||
// Split halves
|
// Split halves
|
||||||
// ─────────────────────────────────────────────────────────────────────────────
|
// ─────────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Read half produced by [`StreamPeer::into_split`].
|
||||||
pub struct StreamReadHalf(pub(crate) FramedReader<BoxReader>);
|
pub struct StreamReadHalf(pub(crate) FramedReader<BoxReader>);
|
||||||
|
|
||||||
impl StreamReadHalf {
|
impl StreamReadHalf {
|
||||||
|
/// Receive the next `SyncMessage` from the boxed reader.
|
||||||
pub async fn recv(&mut self) -> Result<SyncMessage, TransportError> {
|
pub async fn recv(&mut self) -> Result<SyncMessage, TransportError> {
|
||||||
self.0.recv().await
|
self.0.recv().await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Write half produced by [`StreamPeer::into_split`].
|
||||||
pub struct StreamWriteHalf(pub(crate) FramedWriter<BoxWriter>);
|
pub struct StreamWriteHalf(pub(crate) FramedWriter<BoxWriter>);
|
||||||
|
|
||||||
impl StreamWriteHalf {
|
impl StreamWriteHalf {
|
||||||
|
/// Send a `SyncMessage` through the boxed writer.
|
||||||
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
||||||
self.0.send(msg).await
|
self.0.send(msg).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Flush and shut down the boxed writer.
|
||||||
pub async fn shutdown(&mut self) -> Result<(), TransportError> {
|
pub async fn shutdown(&mut self) -> Result<(), TransportError> {
|
||||||
self.0.shutdown().await
|
self.0.shutdown().await
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -27,13 +27,17 @@ use crate::tcp::{TcpConnection, TcpReadHalf, TcpWriteHalf};
|
|||||||
|
|
||||||
/// A unified connection handle for a single sync session (TCP, QUIC, mmap, or stream).
|
/// A unified connection handle for a single sync session (TCP, QUIC, mmap, or stream).
|
||||||
pub enum SyncPeer {
|
pub enum SyncPeer {
|
||||||
|
/// An established async TCP connection.
|
||||||
Tcp(TcpConnection),
|
Tcp(TcpConnection),
|
||||||
|
/// A QUIC connection shared via `Arc` (send + recv are `&self`).
|
||||||
Quic(Arc<QuicConnection>),
|
Quic(Arc<QuicConnection>),
|
||||||
/// Local same-node transport backed by two memory-mapped ring buffers —
|
/// Local same-node transport backed by two memory-mapped ring buffers —
|
||||||
/// one per direction. `send_ch` is the outgoing channel; `recv_ch` is
|
/// one per direction. `send_ch` is the outgoing channel; `recv_ch` is
|
||||||
/// the incoming channel.
|
/// the incoming channel.
|
||||||
Mmap {
|
Mmap {
|
||||||
|
/// Outgoing mmap ring-buffer sender.
|
||||||
send_ch: MmapSender,
|
send_ch: MmapSender,
|
||||||
|
/// Incoming mmap ring-buffer receiver.
|
||||||
recv_ch: MmapReceiver,
|
recv_ch: MmapReceiver,
|
||||||
},
|
},
|
||||||
/// Generic boxed-I/O peer — covers SSH child-process pipes and `--stdio`
|
/// Generic boxed-I/O peer — covers SSH child-process pipes and `--stdio`
|
||||||
@@ -42,6 +46,11 @@ pub enum SyncPeer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl SyncPeer {
|
impl SyncPeer {
|
||||||
|
/// Send a `SyncMessage` to the remote peer.
|
||||||
|
///
|
||||||
|
/// For TCP and Stream peers this performs a framed write (4-byte LE length
|
||||||
|
/// + body). For QUIC, each call opens a new unidirectional stream. For
|
||||||
|
/// Mmap, the call blocks until the ring buffer has space.
|
||||||
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
||||||
match self {
|
match self {
|
||||||
SyncPeer::Tcp(c) => c.send(msg).await,
|
SyncPeer::Tcp(c) => c.send(msg).await,
|
||||||
@@ -51,6 +60,11 @@ impl SyncPeer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Receive the next `SyncMessage` from the remote peer.
|
||||||
|
///
|
||||||
|
/// Blocks until a complete message is available. Returns
|
||||||
|
/// [`TransportError::ConnectionClosed`] when the peer has cleanly closed
|
||||||
|
/// the connection.
|
||||||
pub async fn recv(&mut self) -> Result<SyncMessage, TransportError> {
|
pub async fn recv(&mut self) -> Result<SyncMessage, TransportError> {
|
||||||
match self {
|
match self {
|
||||||
SyncPeer::Tcp(c) => c.recv().await,
|
SyncPeer::Tcp(c) => c.recv().await,
|
||||||
@@ -60,6 +74,7 @@ impl SyncPeer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Gracefully shut down the connection, flushing any pending writes.
|
||||||
pub async fn shutdown(self) -> Result<(), TransportError> {
|
pub async fn shutdown(self) -> Result<(), TransportError> {
|
||||||
match self {
|
match self {
|
||||||
SyncPeer::Tcp(mut c) => c.shutdown().await,
|
SyncPeer::Tcp(mut c) => c.shutdown().await,
|
||||||
@@ -136,13 +151,18 @@ impl SyncPeer {
|
|||||||
|
|
||||||
/// Write half for the sliding-window pipeline.
|
/// Write half for the sliding-window pipeline.
|
||||||
pub enum PipeWriteHalf {
|
pub enum PipeWriteHalf {
|
||||||
|
/// Write half of a split `TcpConnection`.
|
||||||
Tcp(TcpWriteHalf),
|
Tcp(TcpWriteHalf),
|
||||||
|
/// QUIC connection `Arc` clone used for sending (QUIC send is `&self`).
|
||||||
Quic(Arc<QuicConnection>),
|
Quic(Arc<QuicConnection>),
|
||||||
|
/// Mmap ring-buffer sender.
|
||||||
Mmap(MmapSender),
|
Mmap(MmapSender),
|
||||||
|
/// Boxed-I/O write half (SSH / stdio).
|
||||||
Stream(StreamWriteHalf),
|
Stream(StreamWriteHalf),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PipeWriteHalf {
|
impl PipeWriteHalf {
|
||||||
|
/// Send a `SyncMessage` through the write half.
|
||||||
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
pub async fn send(&mut self, msg: &SyncMessage) -> Result<(), TransportError> {
|
||||||
match self {
|
match self {
|
||||||
PipeWriteHalf::Tcp(h) => h.send(msg).await,
|
PipeWriteHalf::Tcp(h) => h.send(msg).await,
|
||||||
@@ -152,6 +172,7 @@ impl PipeWriteHalf {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Gracefully shut down the write half.
|
||||||
pub async fn shutdown(self) -> Result<(), TransportError> {
|
pub async fn shutdown(self) -> Result<(), TransportError> {
|
||||||
match self {
|
match self {
|
||||||
PipeWriteHalf::Tcp(mut h) => h.shutdown().await,
|
PipeWriteHalf::Tcp(mut h) => h.shutdown().await,
|
||||||
@@ -184,13 +205,18 @@ impl PipeWriteHalf {
|
|||||||
|
|
||||||
/// Read half for the sliding-window pipeline.
|
/// Read half for the sliding-window pipeline.
|
||||||
pub enum PipeReadHalf {
|
pub enum PipeReadHalf {
|
||||||
|
/// Read half of a split `TcpConnection`.
|
||||||
Tcp(TcpReadHalf),
|
Tcp(TcpReadHalf),
|
||||||
|
/// QUIC connection `Arc` clone used for receiving.
|
||||||
Quic(Arc<QuicConnection>),
|
Quic(Arc<QuicConnection>),
|
||||||
|
/// Mmap ring-buffer receiver.
|
||||||
Mmap(MmapReceiver),
|
Mmap(MmapReceiver),
|
||||||
|
/// Boxed-I/O read half (SSH / stdio).
|
||||||
Stream(StreamReadHalf),
|
Stream(StreamReadHalf),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl PipeReadHalf {
|
impl PipeReadHalf {
|
||||||
|
/// Receive the next `SyncMessage` from the read half.
|
||||||
pub async fn recv(&mut self) -> Result<SyncMessage, TransportError> {
|
pub async fn recv(&mut self) -> Result<SyncMessage, TransportError> {
|
||||||
match self {
|
match self {
|
||||||
PipeReadHalf::Tcp(h) => h.recv().await,
|
PipeReadHalf::Tcp(h) => h.recv().await,
|
||||||
|
|||||||
@@ -120,27 +120,45 @@ pub const MAX_FRAME_SIZE: usize = 256 * 1024 * 1024;
|
|||||||
pub enum SyncMessage {
|
pub enum SyncMessage {
|
||||||
/// Client announces itself and its current HEAD revision.
|
/// Client announces itself and its current HEAD revision.
|
||||||
ManifestRequest {
|
ManifestRequest {
|
||||||
|
/// Logical agent / file identifier (used by the server to look up the sidecar).
|
||||||
agent_id: String,
|
agent_id: String,
|
||||||
|
/// HEAD revision number the client currently holds.
|
||||||
head_revision: u64,
|
head_revision: u64,
|
||||||
/// Revision count (so the server can compute the delta).
|
/// Total revision count (so the server can compute the delta).
|
||||||
revision_count: u64,
|
revision_count: u64,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Server responds with its own manifest.
|
/// Server responds with its own manifest.
|
||||||
ManifestResponse { manifest: ClawSyncManifest },
|
ManifestResponse {
|
||||||
|
/// Full revision manifest of the server's sidecar.
|
||||||
|
manifest: ClawSyncManifest,
|
||||||
|
},
|
||||||
|
|
||||||
/// One revision packet (sent by either side, depending on push/pull).
|
/// One revision packet (sent by either side, depending on push/pull).
|
||||||
LayerPacket { packet: OnionLayerPacket },
|
LayerPacket {
|
||||||
|
/// The serialized revision data (pages, metadata, BLAKE3).
|
||||||
|
packet: OnionLayerPacket,
|
||||||
|
},
|
||||||
|
|
||||||
/// Acknowledgement of a successfully received + verified revision.
|
/// Acknowledgement of a successfully received + verified revision.
|
||||||
Ack { revision: u64 },
|
Ack {
|
||||||
|
/// The revision number that was successfully applied.
|
||||||
|
revision: u64,
|
||||||
|
},
|
||||||
|
|
||||||
/// Request to retransmit a revision (hash verification failed).
|
/// Request to retransmit a revision (hash verification failed).
|
||||||
RetryRequest { revision: u64, reason: String },
|
RetryRequest {
|
||||||
|
/// The revision number that failed integrity verification.
|
||||||
|
revision: u64,
|
||||||
|
/// Human-readable reason for the retry request.
|
||||||
|
reason: String,
|
||||||
|
},
|
||||||
|
|
||||||
/// Transfer is complete.
|
/// Transfer is complete.
|
||||||
SyncComplete {
|
SyncComplete {
|
||||||
|
/// Total number of revisions successfully transferred in this session.
|
||||||
revisions_transferred: u64,
|
revisions_transferred: u64,
|
||||||
|
/// Total compressed bytes transferred in this session.
|
||||||
bytes_transferred: u64,
|
bytes_transferred: u64,
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -148,7 +166,10 @@ pub enum SyncMessage {
|
|||||||
///
|
///
|
||||||
/// If the remote supports IBLT it responds with `IbltResponse`; otherwise
|
/// If the remote supports IBLT it responds with `IbltResponse`; otherwise
|
||||||
/// it responds with `Error` and the client falls back to `ManifestRequest`.
|
/// it responds with `Error` and the client falls back to `ManifestRequest`.
|
||||||
IbltRequest { sketch: IbltManifest },
|
IbltRequest {
|
||||||
|
/// The client's IBLT sketch of its local revision set.
|
||||||
|
sketch: IbltManifest,
|
||||||
|
},
|
||||||
|
|
||||||
/// IBLT pre-flight response: server's sketch + decoded missing revisions.
|
/// IBLT pre-flight response: server's sketch + decoded missing revisions.
|
||||||
///
|
///
|
||||||
@@ -156,12 +177,17 @@ pub enum SyncMessage {
|
|||||||
/// lacks (what to push). The client decodes `sketch` against its own
|
/// lacks (what to push). The client decodes `sketch` against its own
|
||||||
/// sketch to find revisions it needs to pull.
|
/// sketch to find revisions it needs to pull.
|
||||||
IbltResponse {
|
IbltResponse {
|
||||||
|
/// The server's IBLT sketch (for the client to decode what it should pull).
|
||||||
sketch: IbltManifest,
|
sketch: IbltManifest,
|
||||||
|
/// Revision numbers the server lacks (client should push these).
|
||||||
missing_from_remote: Vec<u64>,
|
missing_from_remote: Vec<u64>,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// An error occurred; connection will be closed after this message.
|
/// An error occurred; connection will be closed after this message.
|
||||||
Error { message: String },
|
Error {
|
||||||
|
/// Human-readable error description.
|
||||||
|
message: String,
|
||||||
|
},
|
||||||
|
|
||||||
// ── FS CDC file delta (variants 9–12) ────────────────────────────────────
|
// ── FS CDC file delta (variants 9–12) ────────────────────────────────────
|
||||||
//
|
//
|
||||||
@@ -179,7 +205,9 @@ pub enum SyncMessage {
|
|||||||
/// Originally: client announces a single-file CDC sync, carrying its ordered
|
/// Originally: client announces a single-file CDC sync, carrying its ordered
|
||||||
/// chunk descriptors. Superseded by the server chunk embedding in `FsDirNeed`.
|
/// chunk descriptors. Superseded by the server chunk embedding in `FsDirNeed`.
|
||||||
FsCdcRequest {
|
FsCdcRequest {
|
||||||
|
/// Relative path of the file being synced.
|
||||||
path: String,
|
path: String,
|
||||||
|
/// Client's ordered CDC chunk descriptors (hash + length per chunk).
|
||||||
chunk_hashes: Vec<FsChunkHash>,
|
chunk_hashes: Vec<FsChunkHash>,
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -188,7 +216,9 @@ pub enum SyncMessage {
|
|||||||
/// Originally: server replies with the indices it needs transferred.
|
/// Originally: server replies with the indices it needs transferred.
|
||||||
/// Superseded by the server chunk embedding in `FsDirNeed`.
|
/// Superseded by the server chunk embedding in `FsDirNeed`.
|
||||||
FsCdcNeed {
|
FsCdcNeed {
|
||||||
|
/// Relative path of the file being synced.
|
||||||
path: String,
|
path: String,
|
||||||
|
/// Indices (into the client's chunk list) that the server needs.
|
||||||
needed_indices: Vec<u32>,
|
needed_indices: Vec<u32>,
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -202,6 +232,7 @@ pub enum SyncMessage {
|
|||||||
/// sync path, while remaining self-contained for single-file use too.
|
/// sync path, while remaining self-contained for single-file use too.
|
||||||
/// Wire overhead is ~12 bytes × N_chunks (negligible relative to data).
|
/// Wire overhead is ~12 bytes × N_chunks (negligible relative to data).
|
||||||
FsCdcData {
|
FsCdcData {
|
||||||
|
/// Relative path of the file being transferred.
|
||||||
path: String,
|
path: String,
|
||||||
/// Full ordered chunk layout of the client's file.
|
/// Full ordered chunk layout of the client's file.
|
||||||
chunk_order: Vec<FsChunkHash>,
|
chunk_order: Vec<FsChunkHash>,
|
||||||
@@ -210,7 +241,10 @@ pub enum SyncMessage {
|
|||||||
},
|
},
|
||||||
|
|
||||||
/// Server confirms it has reconstructed and written the file at `path`.
|
/// Server confirms it has reconstructed and written the file at `path`.
|
||||||
FsFileAck { path: String },
|
FsFileAck {
|
||||||
|
/// Relative path of the file that was successfully written.
|
||||||
|
path: String,
|
||||||
|
},
|
||||||
|
|
||||||
// ── FS directory tree sync (variants 13–16) ───────────────────────────────
|
// ── FS directory tree sync (variants 13–16) ───────────────────────────────
|
||||||
/// Client sends a per-file manifest for an entire directory tree.
|
/// Client sends a per-file manifest for an entire directory tree.
|
||||||
@@ -237,16 +271,23 @@ pub enum SyncMessage {
|
|||||||
|
|
||||||
/// Server reports that all changes have been applied.
|
/// Server reports that all changes have been applied.
|
||||||
FsDirComplete {
|
FsDirComplete {
|
||||||
|
/// Number of files newly created on the server.
|
||||||
files_added: u32,
|
files_added: u32,
|
||||||
|
/// Number of files overwritten on the server.
|
||||||
files_modified: u32,
|
files_modified: u32,
|
||||||
|
/// Number of files deleted from the server.
|
||||||
files_removed: u32,
|
files_removed: u32,
|
||||||
|
/// Total compressed bytes transferred in this session.
|
||||||
bytes_transferred: u64,
|
bytes_transferred: u64,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Dry-run: reports what *would* change without applying anything.
|
/// Dry-run: reports what *would* change without applying anything.
|
||||||
FsDirDryRun {
|
FsDirDryRun {
|
||||||
|
/// Paths that would be created on the server.
|
||||||
would_add: Vec<String>,
|
would_add: Vec<String>,
|
||||||
|
/// Paths that would be overwritten on the server.
|
||||||
would_modify: Vec<String>,
|
would_modify: Vec<String>,
|
||||||
|
/// Paths that would be deleted from the server.
|
||||||
would_remove: Vec<String>,
|
would_remove: Vec<String>,
|
||||||
},
|
},
|
||||||
|
|
||||||
@@ -284,7 +325,9 @@ pub enum SyncMessage {
|
|||||||
/// `dtype` is the `{:?}` format of `clawhdf5::DType` (e.g. `"F64"`).
|
/// `dtype` is the `{:?}` format of `clawhdf5::DType` (e.g. `"F64"`).
|
||||||
/// `shape` is the dataset's dimension sizes.
|
/// `shape` is the dataset's dimension sizes.
|
||||||
Hdf5DataPayload {
|
Hdf5DataPayload {
|
||||||
|
/// HDF5 path of the dataset within the file (e.g., `/group/embeddings`).
|
||||||
path: String,
|
path: String,
|
||||||
|
/// Raw little-endian bytes of the dataset.
|
||||||
data: Vec<u8>,
|
data: Vec<u8>,
|
||||||
/// HDF5 DType debug string, e.g. `"F64"`, `"F32"`, `"I32"`.
|
/// HDF5 DType debug string, e.g. `"F64"`, `"F32"`, `"I32"`.
|
||||||
dtype: String,
|
dtype: String,
|
||||||
@@ -293,13 +336,20 @@ pub enum SyncMessage {
|
|||||||
},
|
},
|
||||||
|
|
||||||
/// Server acknowledges successful reconstruction of a dataset.
|
/// Server acknowledges successful reconstruction of a dataset.
|
||||||
Hdf5DataAck { path: String },
|
Hdf5DataAck {
|
||||||
|
/// HDF5 path of the dataset that was successfully applied.
|
||||||
|
path: String,
|
||||||
|
},
|
||||||
|
|
||||||
/// Server reports that the HDF5 sync is complete.
|
/// Server reports that the HDF5 sync is complete.
|
||||||
Hdf5SyncComplete {
|
Hdf5SyncComplete {
|
||||||
|
/// Number of datasets newly added to the server's file.
|
||||||
datasets_added: u32,
|
datasets_added: u32,
|
||||||
|
/// Number of datasets overwritten on the server.
|
||||||
datasets_modified: u32,
|
datasets_modified: u32,
|
||||||
|
/// Number of datasets removed from the server's file.
|
||||||
datasets_removed: u32,
|
datasets_removed: u32,
|
||||||
|
/// Total compressed bytes transferred in this session.
|
||||||
bytes_transferred: u64,
|
bytes_transferred: u64,
|
||||||
},
|
},
|
||||||
|
|
||||||
|
|||||||
@@ -32,7 +32,9 @@ use crate::protocol::SyncMessage;
|
|||||||
|
|
||||||
/// QUIC endpoint configuration.
|
/// QUIC endpoint configuration.
|
||||||
pub struct QuicConfig {
|
pub struct QuicConfig {
|
||||||
|
/// TLS server configuration for the QUIC server endpoint.
|
||||||
pub server_config: ServerConfig,
|
pub server_config: ServerConfig,
|
||||||
|
/// TLS client configuration for outgoing QUIC connections.
|
||||||
pub client_config: ClientConfig,
|
pub client_config: ClientConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -453,6 +455,10 @@ impl QuicConnection {
|
|||||||
/// A QUIC server endpoint.
|
/// A QUIC server endpoint.
|
||||||
pub struct QuicServer {
|
pub struct QuicServer {
|
||||||
endpoint: Endpoint,
|
endpoint: Endpoint,
|
||||||
|
/// The local `SocketAddr` this server is bound to.
|
||||||
|
///
|
||||||
|
/// Useful when the server was bound to port `0` and the assigned ephemeral
|
||||||
|
/// port must be communicated to clients.
|
||||||
pub local_addr: SocketAddr,
|
pub local_addr: SocketAddr,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -144,6 +144,10 @@ impl TcpWriteHalf {
|
|||||||
/// A listening TCP server that accepts `TcpConnection`s.
|
/// A listening TCP server that accepts `TcpConnection`s.
|
||||||
pub struct TcpServer {
|
pub struct TcpServer {
|
||||||
listener: TcpListener,
|
listener: TcpListener,
|
||||||
|
/// The local `SocketAddr` this server is bound to.
|
||||||
|
///
|
||||||
|
/// Useful for callers that bind to port `0` and need to discover the
|
||||||
|
/// assigned ephemeral port after binding.
|
||||||
pub local_addr: SocketAddr,
|
pub local_addr: SocketAddr,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user