Initial commit: ClawSync v0.1.0
8-crate pure-Rust workspace for revision-aware HDF5 sync. ## Crates - clawhdf5-onion: ClawOnion VFD — page-level versioned HDF5 storage, binary format, writer/reader, branch DAG, GC, snapshots, provenance - clawsync-core: BLAKE3, xxHash3, FastCDC (+ SIMD NEON), zstd/lz4 - clawsync-onion: IBLT sketch, Merkle tree differ, packet differ/merger, ClawSyncManifest, SyncSelector - clawsync-hdf5: dataset-level manifest, differ, patcher, wire payload reconstruction (apply_received_payloads) - clawsync-transport: TCP, QUIC (quinn 0.11/TLS 1.3), SyncPeer abstraction, length-prefixed rkyv wire protocol (21 SyncMessage variants) - clawsync-agent: OnionMemory, SyncScheduler, TcpSyncBackend, PeerCapabilities negotiation - clawsync-fs: CDC-based delta sync for any file type; FsSyncClient/Server, W=16 pipelining, atomic writes - clawsync-cli: push/pull/serve/hdf5-sync/serve-hdf5/sync/serve-fs + all local management commands; --quic on all network commands ## Key features - IBLT pre-flight: O(revision count) vs rsync's O(file size) - W=16 sliding-window push: 13–15x speedup over stop-and-wait at WAN RTT - Dataset-granular HDF5 sync: only modified datasets transferred - CDC delta for any file type: insertion-stable chunk boundaries - Full revision DAG: branch, merge, rollback, export, snapshot, GC - QUIC transport: TLS 1.3, per-message streams via quinn 0.11 ## Tests ~573 passing (default features); ~589 with --features simd-cdc ## Performance (Apple Silicon) - Reconstruct rev=100: 68 µs (target ≤ 1 ms) - BLAKE3 Rayon 1 MB: 10.3 GiB/s (target ≥ 5 GB/s) - GC 500 revisions: 20.6 µs (target ≤ 2 s) - W=16 vs W=1 at 5 ms RTT: 14.8x speedup - No-op pre-flight at 16 MB: 4 ms vs rsync 35 ms (7.8x) Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,176 @@
|
||||
//! AnnotationHeap: variable-length UTF-8 annotation storage.
|
||||
//!
|
||||
//! Stores revision annotations and branch names as length-prefixed
|
||||
//! (`u32` LE + UTF-8 bytes) strings. An offset of `0` in a
|
||||
//! [`RevisionEntry`] or [`BranchEntry`] means "no annotation".
|
||||
//!
|
||||
//! The offset `0` is reserved. The heap always begins with a single
|
||||
//! null byte so that offset 0 is unambiguously "absent".
|
||||
|
||||
/// Variable-length UTF-8 heap for revision annotations and branch names.
|
||||
///
|
||||
/// Layout: each entry is `[len: u32 LE][utf8 bytes]`.
|
||||
/// Offset 0 is reserved (absent). The first real string starts at offset 1.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct AnnotationHeap {
|
||||
data: Vec<u8>,
|
||||
}
|
||||
|
||||
impl AnnotationHeap {
|
||||
/// Create a new heap. The first byte is a reserved null so offset 0
|
||||
/// unambiguously means "no annotation".
|
||||
pub fn new() -> Self {
|
||||
Self { data: vec![0u8] }
|
||||
}
|
||||
|
||||
/// Initialise from raw bytes (e.g., loaded from disk).
|
||||
pub fn from_bytes(bytes: Vec<u8>) -> Self {
|
||||
if bytes.is_empty() {
|
||||
Self::new()
|
||||
} else {
|
||||
Self { data: bytes }
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the raw heap bytes (for serialising to disk).
|
||||
pub fn as_bytes(&self) -> &[u8] {
|
||||
&self.data
|
||||
}
|
||||
|
||||
/// Append a UTF-8 string and return its byte offset within the heap.
|
||||
///
|
||||
/// The returned offset can be stored in a [`RevisionEntry::annotation_off`]
|
||||
/// or [`BranchEntry::name_off`].
|
||||
pub fn push(&mut self, s: &str) -> u64 {
|
||||
let offset = self.data.len() as u64;
|
||||
let len = s.len() as u32;
|
||||
self.data.extend_from_slice(&len.to_le_bytes());
|
||||
self.data.extend_from_slice(s.as_bytes());
|
||||
offset
|
||||
}
|
||||
|
||||
/// Read a string from the heap at the given byte offset.
|
||||
///
|
||||
/// Returns `None` if `offset` is 0 (reserved sentinel) or out of bounds.
|
||||
pub fn get(&self, offset: u64) -> Option<&str> {
|
||||
if offset == 0 {
|
||||
return None;
|
||||
}
|
||||
let off = offset as usize;
|
||||
if off + 4 > self.data.len() {
|
||||
return None;
|
||||
}
|
||||
let len = u32::from_le_bytes(self.data[off..off + 4].try_into().ok()?) as usize;
|
||||
let start = off + 4;
|
||||
let end = start + len;
|
||||
if end > self.data.len() {
|
||||
return None;
|
||||
}
|
||||
std::str::from_utf8(&self.data[start..end]).ok()
|
||||
}
|
||||
|
||||
/// Number of bytes used by the heap (including the reserved null byte).
|
||||
pub fn len(&self) -> usize {
|
||||
self.data.len()
|
||||
}
|
||||
|
||||
/// Returns true if the heap contains only the reserved null byte.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.data.len() <= 1
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn new_heap_has_reserved_null() {
|
||||
let h = AnnotationHeap::new();
|
||||
assert_eq!(h.data.len(), 1);
|
||||
assert_eq!(h.data[0], 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn push_and_get_ascii() {
|
||||
let mut h = AnnotationHeap::new();
|
||||
let off = h.push("hello");
|
||||
assert_eq!(h.get(off), Some("hello"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn push_and_get_unicode() {
|
||||
let mut h = AnnotationHeap::new();
|
||||
let off = h.push("日本語テスト🦀");
|
||||
assert_eq!(h.get(off), Some("日本語テスト🦀"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn push_empty_string() {
|
||||
let mut h = AnnotationHeap::new();
|
||||
let off = h.push("");
|
||||
assert_eq!(h.get(off), Some(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sentinel_zero_returns_none() {
|
||||
let h = AnnotationHeap::new();
|
||||
assert!(h.get(0).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn out_of_bounds_returns_none() {
|
||||
let h = AnnotationHeap::new();
|
||||
assert!(h.get(9999).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiple_entries_independent() {
|
||||
let mut h = AnnotationHeap::new();
|
||||
let o1 = h.push("first");
|
||||
let o2 = h.push("second");
|
||||
let o3 = h.push("third");
|
||||
assert_eq!(h.get(o1), Some("first"));
|
||||
assert_eq!(h.get(o2), Some("second"));
|
||||
assert_eq!(h.get(o3), Some("third"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn offsets_are_strictly_increasing() {
|
||||
let mut h = AnnotationHeap::new();
|
||||
let o1 = h.push("a");
|
||||
let o2 = h.push("b");
|
||||
assert!(o2 > o1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn large_annotation() {
|
||||
let mut h = AnnotationHeap::new();
|
||||
let big = "x".repeat(100_000);
|
||||
let off = h.push(&big);
|
||||
assert_eq!(h.get(off).unwrap().len(), 100_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn roundtrip_via_raw_bytes() {
|
||||
let mut h = AnnotationHeap::new();
|
||||
let o1 = h.push("branch-name");
|
||||
let o2 = h.push("checkpoint v1.0");
|
||||
let raw = h.as_bytes().to_vec();
|
||||
let h2 = AnnotationHeap::from_bytes(raw);
|
||||
assert_eq!(h2.get(o1), Some("branch-name"));
|
||||
assert_eq!(h2.get(o2), Some("checkpoint v1.0"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_empty_false_after_push() {
|
||||
let mut h = AnnotationHeap::new();
|
||||
assert!(h.is_empty());
|
||||
h.push("x");
|
||||
assert!(!h.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
//! High-level onion API — extension functions for opening versioned HDF5 files.
|
||||
//!
|
||||
//! Because `clawhdf5` depends on `clawhdf5-onion` (not the other way around),
|
||||
//! the extension functions that return a [`clawhdf5::File`] live here rather
|
||||
//! than being methods on the `File` type itself.
|
||||
//!
|
||||
//! # Usage
|
||||
//!
|
||||
//! ```rust,ignore
|
||||
//! use clawhdf_onion::api::{open_revision, open_branch};
|
||||
//! use clawhdf_onion::reader::OpenRevision;
|
||||
//!
|
||||
//! // Open a specific historical revision
|
||||
//! let (file_bytes, onion) = open_revision("agent.h5", 7)?;
|
||||
//! let file = clawhdf5::File::from_bytes(file_bytes)?;
|
||||
//!
|
||||
//! // Open the HEAD of a named branch
|
||||
//! let (file_bytes, onion) = open_branch("agent.h5", "experiment-v2")?;
|
||||
//! let file = clawhdf5::File::from_bytes(file_bytes)?;
|
||||
//! ```
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use crate::branch::BranchInfo;
|
||||
use crate::error::OnionError;
|
||||
use crate::reader::OpenRevision;
|
||||
use crate::writer::OnionFile;
|
||||
|
||||
/// Open a specific revision of a versioned HDF5 file.
|
||||
///
|
||||
/// Reads the base `.h5` file and the `.h5.onion` sidecar, then reconstructs
|
||||
/// the logical file state at `revision`.
|
||||
///
|
||||
/// Returns `(reconstructed_bytes, onion_file)`. Callers can construct a
|
||||
/// `clawhdf5::File` from the reconstructed bytes:
|
||||
/// ```rust,ignore
|
||||
/// let (bytes, onion) = open_revision("agent.h5", 7)?;
|
||||
/// let file = clawhdf5::File::from_bytes(bytes)?;
|
||||
/// ```
|
||||
pub fn open_revision(
|
||||
h5_path: &Path,
|
||||
revision: u64,
|
||||
) -> Result<(Vec<u8>, OnionFile), OnionError> {
|
||||
let h5_base = std::fs::read(h5_path)?;
|
||||
let onion = OnionFile::open(h5_path)?;
|
||||
let reconstructed = onion.reconstruct_revision(revision, &h5_base)?;
|
||||
Ok((reconstructed, onion))
|
||||
}
|
||||
|
||||
/// Open the HEAD of a named branch.
|
||||
pub fn open_branch(
|
||||
h5_path: &Path,
|
||||
branch: &str,
|
||||
) -> Result<(Vec<u8>, OnionFile), OnionError> {
|
||||
let h5_base = std::fs::read(h5_path)?;
|
||||
let onion = OnionFile::open(h5_path)?;
|
||||
let reconstructed = onion.open_rev(OpenRevision::Branch(branch.to_owned()), &h5_base)?;
|
||||
Ok((reconstructed, onion))
|
||||
}
|
||||
|
||||
/// Open a specific revision on a named branch.
|
||||
pub fn open_branch_at(
|
||||
h5_path: &Path,
|
||||
branch: &str,
|
||||
revision: u64,
|
||||
) -> Result<(Vec<u8>, OnionFile), OnionError> {
|
||||
let h5_base = std::fs::read(h5_path)?;
|
||||
let onion = OnionFile::open(h5_path)?;
|
||||
let reconstructed =
|
||||
onion.open_rev(OpenRevision::BranchAt(branch.to_owned(), revision), &h5_base)?;
|
||||
Ok((reconstructed, onion))
|
||||
}
|
||||
|
||||
/// List all revisions in a `.onion` sidecar without loading the full HDF5 file.
|
||||
pub fn list_revisions(h5_path: &Path) -> Result<Vec<crate::writer::RevisionSummary>, OnionError> {
|
||||
let onion = OnionFile::open(h5_path)?;
|
||||
Ok(onion.list_revisions())
|
||||
}
|
||||
|
||||
/// List all branches in a `.onion` sidecar.
|
||||
pub fn list_branches(h5_path: &Path) -> Result<Vec<BranchInfo>, OnionError> {
|
||||
let onion = OnionFile::open(h5_path)?;
|
||||
Ok(onion.list_branches())
|
||||
}
|
||||
|
||||
/// Create a new named branch forked from `source` (default `"main"`).
|
||||
///
|
||||
/// Writes the updated sidecar to disk before returning.
|
||||
pub fn create_branch(h5_path: &Path, name: &str, source: &str) -> Result<u32, OnionError> {
|
||||
let mut onion = OnionFile::open(h5_path)?;
|
||||
let id = onion.create_branch(name, source)?;
|
||||
onion.flush()?;
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Delete a named branch (cannot delete `"main"`).
|
||||
///
|
||||
/// Writes the updated sidecar to disk before returning.
|
||||
pub fn delete_branch(h5_path: &Path, name: &str) -> Result<(), OnionError> {
|
||||
let mut onion = OnionFile::open(h5_path)?;
|
||||
onion.delete_branch(name)?;
|
||||
onion.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rename a branch.
|
||||
///
|
||||
/// Writes the updated sidecar to disk before returning.
|
||||
pub fn rename_branch(h5_path: &Path, from: &str, to: &str) -> Result<(), OnionError> {
|
||||
let mut onion = OnionFile::open(h5_path)?;
|
||||
onion.rename_branch(from, to)?;
|
||||
onion.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Roll back an HDF5 file to a specific revision in-place.
|
||||
///
|
||||
/// Reconstructs the file state at `revision` and overwrites the `.h5` file.
|
||||
/// The `.onion` sidecar is unchanged.
|
||||
///
|
||||
/// **Warning:** This is a destructive operation — the caller should make a
|
||||
/// backup if needed.
|
||||
pub fn rollback(h5_path: &Path, revision: u64) -> Result<(), OnionError> {
|
||||
let h5_base = std::fs::read(h5_path)?;
|
||||
let onion = OnionFile::open(h5_path)?;
|
||||
let reconstructed = onion.reconstruct_revision(revision, &h5_base)?;
|
||||
std::fs::write(h5_path, reconstructed)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
fn make_versioned_h5() -> std::path::PathBuf {
|
||||
let f = NamedTempFile::new().unwrap();
|
||||
let h5_path = f.path().with_extension("h5");
|
||||
// Write a minimal "HDF5" base file
|
||||
let base: Vec<u8> = {
|
||||
let mut v = b"\x89HDF\r\n\x1a\n".to_vec();
|
||||
v.extend(vec![0u8; 4096 * 4]);
|
||||
v
|
||||
};
|
||||
std::fs::write(&h5_path, &base).unwrap();
|
||||
|
||||
let mut onion = OnionFile::create(&h5_path, 4096).unwrap();
|
||||
|
||||
// Rev 0
|
||||
let mut s0 = onion.begin_session(None).unwrap();
|
||||
s0.record_page(0, &vec![0xAAu8; 4096]);
|
||||
onion.commit_session(s0, Some("rev 0")).unwrap();
|
||||
|
||||
// Rev 1
|
||||
let mut s1 = onion.begin_session(None).unwrap();
|
||||
s1.record_page(0, &vec![0xBBu8; 4096]);
|
||||
onion.commit_session(s1, Some("rev 1")).unwrap();
|
||||
|
||||
onion.flush().unwrap();
|
||||
h5_path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_revision_rev0() {
|
||||
let h5 = make_versioned_h5();
|
||||
let (bytes, _onion) = open_revision(&h5, 0).unwrap();
|
||||
assert!(bytes[0..4096].iter().all(|&b| b == 0xAA));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_revision_rev1() {
|
||||
let h5 = make_versioned_h5();
|
||||
let (bytes, _onion) = open_revision(&h5, 1).unwrap();
|
||||
assert!(bytes[0..4096].iter().all(|&b| b == 0xBB));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_branch_main() {
|
||||
let h5 = make_versioned_h5();
|
||||
let (bytes, _onion) = open_branch(&h5, "main").unwrap();
|
||||
// HEAD of main is rev 1 (0xBB)
|
||||
assert!(bytes[0..4096].iter().all(|&b| b == 0xBB));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_branch_nonexistent_errors() {
|
||||
let h5 = make_versioned_h5();
|
||||
let err = open_branch(&h5, "does-not-exist").unwrap_err();
|
||||
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_revisions_count() {
|
||||
let h5 = make_versioned_h5();
|
||||
let revs = list_revisions(&h5).unwrap();
|
||||
assert_eq!(revs.len(), 2);
|
||||
assert_eq!(revs[0].annotation, Some("rev 0".to_string()));
|
||||
assert_eq!(revs[1].annotation, Some("rev 1".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_revision_nonexistent_errors() {
|
||||
let h5 = make_versioned_h5();
|
||||
let err = open_revision(&h5, 999).unwrap_err();
|
||||
assert!(matches!(err, OnionError::RevisionNotFound(999)));
|
||||
}
|
||||
|
||||
// ── Branch API ────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn list_branches_returns_main() {
|
||||
let h5 = make_versioned_h5();
|
||||
let branches = list_branches(&h5).unwrap();
|
||||
assert_eq!(branches.len(), 1);
|
||||
assert_eq!(branches[0].name, "main");
|
||||
assert_eq!(branches[0].id, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_branch_forks_and_persists() {
|
||||
let h5 = make_versioned_h5();
|
||||
let id = create_branch(&h5, "experiment", "main").unwrap();
|
||||
assert!(id > 0);
|
||||
|
||||
// Re-open from disk and verify the branch is there.
|
||||
let branches = list_branches(&h5).unwrap();
|
||||
assert_eq!(branches.len(), 2);
|
||||
assert!(branches.iter().any(|b| b.name == "experiment"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_branch_duplicate_errors() {
|
||||
let h5 = make_versioned_h5();
|
||||
create_branch(&h5, "dup", "main").unwrap();
|
||||
let err = create_branch(&h5, "dup", "main").unwrap_err();
|
||||
assert!(matches!(err, OnionError::BranchExists(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_branch_removes_and_persists() {
|
||||
let h5 = make_versioned_h5();
|
||||
create_branch(&h5, "temp", "main").unwrap();
|
||||
|
||||
delete_branch(&h5, "temp").unwrap();
|
||||
|
||||
let branches = list_branches(&h5).unwrap();
|
||||
assert!(!branches.iter().any(|b| b.name == "temp"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_branch_persists() {
|
||||
let h5 = make_versioned_h5();
|
||||
create_branch(&h5, "old", "main").unwrap();
|
||||
|
||||
rename_branch(&h5, "old", "new").unwrap();
|
||||
|
||||
let branches = list_branches(&h5).unwrap();
|
||||
assert!(!branches.iter().any(|b| b.name == "old"));
|
||||
assert!(branches.iter().any(|b| b.name == "new"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_nonexistent_branch_errors() {
|
||||
let h5 = make_versioned_h5();
|
||||
let err = delete_branch(&h5, "ghost").unwrap_err();
|
||||
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
||||
}
|
||||
|
||||
// ── open_branch_at ────────────────────────────────────────────────────────
|
||||
|
||||
/// Helper: make an onion with a feature branch that has one extra revision.
|
||||
///
|
||||
/// Layout:
|
||||
/// main: rev 0 (0xAA), rev 1 (0xBB)
|
||||
/// feature (forked after rev 1): rev 2 (0xCC)
|
||||
fn make_branched_h5() -> std::path::PathBuf {
|
||||
let f = NamedTempFile::new().unwrap();
|
||||
let h5_path = f.path().with_extension("h5");
|
||||
let base: Vec<u8> = {
|
||||
let mut v = b"\x89HDF\r\n\x1a\n".to_vec();
|
||||
v.extend(vec![0u8; 4096 * 4]);
|
||||
v
|
||||
};
|
||||
std::fs::write(&h5_path, &base).unwrap();
|
||||
|
||||
let mut onion = OnionFile::create(&h5_path, 4096).unwrap();
|
||||
|
||||
let mut s0 = onion.begin_session(None).unwrap();
|
||||
s0.record_page(0, &vec![0xAAu8; 4096]);
|
||||
onion.commit_session(s0, Some("rev 0")).unwrap();
|
||||
|
||||
let mut s1 = onion.begin_session(None).unwrap();
|
||||
s1.record_page(0, &vec![0xBBu8; 4096]);
|
||||
onion.commit_session(s1, Some("rev 1")).unwrap();
|
||||
|
||||
// Fork feature branch from main HEAD (rev 1).
|
||||
let feat_id = onion.create_branch("feature", "main").unwrap();
|
||||
|
||||
let mut s2 = onion.begin_session(Some(feat_id)).unwrap();
|
||||
s2.record_page(0, &vec![0xCCu8; 4096]);
|
||||
onion.commit_session(s2, Some("feat rev")).unwrap();
|
||||
|
||||
onion.flush().unwrap();
|
||||
h5_path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_branch_at_head_of_feature() {
|
||||
let h5 = make_branched_h5();
|
||||
// HEAD of feature branch is rev 2 (0xCC)
|
||||
let (bytes, _) = open_branch_at(&h5, "feature", 2).unwrap();
|
||||
assert!(
|
||||
bytes[0..4096].iter().all(|&b| b == 0xCC),
|
||||
"feature HEAD should be 0xCC"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_branch_at_main_rev0_returns_correct_content() {
|
||||
let h5 = make_branched_h5();
|
||||
// Rev 0 is on main — should return the 0xAA content.
|
||||
let (bytes, _) = open_branch_at(&h5, "main", 0).unwrap();
|
||||
assert!(
|
||||
bytes[0..4096].iter().all(|&b| b == 0xAA),
|
||||
"main rev 0 should be 0xAA"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_branch_at_nonexistent_branch_errors() {
|
||||
let h5 = make_branched_h5();
|
||||
let err = open_branch_at(&h5, "no-such", 0).unwrap_err();
|
||||
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_branch_at_revision_beyond_range_errors() {
|
||||
let h5 = make_branched_h5();
|
||||
// Revision 999 does not exist on the feature branch.
|
||||
let err = open_branch_at(&h5, "feature", 999).unwrap_err();
|
||||
assert!(
|
||||
matches!(err, OnionError::RevisionNotFound(_)),
|
||||
"expected RevisionNotFound, got {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
// ── rollback ─────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn rollback_overwrites_h5_with_revision_content() {
|
||||
let h5 = make_versioned_h5();
|
||||
// Initially the file content is the base (zeroes after magic bytes).
|
||||
// After rollback to rev 0, the reconstructed content should have 0xAA at offset 0.
|
||||
rollback(&h5, 0).unwrap();
|
||||
let disk_bytes = std::fs::read(&h5).unwrap();
|
||||
assert!(
|
||||
disk_bytes[0..4096].iter().all(|&b| b == 0xAA),
|
||||
"rollback to rev 0 should write 0xAA page at offset 0"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rollback_to_latest_writes_latest_content() {
|
||||
let h5 = make_versioned_h5();
|
||||
rollback(&h5, 1).unwrap();
|
||||
let disk_bytes = std::fs::read(&h5).unwrap();
|
||||
assert!(
|
||||
disk_bytes[0..4096].iter().all(|&b| b == 0xBB),
|
||||
"rollback to rev 1 should write 0xBB page at offset 0"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rollback_to_nonexistent_revision_errors() {
|
||||
let h5 = make_versioned_h5();
|
||||
let err = rollback(&h5, 999).unwrap_err();
|
||||
assert!(matches!(err, OnionError::RevisionNotFound(999)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rollback_does_not_modify_onion_sidecar() {
|
||||
let h5 = make_versioned_h5();
|
||||
let onion_path = {
|
||||
let mut p = h5.clone();
|
||||
let ext = p.extension().unwrap().to_owned();
|
||||
let mut new_ext = ext.clone();
|
||||
new_ext.push(".onion");
|
||||
p.set_extension(&new_ext);
|
||||
p
|
||||
};
|
||||
let before = std::fs::metadata(&onion_path).unwrap().len();
|
||||
rollback(&h5, 0).unwrap();
|
||||
let after = std::fs::metadata(&onion_path).unwrap().len();
|
||||
assert_eq!(before, after, "sidecar should be unchanged after rollback");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,805 @@
|
||||
//! Branch management: fork, merge, lifecycle operations on the DAG.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
|
||||
use crate::compress::decompress_page;
|
||||
use crate::error::OnionError;
|
||||
use crate::format::{BranchEntry, Codec, NO_PARENT};
|
||||
use crate::writer::OnionFile;
|
||||
|
||||
/// Caller-supplied dataset-level merge resolver.
|
||||
///
|
||||
/// Receives `(dataset_path, target_bytes, source_bytes)` and returns the merged bytes.
|
||||
pub type DatasetResolver = dyn Fn(&str, &[u8], &[u8]) -> Vec<u8>;
|
||||
|
||||
/// Merge strategy used when merging one branch into another.
|
||||
pub enum MergeStrategy {
|
||||
/// Page-level last-write-wins: the source branch's pages replace
|
||||
/// the target branch's pages wherever they conflict.
|
||||
LatestWins,
|
||||
/// Dataset-level merge with a caller-supplied resolver function.
|
||||
///
|
||||
/// The resolver receives `(dataset_path, target_bytes, source_bytes)`
|
||||
/// and returns the merged bytes.
|
||||
DatasetLevel(Box<DatasetResolver>),
|
||||
/// Three-way merge: find common ancestor, diff both sides against it.
|
||||
ThreeWay,
|
||||
}
|
||||
|
||||
/// Public summary of a branch.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BranchInfo {
|
||||
pub id: u32,
|
||||
pub name: String,
|
||||
pub head_rev: u64,
|
||||
pub fork_rev: u64,
|
||||
}
|
||||
|
||||
impl OnionFile {
|
||||
// ── Fork ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Create a new named branch forked from the current HEAD of `source_branch`.
|
||||
///
|
||||
/// Returns the new branch ID.
|
||||
pub fn create_branch(
|
||||
&mut self,
|
||||
name: &str,
|
||||
source_branch: &str,
|
||||
) -> Result<u32, OnionError> {
|
||||
if self.branch_by_name(name).is_some() {
|
||||
return Err(OnionError::BranchExists(name.to_string()));
|
||||
}
|
||||
let source = self
|
||||
.branch_by_name(source_branch)
|
||||
.ok_or_else(|| OnionError::BranchNotFound(source_branch.to_string()))?;
|
||||
|
||||
let fork_rev = source.head_rev;
|
||||
let new_id = self.branches.len() as u32;
|
||||
let name_off = self.annotations.push(name);
|
||||
let created_at = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs_f64())
|
||||
.unwrap_or(0.0);
|
||||
|
||||
let entry = BranchEntry {
|
||||
id: new_id,
|
||||
_pad_id: [0u8; 4],
|
||||
name_off,
|
||||
head_rev: fork_rev, // new branch starts at same HEAD as source
|
||||
fork_rev,
|
||||
created_at,
|
||||
};
|
||||
self.branches.push(entry);
|
||||
self.header.branch_count = self.branches.len() as u32;
|
||||
Ok(new_id)
|
||||
}
|
||||
|
||||
/// Register a branch received from a remote peer during sync.
|
||||
///
|
||||
/// If a branch with this ID already exists, returns its ID without
|
||||
/// modification (idempotent). Used by the merger when applying packets
|
||||
/// from branches the local file hasn't seen before.
|
||||
pub fn ensure_branch_id(&mut self, branch_id: u32, fork_rev: u64) -> u32 {
|
||||
if self.branch_by_id(branch_id).is_some() {
|
||||
return branch_id;
|
||||
}
|
||||
let name = format!("branch-{branch_id}");
|
||||
let name_off = self.annotations.push(&name);
|
||||
let created_at = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|d| d.as_secs_f64())
|
||||
.unwrap_or(0.0);
|
||||
let entry = BranchEntry {
|
||||
id: branch_id,
|
||||
_pad_id: [0u8; 4],
|
||||
name_off,
|
||||
head_rev: fork_rev,
|
||||
fork_rev,
|
||||
created_at,
|
||||
};
|
||||
self.branches.push(entry);
|
||||
self.header.branch_count = self.branches.len() as u32;
|
||||
branch_id
|
||||
}
|
||||
|
||||
// ── Merge ────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Merge `source_branch` into `target_branch` using the given strategy.
|
||||
///
|
||||
/// Produces a new revision on `target_branch` and returns its revision number.
|
||||
/// `LatestWins` is the only strategy fully implemented in Phase 1;
|
||||
/// `DatasetLevel` and `ThreeWay` are scaffolded for Phase 3.
|
||||
pub fn merge_into(
|
||||
&mut self,
|
||||
source_branch: &str,
|
||||
target_branch: &str,
|
||||
strategy: MergeStrategy,
|
||||
) -> Result<u64, OnionError> {
|
||||
let source_id = self
|
||||
.branch_by_name(source_branch)
|
||||
.ok_or_else(|| OnionError::BranchNotFound(source_branch.to_string()))?
|
||||
.id;
|
||||
let target_id = self
|
||||
.branch_by_name(target_branch)
|
||||
.ok_or_else(|| OnionError::BranchNotFound(target_branch.to_string()))?
|
||||
.id;
|
||||
|
||||
match strategy {
|
||||
MergeStrategy::LatestWins => self.merge_latest_wins(source_id, target_id),
|
||||
MergeStrategy::DatasetLevel(resolver) => {
|
||||
self.merge_dataset_level(source_id, target_id, resolver.as_ref())
|
||||
}
|
||||
MergeStrategy::ThreeWay => self.merge_three_way(source_id, target_id),
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_latest_wins(
|
||||
&mut self,
|
||||
source_id: u32,
|
||||
target_id: u32,
|
||||
) -> Result<u64, OnionError> {
|
||||
let fork_rev = self
|
||||
.branches
|
||||
.iter()
|
||||
.find(|b| b.id == source_id)
|
||||
.map(|b| b.fork_rev)
|
||||
.unwrap_or(NO_PARENT);
|
||||
|
||||
// Revisions on the source branch after the fork point
|
||||
let source_revs: Vec<u64> = self
|
||||
.index
|
||||
.branch_revisions(source_id)
|
||||
.filter(|e| fork_rev == NO_PARENT || e.revision > fork_rev)
|
||||
.map(|e| e.revision)
|
||||
.collect();
|
||||
|
||||
if source_revs.is_empty() {
|
||||
return Err(OnionError::Malformed(
|
||||
"source branch has no new revisions to merge".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// Decompress & collect pages in revision order — later writes win.
|
||||
let mut merged: BTreeMap<u64, Vec<u8>> = BTreeMap::new();
|
||||
for rev in &source_revs {
|
||||
for (h5_off, page_bytes) in self.revision_pages(*rev)? {
|
||||
merged.insert(h5_off, page_bytes);
|
||||
}
|
||||
}
|
||||
|
||||
// Commit the merged pages as a new revision on the target branch.
|
||||
let mut session = self.begin_session(Some(target_id))?;
|
||||
for (h5_off, bytes) in &merged {
|
||||
session.record_page(*h5_off, bytes);
|
||||
}
|
||||
let annotation = format!("merge branch {source_id} → {target_id} [latest-wins]");
|
||||
self.commit_session(session, Some(&annotation))
|
||||
}
|
||||
|
||||
fn merge_dataset_level(
|
||||
&mut self,
|
||||
source_id: u32,
|
||||
target_id: u32,
|
||||
resolver: &DatasetResolver,
|
||||
) -> Result<u64, OnionError> {
|
||||
let fork_rev = self
|
||||
.branches
|
||||
.iter()
|
||||
.find(|b| b.id == source_id)
|
||||
.map(|b| b.fork_rev)
|
||||
.unwrap_or(NO_PARENT);
|
||||
|
||||
let source_revs: Vec<u64> = self
|
||||
.index
|
||||
.branch_revisions(source_id)
|
||||
.filter(|e| fork_rev == NO_PARENT || e.revision > fork_rev)
|
||||
.map(|e| e.revision)
|
||||
.collect();
|
||||
|
||||
if source_revs.is_empty() {
|
||||
return Err(OnionError::Malformed(
|
||||
"source branch has no new revisions to merge".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// Collect source delta (latest write per offset)
|
||||
let mut source_delta: BTreeMap<u64, Vec<u8>> = BTreeMap::new();
|
||||
for rev in &source_revs {
|
||||
for (h5_off, bytes) in self.revision_pages(*rev)? {
|
||||
source_delta.insert(h5_off, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
// For each changed offset, find the target branch's current page
|
||||
// (most recent write on target for that offset), then call the resolver.
|
||||
let mut merged: BTreeMap<u64, Vec<u8>> = BTreeMap::new();
|
||||
for (h5_off, source_bytes) in &source_delta {
|
||||
let target_bytes = self
|
||||
.branch_latest_page(target_id, *h5_off)
|
||||
.unwrap_or_else(|| vec![0u8; source_bytes.len()]);
|
||||
// Use the h5_offset as the "dataset path" key (real impl would map offsets to HDF5 paths)
|
||||
let path = format!("page@{h5_off}");
|
||||
let result = resolver(&path, &target_bytes, source_bytes);
|
||||
merged.insert(*h5_off, result);
|
||||
}
|
||||
|
||||
let mut session = self.begin_session(Some(target_id))?;
|
||||
for (h5_off, bytes) in &merged {
|
||||
session.record_page(*h5_off, bytes);
|
||||
}
|
||||
let annotation = format!("merge branch {source_id} → {target_id} [dataset-level]");
|
||||
self.commit_session(session, Some(&annotation))
|
||||
}
|
||||
|
||||
fn merge_three_way(
|
||||
&mut self,
|
||||
source_id: u32,
|
||||
target_id: u32,
|
||||
) -> Result<u64, OnionError> {
|
||||
let source_head = self
|
||||
.index
|
||||
.branch_head(source_id)
|
||||
.ok_or_else(|| OnionError::Malformed(format!("source branch {source_id} has no revisions")))?
|
||||
.revision;
|
||||
|
||||
let target_head = self
|
||||
.index
|
||||
.branch_head(target_id)
|
||||
.ok_or_else(|| OnionError::Malformed(format!("target branch {target_id} has no revisions")))?
|
||||
.revision;
|
||||
|
||||
// Find common ancestor of the two branch HEADs
|
||||
let ancestor_rev = self
|
||||
.index
|
||||
.common_ancestor(source_head, target_head)
|
||||
.ok_or_else(|| {
|
||||
let src_name = self.branches.iter().find(|b| b.id == source_id)
|
||||
.and_then(|b| self.annotations.get(b.name_off))
|
||||
.unwrap_or("?").to_owned();
|
||||
let tgt_name = self.branches.iter().find(|b| b.id == target_id)
|
||||
.and_then(|b| self.annotations.get(b.name_off))
|
||||
.unwrap_or("?").to_owned();
|
||||
OnionError::NoCommonAncestor { a: src_name, b: tgt_name }
|
||||
})?;
|
||||
|
||||
// Collect source delta since ancestor (latest write per offset)
|
||||
let source_revs: Vec<u64> = self
|
||||
.index
|
||||
.branch_revisions(source_id)
|
||||
.filter(|e| e.revision > ancestor_rev)
|
||||
.map(|e| e.revision)
|
||||
.collect();
|
||||
|
||||
let mut source_delta: BTreeMap<u64, Vec<u8>> = BTreeMap::new();
|
||||
for rev in &source_revs {
|
||||
for (h5_off, bytes) in self.revision_pages(*rev)? {
|
||||
source_delta.insert(h5_off, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
// Collect target delta since ancestor (latest write per offset)
|
||||
let target_revs: Vec<u64> = self
|
||||
.index
|
||||
.branch_revisions(target_id)
|
||||
.filter(|e| e.revision > ancestor_rev)
|
||||
.map(|e| e.revision)
|
||||
.collect();
|
||||
|
||||
let mut target_delta: BTreeMap<u64, Vec<u8>> = BTreeMap::new();
|
||||
for rev in &target_revs {
|
||||
for (h5_off, bytes) in self.revision_pages(*rev)? {
|
||||
target_delta.insert(h5_off, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
if source_delta.is_empty() {
|
||||
return Err(OnionError::Malformed(
|
||||
"three-way merge: source has no new changes since the common ancestor".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// Union of all changed page offsets across both deltas
|
||||
let all_offsets: BTreeSet<u64> = source_delta
|
||||
.keys()
|
||||
.chain(target_delta.keys())
|
||||
.copied()
|
||||
.collect();
|
||||
|
||||
let mut merged: BTreeMap<u64, Vec<u8>> = BTreeMap::new();
|
||||
for offset in &all_offsets {
|
||||
let result = match (source_delta.get(offset), target_delta.get(offset)) {
|
||||
// Only source changed this page → use source
|
||||
(Some(s), None) => s.clone(),
|
||||
// Only target changed this page → use target (already on target, no-op)
|
||||
(None, Some(_t)) => continue,
|
||||
// Both changed → source wins (last-write-wins for conflicts)
|
||||
(Some(s), Some(_t)) => s.clone(),
|
||||
(None, None) => unreachable!(),
|
||||
};
|
||||
merged.insert(*offset, result);
|
||||
}
|
||||
|
||||
if merged.is_empty() {
|
||||
return Err(OnionError::Malformed(
|
||||
"three-way merge: no source changes to apply (target already has all changes)".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut session = self.begin_session(Some(target_id))?;
|
||||
for (h5_off, bytes) in &merged {
|
||||
session.record_page(*h5_off, bytes);
|
||||
}
|
||||
let annotation = format!(
|
||||
"3-way merge {source_id} → {target_id} [ancestor rev {ancestor_rev}]"
|
||||
);
|
||||
self.commit_session(session, Some(&annotation))
|
||||
}
|
||||
|
||||
// ── Internal helpers ─────────────────────────────────────────────────────
|
||||
|
||||
/// Return the most recent page bytes written on `branch_id` at `h5_offset`,
|
||||
/// or `None` if that branch has never written to that offset.
|
||||
fn branch_latest_page(&self, branch_id: u32, h5_offset: u64) -> Option<Vec<u8>> {
|
||||
let revs: Vec<u64> = self
|
||||
.index
|
||||
.branch_revisions(branch_id)
|
||||
.map(|e| e.revision)
|
||||
.collect();
|
||||
|
||||
for rev in revs.iter().rev() {
|
||||
if let Some(table) = self.page_tables.get(*rev as usize) {
|
||||
if let Some(pt) = table.iter().find(|pt| pt.h5_offset == h5_offset) {
|
||||
let codec = Codec::from_u8(pt.codec).ok()?;
|
||||
let start = pt.data_offset as usize;
|
||||
let end = start + pt.data_size as usize;
|
||||
if end > self.page_data.len() {
|
||||
return None;
|
||||
}
|
||||
let compressed = &self.page_data[start..end];
|
||||
return decompress_page(compressed, codec, pt.orig_size).ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
// ── Lifecycle ────────────────────────────────────────────────────────────
|
||||
|
||||
/// List all branches with their current HEAD revision.
|
||||
pub fn list_branches(&self) -> Vec<BranchInfo> {
|
||||
self.branches
|
||||
.iter()
|
||||
.map(|b| BranchInfo {
|
||||
id: b.id,
|
||||
name: self
|
||||
.annotations
|
||||
.get(b.name_off)
|
||||
.unwrap_or("?")
|
||||
.to_owned(),
|
||||
head_rev: b.head_rev,
|
||||
fork_rev: b.fork_rev,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Return revision entries for a named branch in chronological order.
|
||||
pub fn branch_history(&self, name: &str) -> Result<Vec<&crate::format::RevisionEntry>, OnionError> {
|
||||
let branch = self
|
||||
.branch_by_name(name)
|
||||
.ok_or_else(|| OnionError::BranchNotFound(name.to_string()))?;
|
||||
Ok(self.index.branch_revisions(branch.id).collect())
|
||||
}
|
||||
|
||||
/// Rename a branch.
|
||||
pub fn rename_branch(&mut self, from: &str, to: &str) -> Result<(), OnionError> {
|
||||
if self.branch_by_name(to).is_some() {
|
||||
return Err(OnionError::BranchExists(to.to_string()));
|
||||
}
|
||||
let id = self
|
||||
.branch_by_name(from)
|
||||
.ok_or_else(|| OnionError::BranchNotFound(from.to_string()))?
|
||||
.id;
|
||||
let new_name_off = self.annotations.push(to);
|
||||
self.branches
|
||||
.iter_mut()
|
||||
.find(|b| b.id == id)
|
||||
.unwrap()
|
||||
.name_off = new_name_off;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Delete a branch. The `main` branch (id 0) cannot be deleted.
|
||||
pub fn delete_branch(&mut self, name: &str) -> Result<(), OnionError> {
|
||||
let branch = self
|
||||
.branch_by_name(name)
|
||||
.ok_or_else(|| OnionError::BranchNotFound(name.to_string()))?;
|
||||
if branch.id == crate::format::BRANCH_MAIN {
|
||||
return Err(OnionError::Malformed("cannot delete the main branch".into()));
|
||||
}
|
||||
let id = branch.id;
|
||||
self.branches.retain(|b| b.id != id);
|
||||
self.header.branch_count = self.branches.len() as u32;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::format::BRANCH_MAIN;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
fn tmp_h5() -> std::path::PathBuf {
|
||||
let f = NamedTempFile::new().unwrap();
|
||||
let path = f.path().with_extension("h5");
|
||||
std::fs::write(&path, b"\x89HDF\r\n\x1a\n").unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_branches_initial() {
|
||||
let h5 = tmp_h5();
|
||||
let onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
let branches = onion.list_branches();
|
||||
assert_eq!(branches.len(), 1);
|
||||
assert_eq!(branches[0].name, "main");
|
||||
assert_eq!(branches[0].id, BRANCH_MAIN);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_branch_from_main() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
// commit something on main first
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
|
||||
let id = onion.create_branch("experiment", "main").unwrap();
|
||||
assert_eq!(id, 1);
|
||||
let branches = onion.list_branches();
|
||||
assert_eq!(branches.len(), 2);
|
||||
assert!(branches.iter().any(|b| b.name == "experiment"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_duplicate_branch_errors() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
onion.create_branch("feat", "main").unwrap();
|
||||
let err = onion.create_branch("feat", "main").unwrap_err();
|
||||
assert!(matches!(err, OnionError::BranchExists(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_branch_from_nonexistent_errors() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
let err = onion.create_branch("feat", "no-such-branch").unwrap_err();
|
||||
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn branch_history_empty_branch() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
|
||||
onion.create_branch("feat", "main").unwrap();
|
||||
let history = onion.branch_history("feat").unwrap();
|
||||
// "feat" branches from main, so no new revisions on it yet
|
||||
assert_eq!(history.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn branch_history_with_commits() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
// main: rev 0
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
|
||||
// create feat branch
|
||||
let feat_id = onion.create_branch("feat", "main").unwrap();
|
||||
|
||||
// commit on feat
|
||||
let mut sf = onion.begin_session(Some(feat_id)).unwrap();
|
||||
sf.record_page(4096, &vec![0xFFu8; 4096]);
|
||||
onion.commit_session(sf, Some("feat commit")).unwrap();
|
||||
|
||||
let history = onion.branch_history("feat").unwrap();
|
||||
assert_eq!(history.len(), 1);
|
||||
assert_eq!(history[0].branch_id, feat_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_branch_succeeds() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
onion.create_branch("old-name", "main").unwrap();
|
||||
onion.rename_branch("old-name", "new-name").unwrap();
|
||||
|
||||
assert!(onion.branch_by_name("new-name").is_some());
|
||||
assert!(onion.branch_by_name("old-name").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_to_existing_name_errors() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
onion.create_branch("a", "main").unwrap();
|
||||
onion.create_branch("b", "main").unwrap();
|
||||
let err = onion.rename_branch("a", "b").unwrap_err();
|
||||
assert!(matches!(err, OnionError::BranchExists(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_branch_removes_it() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
onion.create_branch("temp", "main").unwrap();
|
||||
onion.delete_branch("temp").unwrap();
|
||||
assert!(onion.branch_by_name("temp").is_none());
|
||||
assert_eq!(onion.list_branches().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_main_branch_errors() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
let err = onion.delete_branch("main").unwrap_err();
|
||||
assert!(matches!(err, OnionError::Malformed(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_nonexistent_branch_errors() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
let err = onion.delete_branch("ghost").unwrap_err();
|
||||
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
||||
}
|
||||
|
||||
// ── Merge: LatestWins ────────────────────────────────────────────────────
|
||||
|
||||
/// Helper: main has rev 0 (page 0 = AA), feat forks and writes rev 1
|
||||
/// (page 0 = BB). After merge, main HEAD should have page 0 = BB.
|
||||
fn make_fork_scenario() -> (std::path::PathBuf, OnionFile, u32) {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
// main rev 0
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0xAAu8; 4096]);
|
||||
onion.commit_session(s, Some("main r0")).unwrap();
|
||||
|
||||
// fork feat from main
|
||||
let feat_id = onion.create_branch("feat", "main").unwrap();
|
||||
|
||||
// feat rev 1: overwrite page 0
|
||||
let mut sf = onion.begin_session(Some(feat_id)).unwrap();
|
||||
sf.record_page(0, &vec![0xBBu8; 4096]);
|
||||
onion.commit_session(sf, Some("feat r1")).unwrap();
|
||||
|
||||
(h5, onion, feat_id)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_latest_wins_produces_new_revision() {
|
||||
let (_h5, mut onion, _feat_id) = make_fork_scenario();
|
||||
let pre_count = onion.revision_count();
|
||||
onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap();
|
||||
assert_eq!(onion.revision_count(), pre_count + 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_latest_wins_page_content_correct() {
|
||||
let (h5, mut onion, _feat_id) = make_fork_scenario();
|
||||
onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap();
|
||||
onion.flush().unwrap();
|
||||
|
||||
// Reload from disk and reconstruct main HEAD
|
||||
let onion2 = OnionFile::open(&h5).unwrap();
|
||||
let main_head = onion2.revision_count() - 1;
|
||||
let base = std::fs::read(&h5).unwrap();
|
||||
let state = onion2.reconstruct_revision(main_head, &base).unwrap();
|
||||
// Page 0 should now be BB (from feat)
|
||||
assert!(state[0..4096].iter().all(|&b| b == 0xBB),
|
||||
"page 0 should be BB after latest-wins merge");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_latest_wins_multiple_source_revisions() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
|
||||
// main: rev 0
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0x00u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
|
||||
let feat_id = onion.create_branch("feat", "main").unwrap();
|
||||
|
||||
// feat: revs 1 and 2 (two writes on same page — rev 2 must win)
|
||||
let mut s1 = onion.begin_session(Some(feat_id)).unwrap();
|
||||
s1.record_page(0, &vec![0x11u8; 4096]);
|
||||
onion.commit_session(s1, None).unwrap();
|
||||
|
||||
let mut s2 = onion.begin_session(Some(feat_id)).unwrap();
|
||||
s2.record_page(0, &vec![0x22u8; 4096]);
|
||||
onion.commit_session(s2, None).unwrap();
|
||||
|
||||
onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap();
|
||||
onion.flush().unwrap();
|
||||
|
||||
let onion2 = OnionFile::open(&h5).unwrap();
|
||||
let head = onion2.revision_count() - 1;
|
||||
let base = std::fs::read(&h5).unwrap();
|
||||
let state = onion2.reconstruct_revision(head, &base).unwrap();
|
||||
assert!(state[0..4096].iter().all(|&b| b == 0x22),
|
||||
"latest write (0x22) must win in LatestWins merge");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_source_with_no_new_revisions_errors() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
// fork but make no commits on feat
|
||||
onion.create_branch("feat", "main").unwrap();
|
||||
let err = onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap_err();
|
||||
assert!(matches!(err, OnionError::Malformed(_)));
|
||||
}
|
||||
|
||||
// ── Merge: DatasetLevel ──────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn merge_dataset_level_resolver_called() {
|
||||
let (_h5, mut onion, _) = make_fork_scenario();
|
||||
let called = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let called2 = called.clone();
|
||||
let resolver = move |_path: &str, _target: &[u8], source: &[u8]| -> Vec<u8> {
|
||||
called2.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
source.to_vec() // just return source
|
||||
};
|
||||
onion
|
||||
.merge_into("feat", "main", MergeStrategy::DatasetLevel(Box::new(resolver)))
|
||||
.unwrap();
|
||||
assert!(called.load(std::sync::atomic::Ordering::SeqCst), "resolver must be called");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_dataset_level_resolver_controls_output() {
|
||||
let (h5, mut onion, _feat_id) = make_fork_scenario();
|
||||
// Resolver always returns 0xCC regardless of inputs
|
||||
let resolver = |_path: &str, _target: &[u8], _source: &[u8]| -> Vec<u8> {
|
||||
vec![0xCCu8; 4096]
|
||||
};
|
||||
onion
|
||||
.merge_into("feat", "main", MergeStrategy::DatasetLevel(Box::new(resolver)))
|
||||
.unwrap();
|
||||
onion.flush().unwrap();
|
||||
|
||||
let onion2 = OnionFile::open(&h5).unwrap();
|
||||
let head = onion2.revision_count() - 1;
|
||||
let base = std::fs::read(&h5).unwrap();
|
||||
let state = onion2.reconstruct_revision(head, &base).unwrap();
|
||||
assert!(state[0..4096].iter().all(|&b| b == 0xCC),
|
||||
"resolver output 0xCC should be in merged state");
|
||||
}
|
||||
|
||||
// ── Merge: ThreeWay ──────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn merge_three_way_only_source_change_applied() {
|
||||
// main: 0(AA) → 1(CC on page 4096)
|
||||
// feat: forked at rev 0, writes rev 1(BB on page 0)
|
||||
// 3-way merge feat → main:
|
||||
// - page 0: only feat changed it (vs ancestor rev 0) → use feat (BB)
|
||||
// - page 4096: only main changed it → skip (already on main)
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
|
||||
// main rev 0: page 0 = AA
|
||||
let mut s0 = onion.begin_session(None).unwrap();
|
||||
s0.record_page(0, &vec![0xAAu8; 4096]);
|
||||
onion.commit_session(s0, None).unwrap();
|
||||
|
||||
let feat_id = onion.create_branch("feat", "main").unwrap();
|
||||
|
||||
// main rev 1: writes page 4096 = CC (diverges from feat)
|
||||
let mut sm = onion.begin_session(None).unwrap();
|
||||
sm.record_page(4096, &vec![0xCCu8; 4096]);
|
||||
onion.commit_session(sm, None).unwrap();
|
||||
|
||||
// feat rev 2: writes page 0 = BB
|
||||
let mut sf = onion.begin_session(Some(feat_id)).unwrap();
|
||||
sf.record_page(0, &vec![0xBBu8; 4096]);
|
||||
onion.commit_session(sf, None).unwrap();
|
||||
|
||||
onion.merge_into("feat", "main", MergeStrategy::ThreeWay).unwrap();
|
||||
onion.flush().unwrap();
|
||||
|
||||
let onion2 = OnionFile::open(&h5).unwrap();
|
||||
let head = onion2.revision_count() - 1;
|
||||
let base = std::fs::read(&h5).unwrap();
|
||||
let state = onion2.reconstruct_revision(head, &base).unwrap();
|
||||
|
||||
assert!(state[0..4096].iter().all(|&b| b == 0xBB),
|
||||
"page 0 should be BB (from feat)");
|
||||
assert!(state[4096..8192].iter().all(|&b| b == 0xCC),
|
||||
"page 4096 should stay CC (from main — not overwritten by 3-way)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_three_way_conflict_source_wins() {
|
||||
// Both branches write to page 0 after the fork → source (feat) should win.
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
|
||||
let mut s0 = onion.begin_session(None).unwrap();
|
||||
s0.record_page(0, &vec![0x00u8; 4096]);
|
||||
onion.commit_session(s0, None).unwrap();
|
||||
|
||||
let feat_id = onion.create_branch("feat", "main").unwrap();
|
||||
|
||||
// main writes BB (target change)
|
||||
let mut sm = onion.begin_session(None).unwrap();
|
||||
sm.record_page(0, &vec![0xBBu8; 4096]);
|
||||
onion.commit_session(sm, None).unwrap();
|
||||
|
||||
// feat writes CC (source change)
|
||||
let mut sf = onion.begin_session(Some(feat_id)).unwrap();
|
||||
sf.record_page(0, &vec![0xCCu8; 4096]);
|
||||
onion.commit_session(sf, None).unwrap();
|
||||
|
||||
onion.merge_into("feat", "main", MergeStrategy::ThreeWay).unwrap();
|
||||
onion.flush().unwrap();
|
||||
|
||||
let onion2 = OnionFile::open(&h5).unwrap();
|
||||
let head = onion2.revision_count() - 1;
|
||||
let base = std::fs::read(&h5).unwrap();
|
||||
let state = onion2.reconstruct_revision(head, &base).unwrap();
|
||||
assert!(state[0..4096].iter().all(|&b| b == 0xCC),
|
||||
"on conflict, source (CC) should win");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_nonexistent_source_errors() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
let err = onion.merge_into("no-such", "main", MergeStrategy::LatestWins).unwrap_err();
|
||||
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_nonexistent_target_errors() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
onion.create_branch("feat", "main").unwrap();
|
||||
let err = onion.merge_into("feat", "no-target", MergeStrategy::LatestWins).unwrap_err();
|
||||
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
//! Per-page compression and decompression.
|
||||
//!
|
||||
//! Delegates to `zstd` and `lz4_flex` for the respective codecs.
|
||||
//! Brotli support is a placeholder for Phase 4.
|
||||
|
||||
use crate::error::OnionError;
|
||||
use crate::format::Codec;
|
||||
use crate::tdt;
|
||||
|
||||
/// Compress `data` using the given codec.
|
||||
///
|
||||
/// Returns the compressed bytes, or the original bytes if `codec` is [`Codec::None`].
|
||||
pub fn compress_page(data: &[u8], codec: Codec) -> Result<Vec<u8>, OnionError> {
|
||||
match codec {
|
||||
Codec::None => Ok(data.to_vec()),
|
||||
Codec::Zstd => zstd::bulk::compress(data, 3)
|
||||
.map_err(|e| OnionError::Compress(e.to_string())),
|
||||
Codec::Lz4 => Ok(lz4_flex::compress_prepend_size(data)),
|
||||
Codec::Brotli => Err(OnionError::Compress(
|
||||
"brotli not yet implemented".to_string(),
|
||||
)),
|
||||
Codec::ZstdTdt => {
|
||||
let interleaved = tdt::encode(data, 4);
|
||||
zstd::bulk::compress(&interleaved, 3)
|
||||
.map_err(|e| OnionError::Compress(e.to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decompress `data` using the given codec back to `orig_size` bytes.
|
||||
pub fn decompress_page(
|
||||
data: &[u8],
|
||||
codec: Codec,
|
||||
orig_size: u32,
|
||||
) -> Result<Vec<u8>, OnionError> {
|
||||
match codec {
|
||||
Codec::None => {
|
||||
if data.len() != orig_size as usize {
|
||||
return Err(OnionError::Malformed(format!(
|
||||
"uncompressed page length {} != expected {}",
|
||||
data.len(),
|
||||
orig_size
|
||||
)));
|
||||
}
|
||||
Ok(data.to_vec())
|
||||
}
|
||||
Codec::Zstd => zstd::bulk::decompress(data, orig_size as usize)
|
||||
.map_err(|e| OnionError::Compress(e.to_string())),
|
||||
Codec::Lz4 => lz4_flex::decompress_size_prepended(data)
|
||||
.map_err(|e| OnionError::Compress(e.to_string())),
|
||||
Codec::Brotli => Err(OnionError::Compress(
|
||||
"brotli not yet implemented".to_string(),
|
||||
)),
|
||||
Codec::ZstdTdt => {
|
||||
let interleaved = zstd::bulk::decompress(data, orig_size as usize)
|
||||
.map_err(|e| OnionError::Compress(e.to_string()))?;
|
||||
Ok(tdt::decode(&interleaved, 4, orig_size as usize))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn roundtrip(codec: Codec, data: &[u8]) {
|
||||
let orig_size = data.len() as u32;
|
||||
let compressed = compress_page(data, codec).unwrap();
|
||||
let decompressed = decompress_page(&compressed, codec, orig_size).unwrap();
|
||||
assert_eq!(decompressed, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_none_roundtrip() {
|
||||
let data = b"hello world, this is a test page";
|
||||
roundtrip(Codec::None, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_none_wrong_size_error() {
|
||||
let data = b"short";
|
||||
let err = decompress_page(data, Codec::None, 999).unwrap_err();
|
||||
assert!(matches!(err, OnionError::Malformed(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_zstd_roundtrip_small() {
|
||||
let data = b"the quick brown fox jumps over the lazy dog";
|
||||
roundtrip(Codec::Zstd, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_zstd_roundtrip_4kb() {
|
||||
let data: Vec<u8> = (0..4096).map(|i| (i % 256) as u8).collect();
|
||||
roundtrip(Codec::Zstd, &data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_zstd_compresses_repetitive_data() {
|
||||
// Repetitive data should compress well
|
||||
let data = vec![0xABu8; 4096];
|
||||
let compressed = compress_page(&data, Codec::Zstd).unwrap();
|
||||
assert!(
|
||||
compressed.len() < data.len(),
|
||||
"zstd should compress repetitive data"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_lz4_roundtrip_small() {
|
||||
let data = b"lz4 compression test data";
|
||||
roundtrip(Codec::Lz4, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_lz4_roundtrip_4kb() {
|
||||
let data: Vec<u8> = (0..4096).map(|i| (i % 127) as u8).collect();
|
||||
roundtrip(Codec::Lz4, &data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_lz4_compresses_repetitive_data() {
|
||||
let data = vec![0u8; 4096];
|
||||
let compressed = compress_page(&data, Codec::Lz4).unwrap();
|
||||
assert!(compressed.len() < data.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_brotli_returns_error() {
|
||||
let result = compress_page(b"test", Codec::Brotli);
|
||||
assert!(matches!(result, Err(OnionError::Compress(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_zstd_tdt_roundtrip_4kb() {
|
||||
let mut data = Vec::with_capacity(4096);
|
||||
for i in 0u32..1024 {
|
||||
let v = (i as f32 * 0.001 + 1.0).to_le_bytes();
|
||||
data.extend_from_slice(&v);
|
||||
}
|
||||
roundtrip(Codec::ZstdTdt, &data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_zstd_tdt_roundtrip_random() {
|
||||
let data: Vec<u8> = (0u8..=255).cycle().take(4096).collect();
|
||||
roundtrip(Codec::ZstdTdt, &data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_zstd_tdt_better_ratio_on_smooth_floats() {
|
||||
// Smooth float data: ZstdTdt must compress better than plain Zstd.
|
||||
let mut data = Vec::with_capacity(65536);
|
||||
for i in 0u32..16384 {
|
||||
let v = (1.0f32 + i as f32 * 0.00001).to_le_bytes();
|
||||
data.extend_from_slice(&v);
|
||||
}
|
||||
let orig = data.len() as u32;
|
||||
let zstd_size = compress_page(&data, Codec::Zstd).unwrap().len();
|
||||
let tdt_size = compress_page(&data, Codec::ZstdTdt).unwrap().len();
|
||||
assert!(
|
||||
tdt_size < zstd_size,
|
||||
"ZstdTdt ({tdt_size} B) should beat plain Zstd ({zstd_size} B) on smooth f32 data ({orig} B)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_from_u8_zstd_tdt() {
|
||||
use crate::format::Codec;
|
||||
assert!(matches!(Codec::from_u8(4), Ok(Codec::ZstdTdt)));
|
||||
assert!(matches!(Codec::from_u8(5), Err(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compress_empty_page_none() {
|
||||
let data = b"";
|
||||
roundtrip(Codec::None, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compress_empty_page_zstd() {
|
||||
roundtrip(Codec::Zstd, b"");
|
||||
}
|
||||
|
||||
// ── OnionFile integration: commit with ZstdTdt, flush, reload, reconstruct ─
|
||||
|
||||
#[test]
|
||||
fn onion_with_zstd_tdt_commit_reload_reconstruct() {
|
||||
use crate::writer::OnionFile;
|
||||
use tempfile::TempDir;
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let h5 = dir.path().join("test.h5");
|
||||
let base = b"\x89HDF\r\n\x1a\n";
|
||||
std::fs::write(&h5, base).unwrap();
|
||||
|
||||
// Create file, switch codec to ZstdTdt, commit 2 revisions.
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
onion.set_codec(Codec::ZstdTdt);
|
||||
|
||||
// Float-like page: 1024 f32 values.
|
||||
let page_a: Vec<u8> = (0u32..1024)
|
||||
.flat_map(|i| (i as f32 * 0.001).to_le_bytes())
|
||||
.collect();
|
||||
let page_b: Vec<u8> = (0u32..1024)
|
||||
.flat_map(|i| (i as f32 * 0.002 + 1.0).to_le_bytes())
|
||||
.collect();
|
||||
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &page_a);
|
||||
onion.commit_session(s, Some("rev0")).unwrap();
|
||||
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &page_b);
|
||||
onion.commit_session(s, Some("rev1")).unwrap();
|
||||
|
||||
onion.flush().unwrap();
|
||||
|
||||
// Reload and reconstruct both revisions.
|
||||
let reloaded = OnionFile::open(&h5).unwrap();
|
||||
assert_eq!(reloaded.revision_count(), 2);
|
||||
|
||||
let h5_base = std::fs::read(&h5).unwrap();
|
||||
let rec0 = reloaded.reconstruct_revision(0, &h5_base).unwrap();
|
||||
let rec1 = reloaded.reconstruct_revision(1, &h5_base).unwrap();
|
||||
|
||||
assert_eq!(&rec0[..4096], &page_a[..], "revision 0 page mismatch");
|
||||
assert_eq!(&rec1[..4096], &page_b[..], "revision 1 page mismatch");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
//! Error types for the ClawOnion VFD.
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// All errors that can occur in `clawhdf5-onion`.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum OnionError {
|
||||
/// The `.onion` file does not start with the `CLAWONION` magic bytes.
|
||||
#[error("invalid magic bytes — not a ClawOnion file")]
|
||||
InvalidMagic,
|
||||
|
||||
/// The file carries a `format_version` this reader does not support.
|
||||
#[error("unsupported ClawOnion format version: {0}")]
|
||||
UnknownVersion(u8),
|
||||
|
||||
/// A page table entry uses an unrecognised compression codec byte.
|
||||
#[error("unknown page codec: {0}")]
|
||||
UnknownCodec(u8),
|
||||
|
||||
/// The requested revision does not exist in the index.
|
||||
#[error("revision {0} not found")]
|
||||
RevisionNotFound(u64),
|
||||
|
||||
/// The requested branch does not exist.
|
||||
#[error("branch {0:?} not found")]
|
||||
BranchNotFound(String),
|
||||
|
||||
/// A branch with that name already exists.
|
||||
#[error("branch {0:?} already exists")]
|
||||
BranchExists(String),
|
||||
|
||||
/// Page data BLAKE3 hash does not match stored hash — integrity failure.
|
||||
#[error("BLAKE3 hash mismatch on revision {revision}: stored {stored}, computed {computed}")]
|
||||
HashMismatch {
|
||||
revision: u64,
|
||||
stored: String,
|
||||
computed: String,
|
||||
},
|
||||
|
||||
/// The page size is not a power of two, or is zero.
|
||||
#[error("invalid page size {0}: must be a non-zero power of two")]
|
||||
InvalidPageSize(u32),
|
||||
|
||||
/// The `.onion` file is structurally truncated or otherwise malformed.
|
||||
#[error("malformed ClawOnion file: {0}")]
|
||||
Malformed(String),
|
||||
|
||||
/// A merge was attempted on a branch with no common ancestor.
|
||||
#[error("no common ancestor found between branches {a:?} and {b:?}")]
|
||||
NoCommonAncestor { a: String, b: String },
|
||||
|
||||
/// Compression or decompression failed.
|
||||
#[error("compression error: {0}")]
|
||||
Compress(String),
|
||||
|
||||
/// An underlying I/O error.
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
/// An error from the `clawhdf5` HDF5 parser (e.g. invalid bytes when
|
||||
/// reconstructing a historical revision as a `clawhdf5::File`).
|
||||
#[error("HDF5 parse error: {0}")]
|
||||
Hdf5(String),
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
//! Extension traits for `clawhdf5` types.
|
||||
//!
|
||||
//! `clawhdf5` cannot depend on `clawhdf5-onion` (that would be circular), so
|
||||
//! the integration methods live here as opt-in extension traits. Import them
|
||||
//! with a single `use`:
|
||||
//!
|
||||
//! ```rust,ignore
|
||||
//! use clawhdf_onion::ext::FileBuilderExt;
|
||||
//!
|
||||
//! let mut b = clawhdf5::FileBuilder::new();
|
||||
//! b.create_dataset("data").with_f64_data(&[1.0, 2.0]);
|
||||
//! let vf = b.with_onion("agent.h5", 4096)?;
|
||||
//! // vf is a VersionedFile — commit, branch, revision, etc.
|
||||
//! ```
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use crate::error::OnionError;
|
||||
use crate::versioned_file::VersionedFile;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// FileBuilderExt
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Extension methods for [`clawhdf5::FileBuilder`] that integrate with
|
||||
/// ClawOnion versioning.
|
||||
pub trait FileBuilderExt {
|
||||
/// Write the builder output to `h5_path` and create a versioned
|
||||
/// `.onion` sidecar in one step.
|
||||
///
|
||||
/// Equivalent to:
|
||||
/// ```rust,ignore
|
||||
/// builder.write(h5_path)?;
|
||||
/// VersionedFile::create(h5_path, page_size)
|
||||
/// ```
|
||||
///
|
||||
/// The sidecar is flushed to disk before returning, so `h5_path.onion`
|
||||
/// exists even when no revisions have been committed yet.
|
||||
fn with_onion(
|
||||
self,
|
||||
h5_path: impl AsRef<Path>,
|
||||
page_size: u32,
|
||||
) -> Result<VersionedFile, OnionError>;
|
||||
|
||||
/// Like [`with_onion`](FileBuilderExt::with_onion) but automatically
|
||||
/// chooses the page size from the HDF5 file structure.
|
||||
///
|
||||
/// See [`VersionedFile::create_auto`] for the selection algorithm.
|
||||
fn with_onion_auto(self, h5_path: impl AsRef<Path>) -> Result<VersionedFile, OnionError>;
|
||||
}
|
||||
|
||||
impl FileBuilderExt for clawhdf5::FileBuilder {
|
||||
fn with_onion(
|
||||
self,
|
||||
h5_path: impl AsRef<Path>,
|
||||
page_size: u32,
|
||||
) -> Result<VersionedFile, OnionError> {
|
||||
VersionedFile::from_builder(self, h5_path, page_size)
|
||||
}
|
||||
|
||||
fn with_onion_auto(self, h5_path: impl AsRef<Path>) -> Result<VersionedFile, OnionError> {
|
||||
let h5_path = h5_path.as_ref();
|
||||
self.write(h5_path)
|
||||
.map_err(|e| OnionError::Hdf5(e.to_string()))?;
|
||||
let mut vf = VersionedFile::create_auto(h5_path)?;
|
||||
vf.onion_mut().flush()?;
|
||||
Ok(vf)
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::path::PathBuf;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
fn make_builder(values: &[f64]) -> clawhdf5::FileBuilder {
|
||||
let mut b = clawhdf5::FileBuilder::new();
|
||||
b.create_dataset("data").with_f64_data(values);
|
||||
b
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_onion_creates_versioned_file() {
|
||||
let tmp = NamedTempFile::new().unwrap();
|
||||
let h5_path = tmp.path().with_extension("h5");
|
||||
|
||||
let vf = make_builder(&[1.0, 2.0, 3.0])
|
||||
.with_onion(&h5_path, 4096)
|
||||
.unwrap();
|
||||
|
||||
let f = vf.current().unwrap();
|
||||
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![1.0, 2.0, 3.0]);
|
||||
assert_eq!(vf.onion().revision_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_onion_sidecar_exists_on_disk() {
|
||||
let tmp = NamedTempFile::new().unwrap();
|
||||
let h5_path = tmp.path().with_extension("h5");
|
||||
|
||||
let _ = make_builder(&[1.0]).with_onion(&h5_path, 4096).unwrap();
|
||||
|
||||
let onion_path = PathBuf::from(format!("{}.onion", h5_path.display()));
|
||||
assert!(onion_path.exists(), "sidecar should be flushed by with_onion");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_onion_commit_roundtrip() {
|
||||
let tmp = NamedTempFile::new().unwrap();
|
||||
let h5_path = tmp.path().with_extension("h5");
|
||||
|
||||
let mut vf = make_builder(&[1.0]).with_onion(&h5_path, 4096).unwrap();
|
||||
vf.commit(
|
||||
{
|
||||
let mut b = clawhdf5::FileBuilder::new();
|
||||
b.create_dataset("data").with_f64_data(&[2.0]);
|
||||
b.finish().unwrap()
|
||||
},
|
||||
Some("v1"),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let f = vf.revision(0).unwrap();
|
||||
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![2.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_onion_auto_uses_detected_page_size() {
|
||||
let tmp = NamedTempFile::new().unwrap();
|
||||
let h5_path = tmp.path().with_extension("h5");
|
||||
|
||||
// A small file — auto page size should fall back to the 4 KiB default.
|
||||
let vf = make_builder(&[1.0]).with_onion_auto(&h5_path).unwrap();
|
||||
assert!(
|
||||
vf.page_size().is_power_of_two(),
|
||||
"auto page size must be a power of two"
|
||||
);
|
||||
assert!(vf.page_size() >= 4096, "auto page size must be at least 4 KiB");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,517 @@
|
||||
//! ClawOnion v1 binary format structs.
|
||||
//!
|
||||
//! All on-disk types are `zerocopy`-derived with explicit padding so they
|
||||
//! can be read/written directly from raw bytes without deserialization overhead.
|
||||
//! Every struct uses `#[repr(C)]` and has **no implicit padding** — alignment
|
||||
//! gaps are filled with named `_pad*` fields.
|
||||
//!
|
||||
//! # Layout overview
|
||||
//!
|
||||
//! ```text
|
||||
//! agent.onion
|
||||
//! ├── OnionHeader (128 bytes, fixed, at offset 0)
|
||||
//! ├── RevisionEntry[] (one per revision, fixed-size, at index_offset)
|
||||
//! ├── BranchEntry[] (one per branch, fixed-size, at branch_offset; 0 = absent)
|
||||
//! ├── PageTableEntry[] (one per page per revision, variable position)
|
||||
//! ├── PageData (raw compressed/verbatim page bytes, contiguous)
|
||||
//! └── AnnotationHeap (length-prefixed UTF-8 strings)
|
||||
//! ```
|
||||
//!
|
||||
//! The format magic is `b"CLAWONION"` (9 bytes). Readers must reject files
|
||||
//! with unknown `format_version` values with [`OnionError::UnknownVersion`].
|
||||
|
||||
use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout};
|
||||
|
||||
use crate::error::OnionError;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Constants
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Magic bytes identifying a ClawOnion sidecar file.
|
||||
pub const MAGIC: &[u8; 9] = b"CLAWONION";
|
||||
|
||||
/// Current ClawOnion format version.
|
||||
pub const FORMAT_VERSION: u8 = 1;
|
||||
|
||||
/// Sentinel for "no parent" (root revision or initial branch).
|
||||
pub const NO_PARENT: u64 = u64::MAX;
|
||||
|
||||
/// Branch ID for the default `main` branch.
|
||||
pub const BRANCH_MAIN: u32 = 0;
|
||||
|
||||
/// Header size in bytes (fixed, 128).
|
||||
pub const HEADER_SIZE: usize = 128;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Feature flags
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
pub mod feature_flags {
|
||||
pub const COMPRESSION: u64 = 1 << 0;
|
||||
pub const BRANCHING: u64 = 1 << 1;
|
||||
pub const PROVENANCE: u64 = 1 << 2;
|
||||
pub const SNAPSHOTS: u64 = 1 << 3;
|
||||
}
|
||||
|
||||
pub const DEFAULT_FEATURE_FLAGS: u64 =
|
||||
feature_flags::COMPRESSION | feature_flags::BRANCHING | feature_flags::PROVENANCE;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Codec
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[repr(u8)]
|
||||
pub enum Codec {
|
||||
None = 0,
|
||||
Zstd = 1,
|
||||
Lz4 = 2,
|
||||
Brotli = 3,
|
||||
/// zstd applied after TDT byte-interleaving transform (arXiv:2506.18062).
|
||||
/// Improves compression ratio ~16% for `f32`/`f16` numeric pages.
|
||||
ZstdTdt = 4,
|
||||
}
|
||||
|
||||
impl Codec {
|
||||
pub fn from_u8(v: u8) -> Result<Self, OnionError> {
|
||||
match v {
|
||||
0 => Ok(Codec::None),
|
||||
1 => Ok(Codec::Zstd),
|
||||
2 => Ok(Codec::Lz4),
|
||||
3 => Ok(Codec::Brotli),
|
||||
4 => Ok(Codec::ZstdTdt),
|
||||
other => Err(OnionError::UnknownCodec(other)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// OnionHeader — exactly 128 bytes
|
||||
//
|
||||
// Byte layout (no implicit padding):
|
||||
// [0..9) magic [u8; 9]
|
||||
// [9] format_version u8
|
||||
// [10..16) _pad_align [u8; 6] ← fills gap before u64 @ 16
|
||||
// [16..24) feature_flags u64
|
||||
// [24..28) page_size u32
|
||||
// [28..32) _pad_ps [u8; 4] ← fills gap before u64 @ 32
|
||||
// [32..40) revision_count u64
|
||||
// [40..44) branch_count u32
|
||||
// [44..48) _pad_bc [u8; 4] ← fills gap before u64 @ 48
|
||||
// [48..56) index_offset u64
|
||||
// [56..64) branch_offset u64
|
||||
// [64..72) created_at f64
|
||||
// [72..128) reserved [u8; 56]
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
||||
#[repr(C)]
|
||||
pub struct OnionHeader {
|
||||
pub magic: [u8; 9],
|
||||
pub format_version: u8,
|
||||
pub _pad_align: [u8; 6],
|
||||
pub feature_flags: u64,
|
||||
pub page_size: u32,
|
||||
pub _pad_ps: [u8; 4],
|
||||
pub revision_count: u64,
|
||||
pub branch_count: u32,
|
||||
pub _pad_bc: [u8; 4],
|
||||
pub index_offset: u64,
|
||||
pub branch_offset: u64,
|
||||
pub created_at: f64,
|
||||
pub reserved: [u8; 56],
|
||||
}
|
||||
|
||||
const _: () = assert!(size_of::<OnionHeader>() == HEADER_SIZE);
|
||||
|
||||
impl OnionHeader {
|
||||
pub fn validate(&self) -> Result<(), OnionError> {
|
||||
if &self.magic != MAGIC {
|
||||
return Err(OnionError::InvalidMagic);
|
||||
}
|
||||
if self.format_version != FORMAT_VERSION {
|
||||
return Err(OnionError::UnknownVersion(self.format_version));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn has_feature(&self, flag: u64) -> bool {
|
||||
self.feature_flags & flag != 0
|
||||
}
|
||||
|
||||
pub fn new(page_size: u32, feature_flags: u64, created_at: f64) -> Self {
|
||||
Self {
|
||||
magic: *MAGIC,
|
||||
format_version: FORMAT_VERSION,
|
||||
_pad_align: [0u8; 6],
|
||||
feature_flags,
|
||||
page_size,
|
||||
_pad_ps: [0u8; 4],
|
||||
revision_count: 0,
|
||||
branch_count: 1,
|
||||
_pad_bc: [0u8; 4],
|
||||
index_offset: HEADER_SIZE as u64,
|
||||
branch_offset: 0,
|
||||
created_at,
|
||||
reserved: [0u8; 56],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// RevisionEntry — 112 bytes
|
||||
//
|
||||
// [0..8) revision u64
|
||||
// [8..12) branch_id u32
|
||||
// [12..16) _pad_bi [u8; 4]
|
||||
// [16..24) parent_rev u64
|
||||
// [24..28) page_count u32
|
||||
// [28..32) _pad_pc [u8; 4]
|
||||
// [32..40) page_table_off u64
|
||||
// [40..48) timestamp f64
|
||||
// [48..80) blake3 [u8; 32]
|
||||
// [80..96) session_uuid [u8; 16]
|
||||
// [96..104) annotation_off u64
|
||||
// [104] flags u8
|
||||
// [105..112) _pad_flags [u8; 7]
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
||||
#[repr(C)]
|
||||
pub struct RevisionEntry {
|
||||
pub revision: u64,
|
||||
pub branch_id: u32,
|
||||
pub _pad_bi: [u8; 4],
|
||||
pub parent_rev: u64,
|
||||
pub page_count: u32,
|
||||
pub _pad_pc: [u8; 4],
|
||||
pub page_table_off: u64,
|
||||
pub timestamp: f64,
|
||||
pub blake3: [u8; 32],
|
||||
pub session_uuid: [u8; 16],
|
||||
pub annotation_off: u64,
|
||||
pub flags: u8,
|
||||
pub _pad_flags: [u8; 7],
|
||||
}
|
||||
|
||||
pub const REV_FLAG_SNAPSHOT: u8 = 1 << 0;
|
||||
|
||||
/// Sentinel epoch value written into `RevisionEntry._pad_flags[0..4]` by
|
||||
/// `GcPolicy::EpochFlip` to mark a revision for deferred removal.
|
||||
///
|
||||
/// Revisions with this epoch are compacted on the next `flush()`.
|
||||
/// All new commits have epoch `0` (from zero-initialised `_pad_flags`).
|
||||
pub const EPOCH_DEAD: u32 = u32::MAX;
|
||||
|
||||
impl RevisionEntry {
|
||||
/// Read the epoch tag stored in padding bytes `_pad_flags[0..4]`.
|
||||
///
|
||||
/// `0` means the revision is live (default for all new and legacy entries).
|
||||
/// [`EPOCH_DEAD`] means it is scheduled for deferred GC removal.
|
||||
pub fn epoch(&self) -> u32 {
|
||||
u32::from_le_bytes(self._pad_flags[0..4].try_into().unwrap())
|
||||
}
|
||||
|
||||
/// Write an epoch tag into `_pad_flags[0..4]`.
|
||||
pub fn set_epoch(&mut self, epoch: u32) {
|
||||
self._pad_flags[0..4].copy_from_slice(&epoch.to_le_bytes());
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// BranchEntry — 40 bytes
|
||||
//
|
||||
// [0..4) id u32
|
||||
// [4..8) _pad_id [u8; 4]
|
||||
// [8..16) name_off u64
|
||||
// [16..24) head_rev u64
|
||||
// [24..32) fork_rev u64
|
||||
// [32..40) created_at f64
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
||||
#[repr(C)]
|
||||
pub struct BranchEntry {
|
||||
pub id: u32,
|
||||
pub _pad_id: [u8; 4],
|
||||
pub name_off: u64,
|
||||
pub head_rev: u64,
|
||||
pub fork_rev: u64,
|
||||
pub created_at: f64,
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// PageTableEntry — 32 bytes
|
||||
//
|
||||
// [0..8) h5_offset u64
|
||||
// [8..16) data_offset u64
|
||||
// [16..20) orig_size u32
|
||||
// [20..24) data_size u32
|
||||
// [24] codec u8
|
||||
// [25..32) _pad [u8; 7]
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
|
||||
#[repr(C)]
|
||||
pub struct PageTableEntry {
|
||||
pub h5_offset: u64,
|
||||
pub data_offset: u64,
|
||||
pub orig_size: u32,
|
||||
pub data_size: u32,
|
||||
pub codec: u8,
|
||||
pub _pad: [u8; 7],
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Compile-time size/alignment assertions
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const _: () = assert!(size_of::<OnionHeader>() == 128);
|
||||
const _: () = assert!(size_of::<RevisionEntry>() == 112);
|
||||
const _: () = assert!(size_of::<BranchEntry>() == 40);
|
||||
const _: () = assert!(size_of::<PageTableEntry>() == 32);
|
||||
|
||||
const _: () = assert!(size_of::<RevisionEntry>() % 8 == 0);
|
||||
const _: () = assert!(size_of::<BranchEntry>() % 8 == 0);
|
||||
const _: () = assert!(size_of::<PageTableEntry>() % 8 == 0);
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn header_size_is_128() {
|
||||
assert_eq!(size_of::<OnionHeader>(), 128);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn revision_entry_size() {
|
||||
assert_eq!(size_of::<RevisionEntry>(), 112);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn branch_entry_size() {
|
||||
assert_eq!(size_of::<BranchEntry>(), 40);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_table_entry_size() {
|
||||
assert_eq!(size_of::<PageTableEntry>(), 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_new_validates_ok() {
|
||||
let hdr = OnionHeader::new(4096, DEFAULT_FEATURE_FLAGS, 0.0);
|
||||
assert!(hdr.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_bad_magic_rejected() {
|
||||
let mut hdr = OnionHeader::new(4096, DEFAULT_FEATURE_FLAGS, 0.0);
|
||||
hdr.magic[0] = b'X';
|
||||
assert!(matches!(hdr.validate(), Err(OnionError::InvalidMagic)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_unknown_version_rejected() {
|
||||
let mut hdr = OnionHeader::new(4096, DEFAULT_FEATURE_FLAGS, 0.0);
|
||||
hdr.format_version = 42;
|
||||
assert!(matches!(hdr.validate(), Err(OnionError::UnknownVersion(42))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_feature_flag_check() {
|
||||
let hdr = OnionHeader::new(4096, DEFAULT_FEATURE_FLAGS, 0.0);
|
||||
assert!(hdr.has_feature(feature_flags::COMPRESSION));
|
||||
assert!(hdr.has_feature(feature_flags::BRANCHING));
|
||||
assert!(hdr.has_feature(feature_flags::PROVENANCE));
|
||||
assert!(!hdr.has_feature(feature_flags::SNAPSHOTS));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_roundtrip_bytes() {
|
||||
let hdr = OnionHeader::new(4096, DEFAULT_FEATURE_FLAGS, 1_700_000_000.5);
|
||||
let bytes = hdr.as_bytes();
|
||||
assert_eq!(bytes.len(), HEADER_SIZE);
|
||||
assert_eq!(&bytes[..9], MAGIC);
|
||||
assert_eq!(bytes[9], FORMAT_VERSION);
|
||||
let hdr2 = OnionHeader::read_from_bytes(bytes).unwrap();
|
||||
assert_eq!(hdr2.page_size, 4096);
|
||||
assert_eq!(hdr2.format_version, FORMAT_VERSION);
|
||||
assert!(hdr2.has_feature(feature_flags::BRANCHING));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_reserved_zeroed_on_new() {
|
||||
let hdr = OnionHeader::new(4096, DEFAULT_FEATURE_FLAGS, 0.0);
|
||||
assert!(hdr.reserved.iter().all(|&b| b == 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn revision_entry_roundtrip() {
|
||||
let mut blake3 = [0u8; 32];
|
||||
blake3[0] = 0xAB;
|
||||
blake3[31] = 0xCD;
|
||||
let entry = RevisionEntry {
|
||||
revision: 7,
|
||||
branch_id: 2,
|
||||
_pad_bi: [0u8; 4],
|
||||
parent_rev: 5,
|
||||
page_count: 10,
|
||||
_pad_pc: [0u8; 4],
|
||||
page_table_off: 1024,
|
||||
timestamp: 1_700_000_000.0,
|
||||
blake3,
|
||||
session_uuid: [1u8; 16],
|
||||
annotation_off: 512,
|
||||
flags: 0,
|
||||
_pad_flags: [0u8; 7],
|
||||
};
|
||||
let bytes = entry.as_bytes();
|
||||
let entry2 = RevisionEntry::read_from_bytes(bytes).unwrap();
|
||||
assert_eq!(entry2.revision, 7);
|
||||
assert_eq!(entry2.branch_id, 2);
|
||||
assert_eq!(entry2.parent_rev, 5);
|
||||
assert_eq!(entry2.page_count, 10);
|
||||
assert_eq!(entry2.blake3[0], 0xAB);
|
||||
assert_eq!(entry2.blake3[31], 0xCD);
|
||||
assert_eq!(entry2.annotation_off, 512);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn revision_entry_no_parent() {
|
||||
let entry = RevisionEntry {
|
||||
revision: 0,
|
||||
branch_id: BRANCH_MAIN,
|
||||
_pad_bi: [0; 4],
|
||||
parent_rev: NO_PARENT,
|
||||
page_count: 0,
|
||||
_pad_pc: [0; 4],
|
||||
page_table_off: 0,
|
||||
timestamp: 0.0,
|
||||
blake3: [0u8; 32],
|
||||
session_uuid: [0u8; 16],
|
||||
annotation_off: 0,
|
||||
flags: 0,
|
||||
_pad_flags: [0; 7],
|
||||
};
|
||||
assert_eq!(entry.parent_rev, u64::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn revision_entry_snapshot_flag() {
|
||||
let mut entry = RevisionEntry {
|
||||
revision: 500,
|
||||
branch_id: BRANCH_MAIN,
|
||||
_pad_bi: [0; 4],
|
||||
parent_rev: 499,
|
||||
page_count: 0,
|
||||
_pad_pc: [0; 4],
|
||||
page_table_off: 0,
|
||||
timestamp: 0.0,
|
||||
blake3: [0u8; 32],
|
||||
session_uuid: [0u8; 16],
|
||||
annotation_off: 0,
|
||||
flags: REV_FLAG_SNAPSHOT,
|
||||
_pad_flags: [0; 7],
|
||||
};
|
||||
assert_ne!(entry.flags & REV_FLAG_SNAPSHOT, 0);
|
||||
entry.flags &= !REV_FLAG_SNAPSHOT;
|
||||
assert_eq!(entry.flags & REV_FLAG_SNAPSHOT, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn branch_entry_roundtrip() {
|
||||
let entry = BranchEntry {
|
||||
id: 1,
|
||||
_pad_id: [0; 4],
|
||||
name_off: 128,
|
||||
head_rev: 42,
|
||||
fork_rev: 10,
|
||||
created_at: 1_700_000_000.0,
|
||||
};
|
||||
let bytes = entry.as_bytes();
|
||||
let entry2 = BranchEntry::read_from_bytes(bytes).unwrap();
|
||||
assert_eq!(entry2.id, 1);
|
||||
assert_eq!(entry2.name_off, 128);
|
||||
assert_eq!(entry2.head_rev, 42);
|
||||
assert_eq!(entry2.fork_rev, 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn branch_entry_main_sentinel() {
|
||||
let entry = BranchEntry {
|
||||
id: BRANCH_MAIN,
|
||||
_pad_id: [0; 4],
|
||||
name_off: 0,
|
||||
head_rev: 0,
|
||||
fork_rev: NO_PARENT,
|
||||
created_at: 0.0,
|
||||
};
|
||||
assert_eq!(entry.id, 0);
|
||||
assert_eq!(entry.fork_rev, NO_PARENT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_table_entry_roundtrip() {
|
||||
let entry = PageTableEntry {
|
||||
h5_offset: 8192,
|
||||
data_offset: 4096,
|
||||
orig_size: 4096,
|
||||
data_size: 1024,
|
||||
codec: Codec::Zstd as u8,
|
||||
_pad: [0u8; 7],
|
||||
};
|
||||
let bytes = entry.as_bytes();
|
||||
let entry2 = PageTableEntry::read_from_bytes(bytes).unwrap();
|
||||
assert_eq!(entry2.h5_offset, 8192);
|
||||
assert_eq!(entry2.data_size, 1024);
|
||||
assert_eq!(Codec::from_u8(entry2.codec).unwrap(), Codec::Zstd);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_roundtrip_all_variants() {
|
||||
for &(v, expected) in &[
|
||||
(0u8, Codec::None),
|
||||
(1, Codec::Zstd),
|
||||
(2, Codec::Lz4),
|
||||
(3, Codec::Brotli),
|
||||
] {
|
||||
assert_eq!(Codec::from_u8(v).unwrap(), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codec_unknown_returns_error() {
|
||||
assert!(matches!(Codec::from_u8(99), Err(OnionError::UnknownCodec(99))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn feature_flags_are_distinct_bits() {
|
||||
let flags = [
|
||||
feature_flags::COMPRESSION,
|
||||
feature_flags::BRANCHING,
|
||||
feature_flags::PROVENANCE,
|
||||
feature_flags::SNAPSHOTS,
|
||||
];
|
||||
for i in 0..flags.len() {
|
||||
for j in 0..flags.len() {
|
||||
if i != j {
|
||||
assert_eq!(flags[i] & flags[j], 0, "flags[{i}] and flags[{j}] overlap");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_feature_flags_include_core_bits() {
|
||||
assert!(DEFAULT_FEATURE_FLAGS & feature_flags::COMPRESSION != 0);
|
||||
assert!(DEFAULT_FEATURE_FLAGS & feature_flags::BRANCHING != 0);
|
||||
assert!(DEFAULT_FEATURE_FLAGS & feature_flags::PROVENANCE != 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,605 @@
|
||||
//! Garbage collection: prune old revisions from the `.onion` sidecar.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use crate::compress::compress_page;
|
||||
use crate::error::OnionError;
|
||||
use crate::format::{EPOCH_DEAD, PageTableEntry, REV_FLAG_SNAPSHOT};
|
||||
use crate::writer::OnionFile;
|
||||
|
||||
/// Policy controlling which revisions are retained after GC.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum GcPolicy {
|
||||
/// Keep the N most recent revisions (by revision number).
|
||||
KeepLastN(u64),
|
||||
/// Keep all revisions that have a non-empty annotation.
|
||||
KeepTagged,
|
||||
/// Keep all revisions with a timestamp >= the given Unix epoch value.
|
||||
KeepSince(f64),
|
||||
/// Keep an explicit set of revision numbers (plus their ancestors to
|
||||
/// maintain a valid DAG).
|
||||
KeepRevisions(Vec<u64>),
|
||||
/// **Lazy / epoch-based GC.**
|
||||
///
|
||||
/// Computes the dead set using `inner`, marks those entries with
|
||||
/// [`EPOCH_DEAD`] in their padding bytes, and returns immediately
|
||||
/// without touching `page_data`. The actual compaction is deferred to
|
||||
/// the next [`OnionFile::flush`] call, which runs
|
||||
/// [`OnionFile::compact_dead_epoch_revisions`] before writing.
|
||||
///
|
||||
/// Use this when you want GC to be non-blocking: the mark pass is
|
||||
/// O(revisions) with no I/O; the compaction is amortised into the next
|
||||
/// flush that would happen anyway.
|
||||
EpochFlip(Box<GcPolicy>),
|
||||
}
|
||||
|
||||
/// Statistics returned by a GC run.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct GcStats {
|
||||
/// Number of revisions removed.
|
||||
pub revisions_removed: u64,
|
||||
/// Bytes of page data reclaimed.
|
||||
pub bytes_reclaimed: u64,
|
||||
}
|
||||
|
||||
impl OnionFile {
|
||||
/// Prune revisions according to `policy`.
|
||||
///
|
||||
/// For all policies except [`GcPolicy::EpochFlip`]:
|
||||
/// - The `RevisionIndex` is updated immediately.
|
||||
/// - Page data is compacted in-memory; call [`flush`] to persist.
|
||||
///
|
||||
/// For [`GcPolicy::EpochFlip`]:
|
||||
/// - Dead revisions are *marked* with [`EPOCH_DEAD`] in O(revisions).
|
||||
/// - Page data is **not** touched; compaction is deferred to the next
|
||||
/// [`flush`] call. This makes the GC call itself non-blocking.
|
||||
///
|
||||
/// **Note:** Immediate GC is irreversible. Epoch-flip GC can be
|
||||
/// cancelled by calling `flush_wal()` without `flush()`, but only
|
||||
/// before the next `flush()`.
|
||||
pub fn gc(&mut self, policy: GcPolicy) -> Result<GcStats, OnionError> {
|
||||
let to_remove = self.compute_to_remove(&policy);
|
||||
|
||||
match policy {
|
||||
GcPolicy::EpochFlip(_) => {
|
||||
// ── Lazy path: just mark dead entries ────────────────────
|
||||
let count = to_remove.len() as u64;
|
||||
for &rev in &to_remove {
|
||||
if let Some(entry) = self.index.get_mut(rev) {
|
||||
entry.set_epoch(EPOCH_DEAD);
|
||||
}
|
||||
}
|
||||
Ok(GcStats { revisions_removed: count, bytes_reclaimed: 0 })
|
||||
}
|
||||
_ => {
|
||||
// ── Immediate path: compact now ───────────────────────────
|
||||
self.compact_revisions(to_remove)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the set of revisions to remove given a `policy`.
|
||||
fn compute_to_remove(&self, policy: &GcPolicy) -> HashSet<u64> {
|
||||
let all_revs: Vec<u64> = self.index.entries().iter().map(|e| e.revision).collect();
|
||||
|
||||
let keep: HashSet<u64> = match policy {
|
||||
GcPolicy::KeepLastN(n) => {
|
||||
let start = all_revs.len().saturating_sub(*n as usize);
|
||||
all_revs[start..].iter().copied().collect()
|
||||
}
|
||||
GcPolicy::KeepTagged => all_revs
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|&rev| {
|
||||
self.index
|
||||
.get(rev)
|
||||
.and_then(|e| self.annotations.get(e.annotation_off))
|
||||
.is_some_and(|a| !a.is_empty())
|
||||
})
|
||||
.collect(),
|
||||
GcPolicy::KeepSince(cutoff) => all_revs
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|&rev| {
|
||||
self.index
|
||||
.get(rev)
|
||||
.is_some_and(|e| e.timestamp >= *cutoff)
|
||||
})
|
||||
.collect(),
|
||||
GcPolicy::KeepRevisions(explicit) => {
|
||||
let mut set: HashSet<u64> = explicit.iter().copied().collect();
|
||||
for &rev in explicit {
|
||||
for ancestor in self.index.ancestors(rev) {
|
||||
set.insert(ancestor.revision);
|
||||
}
|
||||
}
|
||||
set
|
||||
}
|
||||
GcPolicy::EpochFlip(inner) => return self.compute_to_remove(inner),
|
||||
};
|
||||
|
||||
all_revs.iter().copied().filter(|rev| !keep.contains(rev)).collect()
|
||||
}
|
||||
|
||||
/// Consolidate + compact the given revision set immediately.
|
||||
///
|
||||
/// Shared by the immediate `gc()` path and the deferred flush path.
|
||||
pub(crate) fn compact_revisions(
|
||||
&mut self,
|
||||
to_remove: HashSet<u64>,
|
||||
) -> Result<GcStats, OnionError> {
|
||||
if to_remove.is_empty() {
|
||||
return Ok(GcStats::default());
|
||||
}
|
||||
|
||||
let revisions_removed = to_remove.len() as u64;
|
||||
|
||||
let mut bytes_reclaimed = 0u64;
|
||||
for &rev in &to_remove {
|
||||
if let Some(table) = self.page_tables.get(rev as usize) {
|
||||
bytes_reclaimed += table.iter().map(|e| e.data_size as u64).sum::<u64>();
|
||||
}
|
||||
}
|
||||
|
||||
// Consolidation: if the oldest surviving revision has ancestors that
|
||||
// will be removed, reconstruct its full state as a root snapshot.
|
||||
let keep: HashSet<u64> = self
|
||||
.index
|
||||
.entries()
|
||||
.iter()
|
||||
.map(|e| e.revision)
|
||||
.filter(|r| !to_remove.contains(r))
|
||||
.collect();
|
||||
|
||||
let mut surviving_sorted: Vec<u64> = keep.iter().copied().collect();
|
||||
surviving_sorted.sort_unstable();
|
||||
|
||||
if let Some(&oldest_surviving) = surviving_sorted.first() {
|
||||
let has_removed_ancestor = self
|
||||
.index
|
||||
.ancestors(oldest_surviving)
|
||||
.skip(1)
|
||||
.any(|e| to_remove.contains(&e.revision));
|
||||
|
||||
if has_removed_ancestor {
|
||||
let h5_path = self.path.with_extension("");
|
||||
let h5_base = std::fs::read(&h5_path).unwrap_or_default();
|
||||
let full_bytes = self.reconstruct_revision(oldest_surviving, &h5_base)?;
|
||||
|
||||
let ps = self.header.page_size as usize;
|
||||
let mut new_table: Vec<PageTableEntry> = Vec::new();
|
||||
for (i, chunk) in full_bytes.chunks(ps).enumerate() {
|
||||
let mut padded = vec![0u8; ps];
|
||||
padded[..chunk.len()].copy_from_slice(chunk);
|
||||
let compressed = compress_page(&padded, self.default_codec)?;
|
||||
let data_offset = self.page_data.len() as u64;
|
||||
self.page_data.extend_from_slice(&compressed);
|
||||
new_table.push(PageTableEntry {
|
||||
h5_offset: (i * ps) as u64,
|
||||
data_offset,
|
||||
orig_size: ps as u32,
|
||||
data_size: compressed.len() as u32,
|
||||
codec: self.default_codec as u8,
|
||||
_pad: [0u8; 7],
|
||||
});
|
||||
}
|
||||
if let Some(table) = self.page_tables.get_mut(oldest_surviving as usize) {
|
||||
*table = new_table;
|
||||
}
|
||||
use crate::format::NO_PARENT;
|
||||
if let Some(entry) = self.index.get_mut(oldest_surviving) {
|
||||
entry.flags |= REV_FLAG_SNAPSHOT;
|
||||
entry.parent_rev = NO_PARENT;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove from index and clear page tables.
|
||||
self.index.remove_revisions(&to_remove);
|
||||
self.header.revision_count = self.index.len() as u64;
|
||||
for &rev in &to_remove {
|
||||
if let Some(table) = self.page_tables.get_mut(rev as usize) {
|
||||
table.clear();
|
||||
}
|
||||
}
|
||||
|
||||
// Compact page_data blob.
|
||||
let surviving_revs: Vec<u64> =
|
||||
self.index.entries().iter().map(|e| e.revision).collect();
|
||||
let mut new_page_data: Vec<u8> = Vec::new();
|
||||
for &rev in &surviving_revs {
|
||||
if let Some(table) = self.page_tables.get_mut(rev as usize) {
|
||||
for entry in table.iter_mut() {
|
||||
let old_start = entry.data_offset as usize;
|
||||
let old_end = old_start + entry.data_size as usize;
|
||||
let new_offset = new_page_data.len() as u64;
|
||||
if old_end <= self.page_data.len() {
|
||||
new_page_data.extend_from_slice(&self.page_data[old_start..old_end]);
|
||||
}
|
||||
entry.data_offset = new_offset;
|
||||
}
|
||||
}
|
||||
}
|
||||
self.page_data = new_page_data;
|
||||
|
||||
Ok(GcStats { revisions_removed, bytes_reclaimed })
|
||||
}
|
||||
|
||||
/// Find all entries marked [`EPOCH_DEAD`] and compact them.
|
||||
///
|
||||
/// Called automatically by [`flush`] when any dead-epoch entries exist.
|
||||
pub(crate) fn compact_dead_epoch_revisions(&mut self) -> Result<(), OnionError> {
|
||||
let dead: HashSet<u64> = self
|
||||
.index
|
||||
.entries()
|
||||
.iter()
|
||||
.filter(|e| e.epoch() == EPOCH_DEAD)
|
||||
.map(|e| e.revision)
|
||||
.collect();
|
||||
if !dead.is_empty() {
|
||||
self.compact_revisions(dead)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
fn tmp_h5() -> std::path::PathBuf {
|
||||
let f = NamedTempFile::new().unwrap();
|
||||
let path = f.path().with_extension("h5");
|
||||
std::fs::write(&path, b"\x89HDF\r\n\x1a\n").unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
fn make_onion_with_n_revisions(n: usize) -> OnionFile {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
for i in 0..n {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![i as u8; 4096]);
|
||||
let annotation = if i % 3 == 0 {
|
||||
Some(format!("tagged-{i}"))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
onion
|
||||
.commit_session(s, annotation.as_deref())
|
||||
.unwrap();
|
||||
}
|
||||
onion
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_keep_last_n_retains_n() {
|
||||
let mut onion = make_onion_with_n_revisions(10);
|
||||
let stats = onion.gc(GcPolicy::KeepLastN(3)).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 7);
|
||||
assert_eq!(onion.revision_count(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_keep_last_n_greater_than_total() {
|
||||
let mut onion = make_onion_with_n_revisions(5);
|
||||
let stats = onion.gc(GcPolicy::KeepLastN(100)).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 0);
|
||||
assert_eq!(onion.revision_count(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_keep_last_zero_clears_all() {
|
||||
let mut onion = make_onion_with_n_revisions(5);
|
||||
let stats = onion.gc(GcPolicy::KeepLastN(0)).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 5);
|
||||
assert_eq!(onion.revision_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_keep_tagged_retains_annotated() {
|
||||
// Revisions 0, 3, 6, 9 are tagged (i % 3 == 0)
|
||||
let mut onion = make_onion_with_n_revisions(10);
|
||||
let stats = onion.gc(GcPolicy::KeepTagged).unwrap();
|
||||
// 4 tagged revisions: 0, 3, 6, 9
|
||||
assert_eq!(onion.revision_count(), 4);
|
||||
assert_eq!(stats.revisions_removed, 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_keep_since_retains_recent() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
// Write some revisions with distinct timestamps
|
||||
for i in 0u8..5 {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![i; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
}
|
||||
// Keep all revisions (cutoff = 0.0 = beginning of time)
|
||||
let stats = onion.gc(GcPolicy::KeepSince(0.0)).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_keep_explicit_revisions_with_ancestors() {
|
||||
let mut onion = make_onion_with_n_revisions(5);
|
||||
// Keep only revision 4; its ancestors (0,1,2,3) must also be kept
|
||||
let stats = onion.gc(GcPolicy::KeepRevisions(vec![4])).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 0, "all are ancestors of rev 4");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_keep_middle_revision_excludes_unrelated() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
// 3 revisions: 0 → 1 → 2
|
||||
for i in 0u8..3 {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![i; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
}
|
||||
// Keep rev 1 and its ancestor (rev 0); rev 2 should be pruned
|
||||
let stats = onion.gc(GcPolicy::KeepRevisions(vec![1])).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 1); // only rev 2 pruned
|
||||
assert!(onion.index.get(2).is_none());
|
||||
assert!(onion.index.get(0).is_some());
|
||||
assert!(onion.index.get(1).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_empty_file_noop() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
let stats = onion.gc(GcPolicy::KeepLastN(10)).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 0);
|
||||
assert_eq!(onion.revision_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_reports_bytes_reclaimed() {
|
||||
let mut onion = make_onion_with_n_revisions(5);
|
||||
let before = onion.page_data.len();
|
||||
let stats = onion.gc(GcPolicy::KeepLastN(1)).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 4);
|
||||
let after = onion.page_data.len();
|
||||
// page_data must be smaller (or at most equal for all-same pages that compress to 0)
|
||||
assert!(after <= before, "page_data must shrink after GC: {before} -> {after}");
|
||||
}
|
||||
|
||||
/// After GC + flush, the on-disk file is smaller than before.
|
||||
#[test]
|
||||
fn gc_flush_reduces_file_size() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
|
||||
// Write 10 revisions with distinct page data so pages don't compress away.
|
||||
for i in 0..10u8 {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
// Use varying patterns to ensure pages are different.
|
||||
let page: Vec<u8> = (0..4096).map(|j| (i ^ (j as u8)).wrapping_add(i)).collect();
|
||||
s.record_page(0, &page);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
}
|
||||
onion.flush().unwrap();
|
||||
let sidecar = OnionFile::sidecar_path_pub(&h5);
|
||||
let size_before = std::fs::metadata(&sidecar).unwrap().len();
|
||||
|
||||
// GC down to 1 revision, then flush.
|
||||
onion.gc(GcPolicy::KeepLastN(1)).unwrap();
|
||||
onion.flush().unwrap();
|
||||
let size_after = std::fs::metadata(&sidecar).unwrap().len();
|
||||
|
||||
assert!(
|
||||
size_after < size_before,
|
||||
"sidecar should shrink after GC+flush: {size_before} → {size_after}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Surviving revisions can still be reconstructed after GC.
|
||||
#[test]
|
||||
fn gc_surviving_revisions_still_readable() {
|
||||
let h5 = tmp_h5();
|
||||
let h5_base = std::fs::read(&h5).unwrap();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
|
||||
for i in 0..6u8 {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![i; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
}
|
||||
|
||||
// Keep last 3 revisions (3, 4, 5).
|
||||
onion.gc(GcPolicy::KeepLastN(3)).unwrap();
|
||||
assert_eq!(onion.revision_count(), 3);
|
||||
|
||||
// The three surviving revisions must still reconstruct without error.
|
||||
for rev in 3..6u64 {
|
||||
let bytes = onion.reconstruct_revision(rev, &h5_base).unwrap();
|
||||
assert!(!bytes.is_empty(), "rev {rev} should produce non-empty bytes");
|
||||
}
|
||||
}
|
||||
|
||||
/// GC with diff-based commits (each revision writes a DIFFERENT page) must
|
||||
/// still produce correct reconstructions after ancestor pruning.
|
||||
///
|
||||
/// Before the consolidation fix, `KeepLastN` would lose pages written by
|
||||
/// pruned revisions, causing reconstructions to fall back to the empty
|
||||
/// h5_base for those pages.
|
||||
#[test]
|
||||
fn gc_diff_based_commits_reconstruct_correctly_after_prune() {
|
||||
let h5 = tmp_h5();
|
||||
let h5_base = std::fs::read(&h5).unwrap();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
|
||||
// Rev 0: only page 0
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0xAA_u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
|
||||
// Rev 1: only page 1 (page 0 unchanged from rev 0)
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(4096, &vec![0xBB_u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
|
||||
// Rev 2: only page 2 (pages 0 and 1 unchanged)
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(8192, &vec![0xCC_u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
|
||||
// Verify reconstruction before GC
|
||||
let before_gc = onion.reconstruct_revision(2, &h5_base).unwrap();
|
||||
assert_eq!(&before_gc[0..4096], &vec![0xAA_u8; 4096], "pre-GC page0");
|
||||
assert_eq!(&before_gc[4096..8192], &vec![0xBB_u8; 4096], "pre-GC page1");
|
||||
assert_eq!(&before_gc[8192..12288], &vec![0xCC_u8; 4096], "pre-GC page2");
|
||||
|
||||
// GC: keep only rev 2 — revs 0 and 1 are ancestors that will be pruned.
|
||||
onion.gc(GcPolicy::KeepLastN(1)).unwrap();
|
||||
assert_eq!(onion.revision_count(), 1);
|
||||
|
||||
// After GC, rev 2 must still reconstruct with all three pages intact.
|
||||
// The oldest surviving revision (rev 2) should have been consolidated
|
||||
// into a root snapshot that captures all pages.
|
||||
let after_gc = onion.reconstruct_revision(2, &h5_base).unwrap();
|
||||
assert_eq!(
|
||||
&after_gc[0..4096],
|
||||
&vec![0xAA_u8; 4096],
|
||||
"post-GC page0 must come from consolidation"
|
||||
);
|
||||
assert_eq!(
|
||||
&after_gc[4096..8192],
|
||||
&vec![0xBB_u8; 4096],
|
||||
"post-GC page1 must come from consolidation"
|
||||
);
|
||||
assert_eq!(
|
||||
&after_gc[8192..12288],
|
||||
&vec![0xCC_u8; 4096],
|
||||
"post-GC page2 must come from consolidation"
|
||||
);
|
||||
}
|
||||
|
||||
// ── Epoch-flip (lazy) GC tests ────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn gc_epoch_flip_marks_entries_dead_without_compacting() {
|
||||
let mut onion = make_onion_with_n_revisions(10);
|
||||
let page_data_len_before = onion.page_data.len();
|
||||
|
||||
let stats = onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(3)))).unwrap();
|
||||
|
||||
// Reports the count that will be removed.
|
||||
assert_eq!(stats.revisions_removed, 7);
|
||||
// bytes_reclaimed is 0 until flush compacts.
|
||||
assert_eq!(stats.bytes_reclaimed, 0);
|
||||
// page_data must NOT have changed yet.
|
||||
assert_eq!(onion.page_data.len(), page_data_len_before,
|
||||
"epoch flip must not compact page_data immediately");
|
||||
// Index still has all 10 entries (removal deferred).
|
||||
assert_eq!(onion.revision_count(), 10);
|
||||
// Entries 0..6 should be marked EPOCH_DEAD.
|
||||
for rev in 0u64..7 {
|
||||
let entry = onion.index.get(rev).unwrap();
|
||||
assert_eq!(entry.epoch(), EPOCH_DEAD, "rev {rev} should be EPOCH_DEAD");
|
||||
}
|
||||
// Entries 7..9 should be live (epoch = 0).
|
||||
for rev in 7u64..10 {
|
||||
let entry = onion.index.get(rev).unwrap();
|
||||
assert_eq!(entry.epoch(), 0, "rev {rev} should be live (epoch=0)");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_epoch_flip_compact_on_flush() {
|
||||
let h5 = tmp_h5();
|
||||
let h5_base = std::fs::read(&h5).unwrap();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
for i in 0..5u8 {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![i; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
}
|
||||
|
||||
// Epoch flip: defer compaction.
|
||||
onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(2)))).unwrap();
|
||||
assert_eq!(onion.revision_count(), 5, "not compacted yet");
|
||||
|
||||
// After flush, deferred compaction runs.
|
||||
onion.flush().unwrap();
|
||||
|
||||
// Reload and verify.
|
||||
let reloaded = OnionFile::open(&h5).unwrap();
|
||||
assert_eq!(reloaded.revision_count(), 2, "2 revisions survive after flush");
|
||||
|
||||
// Surviving revisions are still reconstructable.
|
||||
for rev in 3u64..5 {
|
||||
let bytes = reloaded.reconstruct_revision(rev, &h5_base).unwrap();
|
||||
assert!(!bytes.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_epoch_flip_backward_compat_old_entries() {
|
||||
// Old entries have _pad_flags all zero → epoch() == 0, never dead.
|
||||
let mut onion = make_onion_with_n_revisions(5);
|
||||
// All entries should have epoch=0.
|
||||
for rev in 0u64..5 {
|
||||
assert_eq!(onion.index.get(rev).unwrap().epoch(), 0);
|
||||
}
|
||||
// Immediate GC should still work on files without epoch marks.
|
||||
let stats = onion.gc(GcPolicy::KeepLastN(3)).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 2);
|
||||
assert_eq!(onion.revision_count(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_epoch_flip_nested_policy_keep_tagged() {
|
||||
let mut onion = make_onion_with_n_revisions(9); // tagged at 0,3,6
|
||||
let stats = onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepTagged))).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 6); // keeps 0,3,6
|
||||
assert_eq!(onion.revision_count(), 9, "deferred — index intact");
|
||||
// 0,3,6 are live; rest are dead.
|
||||
for rev in [0u64, 3, 6] {
|
||||
assert_eq!(onion.index.get(rev).unwrap().epoch(), 0, "tagged rev {rev} must be live");
|
||||
}
|
||||
for rev in [1u64, 2, 4, 5, 7, 8] {
|
||||
assert_eq!(onion.index.get(rev).unwrap().epoch(), EPOCH_DEAD,
|
||||
"untagged rev {rev} must be EPOCH_DEAD");
|
||||
}
|
||||
}
|
||||
|
||||
/// After consolidation, the oldest surviving revision is marked as a snapshot.
|
||||
#[test]
|
||||
fn gc_oldest_surviving_becomes_snapshot() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
|
||||
for i in 0..5u8 {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![i; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
}
|
||||
|
||||
// Keep last 2 (revs 3 and 4); revs 0,1,2 are ancestors that will be pruned.
|
||||
onion.gc(GcPolicy::KeepLastN(2)).unwrap();
|
||||
|
||||
// The oldest surviving revision (rev 3) must be flagged as snapshot.
|
||||
let entry = onion.index.get(3).expect("rev 3 must survive");
|
||||
assert_ne!(
|
||||
entry.flags & crate::format::REV_FLAG_SNAPSHOT,
|
||||
0,
|
||||
"oldest surviving rev must be a snapshot after consolidation"
|
||||
);
|
||||
// Rev 4 should NOT be flagged as a snapshot (it was not consolidated).
|
||||
let entry4 = onion.index.get(4).expect("rev 4 must survive");
|
||||
assert_eq!(
|
||||
entry4.flags & crate::format::REV_FLAG_SNAPSHOT,
|
||||
0,
|
||||
"non-consolidated rev must not be a snapshot"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
//! RevisionIndex: O(1) lookup + branch-filtered iteration.
|
||||
|
||||
use crate::format::{RevisionEntry, NO_PARENT};
|
||||
|
||||
/// In-memory index of all revision entries.
|
||||
///
|
||||
/// Entries are sorted by `revision` number (monotonically increasing).
|
||||
/// Lookup by revision number is O(log n) via binary search.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct RevisionIndex {
|
||||
entries: Vec<RevisionEntry>,
|
||||
}
|
||||
|
||||
impl RevisionIndex {
|
||||
/// Create an empty index.
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Load from an existing slice of entries (e.g., deserialized from disk).
|
||||
pub fn from_entries(entries: Vec<RevisionEntry>) -> Self {
|
||||
Self { entries }
|
||||
}
|
||||
|
||||
/// Return all entries as a slice.
|
||||
pub fn entries(&self) -> &[RevisionEntry] {
|
||||
&self.entries
|
||||
}
|
||||
|
||||
/// Total number of revisions.
|
||||
pub fn len(&self) -> usize {
|
||||
self.entries.len()
|
||||
}
|
||||
|
||||
/// Returns true if there are no revisions.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.entries.is_empty()
|
||||
}
|
||||
|
||||
/// Look up a revision by number. O(log n).
|
||||
pub fn get(&self, rev: u64) -> Option<&RevisionEntry> {
|
||||
let pos = self.entries.partition_point(|e| e.revision < rev);
|
||||
self.entries.get(pos).filter(|e| e.revision == rev)
|
||||
}
|
||||
|
||||
/// Mutable lookup by revision number. O(log n).
|
||||
pub fn get_mut(&mut self, rev: u64) -> Option<&mut RevisionEntry> {
|
||||
let pos = self.entries.partition_point(|e| e.revision < rev);
|
||||
self.entries.get_mut(pos).filter(|e| e.revision == rev)
|
||||
}
|
||||
|
||||
/// Append a new revision entry. The caller must ensure `revision` is
|
||||
/// monotonically greater than the current maximum.
|
||||
pub fn append(&mut self, entry: RevisionEntry) {
|
||||
debug_assert!(
|
||||
self.entries.last().is_none_or(|e| e.revision < entry.revision),
|
||||
"revision numbers must be monotonically increasing"
|
||||
);
|
||||
self.entries.push(entry);
|
||||
}
|
||||
|
||||
/// Iterate over all revisions on a specific branch (by `branch_id`).
|
||||
pub fn branch_revisions(&self, branch_id: u32) -> impl Iterator<Item = &RevisionEntry> {
|
||||
self.entries.iter().filter(move |e| e.branch_id == branch_id)
|
||||
}
|
||||
|
||||
/// Return the HEAD revision entry for a branch (highest revision number).
|
||||
pub fn branch_head(&self, branch_id: u32) -> Option<&RevisionEntry> {
|
||||
self.entries
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|e| e.branch_id == branch_id)
|
||||
}
|
||||
|
||||
/// Walk the DAG from `start_rev` to the root, following `parent_rev`.
|
||||
pub fn ancestors(&self, start_rev: u64) -> impl Iterator<Item = &RevisionEntry> {
|
||||
AncestorIter {
|
||||
index: self,
|
||||
current: Some(start_rev),
|
||||
}
|
||||
}
|
||||
|
||||
/// Find the lowest-common ancestor of two revisions.
|
||||
///
|
||||
/// Returns `None` if the two revisions have no common ancestor (which
|
||||
/// should not happen in a well-formed file — all revisions ultimately
|
||||
/// descend from rev 0).
|
||||
pub fn common_ancestor(&self, rev_a: u64, rev_b: u64) -> Option<u64> {
|
||||
let ancestors_a: std::collections::HashSet<u64> =
|
||||
self.ancestors(rev_a).map(|e| e.revision).collect();
|
||||
self.ancestors(rev_b)
|
||||
.find(|e| ancestors_a.contains(&e.revision))
|
||||
.map(|e| e.revision)
|
||||
}
|
||||
|
||||
/// Remove a set of revisions from the index (used by GC).
|
||||
/// The caller is responsible for also removing the corresponding page data.
|
||||
pub fn remove_revisions(&mut self, to_remove: &std::collections::HashSet<u64>) {
|
||||
self.entries.retain(|e| !to_remove.contains(&e.revision));
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterator that walks the revision DAG from a starting revision to the root.
|
||||
struct AncestorIter<'a> {
|
||||
index: &'a RevisionIndex,
|
||||
current: Option<u64>,
|
||||
}
|
||||
|
||||
impl<'a> Iterator for AncestorIter<'a> {
|
||||
type Item = &'a RevisionEntry;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
let rev = self.current?;
|
||||
let entry = self.index.get(rev)?;
|
||||
self.current = if entry.parent_rev == NO_PARENT {
|
||||
None
|
||||
} else {
|
||||
Some(entry.parent_rev)
|
||||
};
|
||||
Some(entry)
|
||||
}
|
||||
}
|
||||
|
||||
/// Summary of branch head state, used by public query APIs.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BranchHeadInfo {
|
||||
pub branch_id: u32,
|
||||
pub head_rev: u64,
|
||||
pub revision_count: usize,
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::format::{BRANCH_MAIN, NO_PARENT, REV_FLAG_SNAPSHOT};
|
||||
|
||||
fn make_entry(revision: u64, branch_id: u32, parent_rev: u64) -> RevisionEntry {
|
||||
RevisionEntry {
|
||||
revision,
|
||||
branch_id,
|
||||
_pad_bi: [0u8; 4],
|
||||
parent_rev,
|
||||
page_count: 1,
|
||||
_pad_pc: [0u8; 4],
|
||||
page_table_off: revision * 256,
|
||||
timestamp: revision as f64,
|
||||
blake3: [0u8; 32],
|
||||
session_uuid: [0u8; 16],
|
||||
annotation_off: 0,
|
||||
flags: 0,
|
||||
_pad_flags: [0u8; 7],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_index() {
|
||||
let idx = RevisionIndex::new();
|
||||
assert!(idx.is_empty());
|
||||
assert_eq!(idx.len(), 0);
|
||||
assert!(idx.get(0).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn append_and_get() {
|
||||
let mut idx = RevisionIndex::new();
|
||||
idx.append(make_entry(0, BRANCH_MAIN, NO_PARENT));
|
||||
idx.append(make_entry(1, BRANCH_MAIN, 0));
|
||||
idx.append(make_entry(2, BRANCH_MAIN, 1));
|
||||
assert_eq!(idx.len(), 3);
|
||||
assert_eq!(idx.get(0).unwrap().revision, 0);
|
||||
assert_eq!(idx.get(2).unwrap().revision, 2);
|
||||
assert!(idx.get(99).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn branch_revisions_filter() {
|
||||
let mut idx = RevisionIndex::new();
|
||||
idx.append(make_entry(0, BRANCH_MAIN, NO_PARENT));
|
||||
idx.append(make_entry(1, BRANCH_MAIN, 0));
|
||||
idx.append(make_entry(2, 1, 1)); // branch 1
|
||||
idx.append(make_entry(3, 1, 2));
|
||||
idx.append(make_entry(4, BRANCH_MAIN, 1));
|
||||
|
||||
let main_revs: Vec<u64> = idx
|
||||
.branch_revisions(BRANCH_MAIN)
|
||||
.map(|e| e.revision)
|
||||
.collect();
|
||||
assert_eq!(main_revs, vec![0, 1, 4]);
|
||||
|
||||
let branch1_revs: Vec<u64> =
|
||||
idx.branch_revisions(1).map(|e| e.revision).collect();
|
||||
assert_eq!(branch1_revs, vec![2, 3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn branch_head_returns_latest() {
|
||||
let mut idx = RevisionIndex::new();
|
||||
idx.append(make_entry(0, BRANCH_MAIN, NO_PARENT));
|
||||
idx.append(make_entry(1, BRANCH_MAIN, 0));
|
||||
idx.append(make_entry(2, 1, 1));
|
||||
idx.append(make_entry(3, BRANCH_MAIN, 1));
|
||||
assert_eq!(idx.branch_head(BRANCH_MAIN).unwrap().revision, 3);
|
||||
assert_eq!(idx.branch_head(1).unwrap().revision, 2);
|
||||
assert!(idx.branch_head(99).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ancestors_linear() {
|
||||
let mut idx = RevisionIndex::new();
|
||||
idx.append(make_entry(0, BRANCH_MAIN, NO_PARENT));
|
||||
idx.append(make_entry(1, BRANCH_MAIN, 0));
|
||||
idx.append(make_entry(2, BRANCH_MAIN, 1));
|
||||
let revs: Vec<u64> = idx.ancestors(2).map(|e| e.revision).collect();
|
||||
assert_eq!(revs, vec![2, 1, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ancestors_stops_at_root() {
|
||||
let mut idx = RevisionIndex::new();
|
||||
idx.append(make_entry(0, BRANCH_MAIN, NO_PARENT));
|
||||
idx.append(make_entry(1, BRANCH_MAIN, 0));
|
||||
let revs: Vec<u64> = idx.ancestors(1).map(|e| e.revision).collect();
|
||||
assert_eq!(revs, vec![1, 0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn common_ancestor_simple() {
|
||||
let mut idx = RevisionIndex::new();
|
||||
// main: 0 → 1 → 2
|
||||
// branch: fork from 1 → 3 → 4
|
||||
idx.append(make_entry(0, BRANCH_MAIN, NO_PARENT));
|
||||
idx.append(make_entry(1, BRANCH_MAIN, 0));
|
||||
idx.append(make_entry(2, BRANCH_MAIN, 1));
|
||||
idx.append(make_entry(3, 1, 1)); // fork from rev 1
|
||||
idx.append(make_entry(4, 1, 3));
|
||||
assert_eq!(idx.common_ancestor(2, 4), Some(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn common_ancestor_same_revision() {
|
||||
let mut idx = RevisionIndex::new();
|
||||
idx.append(make_entry(0, BRANCH_MAIN, NO_PARENT));
|
||||
idx.append(make_entry(1, BRANCH_MAIN, 0));
|
||||
assert_eq!(idx.common_ancestor(1, 1), Some(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_revisions() {
|
||||
let mut idx = RevisionIndex::new();
|
||||
for i in 0u64..5 {
|
||||
idx.append(make_entry(i, BRANCH_MAIN, if i == 0 { NO_PARENT } else { i - 1 }));
|
||||
}
|
||||
let to_remove = [1u64, 2].into_iter().collect();
|
||||
idx.remove_revisions(&to_remove);
|
||||
assert_eq!(idx.len(), 3);
|
||||
assert!(idx.get(1).is_none());
|
||||
assert!(idx.get(2).is_none());
|
||||
assert!(idx.get(0).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_flag_visible_via_index() {
|
||||
let mut idx = RevisionIndex::new();
|
||||
let mut e = make_entry(500, BRANCH_MAIN, 499);
|
||||
e.flags = REV_FLAG_SNAPSHOT;
|
||||
idx.append(e);
|
||||
assert_ne!(idx.get(500).unwrap().flags & REV_FLAG_SNAPSHOT, 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
//! # clawhdf5-onion — ClawOnion VFD
|
||||
//!
|
||||
//! Pure-Rust revision-layered HDF5 versioning using the onion metaphor:
|
||||
//! the original file state is the core, each write session adds a new
|
||||
//! "skin" of page-level changes on top.
|
||||
//!
|
||||
//! ## Format
|
||||
//!
|
||||
//! The on-disk representation is a `.onion` sidecar file alongside the
|
||||
//! primary `.h5` file. See [`format`] for the full binary layout.
|
||||
//!
|
||||
//! ## Features
|
||||
//!
|
||||
//! | Feature | Default | Description |
|
||||
//! |---|---|---|
|
||||
//! | `provenance` | on | BLAKE3 per-revision page hashing |
|
||||
//! | `compress` | on | per-page zstd / lz4 compression |
|
||||
//! | `parallel` | off | Rayon parallel page hashing |
|
||||
//! | `async` | off | Tokio async flush |
|
||||
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
pub mod annotation;
|
||||
pub mod api;
|
||||
pub mod branch;
|
||||
pub mod compress;
|
||||
pub mod error;
|
||||
pub mod ext;
|
||||
pub mod format;
|
||||
pub mod gc;
|
||||
pub mod index;
|
||||
pub mod merkle;
|
||||
pub mod provenance;
|
||||
pub mod reader;
|
||||
pub mod tdt;
|
||||
pub mod versioned_file;
|
||||
pub mod writer;
|
||||
|
||||
pub use error::OnionError;
|
||||
pub use format::{
|
||||
BranchEntry, Codec, OnionHeader, PageTableEntry, RevisionEntry,
|
||||
BRANCH_MAIN, DEFAULT_FEATURE_FLAGS, EPOCH_DEAD, FORMAT_VERSION, HEADER_SIZE, MAGIC, NO_PARENT,
|
||||
REV_FLAG_SNAPSHOT, feature_flags,
|
||||
};
|
||||
pub use api::{
|
||||
open_revision, open_branch, open_branch_at,
|
||||
list_revisions, rollback,
|
||||
list_branches, create_branch, delete_branch, rename_branch,
|
||||
};
|
||||
pub use branch::{BranchInfo, DatasetResolver, MergeStrategy};
|
||||
pub use merkle::{MerkleError, MerkleNode, RevisionMerkleTree, WalkStep};
|
||||
pub use ext::FileBuilderExt;
|
||||
pub use versioned_file::{VersionedFile, open_at_revision, open_at_branch};
|
||||
pub use writer::eof_to_page_size;
|
||||
@@ -0,0 +1,555 @@
|
||||
//! Content-defined Merkle tree over OnionFile revision BLAKE3 hashes.
|
||||
//!
|
||||
//! Enables O(log N) identification of missing revisions between two nodes,
|
||||
//! compared to O(N) for a flat revision-set comparison.
|
||||
//!
|
||||
//! ## Layout
|
||||
//!
|
||||
//! The tree is a balanced binary tree stored in 1-indexed BFS (heap) order:
|
||||
//!
|
||||
//! ```text
|
||||
//! capacity = next_power_of_two(leaf_count)
|
||||
//! root → index 1
|
||||
//! left child(i) → index 2i
|
||||
//! right child(i) → index 2i+1
|
||||
//! leaf k → index capacity + k (k = 0-based ordinal)
|
||||
//! ```
|
||||
//!
|
||||
//! Padding leaves (when `leaf_count < capacity`) have all-zero hashes.
|
||||
//! Internal node hash = `BLAKE3(left_child_hash || right_child_hash)`.
|
||||
//!
|
||||
//! ## Serialisation wire format
|
||||
//!
|
||||
//! ```text
|
||||
//! magic: [u8; 4] = b"MERK"
|
||||
//! version: u8 = 1
|
||||
//! leaf_count: u64 LE
|
||||
//! entries: leaf_count × { revision: u64 LE, blake3: [u8; 32] }
|
||||
//! ```
|
||||
//!
|
||||
//! Size: `5 + 8 + N × 40` bytes. For N = 1 000 this is ~40 KB, vs the
|
||||
//! O(N × 60 B) flat `ClawSyncManifest`.
|
||||
//!
|
||||
//! ## References
|
||||
//!
|
||||
//! arXiv 2104.02158 — content-defined Merkle trees for versioned storage.
|
||||
|
||||
#![forbid(unsafe_code)]
|
||||
|
||||
use blake3::Hasher as Blake3Hasher;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Wire constants
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
const MAGIC: &[u8; 4] = b"MERK";
|
||||
const VERSION: u8 = 1;
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Public types
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// A single node of the revision Merkle tree.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct MerkleNode {
|
||||
/// BLAKE3 hash of this node's subtree.
|
||||
pub hash: [u8; 32],
|
||||
/// Lowest revision number covered by this subtree.
|
||||
pub rev_lo: u64,
|
||||
/// Highest revision number covered by this subtree.
|
||||
pub rev_hi: u64,
|
||||
/// `true` when this node is a leaf (single revision).
|
||||
pub is_leaf: bool,
|
||||
}
|
||||
|
||||
/// Outcome of one step in a Merkle tree diff walk.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum WalkStep {
|
||||
/// Subtree [rev_lo, rev_hi] is identical on both sides — skip.
|
||||
Skip { rev_lo: u64, rev_hi: u64 },
|
||||
/// Subtree differs; descend into children.
|
||||
Descend { node: MerkleNode },
|
||||
/// Leaf differs; this revision needs to be transferred.
|
||||
Leaf { revision: u64, hash: [u8; 32] },
|
||||
}
|
||||
|
||||
/// Balanced binary Merkle tree over OnionFile revision BLAKE3 hashes.
|
||||
///
|
||||
/// Built from a sorted list of `(revision_number, blake3_hash)` pairs.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct RevisionMerkleTree {
|
||||
/// 1-indexed BFS hashes. Index 0 is unused.
|
||||
/// Size: 2 * capacity + 1.
|
||||
hashes: Vec<[u8; 32]>,
|
||||
/// Revision number for each leaf (0-based ordinal).
|
||||
leaf_revisions: Vec<u64>,
|
||||
/// Number of actual revisions (leaves with real hashes).
|
||||
pub leaf_count: usize,
|
||||
/// Smallest power of 2 ≥ leaf_count.
|
||||
pub capacity: usize,
|
||||
}
|
||||
|
||||
impl RevisionMerkleTree {
|
||||
// ── Construction ──────────────────────────────────────────────────────────
|
||||
|
||||
/// Build from a sorted slice of `(revision, blake3_hash)` pairs.
|
||||
///
|
||||
/// The slice **must** be sorted by `revision` in ascending order.
|
||||
pub fn build(entries: &[(u64, [u8; 32])]) -> Self {
|
||||
let leaf_count = entries.len();
|
||||
let capacity = if leaf_count == 0 { 1 } else { leaf_count.next_power_of_two() };
|
||||
|
||||
// 1-indexed BFS array; index 0 unused.
|
||||
let mut hashes = vec![[0u8; 32]; 2 * capacity + 1];
|
||||
let mut leaf_revisions = Vec::with_capacity(leaf_count);
|
||||
|
||||
// Place leaf hashes
|
||||
for (i, (rev, hash)) in entries.iter().enumerate() {
|
||||
hashes[capacity + i] = *hash;
|
||||
leaf_revisions.push(*rev);
|
||||
}
|
||||
// Indices [capacity + leaf_count, 2*capacity) are padding zeros
|
||||
|
||||
// Build internal nodes bottom-up
|
||||
build_internal(&mut hashes, capacity);
|
||||
|
||||
RevisionMerkleTree { hashes, leaf_revisions, leaf_count, capacity }
|
||||
}
|
||||
|
||||
// ── Queries ───────────────────────────────────────────────────────────────
|
||||
|
||||
/// Root hash of the tree. Two trees with the same root hash have identical
|
||||
/// revision sets (assuming no BLAKE3 collisions).
|
||||
pub fn root_hash(&self) -> [u8; 32] {
|
||||
if self.leaf_count == 0 {
|
||||
[0u8; 32]
|
||||
} else {
|
||||
self.hashes[1]
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the revision numbers that `self` has but `other` lacks (or
|
||||
/// has a different hash for).
|
||||
///
|
||||
/// The returned list is sorted in ascending revision order.
|
||||
///
|
||||
/// Algorithm: iterative BFS. At each node, if hashes agree the subtree is
|
||||
/// skipped (O(1)). Only differing subtrees are descended into. Total work
|
||||
/// is O((D + 1) × log N) for D differing revisions.
|
||||
pub fn diff_missing_revisions(&self, other: &RevisionMerkleTree) -> Vec<u64> {
|
||||
if self.leaf_count == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
// Expand other to self's capacity so BFS indices align.
|
||||
let other_hashes = other.padded_hashes(self.capacity);
|
||||
|
||||
let mut result = Vec::new();
|
||||
|
||||
// Quick exit if roots match
|
||||
if self.hashes[1] == other_hashes[1] {
|
||||
return result;
|
||||
}
|
||||
|
||||
// Iterative BFS walk
|
||||
let mut queue: std::collections::VecDeque<usize> = std::collections::VecDeque::new();
|
||||
queue.push_back(1);
|
||||
|
||||
while let Some(node_idx) = queue.pop_front() {
|
||||
if self.hashes[node_idx] == other_hashes[node_idx] {
|
||||
// Subtree matches — skip entirely
|
||||
continue;
|
||||
}
|
||||
|
||||
if node_idx >= self.capacity {
|
||||
// Leaf node
|
||||
let leaf_ord = node_idx - self.capacity;
|
||||
if leaf_ord < self.leaf_count {
|
||||
result.push(self.leaf_revisions[leaf_ord]);
|
||||
}
|
||||
} else {
|
||||
// Internal node — descend
|
||||
queue.push_back(2 * node_idx);
|
||||
queue.push_back(2 * node_idx + 1);
|
||||
}
|
||||
}
|
||||
|
||||
result.sort_unstable();
|
||||
result
|
||||
}
|
||||
|
||||
/// Return a `MerkleNode` for the root.
|
||||
pub fn root_node(&self) -> Option<MerkleNode> {
|
||||
if self.leaf_count == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(self.node_at(1))
|
||||
}
|
||||
|
||||
// ── Serialisation ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Serialise into the compact wire format:
|
||||
/// `MERK | version(1) | leaf_count(u64 LE) | entries[]`
|
||||
pub fn serialise(&self) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(5 + 8 + self.leaf_count * 40);
|
||||
out.extend_from_slice(MAGIC);
|
||||
out.push(VERSION);
|
||||
out.extend_from_slice(&(self.leaf_count as u64).to_le_bytes());
|
||||
for (i, &rev) in self.leaf_revisions.iter().enumerate() {
|
||||
out.extend_from_slice(&rev.to_le_bytes());
|
||||
out.extend_from_slice(&self.hashes[self.capacity + i]);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Deserialise from wire format.
|
||||
pub fn deserialise(data: &[u8]) -> Result<Self, MerkleError> {
|
||||
if data.len() < 5 + 8 {
|
||||
return Err(MerkleError::TruncatedHeader);
|
||||
}
|
||||
if &data[0..4] != MAGIC {
|
||||
return Err(MerkleError::BadMagic);
|
||||
}
|
||||
if data[4] != VERSION {
|
||||
return Err(MerkleError::UnknownVersion(data[4]));
|
||||
}
|
||||
let leaf_count = u64::from_le_bytes(data[5..13].try_into().unwrap()) as usize;
|
||||
let expected_len = 5 + 8 + leaf_count * 40;
|
||||
if data.len() < expected_len {
|
||||
return Err(MerkleError::TruncatedPayload {
|
||||
expected: expected_len,
|
||||
got: data.len(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut entries = Vec::with_capacity(leaf_count);
|
||||
let mut off = 13usize;
|
||||
for _ in 0..leaf_count {
|
||||
let rev = u64::from_le_bytes(data[off..off + 8].try_into().unwrap());
|
||||
let mut hash = [0u8; 32];
|
||||
hash.copy_from_slice(&data[off + 8..off + 40]);
|
||||
entries.push((rev, hash));
|
||||
off += 40;
|
||||
}
|
||||
|
||||
Ok(Self::build(&entries))
|
||||
}
|
||||
|
||||
// ── Internal helpers ──────────────────────────────────────────────────────
|
||||
|
||||
/// Return a copy of this tree's BFS hash array padded (or already matching)
|
||||
/// to `target_capacity`.
|
||||
///
|
||||
/// If `target_capacity > self.capacity`, the leaves are placed at the same
|
||||
/// ordinal positions in the larger tree and internal nodes are recomputed.
|
||||
/// If `target_capacity == self.capacity`, a cheap clone is returned.
|
||||
pub(crate) fn padded_hashes(&self, target_capacity: usize) -> Vec<[u8; 32]> {
|
||||
if target_capacity == self.capacity {
|
||||
return self.hashes.clone();
|
||||
}
|
||||
let mut h = vec![[0u8; 32]; 2 * target_capacity + 1];
|
||||
// Only copy leaves that fit within target_capacity (handles both
|
||||
// expanding a smaller tree and projecting a larger tree down).
|
||||
let copyable = self.leaf_count.min(target_capacity);
|
||||
for i in 0..copyable {
|
||||
h[target_capacity + i] = self.hashes[self.capacity + i];
|
||||
}
|
||||
build_internal(&mut h, target_capacity);
|
||||
h
|
||||
}
|
||||
|
||||
/// Retrieve the `MerkleNode` metadata for BFS index `i`.
|
||||
fn node_at(&self, i: usize) -> MerkleNode {
|
||||
let (lo, hi, is_leaf) = self.node_range(i);
|
||||
MerkleNode {
|
||||
hash: self.hashes[i],
|
||||
rev_lo: lo,
|
||||
rev_hi: hi,
|
||||
is_leaf,
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the [rev_lo, rev_hi] range and leaf flag for BFS index `i`.
|
||||
fn node_range(&self, i: usize) -> (u64, u64, bool) {
|
||||
if i >= self.capacity {
|
||||
// Leaf
|
||||
let ord = i - self.capacity;
|
||||
let rev = if ord < self.leaf_count {
|
||||
self.leaf_revisions[ord]
|
||||
} else {
|
||||
u64::MAX
|
||||
};
|
||||
(rev, rev, true)
|
||||
} else {
|
||||
// Internal: find leftmost/rightmost leaf under this node
|
||||
let depth = (i.ilog2()) as usize;
|
||||
let span = self.capacity >> depth;
|
||||
let lo_ord = (i - (1 << depth)) * span;
|
||||
let hi_ord = lo_ord + span - 1;
|
||||
let lo_rev = if lo_ord < self.leaf_count {
|
||||
self.leaf_revisions[lo_ord]
|
||||
} else {
|
||||
u64::MAX
|
||||
};
|
||||
let hi_rev = if hi_ord < self.leaf_count {
|
||||
self.leaf_revisions[hi_ord]
|
||||
} else if lo_ord < self.leaf_count {
|
||||
*self.leaf_revisions.last().unwrap()
|
||||
} else {
|
||||
u64::MAX
|
||||
};
|
||||
(lo_rev, hi_rev, false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Internal helpers
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Build internal nodes of a 1-indexed BFS tree bottom-up.
|
||||
/// `hashes` must have length `2 * capacity + 1`.
|
||||
fn build_internal(hashes: &mut [[u8; 32]], capacity: usize) {
|
||||
for i in (1..capacity).rev() {
|
||||
let left = hashes[2 * i];
|
||||
let right = hashes[2 * i + 1];
|
||||
hashes[i] = node_hash(&left, &right);
|
||||
}
|
||||
}
|
||||
|
||||
/// Hash two child hashes together: `BLAKE3(left || right)`.
|
||||
#[inline]
|
||||
fn node_hash(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] {
|
||||
let mut h = Blake3Hasher::new();
|
||||
h.update(left);
|
||||
h.update(right);
|
||||
*h.finalize().as_bytes()
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Error type
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Errors from Merkle tree serialisation/deserialisation.
|
||||
#[derive(Debug, thiserror::Error, PartialEq, Eq)]
|
||||
pub enum MerkleError {
|
||||
#[error("truncated header (< 13 bytes)")]
|
||||
TruncatedHeader,
|
||||
#[error("bad magic bytes")]
|
||||
BadMagic,
|
||||
#[error("unknown format version {0}")]
|
||||
UnknownVersion(u8),
|
||||
#[error("truncated payload: expected {expected} bytes, got {got}")]
|
||||
TruncatedPayload { expected: usize, got: usize },
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use proptest::prelude::*;
|
||||
|
||||
fn rev_hash(r: u64) -> [u8; 32] {
|
||||
*blake3::hash(&r.to_le_bytes()).as_bytes()
|
||||
}
|
||||
|
||||
fn build_n(n: usize) -> RevisionMerkleTree {
|
||||
let entries: Vec<(u64, [u8; 32])> = (0..n as u64).map(|r| (r, rev_hash(r))).collect();
|
||||
RevisionMerkleTree::build(&entries)
|
||||
}
|
||||
|
||||
// ── basic structure ───────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tree_empty() {
|
||||
let t = RevisionMerkleTree::build(&[]);
|
||||
assert_eq!(t.leaf_count, 0);
|
||||
assert_eq!(t.capacity, 1);
|
||||
assert_eq!(t.root_hash(), [0u8; 32]);
|
||||
assert!(t.root_node().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tree_single_leaf() {
|
||||
let h = rev_hash(42);
|
||||
let t = RevisionMerkleTree::build(&[(42, h)]);
|
||||
assert_eq!(t.leaf_count, 1);
|
||||
assert_eq!(t.capacity, 1);
|
||||
assert_eq!(t.root_hash(), h);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tree_two_leaves() {
|
||||
let t = build_n(2);
|
||||
assert_eq!(t.capacity, 2);
|
||||
let expected_root = node_hash(&rev_hash(0), &rev_hash(1));
|
||||
assert_eq!(t.root_hash(), expected_root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tree_depth_is_log_n() {
|
||||
// For 1000 leaves, depth should be ≤ ceil(log2(1000)) = 10
|
||||
let t = build_n(1000);
|
||||
assert_eq!(t.capacity, 1024); // next power of 2
|
||||
// Depth = log2(capacity) = 10
|
||||
let depth = t.capacity.ilog2() as usize;
|
||||
assert!(depth <= 10, "depth {depth} > 10");
|
||||
// Also verify no root is all zeros (would mean build failed)
|
||||
assert_ne!(t.root_hash(), [0u8; 32]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tree_identical_trees_no_diff() {
|
||||
let t = build_n(50);
|
||||
let diff = t.diff_missing_revisions(&t.clone());
|
||||
assert!(diff.is_empty(), "identical trees should have empty diff");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tree_one_new_revision() {
|
||||
let base = build_n(5);
|
||||
let extended = build_n(6); // has rev 5 that base lacks
|
||||
let diff = extended.diff_missing_revisions(&base);
|
||||
assert_eq!(diff, vec![5]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tree_multiple_new_revisions() {
|
||||
let base = build_n(3);
|
||||
let extended = build_n(7);
|
||||
let diff = extended.diff_missing_revisions(&base);
|
||||
assert_eq!(diff, vec![3, 4, 5, 6]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tree_remote_ahead_is_empty_diff() {
|
||||
// If remote has MORE revisions than local, from local's perspective
|
||||
// there's nothing "missing on remote" beyond local's range
|
||||
let local = build_n(5);
|
||||
let remote = build_n(10);
|
||||
// local.diff_missing_revisions(remote) → what local has that remote doesn't
|
||||
// local has 0..4, remote has 0..9: local has nothing remote lacks
|
||||
let diff = local.diff_missing_revisions(&remote);
|
||||
assert!(diff.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tree_changed_hash_mid_range() {
|
||||
// One revision in the middle has a different hash (e.g., corruption)
|
||||
let mut entries: Vec<(u64, [u8; 32])> = (0..10u64).map(|r| (r, rev_hash(r))).collect();
|
||||
let t1 = RevisionMerkleTree::build(&entries);
|
||||
entries[5].1 = [0xFFu8; 32]; // alter rev 5
|
||||
let t2 = RevisionMerkleTree::build(&entries);
|
||||
let diff = t1.diff_missing_revisions(&t2);
|
||||
assert_eq!(diff, vec![5]);
|
||||
}
|
||||
|
||||
// ── serialisation ─────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn tree_roundtrip_serialisation() {
|
||||
let t = build_n(100);
|
||||
let bytes = t.serialise();
|
||||
let t2 = RevisionMerkleTree::deserialise(&bytes).unwrap();
|
||||
assert_eq!(t2.leaf_count, 100);
|
||||
assert_eq!(t2.root_hash(), t.root_hash());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tree_serialise_size_linear() {
|
||||
let t = build_n(1000);
|
||||
let bytes = t.serialise();
|
||||
// 5 (header) + 8 (count) + 1000 * 40 (entries)
|
||||
assert_eq!(bytes.len(), 5 + 8 + 1000 * 40);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserialise_bad_magic_errors() {
|
||||
let mut bytes = build_n(5).serialise();
|
||||
bytes[0] = 0xFF;
|
||||
assert_eq!(
|
||||
RevisionMerkleTree::deserialise(&bytes),
|
||||
Err(MerkleError::BadMagic)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserialise_truncated_errors() {
|
||||
// Provide the 13-byte header (MERK + version + leaf_count=5) but no
|
||||
// entry bytes — this gets past the header check and hits TruncatedPayload.
|
||||
let full = build_n(5).serialise();
|
||||
let header_only = &full[..13]; // 5 + 8 bytes
|
||||
assert_eq!(
|
||||
RevisionMerkleTree::deserialise(header_only),
|
||||
Err(MerkleError::TruncatedPayload { expected: 5 + 8 + 5 * 40, got: 13 })
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deserialise_unknown_version_errors() {
|
||||
let mut bytes = build_n(3).serialise();
|
||||
bytes[4] = 99;
|
||||
assert_eq!(
|
||||
RevisionMerkleTree::deserialise(&bytes),
|
||||
Err(MerkleError::UnknownVersion(99))
|
||||
);
|
||||
}
|
||||
|
||||
// ── walk correctness ──────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn diff_empty_local_vs_empty_remote() {
|
||||
let t = RevisionMerkleTree::build(&[]);
|
||||
assert!(t.diff_missing_revisions(&t.clone()).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diff_empty_local_vs_nonempty_remote() {
|
||||
let local = RevisionMerkleTree::build(&[]);
|
||||
let remote = build_n(5);
|
||||
// local has nothing, so nothing is "missing on remote"
|
||||
let diff = local.diff_missing_revisions(&remote);
|
||||
assert!(diff.is_empty());
|
||||
}
|
||||
|
||||
// ── proptest ──────────────────────────────────────────────────────────────
|
||||
|
||||
proptest! {
|
||||
/// For any two random revision sets A ⊇ B (A = local, B = remote),
|
||||
/// `diff_missing_revisions` correctly identifies A \ B.
|
||||
#[test]
|
||||
fn prop_diff_identifies_all_differences(
|
||||
total in 1usize..500,
|
||||
keep_frac in 0.0f64..1.0,
|
||||
) {
|
||||
let all_entries: Vec<(u64, [u8; 32])> = (0..total as u64)
|
||||
.map(|r| (r, rev_hash(r)))
|
||||
.collect();
|
||||
let keep_count = ((total as f64 * keep_frac) as usize).max(0);
|
||||
let remote_entries = &all_entries[..keep_count];
|
||||
|
||||
let local = RevisionMerkleTree::build(&all_entries);
|
||||
let remote = RevisionMerkleTree::build(remote_entries);
|
||||
|
||||
let diff = local.diff_missing_revisions(&remote);
|
||||
|
||||
// Expected: revisions keep_count..total
|
||||
let expected: Vec<u64> = (keep_count as u64..total as u64).collect();
|
||||
prop_assert_eq!(diff, expected);
|
||||
}
|
||||
|
||||
/// Roundtrip: serialise then deserialise preserves root hash and leaf count.
|
||||
#[test]
|
||||
fn prop_serialise_roundtrip(n in 0usize..300) {
|
||||
let entries: Vec<(u64, [u8; 32])> = (0..n as u64).map(|r| (r, rev_hash(r))).collect();
|
||||
let t = RevisionMerkleTree::build(&entries);
|
||||
let bytes = t.serialise();
|
||||
let t2 = RevisionMerkleTree::deserialise(&bytes).unwrap();
|
||||
prop_assert_eq!(t2.root_hash(), t.root_hash());
|
||||
prop_assert_eq!(t2.leaf_count, t.leaf_count);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
//! BLAKE3 provenance: per-revision page hashing and UUIDv7 session IDs.
|
||||
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Threshold above which we use Rayon-based tree hashing per page.
|
||||
/// Below this the Rayon dispatch overhead exceeds the benefit.
|
||||
const RAYON_PAGE_THRESHOLD: usize = 128 * 1024; // 128 KB
|
||||
|
||||
/// Compute the BLAKE3 hash over a set of `(h5_offset, page_bytes)` pairs,
|
||||
/// processed in ascending `h5_offset` order.
|
||||
///
|
||||
/// This is the canonical per-revision hash stored in [`RevisionEntry::blake3`].
|
||||
/// For pages ≥ 128 KB each, the page data is hashed using Rayon tree
|
||||
/// parallelism; the offset bytes always use the single-threaded path.
|
||||
pub fn hash_pages(pages: &[(u64, &[u8])]) -> [u8; 32] {
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
// Sort by h5_offset to ensure deterministic ordering regardless of
|
||||
// the order pages were written during the session.
|
||||
let mut sorted: Vec<(u64, &[u8])> = pages.to_vec();
|
||||
sorted.sort_unstable_by_key(|(off, _)| *off);
|
||||
for (offset, data) in &sorted {
|
||||
hasher.update(&offset.to_le_bytes());
|
||||
if data.len() >= RAYON_PAGE_THRESHOLD {
|
||||
hasher.update_rayon(data);
|
||||
} else {
|
||||
hasher.update(data);
|
||||
}
|
||||
}
|
||||
*hasher.finalize().as_bytes()
|
||||
}
|
||||
|
||||
/// Encode a 32-byte BLAKE3 hash as a lowercase hex string.
|
||||
pub fn to_hex(hash: &[u8; 32]) -> String {
|
||||
let mut s = String::with_capacity(64);
|
||||
for b in hash {
|
||||
s.push_str(&format!("{b:02x}"));
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
/// A write-session identifier backed by a UUIDv7 (time-ordered, globally unique).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct SessionId(pub [u8; 16]);
|
||||
|
||||
impl SessionId {
|
||||
/// Generate a fresh session ID using UUIDv7.
|
||||
pub fn new() -> Self {
|
||||
Self(*Uuid::now_v7().as_bytes())
|
||||
}
|
||||
|
||||
/// Construct from raw bytes (e.g., loaded from a [`RevisionEntry`]).
|
||||
pub fn from_bytes(bytes: [u8; 16]) -> Self {
|
||||
Self(bytes)
|
||||
}
|
||||
|
||||
/// Return the raw 16-byte representation.
|
||||
pub fn as_bytes(&self) -> &[u8; 16] {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SessionId {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn hash_empty_pages() {
|
||||
let h = hash_pages(&[]);
|
||||
// BLAKE3 of empty input has a fixed value
|
||||
let expected = *blake3::hash(b"").as_bytes();
|
||||
assert_eq!(h, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_single_page() {
|
||||
let data = b"test page data";
|
||||
let pages = vec![(0u64, data.as_ref())];
|
||||
let h = hash_pages(&pages);
|
||||
assert_eq!(h.len(), 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_order_independent_of_input_order() {
|
||||
let a = vec![1u8; 4096];
|
||||
let b = vec![2u8; 4096];
|
||||
// Same pages, different insertion order
|
||||
let h1 = hash_pages(&[(0, a.as_ref()), (4096, b.as_ref())]);
|
||||
let h2 = hash_pages(&[(4096, b.as_ref()), (0, a.as_ref())]);
|
||||
assert_eq!(h1, h2, "hash must be order-independent (sorted by h5_offset)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_different_offsets_different_hash() {
|
||||
let data = b"same data";
|
||||
let h1 = hash_pages(&[(0u64, data.as_ref())]);
|
||||
let h2 = hash_pages(&[(4096u64, data.as_ref())]);
|
||||
assert_ne!(h1, h2, "offset is included in hash input");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hash_tampered_page_differs() {
|
||||
let data1 = vec![0xABu8; 4096];
|
||||
let mut data2 = data1.clone();
|
||||
data2[100] ^= 0xFF; // flip a byte
|
||||
let h1 = hash_pages(&[(0, data1.as_ref())]);
|
||||
let h2 = hash_pages(&[(0, data2.as_ref())]);
|
||||
assert_ne!(h1, h2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn to_hex_length_and_chars() {
|
||||
let hash = [0xABu8; 32];
|
||||
let hex = to_hex(&hash);
|
||||
assert_eq!(hex.len(), 64);
|
||||
assert!(hex.chars().all(|c| matches!(c, '0'..='9' | 'a'..='f')));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn to_hex_known_value() {
|
||||
let hash = [0u8; 32];
|
||||
let hex = to_hex(&hash);
|
||||
assert_eq!(hex, "0".repeat(64));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_id_new_is_unique() {
|
||||
let s1 = SessionId::new();
|
||||
let s2 = SessionId::new();
|
||||
// UUIDv7 — astronomically unlikely to collide
|
||||
assert_ne!(s1.0, s2.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_id_roundtrip() {
|
||||
let s = SessionId::new();
|
||||
let bytes = *s.as_bytes();
|
||||
let s2 = SessionId::from_bytes(bytes);
|
||||
assert_eq!(s.0, s2.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_id_is_16_bytes() {
|
||||
let s = SessionId::new();
|
||||
assert_eq!(s.as_bytes().len(), 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiple_pages_hash_deterministic() {
|
||||
let pages: Vec<Vec<u8>> = (0..8).map(|i| vec![i as u8; 4096]).collect();
|
||||
let pairs: Vec<(u64, &[u8])> = pages
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, p)| (i as u64 * 4096, p.as_ref()))
|
||||
.collect();
|
||||
let h1 = hash_pages(&pairs);
|
||||
let h2 = hash_pages(&pairs);
|
||||
assert_eq!(h1, h2, "hash must be deterministic");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
//! Revision reconstruction: merging page layers from rev 0..=K.
|
||||
//!
|
||||
//! Opening a historical revision reads only the `.onion` sidecar —
|
||||
//! the primary `.h5` file is always at the latest committed state.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use crate::compress::decompress_page;
|
||||
use crate::error::OnionError;
|
||||
use crate::format::Codec;
|
||||
use crate::writer::OnionFile;
|
||||
|
||||
/// Selector for which revision to open.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum OpenRevision {
|
||||
/// The most recent revision on the current branch.
|
||||
Head,
|
||||
/// A specific revision number (absolute, any branch).
|
||||
At(u64),
|
||||
/// The HEAD of a named branch.
|
||||
Branch(String),
|
||||
/// A specific revision number on a named branch.
|
||||
BranchAt(String, u64),
|
||||
}
|
||||
|
||||
impl OnionFile {
|
||||
/// Reconstruct the logical `.h5` file bytes as they were at revision `rev`.
|
||||
///
|
||||
/// This merges all page tables from rev 0 through `rev` (inclusive),
|
||||
/// with later revisions winning on conflicts. The result is a complete
|
||||
/// in-memory snapshot of the HDF5 file at that point in history.
|
||||
///
|
||||
/// # Complexity
|
||||
///
|
||||
/// O(K · P) where K is the revision depth and P is the average number
|
||||
/// of changed pages per revision.
|
||||
pub fn reconstruct_revision(&self, rev: u64, h5_base: &[u8]) -> Result<Vec<u8>, OnionError> {
|
||||
use crate::format::REV_FLAG_SNAPSHOT;
|
||||
|
||||
// Collect all ancestor revisions in order from oldest → newest
|
||||
let ancestors: Vec<u64> = {
|
||||
let mut chain = self.index.ancestors(rev).map(|e| e.revision).collect::<Vec<_>>();
|
||||
chain.reverse(); // oldest first
|
||||
chain
|
||||
};
|
||||
|
||||
if ancestors.is_empty() {
|
||||
return Err(OnionError::RevisionNotFound(rev));
|
||||
}
|
||||
|
||||
// Optimisation: find the newest snapshot in the ancestor chain and
|
||||
// start from there instead of from `h5_base`. This bounds
|
||||
// reconstruction depth to O(N_since_snapshot · P).
|
||||
let snapshot_start_idx = ancestors
|
||||
.iter()
|
||||
.rposition(|&r| {
|
||||
self.index
|
||||
.get(r)
|
||||
.is_some_and(|e| e.flags & REV_FLAG_SNAPSHOT != 0)
|
||||
});
|
||||
|
||||
let (start_idx, mut file_bytes) = match snapshot_start_idx {
|
||||
Some(idx) => {
|
||||
// Start with an empty base (snapshot contains ALL pages)
|
||||
(idx, Vec::new())
|
||||
}
|
||||
None => (0, h5_base.to_vec()),
|
||||
};
|
||||
|
||||
// Apply page layers in chronological order from start_idx
|
||||
let mut page_map: BTreeMap<u64, Vec<u8>> = BTreeMap::new();
|
||||
|
||||
for ancestor_rev in &ancestors[start_idx..] {
|
||||
let table = self
|
||||
.page_tables
|
||||
.get(*ancestor_rev as usize)
|
||||
.ok_or_else(|| OnionError::Malformed(format!("missing page table for rev {ancestor_rev}")))?;
|
||||
|
||||
for pt_entry in table {
|
||||
let codec = Codec::from_u8(pt_entry.codec)?;
|
||||
// data_offset in the in-memory page_tables is relative to
|
||||
// the start of self.page_data (not file-absolute).
|
||||
let compressed_start = pt_entry.data_offset as usize;
|
||||
let compressed_end = compressed_start + pt_entry.data_size as usize;
|
||||
|
||||
if compressed_end > self.page_data.len() {
|
||||
return Err(OnionError::Malformed(format!(
|
||||
"page data out of bounds for rev {ancestor_rev} offset {}",
|
||||
pt_entry.h5_offset
|
||||
)));
|
||||
}
|
||||
let compressed = &self.page_data[compressed_start..compressed_end];
|
||||
let page = decompress_page(compressed, codec, pt_entry.orig_size)?;
|
||||
page_map.insert(pt_entry.h5_offset, page);
|
||||
}
|
||||
}
|
||||
|
||||
// Apply page map onto file bytes
|
||||
for (h5_off, page_bytes) in &page_map {
|
||||
let start = *h5_off as usize;
|
||||
let end = start + page_bytes.len();
|
||||
if end > file_bytes.len() {
|
||||
file_bytes.resize(end, 0);
|
||||
}
|
||||
file_bytes[start..end].copy_from_slice(page_bytes);
|
||||
}
|
||||
|
||||
Ok(file_bytes)
|
||||
}
|
||||
|
||||
/// Open a specific revision using an [`OpenRevision`] selector.
|
||||
pub fn open_rev(
|
||||
&self,
|
||||
selector: OpenRevision,
|
||||
h5_base: &[u8],
|
||||
) -> Result<Vec<u8>, OnionError> {
|
||||
let rev = self.resolve_selector(selector)?;
|
||||
self.reconstruct_revision(rev, h5_base)
|
||||
}
|
||||
|
||||
/// Resolve an [`OpenRevision`] selector to a concrete revision number.
|
||||
pub fn resolve_selector(&self, selector: OpenRevision) -> Result<u64, OnionError> {
|
||||
match selector {
|
||||
OpenRevision::Head => {
|
||||
// HEAD of main branch
|
||||
self.branch_head_rev(crate::format::BRANCH_MAIN)
|
||||
.ok_or(OnionError::RevisionNotFound(0))
|
||||
}
|
||||
OpenRevision::At(rev) => {
|
||||
self.index
|
||||
.get(rev)
|
||||
.map(|_| rev)
|
||||
.ok_or(OnionError::RevisionNotFound(rev))
|
||||
}
|
||||
OpenRevision::Branch(name) => {
|
||||
let branch = self
|
||||
.branch_by_name(&name)
|
||||
.ok_or_else(|| OnionError::BranchNotFound(name.clone()))?;
|
||||
if branch.head_rev == crate::format::NO_PARENT {
|
||||
Err(OnionError::RevisionNotFound(0))
|
||||
} else {
|
||||
Ok(branch.head_rev)
|
||||
}
|
||||
}
|
||||
OpenRevision::BranchAt(name, rev) => {
|
||||
// Verify this revision exists on the named branch
|
||||
let branch = self
|
||||
.branch_by_name(&name)
|
||||
.ok_or_else(|| OnionError::BranchNotFound(name.clone()))?;
|
||||
let entry = self
|
||||
.index
|
||||
.get(rev)
|
||||
.ok_or(OnionError::RevisionNotFound(rev))?;
|
||||
if entry.branch_id != branch.id {
|
||||
return Err(OnionError::Malformed(format!(
|
||||
"revision {rev} is not on branch {:?}",
|
||||
self.annotations.get(branch.name_off).unwrap_or("?")
|
||||
)));
|
||||
}
|
||||
Ok(rev)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the decompressed pages that were changed *only in revision `rev`*
|
||||
/// (not the accumulated file state — use [`reconstruct_revision`] for that).
|
||||
///
|
||||
/// Each element is `(h5_offset, uncompressed_page_bytes)`.
|
||||
/// This is used by `clawsync-onion` to build transfer packets.
|
||||
pub fn revision_pages(&self, rev: u64) -> Result<Vec<(u64, Vec<u8>)>, OnionError> {
|
||||
let table = self
|
||||
.page_tables
|
||||
.get(rev as usize)
|
||||
.ok_or(OnionError::RevisionNotFound(rev))?;
|
||||
|
||||
let mut pages = Vec::with_capacity(table.len());
|
||||
for pt in table {
|
||||
let codec = crate::format::Codec::from_u8(pt.codec)?;
|
||||
let start = pt.data_offset as usize;
|
||||
let end = start + pt.data_size as usize;
|
||||
if end > self.page_data.len() {
|
||||
return Err(OnionError::Malformed(format!(
|
||||
"page data out of bounds for rev {rev} offset {}",
|
||||
pt.h5_offset
|
||||
)));
|
||||
}
|
||||
let compressed = &self.page_data[start..end];
|
||||
let raw = crate::compress::decompress_page(compressed, codec, pt.orig_size)?;
|
||||
pages.push((pt.h5_offset, raw));
|
||||
}
|
||||
Ok(pages)
|
||||
}
|
||||
|
||||
/// Return the raw **compressed** page bytes for a revision together with
|
||||
/// the codec and original (uncompressed) size.
|
||||
///
|
||||
/// Compared to [`revision_pages`] this skips the decompression step, so
|
||||
/// callers that intend to send the data over the network can ship the
|
||||
/// compressed bytes directly and let the receiver decompress.
|
||||
///
|
||||
/// Returns `(h5_offset, compressed_data, codec_byte, orig_size)`.
|
||||
pub fn revision_pages_raw(
|
||||
&self,
|
||||
rev: u64,
|
||||
) -> Result<Vec<(u64, Vec<u8>, u8, u32)>, OnionError> {
|
||||
let table = self
|
||||
.page_tables
|
||||
.get(rev as usize)
|
||||
.ok_or(OnionError::RevisionNotFound(rev))?;
|
||||
|
||||
let mut pages = Vec::with_capacity(table.len());
|
||||
for pt in table {
|
||||
let start = pt.data_offset as usize;
|
||||
let end = start + pt.data_size as usize;
|
||||
if end > self.page_data.len() {
|
||||
return Err(OnionError::Malformed(format!(
|
||||
"page data out of bounds (raw) for rev {rev} offset {}",
|
||||
pt.h5_offset
|
||||
)));
|
||||
}
|
||||
let data = self.page_data[start..end].to_vec();
|
||||
pages.push((pt.h5_offset, data, pt.codec, pt.orig_size));
|
||||
}
|
||||
Ok(pages)
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
fn h5_base() -> Vec<u8> {
|
||||
// Minimal "HDF5" file: 8-byte signature + 4KB of zeros
|
||||
let mut v = b"\x89HDF\r\n\x1a\n".to_vec();
|
||||
v.extend(vec![0u8; 4096 * 10]);
|
||||
v
|
||||
}
|
||||
|
||||
fn tmp_h5() -> std::path::PathBuf {
|
||||
let f = NamedTempFile::new().unwrap();
|
||||
let path = f.path().with_extension("h5");
|
||||
let base = h5_base();
|
||||
std::fs::write(&path, &base).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reconstruct_head_after_one_write() {
|
||||
let h5_path = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5_path, 4096).unwrap();
|
||||
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
// Write a page at h5_offset=0 with known content
|
||||
let new_page = vec![0xFFu8; 4096];
|
||||
s.record_page(0, &new_page);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
|
||||
let base = h5_base();
|
||||
let reconstructed = onion.reconstruct_revision(0, &base).unwrap();
|
||||
assert_eq!(&reconstructed[0..4096], new_page.as_slice());
|
||||
// Rest should match base
|
||||
assert_eq!(&reconstructed[4096..], &base[4096..]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reconstruct_applies_layers_in_order() {
|
||||
let h5_path = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5_path, 4096).unwrap();
|
||||
|
||||
// Rev 0: write page at offset 0
|
||||
let mut s0 = onion.begin_session(None).unwrap();
|
||||
s0.record_page(0, &vec![0xAAu8; 4096]);
|
||||
onion.commit_session(s0, None).unwrap();
|
||||
|
||||
// Rev 1: overwrite same page
|
||||
let mut s1 = onion.begin_session(None).unwrap();
|
||||
s1.record_page(0, &vec![0xBBu8; 4096]);
|
||||
onion.commit_session(s1, None).unwrap();
|
||||
|
||||
let base = h5_base();
|
||||
// At rev 0, page should be 0xAA
|
||||
let r0 = onion.reconstruct_revision(0, &base).unwrap();
|
||||
assert!(r0[0..4096].iter().all(|&b| b == 0xAA));
|
||||
|
||||
// At rev 1, page should be 0xBB (later write wins)
|
||||
let r1 = onion.reconstruct_revision(1, &base).unwrap();
|
||||
assert!(r1[0..4096].iter().all(|&b| b == 0xBB));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reconstruct_multiple_pages() {
|
||||
let h5_path = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5_path, 4096).unwrap();
|
||||
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0x11u8; 4096]);
|
||||
s.record_page(4096, &vec![0x22u8; 4096]);
|
||||
s.record_page(8192, &vec![0x33u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
|
||||
let base = h5_base();
|
||||
let r = onion.reconstruct_revision(0, &base).unwrap();
|
||||
assert!(r[0..4096].iter().all(|&b| b == 0x11));
|
||||
assert!(r[4096..8192].iter().all(|&b| b == 0x22));
|
||||
assert!(r[8192..12288].iter().all(|&b| b == 0x33));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_rev_at_selector() {
|
||||
let h5_path = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5_path, 4096).unwrap();
|
||||
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0xDDu8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
|
||||
let base = h5_base();
|
||||
let r = onion.open_rev(OpenRevision::At(0), &base).unwrap();
|
||||
assert!(r[0..4096].iter().all(|&b| b == 0xDD));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_rev_head_selector() {
|
||||
let h5_path = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5_path, 4096).unwrap();
|
||||
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0xAAu8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
|
||||
let mut s2 = onion.begin_session(None).unwrap();
|
||||
s2.record_page(0, &vec![0xBBu8; 4096]);
|
||||
onion.commit_session(s2, None).unwrap();
|
||||
|
||||
let base = h5_base();
|
||||
// HEAD should give us rev 1 (0xBB)
|
||||
let r = onion.open_rev(OpenRevision::Head, &base).unwrap();
|
||||
assert!(r[0..4096].iter().all(|&b| b == 0xBB));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_rev_nonexistent_returns_error() {
|
||||
let h5_path = tmp_h5();
|
||||
let onion = OnionFile::create(&h5_path, 4096).unwrap();
|
||||
let base = h5_base();
|
||||
let err = onion.open_rev(OpenRevision::At(999), &base).unwrap_err();
|
||||
assert!(matches!(err, OnionError::RevisionNotFound(999)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_rev_branch_selector_nonexistent() {
|
||||
let h5_path = tmp_h5();
|
||||
let onion = OnionFile::create(&h5_path, 4096).unwrap();
|
||||
let base = h5_base();
|
||||
let err = onion
|
||||
.open_rev(OpenRevision::Branch("nonexistent".to_string()), &base)
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, OnionError::BranchNotFound(_)));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
//! Byte-interleaving (TDT) transform for improved compression of numeric data.
|
||||
//!
|
||||
//! Based on arXiv:2506.18062 — "Floating-Point Data Transformation for Lossless
|
||||
//! Compression". Reorders bytes so all byte-N of each element are grouped together
|
||||
//! before byte-(N+1), exposing redundancy in sign/exponent bytes that zstd can
|
||||
//! then exploit.
|
||||
//!
|
||||
//! For `f32` data (`element_width = 4`) with N elements:
|
||||
//!
|
||||
//! ```text
|
||||
//! Input: [a0 a1 a2 a3 | b0 b1 b2 b3 | c0 c1 c2 c3 | ...]
|
||||
//! Output: [a0 b0 c0 ... | a1 b1 c1 ... | a2 b2 c2 ... | a3 b3 c3 ...]
|
||||
//! ```
|
||||
//!
|
||||
//! Trailing bytes (when `data.len() % element_width != 0`) are appended verbatim
|
||||
//! after the interleaved region so round-trips always reproduce the original length.
|
||||
//!
|
||||
//! # Choosing element_width
|
||||
//!
|
||||
//! | HDF5 dtype | element_width |
|
||||
//! |------------|---------------|
|
||||
//! | `f32` | 4 |
|
||||
//! | `f64` | 8 |
|
||||
//! | `f16` | 2 |
|
||||
//! | `i32`/`u32` | 4 |
|
||||
//! | `i16`/`u16` | 2 |
|
||||
//! | `i8`/`u8` | 1 (no-op) |
|
||||
|
||||
/// Apply the TDT byte-interleaving transform.
|
||||
///
|
||||
/// Groups bytes by their position within each `element_width`-byte element.
|
||||
/// Returns a copy of `data` reordered for better zstd compression.
|
||||
///
|
||||
/// When `element_width <= 1` the input is returned unchanged (byte data is
|
||||
/// already in the best form for the compressor).
|
||||
pub fn encode(data: &[u8], element_width: usize) -> Vec<u8> {
|
||||
if element_width <= 1 || data.is_empty() {
|
||||
return data.to_vec();
|
||||
}
|
||||
let n_full = data.len() / element_width;
|
||||
let tail_start = n_full * element_width;
|
||||
|
||||
let mut out = Vec::with_capacity(data.len());
|
||||
for byte_pos in 0..element_width {
|
||||
for elem in 0..n_full {
|
||||
out.push(data[elem * element_width + byte_pos]);
|
||||
}
|
||||
}
|
||||
out.extend_from_slice(&data[tail_start..]);
|
||||
out
|
||||
}
|
||||
|
||||
/// Reverse the TDT byte-interleaving transform.
|
||||
///
|
||||
/// `orig_len` must equal the `data.len()` that was passed to [`encode`].
|
||||
pub fn decode(data: &[u8], element_width: usize, orig_len: usize) -> Vec<u8> {
|
||||
if element_width <= 1 || data.is_empty() {
|
||||
return data[..orig_len.min(data.len())].to_vec();
|
||||
}
|
||||
let n_full = orig_len / element_width;
|
||||
let tail_len = orig_len - n_full * element_width;
|
||||
|
||||
let mut out = vec![0u8; orig_len];
|
||||
for byte_pos in 0..element_width {
|
||||
for elem in 0..n_full {
|
||||
out[elem * element_width + byte_pos] = data[byte_pos * n_full + elem];
|
||||
}
|
||||
}
|
||||
// Tail bytes sit after the n_full * element_width interleaved bytes.
|
||||
let tail_src = n_full * element_width;
|
||||
out[orig_len - tail_len..].copy_from_slice(&data[tail_src..tail_src + tail_len]);
|
||||
out
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn roundtrip(data: &[u8], width: usize) {
|
||||
let encoded = encode(data, width);
|
||||
assert_eq!(encoded.len(), data.len(), "encode must preserve length");
|
||||
let decoded = decode(&encoded, width, data.len());
|
||||
assert_eq!(decoded, data, "roundtrip must reproduce original bytes");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_encode_decode_f32_roundtrip_4kb() {
|
||||
// Smooth-ish f32 data: sine-wave pattern cast to bytes.
|
||||
let mut data = Vec::with_capacity(4096);
|
||||
for i in 0u32..1024 {
|
||||
let v = ((i as f32 / 128.0).sin() * 1000.0) as f32;
|
||||
data.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
roundtrip(&data, 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_encode_decode_f32_roundtrip_64kb() {
|
||||
let mut data = Vec::with_capacity(65536);
|
||||
for i in 0u32..16384 {
|
||||
let v = i as f32 * 0.001;
|
||||
data.extend_from_slice(&v.to_le_bytes());
|
||||
}
|
||||
roundtrip(&data, 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_encode_4_elements_manual() {
|
||||
// 4 × f32 = 16 bytes: a=[0,1,2,3] b=[4,5,6,7] c=[8,9,10,11] d=[12,13,14,15]
|
||||
let data: Vec<u8> = (0u8..16).collect();
|
||||
let enc = encode(&data, 4);
|
||||
// byte_pos=0: elements 0,4,8,12
|
||||
// byte_pos=1: elements 1,5,9,13
|
||||
// byte_pos=2: elements 2,6,10,14
|
||||
// byte_pos=3: elements 3,7,11,15
|
||||
assert_eq!(enc, vec![0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15]);
|
||||
let dec = decode(&enc, 4, 16);
|
||||
assert_eq!(dec, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_encode_f16_width2() {
|
||||
// 4 × f16 = 8 bytes
|
||||
let data: Vec<u8> = (0u8..8).collect();
|
||||
let enc = encode(&data, 2);
|
||||
// byte_pos=0: [0,2,4,6], byte_pos=1: [1,3,5,7]
|
||||
assert_eq!(enc, vec![0, 2, 4, 6, 1, 3, 5, 7]);
|
||||
roundtrip(&data, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_encode_non_aligned_tail() {
|
||||
// 9 bytes with element_width=4: 2 full elements (8 bytes) + 1 tail byte
|
||||
let data: Vec<u8> = (0u8..9).collect();
|
||||
let enc = encode(&data, 4);
|
||||
// byte_pos=0: [0, 4], byte_pos=1: [1, 5], byte_pos=2: [2, 6], byte_pos=3: [3, 7]
|
||||
// tail: [8]
|
||||
assert_eq!(enc, vec![0, 4, 1, 5, 2, 6, 3, 7, 8]);
|
||||
let dec = decode(&enc, 4, 9);
|
||||
assert_eq!(dec, data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_encode_width_1_is_noop() {
|
||||
let data: Vec<u8> = (0u8..64).collect();
|
||||
assert_eq!(encode(&data, 1), data);
|
||||
assert_eq!(decode(&data, 1, data.len()), data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_encode_empty_is_noop() {
|
||||
assert_eq!(encode(&[], 4), &[] as &[u8]);
|
||||
assert_eq!(decode(&[], 4, 0), &[] as &[u8]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_encode_single_element() {
|
||||
let data = vec![0u8, 1, 2, 3];
|
||||
// Single f32: n_full=1, no reordering possible — output equals input
|
||||
assert_eq!(encode(&data, 4), data);
|
||||
roundtrip(&data, 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_encode_improves_compressibility_float() {
|
||||
// Generate float data with varying mantissas but identical exponents —
|
||||
// TDT should group identical exponent bytes together, helping zstd.
|
||||
let mut data = Vec::with_capacity(4096);
|
||||
for i in 0u32..1024 {
|
||||
let v = (1.0f32 + i as f32 * 0.0001).to_le_bytes();
|
||||
data.extend_from_slice(&v);
|
||||
}
|
||||
let raw_compressed = zstd::bulk::compress(&data, 3).unwrap();
|
||||
let transformed = encode(&data, 4);
|
||||
let tdt_compressed = zstd::bulk::compress(&transformed, 3).unwrap();
|
||||
assert!(
|
||||
tdt_compressed.len() <= raw_compressed.len(),
|
||||
"TDT+zstd ({} bytes) should be no worse than raw zstd ({} bytes) on structured float data",
|
||||
tdt_compressed.len(), raw_compressed.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_roundtrip_random_bytes() {
|
||||
// Random bytes: TDT is neutral (neither helps nor hurts correctness).
|
||||
let data: Vec<u8> = (0u8..=255).cycle().take(4096).collect();
|
||||
roundtrip(&data, 4);
|
||||
roundtrip(&data, 2);
|
||||
roundtrip(&data, 8);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user