Initial commit: ClawSync v0.1.0
8-crate pure-Rust workspace for revision-aware HDF5 sync. ## Crates - clawhdf5-onion: ClawOnion VFD — page-level versioned HDF5 storage, binary format, writer/reader, branch DAG, GC, snapshots, provenance - clawsync-core: BLAKE3, xxHash3, FastCDC (+ SIMD NEON), zstd/lz4 - clawsync-onion: IBLT sketch, Merkle tree differ, packet differ/merger, ClawSyncManifest, SyncSelector - clawsync-hdf5: dataset-level manifest, differ, patcher, wire payload reconstruction (apply_received_payloads) - clawsync-transport: TCP, QUIC (quinn 0.11/TLS 1.3), SyncPeer abstraction, length-prefixed rkyv wire protocol (21 SyncMessage variants) - clawsync-agent: OnionMemory, SyncScheduler, TcpSyncBackend, PeerCapabilities negotiation - clawsync-fs: CDC-based delta sync for any file type; FsSyncClient/Server, W=16 pipelining, atomic writes - clawsync-cli: push/pull/serve/hdf5-sync/serve-hdf5/sync/serve-fs + all local management commands; --quic on all network commands ## Key features - IBLT pre-flight: O(revision count) vs rsync's O(file size) - W=16 sliding-window push: 13–15x speedup over stop-and-wait at WAN RTT - Dataset-granular HDF5 sync: only modified datasets transferred - CDC delta for any file type: insertion-stable chunk boundaries - Full revision DAG: branch, merge, rollback, export, snapshot, GC - QUIC transport: TLS 1.3, per-message streams via quinn 0.11 ## Tests ~573 passing (default features); ~589 with --features simd-cdc ## Performance (Apple Silicon) - Reconstruct rev=100: 68 µs (target ≤ 1 ms) - BLAKE3 Rayon 1 MB: 10.3 GiB/s (target ≥ 5 GB/s) - GC 500 revisions: 20.6 µs (target ≤ 2 s) - W=16 vs W=1 at 5 ms RTT: 14.8x speedup - No-op pre-flight at 16 MB: 4 ms vs rsync 35 ms (7.8x) Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
osobh
@@ -0,0 +1,605 @@
|
||||
//! Garbage collection: prune old revisions from the `.onion` sidecar.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use crate::compress::compress_page;
|
||||
use crate::error::OnionError;
|
||||
use crate::format::{EPOCH_DEAD, PageTableEntry, REV_FLAG_SNAPSHOT};
|
||||
use crate::writer::OnionFile;
|
||||
|
||||
/// Policy controlling which revisions are retained after GC.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum GcPolicy {
|
||||
/// Keep the N most recent revisions (by revision number).
|
||||
KeepLastN(u64),
|
||||
/// Keep all revisions that have a non-empty annotation.
|
||||
KeepTagged,
|
||||
/// Keep all revisions with a timestamp >= the given Unix epoch value.
|
||||
KeepSince(f64),
|
||||
/// Keep an explicit set of revision numbers (plus their ancestors to
|
||||
/// maintain a valid DAG).
|
||||
KeepRevisions(Vec<u64>),
|
||||
/// **Lazy / epoch-based GC.**
|
||||
///
|
||||
/// Computes the dead set using `inner`, marks those entries with
|
||||
/// [`EPOCH_DEAD`] in their padding bytes, and returns immediately
|
||||
/// without touching `page_data`. The actual compaction is deferred to
|
||||
/// the next [`OnionFile::flush`] call, which runs
|
||||
/// [`OnionFile::compact_dead_epoch_revisions`] before writing.
|
||||
///
|
||||
/// Use this when you want GC to be non-blocking: the mark pass is
|
||||
/// O(revisions) with no I/O; the compaction is amortised into the next
|
||||
/// flush that would happen anyway.
|
||||
EpochFlip(Box<GcPolicy>),
|
||||
}
|
||||
|
||||
/// Statistics returned by a GC run.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct GcStats {
|
||||
/// Number of revisions removed.
|
||||
pub revisions_removed: u64,
|
||||
/// Bytes of page data reclaimed.
|
||||
pub bytes_reclaimed: u64,
|
||||
}
|
||||
|
||||
impl OnionFile {
|
||||
/// Prune revisions according to `policy`.
|
||||
///
|
||||
/// For all policies except [`GcPolicy::EpochFlip`]:
|
||||
/// - The `RevisionIndex` is updated immediately.
|
||||
/// - Page data is compacted in-memory; call [`flush`] to persist.
|
||||
///
|
||||
/// For [`GcPolicy::EpochFlip`]:
|
||||
/// - Dead revisions are *marked* with [`EPOCH_DEAD`] in O(revisions).
|
||||
/// - Page data is **not** touched; compaction is deferred to the next
|
||||
/// [`flush`] call. This makes the GC call itself non-blocking.
|
||||
///
|
||||
/// **Note:** Immediate GC is irreversible. Epoch-flip GC can be
|
||||
/// cancelled by calling `flush_wal()` without `flush()`, but only
|
||||
/// before the next `flush()`.
|
||||
pub fn gc(&mut self, policy: GcPolicy) -> Result<GcStats, OnionError> {
|
||||
let to_remove = self.compute_to_remove(&policy);
|
||||
|
||||
match policy {
|
||||
GcPolicy::EpochFlip(_) => {
|
||||
// ── Lazy path: just mark dead entries ────────────────────
|
||||
let count = to_remove.len() as u64;
|
||||
for &rev in &to_remove {
|
||||
if let Some(entry) = self.index.get_mut(rev) {
|
||||
entry.set_epoch(EPOCH_DEAD);
|
||||
}
|
||||
}
|
||||
Ok(GcStats { revisions_removed: count, bytes_reclaimed: 0 })
|
||||
}
|
||||
_ => {
|
||||
// ── Immediate path: compact now ───────────────────────────
|
||||
self.compact_revisions(to_remove)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the set of revisions to remove given a `policy`.
|
||||
fn compute_to_remove(&self, policy: &GcPolicy) -> HashSet<u64> {
|
||||
let all_revs: Vec<u64> = self.index.entries().iter().map(|e| e.revision).collect();
|
||||
|
||||
let keep: HashSet<u64> = match policy {
|
||||
GcPolicy::KeepLastN(n) => {
|
||||
let start = all_revs.len().saturating_sub(*n as usize);
|
||||
all_revs[start..].iter().copied().collect()
|
||||
}
|
||||
GcPolicy::KeepTagged => all_revs
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|&rev| {
|
||||
self.index
|
||||
.get(rev)
|
||||
.and_then(|e| self.annotations.get(e.annotation_off))
|
||||
.is_some_and(|a| !a.is_empty())
|
||||
})
|
||||
.collect(),
|
||||
GcPolicy::KeepSince(cutoff) => all_revs
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|&rev| {
|
||||
self.index
|
||||
.get(rev)
|
||||
.is_some_and(|e| e.timestamp >= *cutoff)
|
||||
})
|
||||
.collect(),
|
||||
GcPolicy::KeepRevisions(explicit) => {
|
||||
let mut set: HashSet<u64> = explicit.iter().copied().collect();
|
||||
for &rev in explicit {
|
||||
for ancestor in self.index.ancestors(rev) {
|
||||
set.insert(ancestor.revision);
|
||||
}
|
||||
}
|
||||
set
|
||||
}
|
||||
GcPolicy::EpochFlip(inner) => return self.compute_to_remove(inner),
|
||||
};
|
||||
|
||||
all_revs.iter().copied().filter(|rev| !keep.contains(rev)).collect()
|
||||
}
|
||||
|
||||
/// Consolidate + compact the given revision set immediately.
|
||||
///
|
||||
/// Shared by the immediate `gc()` path and the deferred flush path.
|
||||
pub(crate) fn compact_revisions(
|
||||
&mut self,
|
||||
to_remove: HashSet<u64>,
|
||||
) -> Result<GcStats, OnionError> {
|
||||
if to_remove.is_empty() {
|
||||
return Ok(GcStats::default());
|
||||
}
|
||||
|
||||
let revisions_removed = to_remove.len() as u64;
|
||||
|
||||
let mut bytes_reclaimed = 0u64;
|
||||
for &rev in &to_remove {
|
||||
if let Some(table) = self.page_tables.get(rev as usize) {
|
||||
bytes_reclaimed += table.iter().map(|e| e.data_size as u64).sum::<u64>();
|
||||
}
|
||||
}
|
||||
|
||||
// Consolidation: if the oldest surviving revision has ancestors that
|
||||
// will be removed, reconstruct its full state as a root snapshot.
|
||||
let keep: HashSet<u64> = self
|
||||
.index
|
||||
.entries()
|
||||
.iter()
|
||||
.map(|e| e.revision)
|
||||
.filter(|r| !to_remove.contains(r))
|
||||
.collect();
|
||||
|
||||
let mut surviving_sorted: Vec<u64> = keep.iter().copied().collect();
|
||||
surviving_sorted.sort_unstable();
|
||||
|
||||
if let Some(&oldest_surviving) = surviving_sorted.first() {
|
||||
let has_removed_ancestor = self
|
||||
.index
|
||||
.ancestors(oldest_surviving)
|
||||
.skip(1)
|
||||
.any(|e| to_remove.contains(&e.revision));
|
||||
|
||||
if has_removed_ancestor {
|
||||
let h5_path = self.path.with_extension("");
|
||||
let h5_base = std::fs::read(&h5_path).unwrap_or_default();
|
||||
let full_bytes = self.reconstruct_revision(oldest_surviving, &h5_base)?;
|
||||
|
||||
let ps = self.header.page_size as usize;
|
||||
let mut new_table: Vec<PageTableEntry> = Vec::new();
|
||||
for (i, chunk) in full_bytes.chunks(ps).enumerate() {
|
||||
let mut padded = vec![0u8; ps];
|
||||
padded[..chunk.len()].copy_from_slice(chunk);
|
||||
let compressed = compress_page(&padded, self.default_codec)?;
|
||||
let data_offset = self.page_data.len() as u64;
|
||||
self.page_data.extend_from_slice(&compressed);
|
||||
new_table.push(PageTableEntry {
|
||||
h5_offset: (i * ps) as u64,
|
||||
data_offset,
|
||||
orig_size: ps as u32,
|
||||
data_size: compressed.len() as u32,
|
||||
codec: self.default_codec as u8,
|
||||
_pad: [0u8; 7],
|
||||
});
|
||||
}
|
||||
if let Some(table) = self.page_tables.get_mut(oldest_surviving as usize) {
|
||||
*table = new_table;
|
||||
}
|
||||
use crate::format::NO_PARENT;
|
||||
if let Some(entry) = self.index.get_mut(oldest_surviving) {
|
||||
entry.flags |= REV_FLAG_SNAPSHOT;
|
||||
entry.parent_rev = NO_PARENT;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove from index and clear page tables.
|
||||
self.index.remove_revisions(&to_remove);
|
||||
self.header.revision_count = self.index.len() as u64;
|
||||
for &rev in &to_remove {
|
||||
if let Some(table) = self.page_tables.get_mut(rev as usize) {
|
||||
table.clear();
|
||||
}
|
||||
}
|
||||
|
||||
// Compact page_data blob.
|
||||
let surviving_revs: Vec<u64> =
|
||||
self.index.entries().iter().map(|e| e.revision).collect();
|
||||
let mut new_page_data: Vec<u8> = Vec::new();
|
||||
for &rev in &surviving_revs {
|
||||
if let Some(table) = self.page_tables.get_mut(rev as usize) {
|
||||
for entry in table.iter_mut() {
|
||||
let old_start = entry.data_offset as usize;
|
||||
let old_end = old_start + entry.data_size as usize;
|
||||
let new_offset = new_page_data.len() as u64;
|
||||
if old_end <= self.page_data.len() {
|
||||
new_page_data.extend_from_slice(&self.page_data[old_start..old_end]);
|
||||
}
|
||||
entry.data_offset = new_offset;
|
||||
}
|
||||
}
|
||||
}
|
||||
self.page_data = new_page_data;
|
||||
|
||||
Ok(GcStats { revisions_removed, bytes_reclaimed })
|
||||
}
|
||||
|
||||
/// Find all entries marked [`EPOCH_DEAD`] and compact them.
|
||||
///
|
||||
/// Called automatically by [`flush`] when any dead-epoch entries exist.
|
||||
pub(crate) fn compact_dead_epoch_revisions(&mut self) -> Result<(), OnionError> {
|
||||
let dead: HashSet<u64> = self
|
||||
.index
|
||||
.entries()
|
||||
.iter()
|
||||
.filter(|e| e.epoch() == EPOCH_DEAD)
|
||||
.map(|e| e.revision)
|
||||
.collect();
|
||||
if !dead.is_empty() {
|
||||
self.compact_revisions(dead)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
fn tmp_h5() -> std::path::PathBuf {
|
||||
let f = NamedTempFile::new().unwrap();
|
||||
let path = f.path().with_extension("h5");
|
||||
std::fs::write(&path, b"\x89HDF\r\n\x1a\n").unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
fn make_onion_with_n_revisions(n: usize) -> OnionFile {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
for i in 0..n {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![i as u8; 4096]);
|
||||
let annotation = if i % 3 == 0 {
|
||||
Some(format!("tagged-{i}"))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
onion
|
||||
.commit_session(s, annotation.as_deref())
|
||||
.unwrap();
|
||||
}
|
||||
onion
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_keep_last_n_retains_n() {
|
||||
let mut onion = make_onion_with_n_revisions(10);
|
||||
let stats = onion.gc(GcPolicy::KeepLastN(3)).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 7);
|
||||
assert_eq!(onion.revision_count(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_keep_last_n_greater_than_total() {
|
||||
let mut onion = make_onion_with_n_revisions(5);
|
||||
let stats = onion.gc(GcPolicy::KeepLastN(100)).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 0);
|
||||
assert_eq!(onion.revision_count(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_keep_last_zero_clears_all() {
|
||||
let mut onion = make_onion_with_n_revisions(5);
|
||||
let stats = onion.gc(GcPolicy::KeepLastN(0)).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 5);
|
||||
assert_eq!(onion.revision_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_keep_tagged_retains_annotated() {
|
||||
// Revisions 0, 3, 6, 9 are tagged (i % 3 == 0)
|
||||
let mut onion = make_onion_with_n_revisions(10);
|
||||
let stats = onion.gc(GcPolicy::KeepTagged).unwrap();
|
||||
// 4 tagged revisions: 0, 3, 6, 9
|
||||
assert_eq!(onion.revision_count(), 4);
|
||||
assert_eq!(stats.revisions_removed, 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_keep_since_retains_recent() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
// Write some revisions with distinct timestamps
|
||||
for i in 0u8..5 {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![i; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
}
|
||||
// Keep all revisions (cutoff = 0.0 = beginning of time)
|
||||
let stats = onion.gc(GcPolicy::KeepSince(0.0)).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_keep_explicit_revisions_with_ancestors() {
|
||||
let mut onion = make_onion_with_n_revisions(5);
|
||||
// Keep only revision 4; its ancestors (0,1,2,3) must also be kept
|
||||
let stats = onion.gc(GcPolicy::KeepRevisions(vec![4])).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 0, "all are ancestors of rev 4");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_keep_middle_revision_excludes_unrelated() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
// 3 revisions: 0 → 1 → 2
|
||||
for i in 0u8..3 {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![i; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
}
|
||||
// Keep rev 1 and its ancestor (rev 0); rev 2 should be pruned
|
||||
let stats = onion.gc(GcPolicy::KeepRevisions(vec![1])).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 1); // only rev 2 pruned
|
||||
assert!(onion.index.get(2).is_none());
|
||||
assert!(onion.index.get(0).is_some());
|
||||
assert!(onion.index.get(1).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_empty_file_noop() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
let stats = onion.gc(GcPolicy::KeepLastN(10)).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 0);
|
||||
assert_eq!(onion.revision_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_reports_bytes_reclaimed() {
|
||||
let mut onion = make_onion_with_n_revisions(5);
|
||||
let before = onion.page_data.len();
|
||||
let stats = onion.gc(GcPolicy::KeepLastN(1)).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 4);
|
||||
let after = onion.page_data.len();
|
||||
// page_data must be smaller (or at most equal for all-same pages that compress to 0)
|
||||
assert!(after <= before, "page_data must shrink after GC: {before} -> {after}");
|
||||
}
|
||||
|
||||
/// After GC + flush, the on-disk file is smaller than before.
|
||||
#[test]
|
||||
fn gc_flush_reduces_file_size() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
|
||||
// Write 10 revisions with distinct page data so pages don't compress away.
|
||||
for i in 0..10u8 {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
// Use varying patterns to ensure pages are different.
|
||||
let page: Vec<u8> = (0..4096).map(|j| (i ^ (j as u8)).wrapping_add(i)).collect();
|
||||
s.record_page(0, &page);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
}
|
||||
onion.flush().unwrap();
|
||||
let sidecar = OnionFile::sidecar_path_pub(&h5);
|
||||
let size_before = std::fs::metadata(&sidecar).unwrap().len();
|
||||
|
||||
// GC down to 1 revision, then flush.
|
||||
onion.gc(GcPolicy::KeepLastN(1)).unwrap();
|
||||
onion.flush().unwrap();
|
||||
let size_after = std::fs::metadata(&sidecar).unwrap().len();
|
||||
|
||||
assert!(
|
||||
size_after < size_before,
|
||||
"sidecar should shrink after GC+flush: {size_before} → {size_after}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Surviving revisions can still be reconstructed after GC.
|
||||
#[test]
|
||||
fn gc_surviving_revisions_still_readable() {
|
||||
let h5 = tmp_h5();
|
||||
let h5_base = std::fs::read(&h5).unwrap();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
|
||||
for i in 0..6u8 {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![i; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
}
|
||||
|
||||
// Keep last 3 revisions (3, 4, 5).
|
||||
onion.gc(GcPolicy::KeepLastN(3)).unwrap();
|
||||
assert_eq!(onion.revision_count(), 3);
|
||||
|
||||
// The three surviving revisions must still reconstruct without error.
|
||||
for rev in 3..6u64 {
|
||||
let bytes = onion.reconstruct_revision(rev, &h5_base).unwrap();
|
||||
assert!(!bytes.is_empty(), "rev {rev} should produce non-empty bytes");
|
||||
}
|
||||
}
|
||||
|
||||
/// GC with diff-based commits (each revision writes a DIFFERENT page) must
|
||||
/// still produce correct reconstructions after ancestor pruning.
|
||||
///
|
||||
/// Before the consolidation fix, `KeepLastN` would lose pages written by
|
||||
/// pruned revisions, causing reconstructions to fall back to the empty
|
||||
/// h5_base for those pages.
|
||||
#[test]
|
||||
fn gc_diff_based_commits_reconstruct_correctly_after_prune() {
|
||||
let h5 = tmp_h5();
|
||||
let h5_base = std::fs::read(&h5).unwrap();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
|
||||
// Rev 0: only page 0
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![0xAA_u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
|
||||
// Rev 1: only page 1 (page 0 unchanged from rev 0)
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(4096, &vec![0xBB_u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
|
||||
// Rev 2: only page 2 (pages 0 and 1 unchanged)
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(8192, &vec![0xCC_u8; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
|
||||
// Verify reconstruction before GC
|
||||
let before_gc = onion.reconstruct_revision(2, &h5_base).unwrap();
|
||||
assert_eq!(&before_gc[0..4096], &vec![0xAA_u8; 4096], "pre-GC page0");
|
||||
assert_eq!(&before_gc[4096..8192], &vec![0xBB_u8; 4096], "pre-GC page1");
|
||||
assert_eq!(&before_gc[8192..12288], &vec![0xCC_u8; 4096], "pre-GC page2");
|
||||
|
||||
// GC: keep only rev 2 — revs 0 and 1 are ancestors that will be pruned.
|
||||
onion.gc(GcPolicy::KeepLastN(1)).unwrap();
|
||||
assert_eq!(onion.revision_count(), 1);
|
||||
|
||||
// After GC, rev 2 must still reconstruct with all three pages intact.
|
||||
// The oldest surviving revision (rev 2) should have been consolidated
|
||||
// into a root snapshot that captures all pages.
|
||||
let after_gc = onion.reconstruct_revision(2, &h5_base).unwrap();
|
||||
assert_eq!(
|
||||
&after_gc[0..4096],
|
||||
&vec![0xAA_u8; 4096],
|
||||
"post-GC page0 must come from consolidation"
|
||||
);
|
||||
assert_eq!(
|
||||
&after_gc[4096..8192],
|
||||
&vec![0xBB_u8; 4096],
|
||||
"post-GC page1 must come from consolidation"
|
||||
);
|
||||
assert_eq!(
|
||||
&after_gc[8192..12288],
|
||||
&vec![0xCC_u8; 4096],
|
||||
"post-GC page2 must come from consolidation"
|
||||
);
|
||||
}
|
||||
|
||||
// ── Epoch-flip (lazy) GC tests ────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn gc_epoch_flip_marks_entries_dead_without_compacting() {
|
||||
let mut onion = make_onion_with_n_revisions(10);
|
||||
let page_data_len_before = onion.page_data.len();
|
||||
|
||||
let stats = onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(3)))).unwrap();
|
||||
|
||||
// Reports the count that will be removed.
|
||||
assert_eq!(stats.revisions_removed, 7);
|
||||
// bytes_reclaimed is 0 until flush compacts.
|
||||
assert_eq!(stats.bytes_reclaimed, 0);
|
||||
// page_data must NOT have changed yet.
|
||||
assert_eq!(onion.page_data.len(), page_data_len_before,
|
||||
"epoch flip must not compact page_data immediately");
|
||||
// Index still has all 10 entries (removal deferred).
|
||||
assert_eq!(onion.revision_count(), 10);
|
||||
// Entries 0..6 should be marked EPOCH_DEAD.
|
||||
for rev in 0u64..7 {
|
||||
let entry = onion.index.get(rev).unwrap();
|
||||
assert_eq!(entry.epoch(), EPOCH_DEAD, "rev {rev} should be EPOCH_DEAD");
|
||||
}
|
||||
// Entries 7..9 should be live (epoch = 0).
|
||||
for rev in 7u64..10 {
|
||||
let entry = onion.index.get(rev).unwrap();
|
||||
assert_eq!(entry.epoch(), 0, "rev {rev} should be live (epoch=0)");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_epoch_flip_compact_on_flush() {
|
||||
let h5 = tmp_h5();
|
||||
let h5_base = std::fs::read(&h5).unwrap();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
for i in 0..5u8 {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![i; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
}
|
||||
|
||||
// Epoch flip: defer compaction.
|
||||
onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(2)))).unwrap();
|
||||
assert_eq!(onion.revision_count(), 5, "not compacted yet");
|
||||
|
||||
// After flush, deferred compaction runs.
|
||||
onion.flush().unwrap();
|
||||
|
||||
// Reload and verify.
|
||||
let reloaded = OnionFile::open(&h5).unwrap();
|
||||
assert_eq!(reloaded.revision_count(), 2, "2 revisions survive after flush");
|
||||
|
||||
// Surviving revisions are still reconstructable.
|
||||
for rev in 3u64..5 {
|
||||
let bytes = reloaded.reconstruct_revision(rev, &h5_base).unwrap();
|
||||
assert!(!bytes.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_epoch_flip_backward_compat_old_entries() {
|
||||
// Old entries have _pad_flags all zero → epoch() == 0, never dead.
|
||||
let mut onion = make_onion_with_n_revisions(5);
|
||||
// All entries should have epoch=0.
|
||||
for rev in 0u64..5 {
|
||||
assert_eq!(onion.index.get(rev).unwrap().epoch(), 0);
|
||||
}
|
||||
// Immediate GC should still work on files without epoch marks.
|
||||
let stats = onion.gc(GcPolicy::KeepLastN(3)).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 2);
|
||||
assert_eq!(onion.revision_count(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gc_epoch_flip_nested_policy_keep_tagged() {
|
||||
let mut onion = make_onion_with_n_revisions(9); // tagged at 0,3,6
|
||||
let stats = onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepTagged))).unwrap();
|
||||
assert_eq!(stats.revisions_removed, 6); // keeps 0,3,6
|
||||
assert_eq!(onion.revision_count(), 9, "deferred — index intact");
|
||||
// 0,3,6 are live; rest are dead.
|
||||
for rev in [0u64, 3, 6] {
|
||||
assert_eq!(onion.index.get(rev).unwrap().epoch(), 0, "tagged rev {rev} must be live");
|
||||
}
|
||||
for rev in [1u64, 2, 4, 5, 7, 8] {
|
||||
assert_eq!(onion.index.get(rev).unwrap().epoch(), EPOCH_DEAD,
|
||||
"untagged rev {rev} must be EPOCH_DEAD");
|
||||
}
|
||||
}
|
||||
|
||||
/// After consolidation, the oldest surviving revision is marked as a snapshot.
|
||||
#[test]
|
||||
fn gc_oldest_surviving_becomes_snapshot() {
|
||||
let h5 = tmp_h5();
|
||||
let mut onion = OnionFile::create(&h5, 4096).unwrap();
|
||||
|
||||
for i in 0..5u8 {
|
||||
let mut s = onion.begin_session(None).unwrap();
|
||||
s.record_page(0, &vec![i; 4096]);
|
||||
onion.commit_session(s, None).unwrap();
|
||||
}
|
||||
|
||||
// Keep last 2 (revs 3 and 4); revs 0,1,2 are ancestors that will be pruned.
|
||||
onion.gc(GcPolicy::KeepLastN(2)).unwrap();
|
||||
|
||||
// The oldest surviving revision (rev 3) must be flagged as snapshot.
|
||||
let entry = onion.index.get(3).expect("rev 3 must survive");
|
||||
assert_ne!(
|
||||
entry.flags & crate::format::REV_FLAG_SNAPSHOT,
|
||||
0,
|
||||
"oldest surviving rev must be a snapshot after consolidation"
|
||||
);
|
||||
// Rev 4 should NOT be flagged as a snapshot (it was not consolidated).
|
||||
let entry4 = onion.index.get(4).expect("rev 4 must survive");
|
||||
assert_eq!(
|
||||
entry4.flags & crate::format::REV_FLAG_SNAPSHOT,
|
||||
0,
|
||||
"non-consolidated rev must not be a snapshot"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user