redclaw/clawsync
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply rustfmt to entire workspace

Browse Source 
Runs cargo fmt --all; all 573 tests still passing, clippy still clean.
No logic changes — formatting only.

Co-Authored-By: Claude Sonnet 4.6 <[email protected]>
This commit is contained in:
osobh
2026-04-04 20:31:33 -05:00
co-authored by Claude Sonnet 4.6
parent 3d524e2d63
commit 1c107fe58a
69 changed files with 2790 additions and 1325 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-onion/benches/epoch_gc_bench.rs
+13 -7
View File
@@ -46,7 +46,9 @@ fn bench_epoch_gc(c: &mut Criterion) {
b.iter_with_setup(
|| make_onion(n),
|(_tmp, _h5, mut onion)| {
let _ = onion.gc(black_box(GcPolicy::KeepLastN((n / 2) as u64))).unwrap();
let _ = onion
.gc(black_box(GcPolicy::KeepLastN((n / 2) as u64)))
.unwrap();
},
);
});
@@ -60,9 +62,11 @@ fn bench_epoch_gc(c: &mut Criterion) {
b.iter_with_setup(
|| make_onion(n),
|(_tmp, _h5, mut onion)| {
let _ = onion.gc(black_box(GcPolicy::EpochFlip(Box::new(
GcPolicy::KeepLastN((n / 2) as u64),
)))).unwrap();
let _ = onion
.gc(black_box(GcPolicy::EpochFlip(Box::new(
GcPolicy::KeepLastN((n / 2) as u64),
))))
.unwrap();
},
);
});
@@ -76,9 +80,11 @@ fn bench_epoch_gc(c: &mut Criterion) {
b.iter_with_setup(
|| {
let (tmp, h5, mut onion) = make_onion(n);
onion.gc(GcPolicy::EpochFlip(Box::new(
GcPolicy::KeepLastN((n / 2) as u64),
))).unwrap();
onion
.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(
(n / 2) as u64,
))))
.unwrap();
(tmp, h5, onion)
},
|(_tmp, _h5, mut onion)| {
crates/clawhdf5-onion/benches/merkle_bench.rs
+34 -15
View File
@@ -51,21 +51,31 @@ fn bench_merkle(c: &mut Criterion) {
for &n in &ns {
let full = RevisionMerkleTree::build(&make_entries(n));
let minus1 = RevisionMerkleTree::build(&make_entries(n - 1));
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, minus1), |b, (f, m)| {
b.iter(|| f.diff_missing_revisions(black_box(m)));
});
group.bench_with_input(
BenchmarkId::from_parameter(n),
&(full, minus1),
|b, (f, m)| {
b.iter(|| f.diff_missing_revisions(black_box(m)));
},
);
}
group.finish();
// ── Diff walk: D=10 ──────────────────────────────────────────────────────
let mut group = c.benchmark_group("merkle_index/diff_walk_d10");
for &n in &ns {
if n < 20 { continue; }
let full = RevisionMerkleTree::build(&make_entries(n));
let base = RevisionMerkleTree::build(&make_entries(n - 10));
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, base), |b, (f, base)| {
b.iter(|| f.diff_missing_revisions(black_box(base)));
});
if n < 20 {
continue;
}
let full = RevisionMerkleTree::build(&make_entries(n));
let base = RevisionMerkleTree::build(&make_entries(n - 10));
group.bench_with_input(
BenchmarkId::from_parameter(n),
&(full, base),
|b, (f, base)| {
b.iter(|| f.diff_missing_revisions(black_box(base)));
},
);
}
group.finish();
@@ -74,9 +84,13 @@ fn bench_merkle(c: &mut Criterion) {
for &n in &[1_000usize, 10_000] {
let full = RevisionMerkleTree::build(&make_entries(n));
let base = RevisionMerkleTree::build(&make_entries(n - 100));
group.bench_with_input(BenchmarkId::from_parameter(n), &(full, base), |b, (f, base)| {
b.iter(|| f.diff_missing_revisions(black_box(base)));
});
group.bench_with_input(
BenchmarkId::from_parameter(n),
&(full, base),
|b, (f, base)| {
b.iter(|| f.diff_missing_revisions(black_box(base)));
},
);
}
group.finish();
@@ -102,14 +116,19 @@ fn bench_merkle(c: &mut Criterion) {
// ── Serialised size report (not timed) ───────────────────────────────────
println!("\n=== Serialised size vs flat manifest (N × 60 B) ===");
println!("{:>8} {:>12} {:>12} {:>10}", "N", "Merkle (B)", "Flat (B)", "ratio");
println!(
"{:>8} {:>12} {:>12} {:>10}",
"N", "Merkle (B)", "Flat (B)", "ratio"
);
for &n in &[10usize, 100, 1_000, 10_000] {
let tree = RevisionMerkleTree::build(&make_entries(n));
let merkle_sz = tree.serialise().len();
let flat_sz = n * 60;
let flat_sz = n * 60;
println!(
"{:>8} {:>12} {:>12} {:>10.2}x",
n, merkle_sz, flat_sz,
n,
merkle_sz,
flat_sz,
flat_sz as f64 / merkle_sz as f64
);
}
crates/clawhdf5-onion/benches/onion_bench.rs
+22 -19
View File
@@ -66,7 +66,10 @@ fn bench_write_one_revision(c: &mut Criterion) {
b.iter(|| {
let mut s = onion.begin_session(None).unwrap();
for i in 0u64..4 {
s.record_page(i * PAGE_SIZE as u64, black_box(&vec![0xCDu8; PAGE_SIZE as usize]));
s.record_page(
i * PAGE_SIZE as u64,
black_box(&vec![0xCDu8; PAGE_SIZE as usize]),
);
}
black_box(onion.commit_session(s, None).unwrap());
});
@@ -83,15 +86,15 @@ fn bench_reconstruct_revision(c: &mut Criterion) {
let (_f, _h5, onion, base) = tmp_onion_with_n(depth);
let target_rev = (depth - 1) as u64;
group.bench_with_input(
BenchmarkId::new("no_snapshot", depth),
&depth,
|b, _| {
b.iter(|| {
black_box(onion.reconstruct_revision(target_rev, black_box(&base)).unwrap());
});
},
);
group.bench_with_input(BenchmarkId::new("no_snapshot", depth), &depth, |b, _| {
b.iter(|| {
black_box(
onion
.reconstruct_revision(target_rev, black_box(&base))
.unwrap(),
);
});
});
}
// With snapshot: create a real snapshot at midpoint via create_snapshot()
@@ -107,15 +110,15 @@ fn bench_reconstruct_revision(c: &mut Criterion) {
}
let target_rev = onion.revision_count() - 1;
group.bench_with_input(
BenchmarkId::new("with_snapshot", depth),
&depth,
|b, _| {
b.iter(|| {
black_box(onion.reconstruct_revision(target_rev, black_box(&base)).unwrap());
});
},
);
group.bench_with_input(BenchmarkId::new("with_snapshot", depth), &depth, |b, _| {
b.iter(|| {
black_box(
onion
.reconstruct_revision(target_rev, black_box(&base))
.unwrap(),
);
});
});
}
group.finish();
crates/clawhdf5-onion/benches/tdt_bench.rs
+21 -17
View File
@@ -35,7 +35,9 @@ fn make_f32_random(n_bytes: usize) -> Vec<u8> {
let mut state = 0x_dead_beef_u64;
let mut out = Vec::with_capacity(n_bytes);
while out.len() < n_bytes {
state = state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
state = state
.wrapping_mul(6364136223846793005)
.wrapping_add(1442695040888963407);
out.extend_from_slice(&(state as u32).to_le_bytes());
}
out.truncate(n_bytes);
@@ -72,16 +74,13 @@ fn bench_tdt_compress(c: &mut Criterion) {
let sizes = [4096usize, 65536];
let datasets: &[(&str, fn(usize) -> Vec<u8>, usize)] = &[
("f32_smooth", make_f32_smooth, 4),
("f32_random", make_f32_random, 4),
("f32_smooth", make_f32_smooth, 4),
("f32_random", make_f32_random, 4),
("int32_random", make_int32_random, 4),
("int8_seq", make_int8_seq, 1),
("int8_seq", make_int8_seq, 1),
];
let codecs = [
("zstd", Codec::Zstd),
("zstd_tdt", Codec::ZstdTdt),
];
let codecs = [("zstd", Codec::Zstd), ("zstd_tdt", Codec::ZstdTdt)];
// ── Throughput benchmark ──────────────────────────────────────────────────
let mut group = c.benchmark_group("tdt_compress/throughput");
@@ -90,10 +89,7 @@ fn bench_tdt_compress(c: &mut Criterion) {
let data = make(size);
group.throughput(Throughput::Bytes(size as u64));
for &(codec_name, codec) in &codecs {
let id = BenchmarkId::new(
format!("{codec_name}/{dtype}"),
format!("{size}B"),
);
let id = BenchmarkId::new(format!("{codec_name}/{dtype}"), format!("{size}B"));
group.bench_with_input(id, &data, |b, d| {
b.iter(|| bench_compress(d, codec));
});
@@ -126,16 +122,24 @@ fn bench_tdt_compress(c: &mut Criterion) {
// ── Compression ratio summary (printed, not timed) ───────────────────────
// Run once outside Criterion to print ratio comparison.
println!("\n─── TDT compression ratio summary ───");
println!("{:<20} {:>8} {:>10} {:>10} {:>8}",
"dataset/size", "orig", "zstd", "zstd_tdt", "savings");
println!(
"{:<20} {:>8} {:>10} {:>10} {:>8}",
"dataset/size", "orig", "zstd", "zstd_tdt", "savings"
);
for &size in &sizes {
for &(dtype, make, _w) in datasets {
let data = make(size);
let zstd_size = compress_page(&data, Codec::Zstd).unwrap().len();
let tdt_size = compress_page(&data, Codec::ZstdTdt).unwrap().len();
let tdt_size = compress_page(&data, Codec::ZstdTdt).unwrap().len();
let savings_pct = 100.0 * (1.0 - tdt_size as f64 / zstd_size as f64);
println!("{:<20} {:>8} {:>10} {:>10} {:>7.1}%",
format!("{dtype}/{size}B"), size, zstd_size, tdt_size, savings_pct);
println!(
"{:<20} {:>8} {:>10} {:>10} {:>7.1}%",
format!("{dtype}/{size}B"),
size,
zstd_size,
tdt_size,
savings_pct
);
}
}
println!();
crates/clawhdf5-onion/src/api.rs
+6 -10
View File
@@ -37,10 +37,7 @@ use crate::writer::OnionFile;
/// let (bytes, onion) = open_revision("agent.h5", 7)?;
/// let file = clawhdf5::File::from_bytes(bytes)?;
/// ```
pub fn open_revision(
h5_path: &Path,
revision: u64,
) -> Result<(Vec<u8>, OnionFile), OnionError> {
pub fn open_revision(h5_path: &Path, revision: u64) -> Result<(Vec<u8>, OnionFile), OnionError> {
let h5_base = std::fs::read(h5_path)?;
let onion = OnionFile::open(h5_path)?;
let reconstructed = onion.reconstruct_revision(revision, &h5_base)?;
@@ -48,10 +45,7 @@ pub fn open_revision(
}
/// Open the HEAD of a named branch.
pub fn open_branch(
h5_path: &Path,
branch: &str,
) -> Result<(Vec<u8>, OnionFile), OnionError> {
pub fn open_branch(h5_path: &Path, branch: &str) -> Result<(Vec<u8>, OnionFile), OnionError> {
let h5_base = std::fs::read(h5_path)?;
let onion = OnionFile::open(h5_path)?;
let reconstructed = onion.open_rev(OpenRevision::Branch(branch.to_owned()), &h5_base)?;
@@ -66,8 +60,10 @@ pub fn open_branch_at(
) -> Result<(Vec<u8>, OnionFile), OnionError> {
let h5_base = std::fs::read(h5_path)?;
let onion = OnionFile::open(h5_path)?;
let reconstructed =
onion.open_rev(OpenRevision::BranchAt(branch.to_owned(), revision), &h5_base)?;
let reconstructed = onion.open_rev(
OpenRevision::BranchAt(branch.to_owned(), revision),
&h5_base,
)?;
Ok((reconstructed, onion))
}
crates/clawhdf5-onion/src/branch.rs
+101 -59
View File
@@ -41,11 +41,7 @@ impl OnionFile {
/// Create a new named branch forked from the current HEAD of `source_branch`.
///
/// Returns the new branch ID.
pub fn create_branch(
&mut self,
name: &str,
source_branch: &str,
) -> Result<u32, OnionError> {
pub fn create_branch(&mut self, name: &str, source_branch: &str) -> Result<u32, OnionError> {
if self.branch_by_name(name).is_some() {
return Err(OnionError::BranchExists(name.to_string()));
}
@@ -133,11 +129,7 @@ impl OnionFile {
}
}
fn merge_latest_wins(
&mut self,
source_id: u32,
target_id: u32,
) -> Result<u64, OnionError> {
fn merge_latest_wins(&mut self, source_id: u32, target_id: u32) -> Result<u64, OnionError> {
let fork_rev = self
.branches
.iter()
@@ -231,21 +223,21 @@ impl OnionFile {
self.commit_session(session, Some(&annotation))
}
fn merge_three_way(
&mut self,
source_id: u32,
target_id: u32,
) -> Result<u64, OnionError> {
fn merge_three_way(&mut self, source_id: u32, target_id: u32) -> Result<u64, OnionError> {
let source_head = self
.index
.branch_head(source_id)
.ok_or_else(|| OnionError::Malformed(format!("source branch {source_id} has no revisions")))?
.ok_or_else(|| {
OnionError::Malformed(format!("source branch {source_id} has no revisions"))
})?
.revision;
let target_head = self
.index
.branch_head(target_id)
.ok_or_else(|| OnionError::Malformed(format!("target branch {target_id} has no revisions")))?
.ok_or_else(|| {
OnionError::Malformed(format!("target branch {target_id} has no revisions"))
})?
.revision;
// Find common ancestor of the two branch HEADs
@@ -253,13 +245,24 @@ impl OnionFile {
.index
.common_ancestor(source_head, target_head)
.ok_or_else(|| {
let src_name = self.branches.iter().find(|b| b.id == source_id)
let src_name = self
.branches
.iter()
.find(|b| b.id == source_id)
.and_then(|b| self.annotations.get(b.name_off))
.unwrap_or("?").to_owned();
let tgt_name = self.branches.iter().find(|b| b.id == target_id)
.unwrap_or("?")
.to_owned();
let tgt_name = self
.branches
.iter()
.find(|b| b.id == target_id)
.and_then(|b| self.annotations.get(b.name_off))
.unwrap_or("?").to_owned();
OnionError::NoCommonAncestor { a: src_name, b: tgt_name }
.unwrap_or("?")
.to_owned();
OnionError::NoCommonAncestor {
a: src_name,
b: tgt_name,
}
})?;
// Collect source delta since ancestor (latest write per offset)
@@ -321,7 +324,8 @@ impl OnionFile {
if merged.is_empty() {
return Err(OnionError::Malformed(
"three-way merge: no source changes to apply (target already has all changes)".to_string(),
"three-way merge: no source changes to apply (target already has all changes)"
.to_string(),
));
}
@@ -329,9 +333,8 @@ impl OnionFile {
for (h5_off, bytes) in &merged {
session.record_page(*h5_off, bytes);
}
let annotation = format!(
"3-way merge {source_id} → {target_id} [ancestor rev {ancestor_rev}]"
);
let annotation =
format!("3-way merge {source_id} → {target_id} [ancestor rev {ancestor_rev}]");
self.commit_session(session, Some(&annotation))
}
@@ -371,11 +374,7 @@ impl OnionFile {
.iter()
.map(|b| BranchInfo {
id: b.id,
name: self
.annotations
.get(b.name_off)
.unwrap_or("?")
.to_owned(),
name: self.annotations.get(b.name_off).unwrap_or("?").to_owned(),
head_rev: b.head_rev,
fork_rev: b.fork_rev,
})
@@ -383,7 +382,10 @@ impl OnionFile {
}
/// Return revision entries for a named branch in chronological order.
pub fn branch_history(&self, name: &str) -> Result<Vec<&crate::format::RevisionEntry>, OnionError> {
pub fn branch_history(
&self,
name: &str,
) -> Result<Vec<&crate::format::RevisionEntry>, OnionError> {
let branch = self
.branch_by_name(name)
.ok_or_else(|| OnionError::BranchNotFound(name.to_string()))?;
@@ -414,7 +416,9 @@ impl OnionFile {
.branch_by_name(name)
.ok_or_else(|| OnionError::BranchNotFound(name.to_string()))?;
if branch.id == crate::format::BRANCH_MAIN {
return Err(OnionError::Malformed("cannot delete the main branch".into()));
return Err(OnionError::Malformed(
"cannot delete the main branch".into(),
));
}
let id = branch.id;
self.branches.retain(|b| b.id != id);
@@ -605,14 +609,18 @@ mod tests {
fn merge_latest_wins_produces_new_revision() {
let (_h5, mut onion, _feat_id) = make_fork_scenario();
let pre_count = onion.revision_count();
onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap();
onion
.merge_into("feat", "main", MergeStrategy::LatestWins)
.unwrap();
assert_eq!(onion.revision_count(), pre_count + 1);
}
#[test]
fn merge_latest_wins_page_content_correct() {
let (h5, mut onion, _feat_id) = make_fork_scenario();
onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap();
onion
.merge_into("feat", "main", MergeStrategy::LatestWins)
.unwrap();
onion.flush().unwrap();
// Reload from disk and reconstruct main HEAD
@@ -621,8 +629,10 @@ mod tests {
let base = std::fs::read(&h5).unwrap();
let state = onion2.reconstruct_revision(main_head, &base).unwrap();
// Page 0 should now be BB (from feat)
assert!(state[0..4096].iter().all(|&b| b == 0xBB),
"page 0 should be BB after latest-wins merge");
assert!(
state[0..4096].iter().all(|&b| b == 0xBB),
"page 0 should be BB after latest-wins merge"
);
}
#[test]
@@ -646,15 +656,19 @@ mod tests {
s2.record_page(0, &vec![0x22u8; 4096]);
onion.commit_session(s2, None).unwrap();
onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap();
onion
.merge_into("feat", "main", MergeStrategy::LatestWins)
.unwrap();
onion.flush().unwrap();
let onion2 = OnionFile::open(&h5).unwrap();
let head = onion2.revision_count() - 1;
let base = std::fs::read(&h5).unwrap();
let state = onion2.reconstruct_revision(head, &base).unwrap();
assert!(state[0..4096].iter().all(|&b| b == 0x22),
"latest write (0x22) must win in LatestWins merge");
assert!(
state[0..4096].iter().all(|&b| b == 0x22),
"latest write (0x22) must win in LatestWins merge"
);
}
#[test]
@@ -666,7 +680,9 @@ mod tests {
onion.commit_session(s, None).unwrap();
// fork but make no commits on feat
onion.create_branch("feat", "main").unwrap();
let err = onion.merge_into("feat", "main", MergeStrategy::LatestWins).unwrap_err();
let err = onion
.merge_into("feat", "main", MergeStrategy::LatestWins)
.unwrap_err();
assert!(matches!(err, OnionError::Malformed(_)));
}
@@ -682,20 +698,30 @@ mod tests {
source.to_vec() // just return source
};
onion
.merge_into("feat", "main", MergeStrategy::DatasetLevel(Box::new(resolver)))
.merge_into(
"feat",
"main",
MergeStrategy::DatasetLevel(Box::new(resolver)),
)
.unwrap();
assert!(called.load(std::sync::atomic::Ordering::SeqCst), "resolver must be called");
assert!(
called.load(std::sync::atomic::Ordering::SeqCst),
"resolver must be called"
);
}
#[test]
fn merge_dataset_level_resolver_controls_output() {
let (h5, mut onion, _feat_id) = make_fork_scenario();
// Resolver always returns 0xCC regardless of inputs
let resolver = |_path: &str, _target: &[u8], _source: &[u8]| -> Vec<u8> {
vec![0xCCu8; 4096]
};
let resolver =
|_path: &str, _target: &[u8], _source: &[u8]| -> Vec<u8> { vec![0xCCu8; 4096] };
onion
.merge_into("feat", "main", MergeStrategy::DatasetLevel(Box::new(resolver)))
.merge_into(
"feat",
"main",
MergeStrategy::DatasetLevel(Box::new(resolver)),
)
.unwrap();
onion.flush().unwrap();
@@ -703,8 +729,10 @@ mod tests {
let head = onion2.revision_count() - 1;
let base = std::fs::read(&h5).unwrap();
let state = onion2.reconstruct_revision(head, &base).unwrap();
assert!(state[0..4096].iter().all(|&b| b == 0xCC),
"resolver output 0xCC should be in merged state");
assert!(
state[0..4096].iter().all(|&b| b == 0xCC),
"resolver output 0xCC should be in merged state"
);
}
// ── Merge: ThreeWay ──────────────────────────────────────────────────────
@@ -736,7 +764,9 @@ mod tests {
sf.record_page(0, &vec![0xBBu8; 4096]);
onion.commit_session(sf, None).unwrap();
onion.merge_into("feat", "main", MergeStrategy::ThreeWay).unwrap();
onion
.merge_into("feat", "main", MergeStrategy::ThreeWay)
.unwrap();
onion.flush().unwrap();
let onion2 = OnionFile::open(&h5).unwrap();
@@ -744,10 +774,14 @@ mod tests {
let base = std::fs::read(&h5).unwrap();
let state = onion2.reconstruct_revision(head, &base).unwrap();
assert!(state[0..4096].iter().all(|&b| b == 0xBB),
"page 0 should be BB (from feat)");
assert!(state[4096..8192].iter().all(|&b| b == 0xCC),
"page 4096 should stay CC (from main — not overwritten by 3-way)");
assert!(
state[0..4096].iter().all(|&b| b == 0xBB),
"page 0 should be BB (from feat)"
);
assert!(
state[4096..8192].iter().all(|&b| b == 0xCC),
"page 4096 should stay CC (from main — not overwritten by 3-way)"
);
}
#[test]
@@ -772,22 +806,28 @@ mod tests {
sf.record_page(0, &vec![0xCCu8; 4096]);
onion.commit_session(sf, None).unwrap();
onion.merge_into("feat", "main", MergeStrategy::ThreeWay).unwrap();
onion
.merge_into("feat", "main", MergeStrategy::ThreeWay)
.unwrap();
onion.flush().unwrap();
let onion2 = OnionFile::open(&h5).unwrap();
let head = onion2.revision_count() - 1;
let base = std::fs::read(&h5).unwrap();
let state = onion2.reconstruct_revision(head, &base).unwrap();
assert!(state[0..4096].iter().all(|&b| b == 0xCC),
"on conflict, source (CC) should win");
assert!(
state[0..4096].iter().all(|&b| b == 0xCC),
"on conflict, source (CC) should win"
);
}
#[test]
fn merge_nonexistent_source_errors() {
let h5 = tmp_h5();
let mut onion = OnionFile::create(&h5, 4096).unwrap();
let err = onion.merge_into("no-such", "main", MergeStrategy::LatestWins).unwrap_err();
let err = onion
.merge_into("no-such", "main", MergeStrategy::LatestWins)
.unwrap_err();
assert!(matches!(err, OnionError::BranchNotFound(_)));
}
@@ -799,7 +839,9 @@ mod tests {
s.record_page(0, &vec![0u8; 4096]);
onion.commit_session(s, None).unwrap();
onion.create_branch("feat", "main").unwrap();
let err = onion.merge_into("feat", "no-target", MergeStrategy::LatestWins).unwrap_err();
let err = onion
.merge_into("feat", "no-target", MergeStrategy::LatestWins)
.unwrap_err();
assert!(matches!(err, OnionError::BranchNotFound(_)));
}
}
crates/clawhdf5-onion/src/compress.rs
+6 -10
View File
@@ -13,26 +13,22 @@ use crate::tdt;
pub fn compress_page(data: &[u8], codec: Codec) -> Result<Vec<u8>, OnionError> {
match codec {
Codec::None => Ok(data.to_vec()),
Codec::Zstd => zstd::bulk::compress(data, 3)
.map_err(|e| OnionError::Compress(e.to_string())),
Codec::Zstd => {
zstd::bulk::compress(data, 3).map_err(|e| OnionError::Compress(e.to_string()))
}
Codec::Lz4 => Ok(lz4_flex::compress_prepend_size(data)),
Codec::Brotli => Err(OnionError::Compress(
"brotli not yet implemented".to_string(),
)),
Codec::ZstdTdt => {
let interleaved = tdt::encode(data, 4);
zstd::bulk::compress(&interleaved, 3)
.map_err(|e| OnionError::Compress(e.to_string()))
zstd::bulk::compress(&interleaved, 3).map_err(|e| OnionError::Compress(e.to_string()))
}
}
}
/// Decompress `data` using the given codec back to `orig_size` bytes.
pub fn decompress_page(
data: &[u8],
codec: Codec,
orig_size: u32,
) -> Result<Vec<u8>, OnionError> {
pub fn decompress_page(data: &[u8], codec: Codec, orig_size: u32) -> Result<Vec<u8>, OnionError> {
match codec {
Codec::None => {
if data.len() != orig_size as usize {
@@ -161,7 +157,7 @@ mod tests {
}
let orig = data.len() as u32;
let zstd_size = compress_page(&data, Codec::Zstd).unwrap().len();
let tdt_size = compress_page(&data, Codec::ZstdTdt).unwrap().len();
let tdt_size = compress_page(&data, Codec::ZstdTdt).unwrap().len();
assert!(
tdt_size < zstd_size,
"ZstdTdt ({tdt_size} B) should beat plain Zstd ({zstd_size} B) on smooth f32 data ({orig} B)"
crates/clawhdf5-onion/src/ext.rs
+12 -3
View File
@@ -94,7 +94,10 @@ mod tests {
.unwrap();
let f = vf.current().unwrap();
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![1.0, 2.0, 3.0]);
assert_eq!(
f.dataset("data").unwrap().read_f64().unwrap(),
vec![1.0, 2.0, 3.0]
);
assert_eq!(vf.onion().revision_count(), 0);
}
@@ -106,7 +109,10 @@ mod tests {
let _ = make_builder(&[1.0]).with_onion(&h5_path, 4096).unwrap();
let onion_path = PathBuf::from(format!("{}.onion", h5_path.display()));
assert!(onion_path.exists(), "sidecar should be flushed by with_onion");
assert!(
onion_path.exists(),
"sidecar should be flushed by with_onion"
);
}
#[test]
@@ -140,6 +146,9 @@ mod tests {
vf.page_size().is_power_of_two(),
"auto page size must be a power of two"
);
assert!(vf.page_size() >= 4096, "auto page size must be at least 4 KiB");
assert!(
vf.page_size() >= 4096,
"auto page size must be at least 4 KiB"
);
}
}
crates/clawhdf5-onion/src/format.rs
+53 -47
View File
@@ -49,9 +49,9 @@ pub const HEADER_SIZE: usize = 128;
pub mod feature_flags {
pub const COMPRESSION: u64 = 1 << 0;
pub const BRANCHING: u64 = 1 << 1;
pub const PROVENANCE: u64 = 1 << 2;
pub const SNAPSHOTS: u64 = 1 << 3;
pub const BRANCHING: u64 = 1 << 1;
pub const PROVENANCE: u64 = 1 << 2;
pub const SNAPSHOTS: u64 = 1 << 3;
}
pub const DEFAULT_FEATURE_FLAGS: u64 =
@@ -64,13 +64,13 @@ pub const DEFAULT_FEATURE_FLAGS: u64 =
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub enum Codec {
None = 0,
Zstd = 1,
Lz4 = 2,
Brotli = 3,
None = 0,
Zstd = 1,
Lz4 = 2,
Brotli = 3,
/// zstd applied after TDT byte-interleaving transform (arXiv:2506.18062).
/// Improves compression ratio ~16% for `f32`/`f16` numeric pages.
ZstdTdt = 4,
ZstdTdt = 4,
}
impl Codec {
@@ -108,19 +108,19 @@ impl Codec {
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
#[repr(C)]
pub struct OnionHeader {
pub magic: [u8; 9],
pub magic: [u8; 9],
pub format_version: u8,
pub _pad_align: [u8; 6],
pub feature_flags: u64,
pub page_size: u32,
pub _pad_ps: [u8; 4],
pub _pad_align: [u8; 6],
pub feature_flags: u64,
pub page_size: u32,
pub _pad_ps: [u8; 4],
pub revision_count: u64,
pub branch_count: u32,
pub _pad_bc: [u8; 4],
pub index_offset: u64,
pub branch_offset: u64,
pub created_at: f64,
pub reserved: [u8; 56],
pub branch_count: u32,
pub _pad_bc: [u8; 4],
pub index_offset: u64,
pub branch_offset: u64,
pub created_at: f64,
pub reserved: [u8; 56],
}
const _: () = assert!(size_of::<OnionHeader>() == HEADER_SIZE);
@@ -180,19 +180,19 @@ impl OnionHeader {
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
#[repr(C)]
pub struct RevisionEntry {
pub revision: u64,
pub branch_id: u32,
pub _pad_bi: [u8; 4],
pub parent_rev: u64,
pub page_count: u32,
pub _pad_pc: [u8; 4],
pub revision: u64,
pub branch_id: u32,
pub _pad_bi: [u8; 4],
pub parent_rev: u64,
pub page_count: u32,
pub _pad_pc: [u8; 4],
pub page_table_off: u64,
pub timestamp: f64,
pub blake3: [u8; 32],
pub session_uuid: [u8; 16],
pub timestamp: f64,
pub blake3: [u8; 32],
pub session_uuid: [u8; 16],
pub annotation_off: u64,
pub flags: u8,
pub _pad_flags: [u8; 7],
pub flags: u8,
pub _pad_flags: [u8; 7],
}
pub const REV_FLAG_SNAPSHOT: u8 = 1 << 0;
@@ -233,11 +233,11 @@ impl RevisionEntry {
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
#[repr(C)]
pub struct BranchEntry {
pub id: u32,
pub _pad_id: [u8; 4],
pub name_off: u64,
pub head_rev: u64,
pub fork_rev: u64,
pub id: u32,
pub _pad_id: [u8; 4],
pub name_off: u64,
pub head_rev: u64,
pub fork_rev: u64,
pub created_at: f64,
}
@@ -255,25 +255,25 @@ pub struct BranchEntry {
#[derive(FromBytes, IntoBytes, KnownLayout, Immutable, Debug, Clone)]
#[repr(C)]
pub struct PageTableEntry {
pub h5_offset: u64,
pub h5_offset: u64,
pub data_offset: u64,
pub orig_size: u32,
pub data_size: u32,
pub codec: u8,
pub _pad: [u8; 7],
pub orig_size: u32,
pub data_size: u32,
pub codec: u8,
pub _pad: [u8; 7],
}
// ─────────────────────────────────────────────────────────────────────────────
// Compile-time size/alignment assertions
// ─────────────────────────────────────────────────────────────────────────────
const _: () = assert!(size_of::<OnionHeader>() == 128);
const _: () = assert!(size_of::<RevisionEntry>() == 112);
const _: () = assert!(size_of::<BranchEntry>() == 40);
const _: () = assert!(size_of::<OnionHeader>() == 128);
const _: () = assert!(size_of::<RevisionEntry>() == 112);
const _: () = assert!(size_of::<BranchEntry>() == 40);
const _: () = assert!(size_of::<PageTableEntry>() == 32);
const _: () = assert!(size_of::<RevisionEntry>() % 8 == 0);
const _: () = assert!(size_of::<BranchEntry>() % 8 == 0);
const _: () = assert!(size_of::<RevisionEntry>() % 8 == 0);
const _: () = assert!(size_of::<BranchEntry>() % 8 == 0);
const _: () = assert!(size_of::<PageTableEntry>() % 8 == 0);
// ─────────────────────────────────────────────────────────────────────────────
@@ -321,7 +321,10 @@ mod tests {
fn header_unknown_version_rejected() {
let mut hdr = OnionHeader::new(4096, DEFAULT_FEATURE_FLAGS, 0.0);
hdr.format_version = 42;
assert!(matches!(hdr.validate(), Err(OnionError::UnknownVersion(42))));
assert!(matches!(
hdr.validate(),
Err(OnionError::UnknownVersion(42))
));
}
#[test]
@@ -488,7 +491,10 @@ mod tests {
#[test]
fn codec_unknown_returns_error() {
assert!(matches!(Codec::from_u8(99), Err(OnionError::UnknownCodec(99))));
assert!(matches!(
Codec::from_u8(99),
Err(OnionError::UnknownCodec(99))
));
}
#[test]
crates/clawhdf5-onion/src/gc.rs
+58 -25
View File
@@ -69,7 +69,10 @@ impl OnionFile {
entry.set_epoch(EPOCH_DEAD);
}
}
Ok(GcStats { revisions_removed: count, bytes_reclaimed: 0 })
Ok(GcStats {
revisions_removed: count,
bytes_reclaimed: 0,
})
}
_ => {
// ── Immediate path: compact now ───────────────────────────
@@ -100,11 +103,7 @@ impl OnionFile {
GcPolicy::KeepSince(cutoff) => all_revs
.iter()
.copied()
.filter(|&rev| {
self.index
.get(rev)
.is_some_and(|e| e.timestamp >= *cutoff)
})
.filter(|&rev| self.index.get(rev).is_some_and(|e| e.timestamp >= *cutoff))
.collect(),
GcPolicy::KeepRevisions(explicit) => {
let mut set: HashSet<u64> = explicit.iter().copied().collect();
@@ -118,7 +117,11 @@ impl OnionFile {
GcPolicy::EpochFlip(inner) => return self.compute_to_remove(inner),
};
all_revs.iter().copied().filter(|rev| !keep.contains(rev)).collect()
all_revs
.iter()
.copied()
.filter(|rev| !keep.contains(rev))
.collect()
}
/// Consolidate + compact the given revision set immediately.
@@ -204,8 +207,7 @@ impl OnionFile {
}
// Compact page_data blob.
let surviving_revs: Vec<u64> =
self.index.entries().iter().map(|e| e.revision).collect();
let surviving_revs: Vec<u64> = self.index.entries().iter().map(|e| e.revision).collect();
let mut new_page_data: Vec<u8> = Vec::new();
for &rev in &surviving_revs {
if let Some(table) = self.page_tables.get_mut(rev as usize) {
@@ -222,7 +224,10 @@ impl OnionFile {
}
self.page_data = new_page_data;
Ok(GcStats { revisions_removed, bytes_reclaimed })
Ok(GcStats {
revisions_removed,
bytes_reclaimed,
})
}
/// Find all entries marked [`EPOCH_DEAD`] and compact them.
@@ -270,9 +275,7 @@ mod tests {
} else {
None
};
onion
.commit_session(s, annotation.as_deref())
.unwrap();
onion.commit_session(s, annotation.as_deref()).unwrap();
}
onion
}
@@ -369,7 +372,10 @@ mod tests {
assert_eq!(stats.revisions_removed, 4);
let after = onion.page_data.len();
// page_data must be smaller (or at most equal for all-same pages that compress to 0)
assert!(after <= before, "page_data must shrink after GC: {before} -> {after}");
assert!(
after <= before,
"page_data must shrink after GC: {before} -> {after}"
);
}
/// After GC + flush, the on-disk file is smaller than before.
@@ -421,7 +427,10 @@ mod tests {
// The three surviving revisions must still reconstruct without error.
for rev in 3..6u64 {
let bytes = onion.reconstruct_revision(rev, &h5_base).unwrap();
assert!(!bytes.is_empty(), "rev {rev} should produce non-empty bytes");
assert!(
!bytes.is_empty(),
"rev {rev} should produce non-empty bytes"
);
}
}
@@ -456,7 +465,11 @@ mod tests {
let before_gc = onion.reconstruct_revision(2, &h5_base).unwrap();
assert_eq!(&before_gc[0..4096], &vec![0xAA_u8; 4096], "pre-GC page0");
assert_eq!(&before_gc[4096..8192], &vec![0xBB_u8; 4096], "pre-GC page1");
assert_eq!(&before_gc[8192..12288], &vec![0xCC_u8; 4096], "pre-GC page2");
assert_eq!(
&before_gc[8192..12288],
&vec![0xCC_u8; 4096],
"pre-GC page2"
);
// GC: keep only rev 2 — revs 0 and 1 are ancestors that will be pruned.
onion.gc(GcPolicy::KeepLastN(1)).unwrap();
@@ -490,15 +503,20 @@ mod tests {
let mut onion = make_onion_with_n_revisions(10);
let page_data_len_before = onion.page_data.len();
let stats = onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(3)))).unwrap();
let stats = onion
.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(3))))
.unwrap();
// Reports the count that will be removed.
assert_eq!(stats.revisions_removed, 7);
// bytes_reclaimed is 0 until flush compacts.
assert_eq!(stats.bytes_reclaimed, 0);
// page_data must NOT have changed yet.
assert_eq!(onion.page_data.len(), page_data_len_before,
"epoch flip must not compact page_data immediately");
assert_eq!(
onion.page_data.len(),
page_data_len_before,
"epoch flip must not compact page_data immediately"
);
// Index still has all 10 entries (removal deferred).
assert_eq!(onion.revision_count(), 10);
// Entries 0..6 should be marked EPOCH_DEAD.
@@ -525,7 +543,9 @@ mod tests {
}
// Epoch flip: defer compaction.
onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(2)))).unwrap();
onion
.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepLastN(2))))
.unwrap();
assert_eq!(onion.revision_count(), 5, "not compacted yet");
// After flush, deferred compaction runs.
@@ -533,7 +553,11 @@ mod tests {
// Reload and verify.
let reloaded = OnionFile::open(&h5).unwrap();
assert_eq!(reloaded.revision_count(), 2, "2 revisions survive after flush");
assert_eq!(
reloaded.revision_count(),
2,
"2 revisions survive after flush"
);
// Surviving revisions are still reconstructable.
for rev in 3u64..5 {
@@ -559,16 +583,25 @@ mod tests {
#[test]
fn gc_epoch_flip_nested_policy_keep_tagged() {
let mut onion = make_onion_with_n_revisions(9); // tagged at 0,3,6
let stats = onion.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepTagged))).unwrap();
let stats = onion
.gc(GcPolicy::EpochFlip(Box::new(GcPolicy::KeepTagged)))
.unwrap();
assert_eq!(stats.revisions_removed, 6); // keeps 0,3,6
assert_eq!(onion.revision_count(), 9, "deferred — index intact");
// 0,3,6 are live; rest are dead.
for rev in [0u64, 3, 6] {
assert_eq!(onion.index.get(rev).unwrap().epoch(), 0, "tagged rev {rev} must be live");
assert_eq!(
onion.index.get(rev).unwrap().epoch(),
0,
"tagged rev {rev} must be live"
);
}
for rev in [1u64, 2, 4, 5, 7, 8] {
assert_eq!(onion.index.get(rev).unwrap().epoch(), EPOCH_DEAD,
"untagged rev {rev} must be EPOCH_DEAD");
assert_eq!(
onion.index.get(rev).unwrap().epoch(),
EPOCH_DEAD,
"untagged rev {rev} must be EPOCH_DEAD"
);
}
}
crates/clawhdf5-onion/src/index.rs
+14 -10
View File
@@ -1,6 +1,6 @@
//! RevisionIndex: O(1) lookup + branch-filtered iteration.
use crate::format::{RevisionEntry, NO_PARENT};
use crate::format::{NO_PARENT, RevisionEntry};
/// In-memory index of all revision entries.
///
@@ -53,7 +53,9 @@ impl RevisionIndex {
/// monotonically greater than the current maximum.
pub fn append(&mut self, entry: RevisionEntry) {
debug_assert!(
self.entries.last().is_none_or(|e| e.revision < entry.revision),
self.entries
.last()
.is_none_or(|e| e.revision < entry.revision),
"revision numbers must be monotonically increasing"
);
self.entries.push(entry);
@@ -61,15 +63,14 @@ impl RevisionIndex {
/// Iterate over all revisions on a specific branch (by `branch_id`).
pub fn branch_revisions(&self, branch_id: u32) -> impl Iterator<Item = &RevisionEntry> {
self.entries.iter().filter(move |e| e.branch_id == branch_id)
self.entries
.iter()
.filter(move |e| e.branch_id == branch_id)
}
/// Return the HEAD revision entry for a branch (highest revision number).
pub fn branch_head(&self, branch_id: u32) -> Option<&RevisionEntry> {
self.entries
.iter()
.rev()
.find(|e| e.branch_id == branch_id)
self.entries.iter().rev().find(|e| e.branch_id == branch_id)
}
/// Walk the DAG from `start_rev` to the root, following `parent_rev`.
@@ -191,8 +192,7 @@ mod tests {
.collect();
assert_eq!(main_revs, vec![0, 1, 4]);
let branch1_revs: Vec<u64> =
idx.branch_revisions(1).map(|e| e.revision).collect();
let branch1_revs: Vec<u64> = idx.branch_revisions(1).map(|e| e.revision).collect();
assert_eq!(branch1_revs, vec![2, 3]);
}
@@ -252,7 +252,11 @@ mod tests {
fn remove_revisions() {
let mut idx = RevisionIndex::new();
for i in 0u64..5 {
idx.append(make_entry(i, BRANCH_MAIN, if i == 0 { NO_PARENT } else { i - 1 }));
idx.append(make_entry(
i,
BRANCH_MAIN,
if i == 0 { NO_PARENT } else { i - 1 },
));
}
let to_remove = [1u64, 2].into_iter().collect();
idx.remove_revisions(&to_remove);
crates/clawhdf5-onion/src/lib.rs
+10 -11
View File
@@ -36,19 +36,18 @@ pub mod tdt;
pub mod versioned_file;
pub mod writer;
pub use error::OnionError;
pub use format::{
BranchEntry, Codec, OnionHeader, PageTableEntry, RevisionEntry,
BRANCH_MAIN, DEFAULT_FEATURE_FLAGS, EPOCH_DEAD, FORMAT_VERSION, HEADER_SIZE, MAGIC, NO_PARENT,
REV_FLAG_SNAPSHOT, feature_flags,
};
pub use api::{
open_revision, open_branch, open_branch_at,
list_revisions, rollback,
list_branches, create_branch, delete_branch, rename_branch,
create_branch, delete_branch, list_branches, list_revisions, open_branch, open_branch_at,
open_revision, rename_branch, rollback,
};
pub use branch::{BranchInfo, DatasetResolver, MergeStrategy};
pub use merkle::{MerkleError, MerkleNode, RevisionMerkleTree, WalkStep};
pub use error::OnionError;
pub use ext::FileBuilderExt;
pub use versioned_file::{VersionedFile, open_at_revision, open_at_branch};
pub use format::{
BRANCH_MAIN, BranchEntry, Codec, DEFAULT_FEATURE_FLAGS, EPOCH_DEAD, FORMAT_VERSION,
HEADER_SIZE, MAGIC, NO_PARENT, OnionHeader, PageTableEntry, REV_FLAG_SNAPSHOT, RevisionEntry,
feature_flags,
};
pub use merkle::{MerkleError, MerkleNode, RevisionMerkleTree, WalkStep};
pub use versioned_file::{VersionedFile, open_at_branch, open_at_revision};
pub use writer::eof_to_page_size;
crates/clawhdf5-onion/src/merkle.rs
+16 -4
View File
@@ -97,7 +97,11 @@ impl RevisionMerkleTree {
/// The slice **must** be sorted by `revision` in ascending order.
pub fn build(entries: &[(u64, [u8; 32])]) -> Self {
let leaf_count = entries.len();
let capacity = if leaf_count == 0 { 1 } else { leaf_count.next_power_of_two() };
let capacity = if leaf_count == 0 {
1
} else {
leaf_count.next_power_of_two()
};
// 1-indexed BFS array; index 0 unused.
let mut hashes = vec![[0u8; 32]; 2 * capacity + 1];
@@ -113,7 +117,12 @@ impl RevisionMerkleTree {
// Build internal nodes bottom-up
build_internal(&mut hashes, capacity);
RevisionMerkleTree { hashes, leaf_revisions, leaf_count, capacity }
RevisionMerkleTree {
hashes,
leaf_revisions,
leaf_count,
capacity,
}
}
// ── Queries ───────────────────────────────────────────────────────────────
@@ -311,7 +320,7 @@ impl RevisionMerkleTree {
/// `hashes` must have length `2 * capacity + 1`.
fn build_internal(hashes: &mut [[u8; 32]], capacity: usize) {
for i in (1..capacity).rev() {
let left = hashes[2 * i];
let left = hashes[2 * i];
let right = hashes[2 * i + 1];
hashes[i] = node_hash(&left, &right);
}
@@ -484,7 +493,10 @@ mod tests {
let header_only = &full[..13]; // 5 + 8 bytes
assert_eq!(
RevisionMerkleTree::deserialise(header_only),
Err(MerkleError::TruncatedPayload { expected: 5 + 8 + 5 * 40, got: 13 })
Err(MerkleError::TruncatedPayload {
expected: 5 + 8 + 5 * 40,
got: 13
})
);
}
crates/clawhdf5-onion/src/provenance.rs
+4 -1
View File
@@ -96,7 +96,10 @@ mod tests {
// Same pages, different insertion order
let h1 = hash_pages(&[(0, a.as_ref()), (4096, b.as_ref())]);
let h2 = hash_pages(&[(4096, b.as_ref()), (0, a.as_ref())]);
assert_eq!(h1, h2, "hash must be order-independent (sorted by h5_offset)");
assert_eq!(
h1, h2,
"hash must be order-independent (sorted by h5_offset)"
);
}
#[test]
crates/clawhdf5-onion/src/reader.rs
+21 -25
View File
@@ -3,11 +3,11 @@
//! Opening a historical revision reads only the `.onion` sidecar —
//! the primary `.h5` file is always at the latest committed state.
use std::collections::BTreeMap;
use crate::compress::decompress_page;
use crate::error::OnionError;
use crate::format::Codec;
use crate::writer::OnionFile;
use std::collections::BTreeMap;
/// `(h5_offset, compressed_data, codec_byte, orig_size)` — raw page as stored.
pub type RawPage = (u64, Vec<u8>, u8, u32);
@@ -41,7 +41,11 @@ impl OnionFile {
// Collect all ancestor revisions in order from oldest → newest
let ancestors: Vec<u64> = {
let mut chain = self.index.ancestors(rev).map(|e| e.revision).collect::<Vec<_>>();
let mut chain = self
.index
.ancestors(rev)
.map(|e| e.revision)
.collect::<Vec<_>>();
chain.reverse(); // oldest first
chain
};
@@ -53,13 +57,11 @@ impl OnionFile {
// Optimisation: find the newest snapshot in the ancestor chain and
// start from there instead of from `h5_base`. This bounds
// reconstruction depth to O(N_since_snapshot · P).
let snapshot_start_idx = ancestors
.iter()
.rposition(|&r| {
self.index
.get(r)
.is_some_and(|e| e.flags & REV_FLAG_SNAPSHOT != 0)
});
let snapshot_start_idx = ancestors.iter().rposition(|&r| {
self.index
.get(r)
.is_some_and(|e| e.flags & REV_FLAG_SNAPSHOT != 0)
});
let (start_idx, mut file_bytes) = match snapshot_start_idx {
Some(idx) => {
@@ -76,7 +78,9 @@ impl OnionFile {
let table = self
.page_tables
.get(*ancestor_rev as usize)
.ok_or_else(|| OnionError::Malformed(format!("missing page table for rev {ancestor_rev}")))?;
.ok_or_else(|| {
OnionError::Malformed(format!("missing page table for rev {ancestor_rev}"))
})?;
for pt_entry in table {
let codec = Codec::from_u8(pt_entry.codec)?;
@@ -111,11 +115,7 @@ impl OnionFile {
}
/// Open a specific revision using an [`OpenRevision`] selector.
pub fn open_rev(
&self,
selector: OpenRevision,
h5_base: &[u8],
) -> Result<Vec<u8>, OnionError> {
pub fn open_rev(&self, selector: OpenRevision, h5_base: &[u8]) -> Result<Vec<u8>, OnionError> {
let rev = self.resolve_selector(selector)?;
self.reconstruct_revision(rev, h5_base)
}
@@ -128,12 +128,11 @@ impl OnionFile {
self.branch_head_rev(crate::format::BRANCH_MAIN)
.ok_or(OnionError::RevisionNotFound(0))
}
OpenRevision::At(rev) => {
self.index
.get(rev)
.map(|_| rev)
.ok_or(OnionError::RevisionNotFound(rev))
}
OpenRevision::At(rev) => self
.index
.get(rev)
.map(|_| rev)
.ok_or(OnionError::RevisionNotFound(rev)),
OpenRevision::Branch(name) => {
let branch = self
.branch_by_name(&name)
@@ -201,10 +200,7 @@ impl OnionFile {
/// compressed bytes directly and let the receiver decompress.
///
/// Returns `(h5_offset, compressed_data, codec_byte, orig_size)`.
pub fn revision_pages_raw(
&self,
rev: u64,
) -> Result<Vec<RawPage>, OnionError> {
pub fn revision_pages_raw(&self, rev: u64) -> Result<Vec<RawPage>, OnionError> {
let table = self
.page_tables
.get(rev as usize)
crates/clawhdf5-onion/src/tdt.rs
+6 -2
View File
@@ -117,7 +117,10 @@ mod tests {
// byte_pos=1: elements 1,5,9,13
// byte_pos=2: elements 2,6,10,14
// byte_pos=3: elements 3,7,11,15
assert_eq!(enc, vec![0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15]);
assert_eq!(
enc,
vec![0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15]
);
let dec = decode(&enc, 4, 16);
assert_eq!(dec, data);
}
@@ -180,7 +183,8 @@ mod tests {
assert!(
tdt_compressed.len() <= raw_compressed.len(),
"TDT+zstd ({} bytes) should be no worse than raw zstd ({} bytes) on structured float data",
tdt_compressed.len(), raw_compressed.len()
tdt_compressed.len(),
raw_compressed.len()
);
}
crates/clawhdf5-onion/src/versioned_file.rs
+167 -53
View File
@@ -57,21 +57,21 @@ use crate::writer::OnionFile;
/// [`export_revision`](VersionedFile::export_revision) to materialise a
/// specific revision to disk.
pub struct VersionedFile {
onion: OnionFile,
h5_path: PathBuf,
onion: OnionFile,
h5_path: PathBuf,
/// Original `.h5` bytes — the reconstruction base. **Never mutated.**
h5_base: Vec<u8>,
h5_base: Vec<u8>,
/// In-memory state of the most recently committed revision on the **default**
/// branch. Kept in sync with `branch_state[BRANCH_MAIN]`.
/// Used by [`snapshot`](VersionedFile::snapshot) and
/// [`auto_snapshot`](VersionedFile::auto_snapshot).
current_h5: Vec<u8>,
current_h5: Vec<u8>,
/// Per-branch diff baseline: `branch_id → last-committed bytes on that branch`.
///
/// Ensures that interleaved commits on different branches always diff against
/// the correct prior state for each branch, not just the globally last commit.
branch_state: HashMap<u32, Vec<u8>>,
page_size: u32,
page_size: u32,
}
impl VersionedFile {
@@ -105,7 +105,14 @@ impl VersionedFile {
.cloned()
.unwrap_or_else(|| h5_base.clone());
Ok(Self { onion, h5_path, h5_base, current_h5, branch_state, page_size })
Ok(Self {
onion,
h5_path,
h5_base,
current_h5,
branch_state,
page_size,
})
}
/// Create a new versioned HDF5 file from a [`clawhdf5::FileBuilder`].
@@ -144,7 +151,14 @@ impl VersionedFile {
let h5_base = std::fs::read(&h5_path)?;
let current_h5 = h5_base.clone();
let onion = OnionFile::create(&h5_path, page_size)?;
Ok(Self { onion, h5_path, h5_base, current_h5, branch_state: HashMap::new(), page_size })
Ok(Self {
onion,
h5_path,
h5_base,
current_h5,
branch_state: HashMap::new(),
page_size,
})
}
/// Create a new versioned HDF5 file, automatically choosing the page size.
@@ -163,7 +177,14 @@ impl VersionedFile {
let current_h5 = h5_base.clone();
let onion = OnionFile::create_auto(&h5_path)?;
let page_size = onion.page_size();
Ok(Self { onion, h5_path, h5_base, current_h5, branch_state: HashMap::new(), page_size })
Ok(Self {
onion,
h5_path,
h5_base,
current_h5,
branch_state: HashMap::new(),
page_size,
})
}
// ── Read API ──────────────────────────────────────────────────────────────
@@ -352,7 +373,9 @@ impl VersionedFile {
/// vf.auto_snapshot(500)?; // snapshot every 500 revisions
/// ```
pub fn auto_snapshot(&mut self, interval: u64) -> Result<Option<u64>, OnionError> {
let result = self.onion.auto_snapshot_if_needed(&self.current_h5, interval)?;
let result = self
.onion
.auto_snapshot_if_needed(&self.current_h5, interval)?;
if result.is_some() {
self.onion.flush()?;
}
@@ -383,7 +406,8 @@ impl VersionedFile {
// Refresh the target branch's cached state so future commits on it
// use the merged content as their diff baseline.
let (target_id, target_head) = {
let entry = self.onion
let entry = self
.onion
.branch_by_name(target)
.ok_or_else(|| OnionError::BranchNotFound(target.to_string()))?;
(entry.id, entry.head_rev)
@@ -435,8 +459,7 @@ impl VersionedFile {
// ── Internal ─────────────────────────────────────────────────────────────
fn hdf5_from_bytes(&self, bytes: Vec<u8>) -> Result<clawhdf5::File, OnionError> {
clawhdf5::File::from_bytes(bytes)
.map_err(|e| OnionError::Hdf5(e.to_string()))
clawhdf5::File::from_bytes(bytes).map_err(|e| OnionError::Hdf5(e.to_string()))
}
}
@@ -510,11 +533,17 @@ mod tests {
// Read back rev 0 — should still be [1.0, 2.0]
let f0 = vf.revision(0).unwrap();
assert_eq!(f0.dataset("data").unwrap().read_f64().unwrap(), vec![1.0, 2.0]);
assert_eq!(
f0.dataset("data").unwrap().read_f64().unwrap(),
vec![1.0, 2.0]
);
// Read back rev 1 — should be [3.0, 4.0]
let f1 = vf.revision(1).unwrap();
assert_eq!(f1.dataset("data").unwrap().read_f64().unwrap(), vec![3.0, 4.0]);
assert_eq!(
f1.dataset("data").unwrap().read_f64().unwrap(),
vec![3.0, 4.0]
);
}
#[test]
@@ -580,9 +609,15 @@ mod tests {
let vf2 = VersionedFile::open(&h5_path).unwrap();
assert_eq!(vf2.onion().revision_count(), 2);
let f0 = vf2.revision(0).unwrap();
assert_eq!(f0.dataset("data").unwrap().read_f64().unwrap(), vec![10.0, 20.0]);
assert_eq!(
f0.dataset("data").unwrap().read_f64().unwrap(),
vec![10.0, 20.0]
);
let f1 = vf2.revision(1).unwrap();
assert_eq!(f1.dataset("data").unwrap().read_f64().unwrap(), vec![30.0, 40.0]);
assert_eq!(
f1.dataset("data").unwrap().read_f64().unwrap(),
vec![30.0, 40.0]
);
}
// ── merge ────────────────────────────────────────────────────────────────
@@ -598,7 +633,8 @@ mod tests {
// Fork feat
let feat_id = vf.onion_mut().create_branch("feat", "main").unwrap();
// Rev 1: feat writes [3.0, 4.0]
vf.commit_on(Some("feat"), make_h5(&[3.0, 4.0]), Some("feat-v1")).unwrap();
vf.commit_on(Some("feat"), make_h5(&[3.0, 4.0]), Some("feat-v1"))
.unwrap();
// Merge feat → main
let merge_rev = vf.merge("feat", "main", MergeStrategy::LatestWins).unwrap();
@@ -610,7 +646,10 @@ mod tests {
// Content at merge revision reflects feat's state (latest wins)
let f = vf.revision(merge_rev).unwrap();
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![3.0, 4.0]);
assert_eq!(
f.dataset("data").unwrap().read_f64().unwrap(),
vec![3.0, 4.0]
);
let _ = feat_id;
}
@@ -622,7 +661,8 @@ mod tests {
vf.commit(make_h5(&[1.0]), Some("main-v1")).unwrap();
vf.onion_mut().create_branch("feat", "main").unwrap();
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1")).unwrap();
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1"))
.unwrap();
vf.merge("feat", "main", MergeStrategy::LatestWins).unwrap();
@@ -640,7 +680,9 @@ mod tests {
let mut vf = VersionedFile::create(&h5_path, 4096).unwrap();
vf.commit(make_h5(&[1.0]), None).unwrap();
let err = vf.merge("ghost", "main", MergeStrategy::LatestWins).unwrap_err();
let err = vf
.merge("ghost", "main", MergeStrategy::LatestWins)
.unwrap_err();
assert!(matches!(err, OnionError::BranchNotFound(_)));
}
@@ -653,11 +695,12 @@ mod tests {
vf.commit(make_h5(&[1.0]), Some("main-v1")).unwrap();
let feat_id = vf.onion_mut().create_branch("feat", "main").unwrap();
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1")).unwrap();
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1"))
.unwrap();
let revs = vf.onion().list_revisions();
assert_eq!(revs.len(), 2);
assert_eq!(revs[0].branch_id, 0, "rev 0 is on main");
assert_eq!(revs[0].branch_id, 0, "rev 0 is on main");
assert_eq!(revs[1].branch_id, feat_id, "rev 1 is on feat");
}
@@ -679,7 +722,9 @@ mod tests {
let (_tmp, h5_path) = tmp_h5(&[1.0]);
let mut vf = VersionedFile::create(&h5_path, 4096).unwrap();
let err = vf.commit_on(Some("ghost"), make_h5(&[2.0]), None).unwrap_err();
let err = vf
.commit_on(Some("ghost"), make_h5(&[2.0]), None)
.unwrap_err();
assert!(matches!(err, OnionError::BranchNotFound(_)));
}
@@ -698,7 +743,8 @@ mod tests {
vf.onion_mut().create_branch("feat", "main").unwrap();
// Commit v2 on feat (rev 1: [2.0])
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1")).unwrap();
vf.commit_on(Some("feat"), make_h5(&[2.0]), Some("feat-v1"))
.unwrap();
// Commit v2 on main (rev 2: [3.0]) — diff must be against main's last
// commit ([1.0]), not feat's last commit ([2.0]).
@@ -706,16 +752,25 @@ mod tests {
// Reconstruct each revision and verify round-trip fidelity
let f0 = vf.revision(0).unwrap();
assert_eq!(f0.dataset("data").unwrap().read_f64().unwrap(), vec![1.0],
"rev 0 (main-v1)");
assert_eq!(
f0.dataset("data").unwrap().read_f64().unwrap(),
vec![1.0],
"rev 0 (main-v1)"
);
let f1 = vf.revision(1).unwrap();
assert_eq!(f1.dataset("data").unwrap().read_f64().unwrap(), vec![2.0],
"rev 1 (feat-v1)");
assert_eq!(
f1.dataset("data").unwrap().read_f64().unwrap(),
vec![2.0],
"rev 1 (feat-v1)"
);
let f2 = vf.revision(2).unwrap();
assert_eq!(f2.dataset("data").unwrap().read_f64().unwrap(), vec![3.0],
"rev 2 (main-v2) — would be wrong if diffed against feat state");
assert_eq!(
f2.dataset("data").unwrap().read_f64().unwrap(),
vec![3.0],
"rev 2 (main-v2) — would be wrong if diffed against feat state"
);
}
/// After a reload, `open()` must restore per-branch state from the sidecar
@@ -730,13 +785,21 @@ mod tests {
let mut s = vf.onion_mut().begin_session(Some(feat_id)).unwrap();
let feat_bytes = make_h5(&[9.0]);
let ps = 4096usize;
let base = vf.onion().reconstruct_revision(0,
&std::fs::read(&h5_path).unwrap()).unwrap();
let base = vf
.onion()
.reconstruct_revision(0, &std::fs::read(&h5_path).unwrap())
.unwrap();
for i in 0..(feat_bytes.len().max(base.len())).div_ceil(ps) {
let start = i * ps;
if start >= feat_bytes.len() { break; }
let ns = &feat_bytes[start..feat_bytes.len().min(start+ps)];
let os: &[u8] = if start < base.len() { &base[start..base.len().min(start+ps)] } else { &[] };
if start >= feat_bytes.len() {
break;
}
let ns = &feat_bytes[start..feat_bytes.len().min(start + ps)];
let os: &[u8] = if start < base.len() {
&base[start..base.len().min(start + ps)]
} else {
&[]
};
if ns != os {
let mut pad = vec![0u8; ps];
pad[..ns.len()].copy_from_slice(ns);
@@ -752,8 +815,11 @@ mod tests {
vf2.commit(make_h5(&[2.0]), Some("main-v2")).unwrap();
let f_main = vf2.revision(2).unwrap();
assert_eq!(f_main.dataset("data").unwrap().read_f64().unwrap(), vec![2.0],
"main-v2 content must be [2.0] after reload");
assert_eq!(
f_main.dataset("data").unwrap().read_f64().unwrap(),
vec![2.0],
"main-v2 content must be [2.0] after reload"
);
}
// ── create_auto ──────────────────────────────────────────────────────────
@@ -767,7 +833,10 @@ mod tests {
"auto page size must be a power of two, got {}",
vf.page_size()
);
assert!(vf.page_size() >= 4096, "auto page size must be at least 4 KiB");
assert!(
vf.page_size() >= 4096,
"auto page size must be at least 4 KiB"
);
}
#[test]
@@ -778,7 +847,10 @@ mod tests {
vf.commit(make_h5(&[7.0, 8.0]), Some("auto-v1")).unwrap();
let f = vf.revision(0).unwrap();
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![7.0, 8.0]);
assert_eq!(
f.dataset("data").unwrap().read_f64().unwrap(),
vec![7.0, 8.0]
);
}
// ── snapshot / auto_snapshot ─────────────────────────────────────────────
@@ -797,7 +869,10 @@ mod tests {
// The snapshot revision must be flagged as a snapshot.
let revs = vf.onion().list_revisions();
let snap = revs.iter().find(|r| r.revision == snap_rev).unwrap();
assert!(snap.is_snapshot, "snapshot revision must have is_snapshot == true");
assert!(
snap.is_snapshot,
"snapshot revision must have is_snapshot == true"
);
}
#[test]
@@ -843,7 +918,10 @@ mod tests {
vf.commit(make_h5(&[i as f64]), None).unwrap();
}
let result = vf.auto_snapshot(5).unwrap();
assert!(result.is_some(), "auto_snapshot should fire when count % interval == 0");
assert!(
result.is_some(),
"auto_snapshot should fire when count % interval == 0"
);
}
#[test]
@@ -855,7 +933,10 @@ mod tests {
vf.commit(make_h5(&[2.0]), None).unwrap();
// 2 revisions, interval 5 — should not trigger
let result = vf.auto_snapshot(5).unwrap();
assert!(result.is_none(), "auto_snapshot should be a no-op when count % interval != 0");
assert!(
result.is_none(),
"auto_snapshot should be a no-op when count % interval != 0"
);
}
#[test]
@@ -908,12 +989,21 @@ mod tests {
// Record a changed page for the feat branch rev
let feat_bytes = make_h5(&[3.0]);
let ps = 4096usize;
let current = vf.onion().reconstruct_revision(1, &std::fs::read(&h5_path).unwrap()).unwrap();
let current = vf
.onion()
.reconstruct_revision(1, &std::fs::read(&h5_path).unwrap())
.unwrap();
for i in 0..(feat_bytes.len().max(current.len())).div_ceil(ps) {
let start = i * ps;
if start >= feat_bytes.len() { break; }
if start >= feat_bytes.len() {
break;
}
let new_sl = &feat_bytes[start..feat_bytes.len().min(start + ps)];
let old_sl: &[u8] = if start < current.len() { &current[start..current.len().min(start + ps)] } else { &[] };
let old_sl: &[u8] = if start < current.len() {
&current[start..current.len().min(start + ps)]
} else {
&[]
};
if new_sl != old_sl {
let mut pad = vec![0u8; ps];
pad[..new_sl.len()].copy_from_slice(new_sl);
@@ -1009,7 +1099,10 @@ mod tests {
p.set_extension(&new_ext);
p
};
assert!(onion_path.exists(), ".onion sidecar should exist after from_builder: checked {onion_path:?}");
assert!(
onion_path.exists(),
".onion sidecar should exist after from_builder: checked {onion_path:?}"
);
}
#[test]
@@ -1026,7 +1119,10 @@ mod tests {
assert_eq!(rev, 0);
let f = vf.revision(0).unwrap();
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![3.0, 4.0]);
assert_eq!(
f.dataset("data").unwrap().read_f64().unwrap(),
vec![3.0, 4.0]
);
}
// ── export_revision ───────────────────────────────────────────────────────
@@ -1046,7 +1142,10 @@ mod tests {
// Re-open the exported file as a raw clawhdf5::File
let bytes = std::fs::read(&export_path).unwrap();
let f = clawhdf5::File::from_bytes(bytes).unwrap();
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![1.0, 2.0]);
assert_eq!(
f.dataset("data").unwrap().read_f64().unwrap(),
vec![1.0, 2.0]
);
}
#[test]
@@ -1075,7 +1174,10 @@ mod tests {
let bytes = std::fs::read(&export_path).unwrap();
let f = clawhdf5::File::from_bytes(bytes).unwrap();
assert_eq!(f.dataset("data").unwrap().read_f64().unwrap(), vec![8.0, 9.0]);
assert_eq!(
f.dataset("data").unwrap().read_f64().unwrap(),
vec![8.0, 9.0]
);
}
// ── multiple datasets ─────────────────────────────────────────────────────
@@ -1098,11 +1200,23 @@ mod tests {
vf.commit(bytes_v2, Some("update x only")).unwrap();
let f0 = vf.revision(0).unwrap();
assert_eq!(f0.dataset("x").unwrap().read_f64().unwrap(), vec![1.0, 2.0, 3.0]);
assert_eq!(f0.dataset("y").unwrap().read_f64().unwrap(), vec![4.0, 5.0, 6.0]);
assert_eq!(
f0.dataset("x").unwrap().read_f64().unwrap(),
vec![1.0, 2.0, 3.0]
);
assert_eq!(
f0.dataset("y").unwrap().read_f64().unwrap(),
vec![4.0, 5.0, 6.0]
);
let f1 = vf.revision(1).unwrap();
assert_eq!(f1.dataset("x").unwrap().read_f64().unwrap(), vec![10.0, 20.0, 30.0]);
assert_eq!(f1.dataset("y").unwrap().read_f64().unwrap(), vec![4.0, 5.0, 6.0]);
assert_eq!(
f1.dataset("x").unwrap().read_f64().unwrap(),
vec![10.0, 20.0, 30.0]
);
assert_eq!(
f1.dataset("y").unwrap().read_f64().unwrap(),
vec![4.0, 5.0, 6.0]
);
}
}
crates/clawhdf5-onion/src/writer.rs
+24 -17
View File
@@ -18,8 +18,8 @@ use crate::annotation::AnnotationHeap;
use crate::compress::compress_page;
use crate::error::OnionError;
use crate::format::{
BranchEntry, Codec, OnionHeader, PageTableEntry, RevisionEntry,
BRANCH_MAIN, DEFAULT_FEATURE_FLAGS, HEADER_SIZE, NO_PARENT,
BRANCH_MAIN, BranchEntry, Codec, DEFAULT_FEATURE_FLAGS, HEADER_SIZE, NO_PARENT, OnionHeader,
PageTableEntry, RevisionEntry,
};
use crate::index::RevisionIndex;
use crate::provenance::{SessionId, hash_pages};
@@ -204,9 +204,7 @@ impl OnionFile {
annotation: Option<&str>,
) -> Result<u64, OnionError> {
let revision = self.index.len() as u64;
let parent_rev = self
.branch_head_rev(session.branch_id)
.unwrap_or(NO_PARENT);
let parent_rev = self.branch_head_rev(session.branch_id).unwrap_or(NO_PARENT);
// Compress pages and build BLAKE3 input
let mut pages: Vec<(u64, Vec<u8>, u32, Codec)> = Vec::new();
@@ -226,9 +224,7 @@ impl OnionFile {
);
// Annotation
let annotation_off = annotation
.map(|a| self.annotations.push(a))
.unwrap_or(0);
let annotation_off = annotation.map(|a| self.annotations.push(a)).unwrap_or(0);
// Append page table entries
let mut table: Vec<PageTableEntry> = Vec::new();
@@ -451,9 +447,9 @@ impl OnionFile {
}
pub fn branch_by_name(&self, name: &str) -> Option<&BranchEntry> {
self.branches.iter().find(|b| {
self.annotations.get(b.name_off) == Some(name)
})
self.branches
.iter()
.find(|b| self.annotations.get(b.name_off) == Some(name))
}
/// Serialise the entire `.onion` file to bytes.
@@ -480,8 +476,7 @@ impl OnionFile {
let branch_offset = index_offset + (n_revisions * rev_entry_size) as u64;
// Calculate PageTable section start
let page_table_section_start =
branch_offset + (n_branches * branch_entry_size) as u64;
let page_table_section_start = branch_offset + (n_branches * branch_entry_size) as u64;
// Calculate per-revision page_table_off values and total page table size
let mut page_table_offsets: Vec<u64> = Vec::with_capacity(n_revisions);
@@ -599,7 +594,11 @@ impl OnionFile {
// 0-4). We extend the Vec with empty entries so that
// page_tables[rev_number] is always valid.
let max_rev = entries.iter().map(|e| e.revision).max().unwrap_or(0);
let table_len = if entries.is_empty() { 0 } else { max_rev as usize + 1 };
let table_len = if entries.is_empty() {
0
} else {
max_rev as usize + 1
};
let mut page_tables: Vec<Vec<PageTableEntry>> = vec![Vec::new(); table_len];
for entry in &entries {
let pt_start = entry.page_table_off as usize;
@@ -792,7 +791,10 @@ mod tests {
fn create_accepts_valid_page_sizes() {
for &size in &[512u32, 1024, 4096, 65536] {
let path = Path::new("/tmp/dummy.h5");
assert!(OnionFile::create(path, size).is_ok(), "size {size} rejected");
assert!(
OnionFile::create(path, size).is_ok(),
"size {size} rejected"
);
}
}
@@ -973,7 +975,9 @@ mod tests {
.list_revisions()
.into_iter()
.filter(|r| {
onion.index.get(r.revision)
onion
.index
.get(r.revision)
.map_or(false, |e| e.flags & crate::format::REV_FLAG_SNAPSHOT != 0)
})
.collect();
@@ -1041,7 +1045,10 @@ mod tests {
fn eof_to_page_size_always_power_of_two() {
for eof in [1, 100, 1_000, 10_000, 100_000, 1_000_000, 100_000_000u64] {
let ps = eof_to_page_size(eof);
assert!(ps.is_power_of_two(), "page_size {ps} is not a power of two for eof={eof}");
assert!(
ps.is_power_of_two(),
"page_size {ps} is not a power of two for eof={eof}"
);
}
}
crates/clawhdf5-onion/tests/golden.rs
+159 -76
View File
@@ -20,23 +20,27 @@ use tempfile::NamedTempFile;
// Layout constants (must match CLAWONION_SPEC.md §2 + §3)
// ─────────────────────────────────────────────────────────────────────────────
const HEADER_SIZE: usize = 128;
const REV_ENTRY_SIZE: usize = 112;
const HEADER_SIZE: usize = 128;
const REV_ENTRY_SIZE: usize = 112;
const BRANCH_ENTRY_SIZE: usize = 40;
const PAGE_ENTRY_SIZE: usize = 32;
const PAGE_SIZE: u32 = 4096;
const PAGE_ENTRY_SIZE: usize = 32;
const PAGE_SIZE: u32 = 4096;
const INDEX_OFFSET: usize = HEADER_SIZE; // 128
const BRANCH_OFFSET: usize = INDEX_OFFSET + REV_ENTRY_SIZE; // 240
const PT_OFFSET: usize = BRANCH_OFFSET + BRANCH_ENTRY_SIZE; // 280
const PD_OFFSET: usize = PT_OFFSET + PAGE_ENTRY_SIZE; // 312
const INDEX_OFFSET: usize = HEADER_SIZE; // 128
const BRANCH_OFFSET: usize = INDEX_OFFSET + REV_ENTRY_SIZE; // 240
const PT_OFFSET: usize = BRANCH_OFFSET + BRANCH_ENTRY_SIZE; // 280
const PD_OFFSET: usize = PT_OFFSET + PAGE_ENTRY_SIZE; // 312
// ─────────────────────────────────────────────────────────────────────────────
// Helpers
// ─────────────────────────────────────────────────────────────────────────────
fn le64(v: u64) -> [u8; 8] { v.to_le_bytes() }
fn le32(v: u32) -> [u8; 4] { v.to_le_bytes() }
fn le64(v: u64) -> [u8; 8] {
v.to_le_bytes()
}
fn le32(v: u32) -> [u8; 4] {
v.to_le_bytes()
}
/// Build the expected BLAKE3 for one 4096-byte page of 0xAB at h5_offset = 0.
///
@@ -82,7 +86,8 @@ fn golden_minimal_onion_byte_layout() {
let ann_heap: &[u8] = &[0x00, 0x04, 0x00, 0x00, 0x00, b'm', b'a', b'i', b'n'];
let expected_total = PD_OFFSET + PAGE_SIZE as usize + ann_heap.len(); // 4417
assert_eq!(
bytes.len(), expected_total,
bytes.len(),
expected_total,
"total file size mismatch (got {}, expected {expected_total})",
bytes.len()
);
@@ -127,10 +132,17 @@ fn golden_minimal_onion_byte_layout() {
// [64..72] created_at — f64, time-dependent; skip exact value check,
// but assert it's non-zero (a real Unix timestamp).
let created_at = f64::from_le_bytes(bytes[64..72].try_into().unwrap());
assert!(created_at > 0.0, "created_at should be a positive Unix timestamp");
assert!(
created_at > 0.0,
"created_at should be a positive Unix timestamp"
);
// [72..128] reserved — 56 bytes, must all be zero
assert_eq!(&bytes[72..128], &[0u8; 56], "reserved header bytes must be zero");
assert_eq!(
&bytes[72..128],
&[0u8; 56],
"reserved header bytes must be zero"
);
// ══════════════════════════════════════════════════════════════════════════
// §3.2 RevisionEntry — 112 bytes @ offset 128
@@ -139,45 +151,69 @@ fn golden_minimal_onion_byte_layout() {
let re = INDEX_OFFSET; // 128
// [re+0..re+8] revision = 0
assert_eq!(&bytes[re..re+8], &le64(0), "revision number");
assert_eq!(&bytes[re..re + 8], &le64(0), "revision number");
// [re+8..re+12] branch_id = 0 (main)
assert_eq!(&bytes[re+8..re+12], &le32(0), "branch_id");
assert_eq!(&bytes[re + 8..re + 12], &le32(0), "branch_id");
// [re+12..re+16] _pad_bi — must be zero
assert_eq!(&bytes[re+12..re+16], &[0u8; 4], "_pad_bi must be zero");
assert_eq!(&bytes[re + 12..re + 16], &[0u8; 4], "_pad_bi must be zero");
// [re+16..re+24] parent_rev = u64::MAX (root has no parent)
assert_eq!(&bytes[re+16..re+24], &le64(u64::MAX), "parent_rev (NO_PARENT sentinel)");
assert_eq!(
&bytes[re + 16..re + 24],
&le64(u64::MAX),
"parent_rev (NO_PARENT sentinel)"
);
// [re+24..re+28] page_count = 1
assert_eq!(&bytes[re+24..re+28], &le32(1), "page_count");
assert_eq!(&bytes[re + 24..re + 28], &le32(1), "page_count");
// [re+28..re+32] _pad_pc — must be zero
assert_eq!(&bytes[re+28..re+32], &[0u8; 4], "_pad_pc must be zero");
assert_eq!(&bytes[re + 28..re + 32], &[0u8; 4], "_pad_pc must be zero");
// [re+32..re+40] page_table_off = 280 (PT_OFFSET)
assert_eq!(&bytes[re+32..re+40], &le64(PT_OFFSET as u64), "page_table_off");
assert_eq!(
&bytes[re + 32..re + 40],
&le64(PT_OFFSET as u64),
"page_table_off"
);
// [re+40..re+48] timestamp — f64, time-dependent; assert positive
let ts = f64::from_le_bytes(bytes[re+40..re+48].try_into().unwrap());
let ts = f64::from_le_bytes(bytes[re + 40..re + 48].try_into().unwrap());
assert!(ts > 0.0, "revision timestamp should be positive");
// [re+48..re+80] blake3 — verify against spec §5 algorithm
let computed_blake3 = expected_blake3(&page_bytes);
assert_eq!(&bytes[re+48..re+80], &computed_blake3, "BLAKE3 hash mismatch");
assert_eq!(
&bytes[re + 48..re + 80],
&computed_blake3,
"BLAKE3 hash mismatch"
);
// [re+80..re+96] session_uuid — 16 bytes UUIDv7; must be non-zero
assert_ne!(&bytes[re+80..re+96], &[0u8; 16], "session_uuid must not be all zeros");
assert_ne!(
&bytes[re + 80..re + 96],
&[0u8; 16],
"session_uuid must not be all zeros"
);
// [re+96..re+104] annotation_off = 0 (no annotation)
assert_eq!(&bytes[re+96..re+104], &le64(0), "annotation_off must be 0 (no annotation)");
assert_eq!(
&bytes[re + 96..re + 104],
&le64(0),
"annotation_off must be 0 (no annotation)"
);
// [re+104] flags = 0 (not a snapshot)
assert_eq!(bytes[re+104], 0x00, "revision flags");
assert_eq!(bytes[re + 104], 0x00, "revision flags");
// [re+105..re+112] _pad_flags — must be zero
assert_eq!(&bytes[re+105..re+112], &[0u8; 7], "_pad_flags must be zero");
assert_eq!(
&bytes[re + 105..re + 112],
&[0u8; 7],
"_pad_flags must be zero"
);
// ══════════════════════════════════════════════════════════════════════════
// §3.3 BranchEntry — 40 bytes @ offset 240
@@ -186,22 +222,34 @@ fn golden_minimal_onion_byte_layout() {
let be = BRANCH_OFFSET; // 240
// [be+0..be+4] id = 0 (main)
assert_eq!(&bytes[be..be+4], &le32(0), "branch id");
assert_eq!(&bytes[be..be + 4], &le32(0), "branch id");
// [be+4..be+8] _pad_id — must be zero
assert_eq!(&bytes[be+4..be+8], &[0u8; 4], "_pad_id must be zero");
assert_eq!(&bytes[be + 4..be + 8], &[0u8; 4], "_pad_id must be zero");
// [be+8..be+16] name_off = 1 (heap-relative; first string after reserved null)
assert_eq!(&bytes[be+8..be+16], &le64(1), "name_off for main branch");
assert_eq!(
&bytes[be + 8..be + 16],
&le64(1),
"name_off for main branch"
);
// [be+16..be+24] head_rev = 0 (updated after commit)
assert_eq!(&bytes[be+16..be+24], &le64(0), "head_rev after first commit");
assert_eq!(
&bytes[be + 16..be + 24],
&le64(0),
"head_rev after first commit"
);
// [be+24..be+32] fork_rev = u64::MAX (main is not forked)
assert_eq!(&bytes[be+24..be+32], &le64(u64::MAX), "fork_rev (NO_PARENT for main)");
assert_eq!(
&bytes[be + 24..be + 32],
&le64(u64::MAX),
"fork_rev (NO_PARENT for main)"
);
// [be+32..be+40] created_at — f64, time-dependent; assert positive
let branch_ts = f64::from_le_bytes(bytes[be+32..be+40].try_into().unwrap());
let branch_ts = f64::from_le_bytes(bytes[be + 32..be + 40].try_into().unwrap());
assert!(branch_ts > 0.0, "branch created_at should be positive");
// ══════════════════════════════════════════════════════════════════════════
@@ -211,36 +259,51 @@ fn golden_minimal_onion_byte_layout() {
let pt = PT_OFFSET; // 280
// [pt+0..pt+8] h5_offset = 0
assert_eq!(&bytes[pt..pt+8], &le64(0), "h5_offset");
assert_eq!(&bytes[pt..pt + 8], &le64(0), "h5_offset");
// [pt+8..pt+16] data_offset = 312 (PD_OFFSET, absolute file position)
assert_eq!(&bytes[pt+8..pt+16], &le64(PD_OFFSET as u64), "data_offset (absolute)");
assert_eq!(
&bytes[pt + 8..pt + 16],
&le64(PD_OFFSET as u64),
"data_offset (absolute)"
);
// [pt+16..pt+20] orig_size = 4096
assert_eq!(&bytes[pt+16..pt+20], &le32(4096), "orig_size");
assert_eq!(&bytes[pt + 16..pt + 20], &le32(4096), "orig_size");
// [pt+20..pt+24] data_size = 4096 (codec=None → no compression)
assert_eq!(&bytes[pt+20..pt+24], &le32(4096), "data_size (codec=None, uncompressed)");
assert_eq!(
&bytes[pt + 20..pt + 24],
&le32(4096),
"data_size (codec=None, uncompressed)"
);
// [pt+24] codec = 0 (None)
assert_eq!(bytes[pt+24], 0x00, "codec = None (0)");
assert_eq!(bytes[pt + 24], 0x00, "codec = None (0)");
// [pt+25..pt+32] _pad — must be zero
assert_eq!(&bytes[pt+25..pt+32], &[0u8; 7], "_pad must be zero");
assert_eq!(&bytes[pt + 25..pt + 32], &[0u8; 7], "_pad must be zero");
// ══════════════════════════════════════════════════════════════════════════
// PageData — 4096 bytes @ offset 312
// ══════════════════════════════════════════════════════════════════════════
assert_eq!(&bytes[PD_OFFSET..PD_OFFSET + 4096], page_bytes.as_slice(),
"page data must equal the uncompressed original bytes");
assert_eq!(
&bytes[PD_OFFSET..PD_OFFSET + 4096],
page_bytes.as_slice(),
"page data must equal the uncompressed original bytes"
);
// ══════════════════════════════════════════════════════════════════════════
// §3.5 AnnotationHeap — tail of file
// ══════════════════════════════════════════════════════════════════════════
// Last 9 bytes: [0x00 reserved] [u32 LE length=4] ["main"]
assert_eq!(&bytes[bytes.len()-9..], ann_heap, "annotation heap content");
assert_eq!(
&bytes[bytes.len() - 9..],
ann_heap,
"annotation heap content"
);
// ══════════════════════════════════════════════════════════════════════════
// §7 Reconstruction — verify round-trip
@@ -270,10 +333,10 @@ fn golden_minimal_onion_byte_layout() {
#[test]
fn byte_exact_format_freeze() {
let re = INDEX_OFFSET; // 128 — RevisionEntry start
let re = INDEX_OFFSET; // 128 — RevisionEntry start
let be = BRANCH_OFFSET; // 240 — BranchEntry start
let pt = PT_OFFSET; // 280 — PageTableEntry start
let pd = PD_OFFSET; // 312 — PageData start
let pt = PT_OFFSET; // 280 — PageTableEntry start
let pd = PD_OFFSET; // 312 — PageData start
// ── Build the actual serialised file ────────────────────────────────────
@@ -298,71 +361,75 @@ fn byte_exact_format_freeze() {
let mut expected = vec![0u8; total];
// §3.1 OnionHeader (128 bytes)
expected[0..9].copy_from_slice(b"CLAWONION"); // magic
expected[9] = 0x01; // format_version
expected[0..9].copy_from_slice(b"CLAWONION"); // magic
expected[9] = 0x01; // format_version
// [10..16] = 0 (_pad_align)
expected[16..24].copy_from_slice(&le64(0x07)); // feature_flags
expected[24..28].copy_from_slice(&le32(4096)); // page_size
// [28..32] = 0 (_pad_ps)
expected[32..40].copy_from_slice(&le64(1)); // revision_count = 1
expected[40..44].copy_from_slice(&le32(1)); // branch_count = 1
expected[32..40].copy_from_slice(&le64(1)); // revision_count = 1
expected[40..44].copy_from_slice(&le32(1)); // branch_count = 1
// [44..48] = 0 (_pad_bc)
expected[48..56].copy_from_slice(&le64(128)); // index_offset
expected[56..64].copy_from_slice(&le64(240)); // branch_offset
expected[48..56].copy_from_slice(&le64(128)); // index_offset
expected[56..64].copy_from_slice(&le64(240)); // branch_offset
// [64..72] = 0 (created_at — zeroed, dynamic)
// [72..128] = 0 (reserved)
// §3.2 RevisionEntry (112 bytes @ 128)
expected[re..re+8] .copy_from_slice(&le64(0)); // revision = 0
expected[re+8..re+12].copy_from_slice(&le32(0)); // branch_id = 0
expected[re..re + 8].copy_from_slice(&le64(0)); // revision = 0
expected[re + 8..re + 12].copy_from_slice(&le32(0)); // branch_id = 0
// [re+12..re+16] = 0 (_pad_bi)
expected[re+16..re+24].copy_from_slice(&le64(u64::MAX)); // parent_rev = NO_PARENT
expected[re+24..re+28].copy_from_slice(&le32(1)); // page_count = 1
expected[re + 16..re + 24].copy_from_slice(&le64(u64::MAX)); // parent_rev = NO_PARENT
expected[re + 24..re + 28].copy_from_slice(&le32(1)); // page_count = 1
// [re+28..re+32] = 0 (_pad_pc)
expected[re+32..re+40].copy_from_slice(&le64(pt as u64)); // page_table_off
expected[re + 32..re + 40].copy_from_slice(&le64(pt as u64)); // page_table_off
// [re+40..re+48] = 0 (timestamp — zeroed, dynamic)
expected[re+48..re+80].copy_from_slice(&expected_blake3(&page_bytes)); // blake3
expected[re + 48..re + 80].copy_from_slice(&expected_blake3(&page_bytes)); // blake3
// [re+80..re+96] = 0 (session_uuid — zeroed, dynamic)
// [re+96..re+104] = 0 (annotation_off = 0, no annotation)
// [re+104] = 0 (flags = 0, not a snapshot)
// [re+105..re+112] = 0 (_pad_flags)
// §3.3 BranchEntry (40 bytes @ 240)
expected[be..be+4] .copy_from_slice(&le32(0)); // id = 0 (main)
expected[be..be + 4].copy_from_slice(&le32(0)); // id = 0 (main)
// [be+4..be+8] = 0 (_pad_id)
expected[be+8..be+16].copy_from_slice(&le64(1)); // name_off = 1 (heap offset of "main")
expected[be+16..be+24].copy_from_slice(&le64(0)); // head_rev = 0
expected[be+24..be+32].copy_from_slice(&le64(u64::MAX)); // fork_rev = NO_PARENT
expected[be + 8..be + 16].copy_from_slice(&le64(1)); // name_off = 1 (heap offset of "main")
expected[be + 16..be + 24].copy_from_slice(&le64(0)); // head_rev = 0
expected[be + 24..be + 32].copy_from_slice(&le64(u64::MAX)); // fork_rev = NO_PARENT
// [be+32..be+40] = 0 (created_at — zeroed, dynamic)
// §3.4 PageTableEntry (32 bytes @ 280)
expected[pt..pt+8] .copy_from_slice(&le64(0)); // h5_offset = 0
expected[pt+8..pt+16].copy_from_slice(&le64(pd as u64)); // data_offset
expected[pt+16..pt+20].copy_from_slice(&le32(4096)); // orig_size
expected[pt+20..pt+24].copy_from_slice(&le32(4096)); // data_size (no compression)
expected[pt..pt + 8].copy_from_slice(&le64(0)); // h5_offset = 0
expected[pt + 8..pt + 16].copy_from_slice(&le64(pd as u64)); // data_offset
expected[pt + 16..pt + 20].copy_from_slice(&le32(4096)); // orig_size
expected[pt + 20..pt + 24].copy_from_slice(&le32(4096)); // data_size (no compression)
// [pt+24] = 0 (codec = None)
// [pt+25..pt+32] = 0 (_pad)
// PageData (4096 bytes @ 312)
expected[pd..pd+4096].fill(0xAB);
expected[pd..pd + 4096].fill(0xAB);
// §3.5 AnnotationHeap (9 bytes at end)
expected[pd+4096..].copy_from_slice(ann_heap);
expected[pd + 4096..].copy_from_slice(ann_heap);
// ── Zero dynamic fields in both buffers ──────────────────────────────────
// OnionHeader.created_at
actual[64..72].fill(0);
// RevisionEntry.timestamp
actual[re+40..re+48].fill(0);
actual[re + 40..re + 48].fill(0);
// RevisionEntry.session_uuid
actual[re+80..re+96].fill(0);
actual[re + 80..re + 96].fill(0);
// BranchEntry.created_at
actual[be+32..be+40].fill(0);
actual[be + 32..be + 40].fill(0);
// ── Compare byte-for-byte ────────────────────────────────────────────────
assert_eq!(actual.len(), expected.len(),
assert_eq!(
actual.len(),
expected.len(),
"total file size mismatch: actual={} expected={}",
actual.len(), expected.len());
actual.len(),
expected.len()
);
// Find the first differing byte for a useful failure message.
if actual != expected {
@@ -387,13 +454,29 @@ fn byte_exact_format_freeze() {
#[test]
fn struct_sizes_match_spec() {
use clawhdf5_onion::format::{BranchEntry, OnionHeader, PageTableEntry, RevisionEntry};
use std::mem::size_of;
use clawhdf5_onion::format::{OnionHeader, RevisionEntry, BranchEntry, PageTableEntry};
assert_eq!(size_of::<OnionHeader>(), 128, "OnionHeader must be 128 bytes (§3.1)");
assert_eq!(size_of::<RevisionEntry>(), 112, "RevisionEntry must be 112 bytes (§3.2)");
assert_eq!(size_of::<BranchEntry>(), 40, "BranchEntry must be 40 bytes (§3.3)");
assert_eq!(size_of::<PageTableEntry>(), 32, "PageTableEntry must be 32 bytes (§3.4)");
assert_eq!(
size_of::<OnionHeader>(),
128,
"OnionHeader must be 128 bytes (§3.1)"
);
assert_eq!(
size_of::<RevisionEntry>(),
112,
"RevisionEntry must be 112 bytes (§3.2)"
);
assert_eq!(
size_of::<BranchEntry>(),
40,
"BranchEntry must be 40 bytes (§3.3)"
);
assert_eq!(
size_of::<PageTableEntry>(),
32,
"PageTableEntry must be 32 bytes (§3.4)"
);
}
// ─────────────────────────────────────────────────────────────────────────────
crates/clawhdf5-onion/tests/proptest_onion.rs
+7 -2
View File
@@ -258,8 +258,13 @@ fn assert_gc_invariants(onion: &OnionFile) {
// 5. blake3_hex has correct length (64 chars)
for rev in &revisions {
assert_eq!(rev.blake3_hex.len(), 64,
"revision {} has blake3_hex of wrong length {}", rev.revision, rev.blake3_hex.len());
assert_eq!(
rev.blake3_hex.len(),
64,
"revision {} has blake3_hex of wrong length {}",
rev.revision,
rev.blake3_hex.len()
);
}
// 6. Pages are accessible for every revision
crates/clawhdf5-onion/tests/proptest_versioned.rs
+8 -6
View File
@@ -48,7 +48,11 @@ struct Round {
fn arb_round() -> impl Strategy<Value = Round> {
(any::<bool>(), 0usize..8usize, 1u8..=255u8).prop_map(|(create_branch, branch_idx, fill)| {
Round { create_branch, branch_idx, fill }
Round {
create_branch,
branch_idx,
fill,
}
})
}
@@ -72,10 +76,7 @@ fn make_vf() -> (TempDir, std::path::PathBuf, VersionedFile) {
/// Apply `rounds` to `vf`, returning a Vec of `(revision_number, expected_page_bytes)`.
///
/// `branches` starts as `["main"]` and grows as new branches are created.
fn apply_rounds(
vf: &mut VersionedFile,
rounds: &[Round],
) -> Vec<(u64, Vec<u8>)> {
fn apply_rounds(vf: &mut VersionedFile, rounds: &[Round]) -> Vec<(u64, Vec<u8>)> {
let mut branches: Vec<String> = vec!["main".to_string()];
let mut expectations: Vec<(u64, Vec<u8>)> = Vec::new();
let mut branch_counter: usize = 0;
@@ -124,7 +125,8 @@ fn verify_expectations(
prop_assert_eq!(
&actual[..page_end],
expected.as_slice(),
"revision {}: page 0 mismatch", rev
"revision {}: page 0 mismatch",
rev
);
}
Ok(())