quantumclaw/clawhdf5
3
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

security(agent): chain WAL entry CRCs and restrict the legacy no-CRC reader

Browse Source 
Two related gaps in the WAL format, both closed:

1. Each entry's CRC32 covered only its own bytes, with no sequence number
   or chaining — entries could be reordered, duplicated, or spliced (e.g.
   a Tombstone moved before/after its target Save) while every individual
   entry still passed its own CRC check, silently changing replayed cache
   state. Bump to WAL_VERSION 3: each entry's CRC32 trailer is now computed
   over its own bytes chained with the previous entry's stored CRC
   (crc32(entry_bytes ++ prev_crc)), seeded at 0 after a truncation. Moving,
   duplicating, or reordering an entry breaks the chain at that point, and
   replay stops there — same handling as a bit-flip or truncation. The
   previous per-entry-CRC-only format becomes WAL_VERSION_CRC_UNCHAINED (2)
   and remains fully readable (not restricted, since it still verifies each
   entry); WalFile::open migrates it to v3 by recreating the file fresh,
   same as the existing v1 migration.

   WalFile::open() on an existing v3 file scans it once to resume the CRC
   chain correctly for further appends — required because a process
   restart without an intervening flush reopens the same (non-truncated)
   WAL and keeps appending to it, so new entries must chain against the
   real last entry already on disk, not restart from 0.

2. WAL_VERSION_LEGACY_NO_CRC (v1, no integrity verification at all) was
   reachable through the public WalFile::read_entries — a version byte
   flipped from 2/3 down to 1 silently downgraded every entry to the
   fully-unverified pre-hardening parser for any caller, not just the
   one-time migration path. Split into WalFile::read_entries (rejects v1
   with a typed error; still reads v2/v3) and the pub(crate)
   read_entries_for_migration (accepts v1 too), used exclusively by
   HDF5Memory::open's migration flow.

INT-09
This commit is contained in:
ClawHDF5 Coding Agent
2026-08-17 01:01:29 +00:00
parent 5db1008eb7
commit 3a30327f35
3 changed files with 335 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/clawhdf5-agent/src/lib.rs
+4 -1
View File
@@ -292,7 +292,10 @@ impl HDF5Memory {
// Replay WAL if present
let wal_path = path.with_extension("h5.wal");
let wal = if wal_path.exists() {
let entries = wal::WalFile::read_entries(&wal_path)?;
// Uses the migration-only reader since this is the one legitimate
// path that may need to read a legacy (pre-CRC) WAL file — see
// WalFile::read_entries_for_migration.
let entries = wal::WalFile::read_entries_for_migration(&wal_path)?;
wal::replay_into_cache(&entries, &mut cache);
Some(wal::WalFile::open(&wal_path)?)
} else if config.wal_enabled {
crates/clawhdf5-agent/src/wal.rs
+324 -62
View File
@@ -13,16 +13,40 @@ use crate::MemoryError;
const WAL_MAGIC: [u8; 4] = [0x45, 0x48, 0x57, 0x4C]; // "EHWL"
/// Current WAL format version: every entry ends with a 4-byte CRC32 trailer
/// (see [`TeeReader`]) so a bit-flip is detected and replay stops there
/// instead of silently accepting corrupted data.
const WAL_VERSION: u8 = 2;
/// Current WAL format version: every entry's CRC32 trailer is computed over
/// its own bytes *chained with the previous entry's stored CRC*
/// (`crc32(entry_bytes ++ prev_crc.to_le_bytes())`, seeded with 0 for the
/// first entry after a truncation). A per-entry CRC alone only detects a
/// bit-flip within that entry; chaining additionally detects entries being
/// reordered, duplicated, or spliced (e.g. a Tombstone moved before/after
/// its target Save) — the moved/inserted entry's stored CRC was computed
/// against a different predecessor than the one now in front of it on disk,
/// so the chain breaks at that point and replay stops there.
const WAL_VERSION: u8 = 3;
/// The only other WAL version this crate still knows how to *read*: no
/// per-entry CRC trailer. Written by versions of this crate before the CRC32
/// hardening. `WalFile::open` migrates a legacy file to [`WAL_VERSION`] by
/// recreating it fresh — safe because every real call site reads existing
/// entries via [`WalFile::read_entries`] before calling `open` (see
/// The previous WAL format version: still a CRC32 per entry (so a bit-flip
/// within one entry is caught), but not chained to the previous entry's CRC
/// (so reordering/splicing whole entries is not detected). Written by
/// versions of this crate before the chaining hardening. Fully supported for
/// reading via [`WalFile::read_entries`] — not restricted like
/// [`WAL_VERSION_LEGACY_NO_CRC`], since it still verifies each entry
/// individually. `WalFile::open` migrates it to [`WAL_VERSION`] by
/// recreating the file fresh, the same as the legacy-no-CRC migration below.
const WAL_VERSION_CRC_UNCHAINED: u8 = 2;
/// The oldest WAL version this crate still knows how to *read*: no
/// per-entry CRC trailer at all, so a bit-flip anywhere is silently
/// accepted. Written by versions of this crate before the CRC32 hardening.
/// Because of that — unlike [`WAL_VERSION_CRC_UNCHAINED`] — this version is
/// deliberately *not* reachable through the public [`WalFile::read_entries`]
/// API; only [`WalFile::read_entries_for_migration`] (used exclusively by
/// `HDF5Memory::open`'s one-time migration path) will parse it. Flipping a
/// version byte from 2/3 down to 1 no longer silently downgrades a file to
/// the fully-unverified parser for an arbitrary caller.
///
/// `WalFile::open` migrates a legacy file to [`WAL_VERSION`] by recreating
/// it fresh — safe because every real call site reads existing entries via
/// [`WalFile::read_entries_for_migration`] before calling `open` (see
/// `HDF5Memory::open`), so no data is lost.
const WAL_VERSION_LEGACY_NO_CRC: u8 = 1;
@@ -77,15 +101,21 @@ pub struct WalFile {
entry_count: u32,
/// Entries written since the last header count update.
pending_header_sync: u32,
/// CRC32 chain state: the previous entry's stored CRC (0 if this file
/// has no entries yet), folded into the next entry's CRC computation.
/// Reset to 0 by `truncate()`/`create_fresh_wal_file`, and re-derived by
/// scanning existing entries when `open()` attaches to a non-empty file.
running_crc: u32,
}
impl WalFile {
/// Open or create a WAL file. If it exists, read the header and entry count.
///
/// A legacy (pre-CRC) WAL file is migrated to the current format by
/// recreating it fresh — see [`WAL_VERSION_LEGACY_NO_CRC`]. Callers that
/// need the legacy file's entries must call [`WalFile::read_entries`]
/// first, before calling `open`.
/// A pre-chaining WAL file ([`WAL_VERSION_CRC_UNCHAINED`] or
/// [`WAL_VERSION_LEGACY_NO_CRC`]) is migrated to the current format by
/// recreating it fresh. Callers that need an existing file's entries must
/// call [`WalFile::read_entries`] (or, for a legacy-no-CRC file,
/// [`WalFile::read_entries_for_migration`]) first, before calling `open`.
pub fn open(path: &Path) -> Result<Self, MemoryError> {
if path.exists() {
// Read existing header
@@ -105,17 +135,28 @@ impl WalFile {
WAL_VERSION => {
let mut count_buf = [0u8; 4];
f.read_exact(&mut count_buf)?;
let entry_count = u32::from_le_bytes(count_buf);
// Seek to end for appending
let header_count = u32::from_le_bytes(count_buf);
// Scan any existing entries to resume the CRC chain
// correctly for further appends (the header's count may
// be stale from deferred group-commit sync, same
// tolerance `read_entries` already has, so the scanned
// count is also the more accurate of the two).
let (entries, running_crc) = read_chained_entries(&mut f, 0);
let entry_count = if entries.is_empty() {
header_count
} else {
entries.len() as u32
};
f.seek(SeekFrom::End(0))?;
Ok(Self {
path: path.to_path_buf(),
file: Some(f),
entry_count,
pending_header_sync: 0,
running_crc,
})
}
WAL_VERSION_LEGACY_NO_CRC => {
WAL_VERSION_CRC_UNCHAINED | WAL_VERSION_LEGACY_NO_CRC => {
drop(f);
let f = create_fresh_wal_file(path)?;
Ok(Self {
@@ -123,6 +164,7 @@ impl WalFile {
file: Some(f),
entry_count: 0,
pending_header_sync: 0,
running_crc: 0,
})
}
v => Err(MemoryError::Schema(format!("unsupported WAL version {v}"))),
@@ -134,6 +176,7 @@ impl WalFile {
file: Some(f),
entry_count: 0,
pending_header_sync: 0,
running_crc: 0,
})
}
}
@@ -168,7 +211,10 @@ impl WalFile {
serialize_str(&mut buf, &entry.session_id);
serialize_str(&mut buf, &entry.tags);
let crc = crc32(&buf);
// Chain this entry's CRC to the previous one's so reordering/
// splicing entries (not just flipping a bit within one) is detected
// on replay — see WAL_VERSION's doc comment.
let crc = chained_crc(&buf, self.running_crc);
buf.extend_from_slice(&crc.to_le_bytes());
let f = self
@@ -177,6 +223,7 @@ impl WalFile {
.ok_or_else(|| MemoryError::Io(std::io::Error::other("WAL file not open")))?;
f.write_all(&buf)?;
self.running_crc = crc;
self.entry_count += 1;
self.pending_header_sync += 1;
if self.pending_header_sync >= GROUP_COMMIT_SIZE {
@@ -191,7 +238,7 @@ impl WalFile {
buf[0] = WalEntryType::Tombstone as u8;
buf[1..9].copy_from_slice(&timestamp.to_le_bytes());
buf[9..13].copy_from_slice(&(index as u32).to_le_bytes());
let crc = crc32(&buf[..13]);
let crc = chained_crc(&buf[..13], self.running_crc);
buf[13..17].copy_from_slice(&crc.to_le_bytes());
let f = self
@@ -200,6 +247,7 @@ impl WalFile {
.ok_or_else(|| MemoryError::Io(std::io::Error::other("WAL file not open")))?;
f.write_all(&buf)?;
self.running_crc = crc;
self.entry_count += 1;
self.pending_header_sync += 1;
if self.pending_header_sync >= GROUP_COMMIT_SIZE {
@@ -214,9 +262,36 @@ impl WalFile {
/// (and may be stale if written with deferred group-commit updates). This
/// tolerates both truncated files (crash mid-write) and stale header counts
/// (crash before the next group-commit header sync). On a `WAL_VERSION`
/// file, a CRC32 mismatch on an entry is treated the same way — replay
/// stops there rather than accepting corrupted data.
/// file, a broken CRC chain (bit-flip, or an entry reordered/duplicated/
/// spliced in) is treated the same way — replay stops there rather than
/// accepting corrupted or tampered data. `WAL_VERSION_CRC_UNCHAINED`
/// files are read the same way minus the chain check (each entry's own
/// CRC is still verified).
///
/// Does **not** read [`WAL_VERSION_LEGACY_NO_CRC`] files — that format has
/// no integrity verification at all, so it's only reachable through
/// [`WalFile::read_entries_for_migration`], used exclusively by
/// `HDF5Memory::open`'s one-time migration path. Calling this on a
/// legacy-no-CRC file returns a typed error instead of silently
/// downgrading to the unverified parser.
pub fn read_entries(path: &Path) -> Result<Vec<WalEntry>, MemoryError> {
Self::read_entries_impl(path, false)
}
/// Like [`WalFile::read_entries`], but also accepts
/// [`WAL_VERSION_LEGACY_NO_CRC`] files (no per-entry integrity check at
/// all). Restricted to `pub(crate)` and named accordingly: the only
/// legitimate caller is `HDF5Memory::open`'s one-time migration of a
/// pre-CRC WAL file, which immediately recreates it in the current
/// format afterward. Do not use this for anything else.
pub(crate) fn read_entries_for_migration(path: &Path) -> Result<Vec<WalEntry>, MemoryError> {
Self::read_entries_impl(path, true)
}
fn read_entries_impl(
path: &Path,
allow_legacy_no_crc: bool,
) -> Result<Vec<WalEntry>, MemoryError> {
if !path.exists() {
return Ok(Vec::new());
}
@@ -229,46 +304,61 @@ impl WalFile {
}
// entry_count is a pre-allocation hint only — we read until EOF.
let entry_count_hint = u32::from_le_bytes([header[5], header[6], header[7], header[8]]);
let mut entries = Vec::with_capacity(entry_count_hint as usize);
match header[4] {
WAL_VERSION => loop {
let raw_and_result = {
let mut tee = TeeReader::new(&mut f);
let result = read_one_entry(&mut tee);
(tee.into_buf(), result)
};
let (raw, result) = raw_and_result;
let entry_opt = match result {
Err(()) => break,
Ok(v) => v,
};
let mut crc_buf = [0u8; 4];
if f.read_exact(&mut crc_buf).is_err() {
break;
}
let stored_crc = u32::from_le_bytes(crc_buf);
if crc32(&raw) != stored_crc {
// Corruption detected — stop replay here, same as a clean
// truncation/EOF, rather than accepting the bad entry.
break;
}
if let Some(entry) = entry_opt {
entries.push(entry);
}
},
WAL_VERSION_LEGACY_NO_CRC => loop {
match read_one_entry(&mut f) {
Err(()) => break,
Ok(Some(entry)) => entries.push(entry),
Ok(None) => {}
}
},
v => {
return Err(MemoryError::Schema(format!("unsupported WAL version {v}")));
WAL_VERSION => {
let (entries, _final_crc) = read_chained_entries(&mut f, 0);
Ok(entries)
}
WAL_VERSION_CRC_UNCHAINED => {
let mut entries = Vec::with_capacity(entry_count_hint as usize);
loop {
let raw_and_result = {
let mut tee = TeeReader::new(&mut f);
let result = read_one_entry(&mut tee);
(tee.into_buf(), result)
};
let (raw, result) = raw_and_result;
let entry_opt = match result {
Err(()) => break,
Ok(v) => v,
};
let mut crc_buf = [0u8; 4];
if f.read_exact(&mut crc_buf).is_err() {
break;
}
let stored_crc = u32::from_le_bytes(crc_buf);
if crc32(&raw) != stored_crc {
// Corruption detected — stop replay here, same as a
// clean truncation/EOF, rather than accepting the bad
// entry.
break;
}
if let Some(entry) = entry_opt {
entries.push(entry);
}
}
Ok(entries)
}
WAL_VERSION_LEGACY_NO_CRC if allow_legacy_no_crc => {
let mut entries = Vec::with_capacity(entry_count_hint as usize);
loop {
match read_one_entry(&mut f) {
Err(()) => break,
Ok(Some(entry)) => entries.push(entry),
Ok(None) => {}
}
}
Ok(entries)
}
WAL_VERSION_LEGACY_NO_CRC => Err(MemoryError::Schema(
"WAL file is in the legacy no-CRC format (version 1), which read_entries() no \
longer accepts — it has no per-entry integrity verification. Only the one-time \
migration path (WalFile::open) can read and upgrade it."
.into(),
)),
v => Err(MemoryError::Schema(format!("unsupported WAL version {v}"))),
}
Ok(entries)
}
/// Truncate the WAL (after merge into .h5).
@@ -279,6 +369,7 @@ impl WalFile {
self.file = Some(f);
self.entry_count = 0;
self.pending_header_sync = 0;
self.running_crc = 0;
Ok(())
}
@@ -373,6 +464,55 @@ fn read_embedding<R: Read>(f: &mut R) -> Result<Vec<f32>, MemoryError> {
Ok(vals)
}
/// Compute the CRC32 trailer for a `WAL_VERSION` entry, chaining in the
/// previous entry's stored CRC (0 for the first entry after a truncation).
fn chained_crc(entry_bytes: &[u8], prev_crc: u32) -> u32 {
let mut chained = Vec::with_capacity(entry_bytes.len() + 4);
chained.extend_from_slice(entry_bytes);
chained.extend_from_slice(&prev_crc.to_le_bytes());
crc32(&chained)
}
/// Read and verify all entries from a `WAL_VERSION` (chained-CRC) stream
/// starting at the reader's current position, given the chain state to
/// resume from (0 for a stream starting at the beginning of a fresh WAL).
///
/// Returns the parsed entries and the final running CRC — the chain state to
/// continue from for further appends. Stops (without erroring) at the first
/// entry that fails to parse or whose stored CRC doesn't match the expected
/// chain value — a bit-flip, truncation/EOF, or an entry having been
/// reordered/duplicated/spliced all produce a chain mismatch at that point,
/// and are all handled the same way: replay stops there.
fn read_chained_entries<R: Read>(f: &mut R, start_crc: u32) -> (Vec<WalEntry>, u32) {
let mut entries = Vec::new();
let mut running_crc = start_crc;
loop {
let raw_and_result = {
let mut tee = TeeReader::new(f);
let result = read_one_entry(&mut tee);
(tee.into_buf(), result)
};
let (raw, result) = raw_and_result;
let entry_opt = match result {
Err(()) => break,
Ok(v) => v,
};
let mut crc_buf = [0u8; 4];
if f.read_exact(&mut crc_buf).is_err() {
break;
}
let stored_crc = u32::from_le_bytes(crc_buf);
if chained_crc(&raw, running_crc) != stored_crc {
break;
}
running_crc = stored_crc;
if let Some(entry) = entry_opt {
entries.push(entry);
}
}
(entries, running_crc)
}
/// Create a fresh WAL file at `path` with the current-version header,
/// truncating/overwriting anything already there.
fn create_fresh_wal_file(path: &Path) -> Result<File, MemoryError> {
@@ -912,16 +1052,113 @@ mod tests {
assert_eq!(entries[0].chunk, "first");
}
/// Reordering two entries on disk must break the CRC chain — the
/// second entry's stored CRC was computed against the first entry's
/// real CRC, not against the chain state a reader sees after swapping
/// them, so replay stops immediately instead of accepting the tampered
/// order (INT-09).
#[test]
fn test_wal_reads_legacy_v1_format_without_crc() {
fn test_wal_detects_reordered_entries() {
let dir = TempDir::new().unwrap();
let wal_path = dir.path().join("legacy.h5.wal");
let wal_path = dir.path().join("test.h5.wal");
let mut wal = WalFile::open(&wal_path).unwrap();
wal.append_save(&make_wal_entry("first", &[1.0, 2.0]))
.unwrap();
let len_after_first = std::fs::metadata(&wal_path).unwrap().len() as usize;
wal.append_save(&make_wal_entry("second", &[3.0, 4.0]))
.unwrap();
let len_after_second = std::fs::metadata(&wal_path).unwrap().len() as usize;
drop(wal);
let bytes = std::fs::read(&wal_path).unwrap();
let header_len = 9usize;
let entry1_bytes = bytes[header_len..len_after_first].to_vec();
let entry2_bytes = bytes[len_after_first..len_after_second].to_vec();
let mut spliced = bytes[..header_len].to_vec();
spliced.extend_from_slice(&entry2_bytes);
spliced.extend_from_slice(&entry1_bytes);
std::fs::write(&wal_path, &spliced).unwrap();
let entries = WalFile::read_entries(&wal_path).unwrap();
assert!(
entries.is_empty(),
"reordered entries must break the CRC chain and stop replay, got {} entries",
entries.len()
);
}
/// Splicing a third-party entry in between two legitimate entries (e.g.
/// moving a Tombstone in front of the Save it's meant to follow) must
/// also break the chain for everything after the splice point.
#[test]
fn test_wal_detects_spliced_entry() {
let dir = TempDir::new().unwrap();
let wal_path = dir.path().join("test.h5.wal");
let mut wal = WalFile::open(&wal_path).unwrap();
wal.append_save(&make_wal_entry("first", &[1.0])).unwrap();
let len_after_first = std::fs::metadata(&wal_path).unwrap().len() as usize;
wal.append_save(&make_wal_entry("second", &[2.0])).unwrap();
let len_after_second = std::fs::metadata(&wal_path).unwrap().len() as usize;
wal.append_save(&make_wal_entry("third", &[3.0])).unwrap();
drop(wal);
let bytes = std::fs::read(&wal_path).unwrap();
let entry2_bytes = bytes[len_after_first..len_after_second].to_vec();
// Duplicate "second" right after itself: [first][second][second][third]
let mut spliced = bytes[..len_after_second].to_vec();
spliced.extend_from_slice(&entry2_bytes);
spliced.extend_from_slice(&bytes[len_after_second..]);
std::fs::write(&wal_path, &spliced).unwrap();
let entries = WalFile::read_entries(&wal_path).unwrap();
assert_eq!(
entries.len(),
2,
"replay must stop at the spliced duplicate, keeping only the entries before it"
);
assert_eq!(entries[0].chunk, "first");
assert_eq!(entries[1].chunk, "second");
}
/// A WAL closed (without truncating) and reopened must continue the CRC
/// chain correctly for newly appended entries — this is the normal
/// crash-restart-without-flush scenario (`HDF5Memory::open` replays
/// existing entries, then reopens the same file for further appends
/// without clearing it), and must not produce a false "reordering"
/// detection for its own legitimately-appended entries.
#[test]
fn test_wal_chain_continues_across_reopen() {
let dir = TempDir::new().unwrap();
let wal_path = dir.path().join("test.h5.wal");
let mut wal = WalFile::open(&wal_path).unwrap();
wal.append_save(&make_wal_entry("first", &[1.0])).unwrap();
drop(wal); // simulate a restart without ever truncating the WAL
let mut wal2 = WalFile::open(&wal_path).unwrap();
wal2.append_save(&make_wal_entry("second", &[2.0]))
.unwrap();
drop(wal2);
let entries = WalFile::read_entries(&wal_path).unwrap();
assert_eq!(
entries.len(),
2,
"both pre- and post-reopen entries must replay cleanly"
);
assert_eq!(entries[0].chunk, "first");
assert_eq!(entries[1].chunk, "second");
}
/// Build a legacy (WAL_VERSION_LEGACY_NO_CRC) WAL file containing one
/// Save entry, with no trailing CRC32.
fn build_legacy_v1_wal_bytes() -> Vec<u8> {
let mut buf = Vec::new();
buf.extend_from_slice(&WAL_MAGIC);
buf.push(WAL_VERSION_LEGACY_NO_CRC);
buf.extend_from_slice(&1u32.to_le_bytes());
// One Save entry in the old format: type + timestamp + fields, with
// no trailing CRC32.
buf.push(WalEntryType::Save as u8);
buf.extend_from_slice(&42.0f64.to_le_bytes());
serialize_str(&mut buf, "legacy-chunk");
@@ -933,14 +1170,39 @@ mod tests {
serialize_str(&mut buf, "chan");
serialize_str(&mut buf, "sess");
serialize_str(&mut buf, "tags");
std::fs::write(&wal_path, &buf).unwrap();
buf
}
let entries = WalFile::read_entries(&wal_path).unwrap();
#[test]
fn test_wal_reads_legacy_v1_format_without_crc() {
let dir = TempDir::new().unwrap();
let wal_path = dir.path().join("legacy.h5.wal");
std::fs::write(&wal_path, build_legacy_v1_wal_bytes()).unwrap();
// Only the migration-only reader may read a legacy no-CRC file.
let entries = WalFile::read_entries_for_migration(&wal_path).unwrap();
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].chunk, "legacy-chunk");
assert_eq!(entries[0].embedding, vec![1.0, 2.0]);
}
/// The public `read_entries` must reject a legacy no-CRC file instead of
/// silently downgrading to the fully-unverified parser (INT-09) — flipping
/// a version byte from 2/3 down to 1 must not be a way to bypass every
/// integrity check for an arbitrary caller of the public API.
#[test]
fn test_wal_read_entries_rejects_legacy_v1_format() {
let dir = TempDir::new().unwrap();
let wal_path = dir.path().join("legacy.h5.wal");
std::fs::write(&wal_path, build_legacy_v1_wal_bytes()).unwrap();
let result = WalFile::read_entries(&wal_path);
assert!(
result.is_err(),
"read_entries() must reject a legacy no-CRC WAL file, not silently parse it"
);
}
#[test]
fn test_wal_open_migrates_legacy_v1_to_current_version() {
let dir = TempDir::new().unwrap();